Rollup merge of #40612 - TimNN:new-netbsd-cross, r=alexcrichton

Use the "official" cross compiler for NetBSD

The current NetBSD cross compiler is lacking, for example `std::thread` is not available (which causes problems for LLVM 4.0). This PR uses the official netbsd build system to compiler the cross compiler.

@alexcrichton: Can you please mirror `ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/source/sets/{src,gnusrc,sharesrc,syssrc}.tgz`. (Optionally you may want to use NetBSD versions 7.0.2 or 7.1, in that case you'll probably want to update the binary downloads used today as well).

I'll update the URL's afterwards (or feel free to use "allow edits from maintainers").

r? @alexcrichton

2 files changed, 70 insertions, 107 deletions

diff --git a/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile b/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile
index fd67f185162..c68f30a6e56 100644
--- a/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile
+++ b/src/ci/docker/dist-s390x-linux-netbsd/Dockerfile
@@ -64,17 +64,17 @@ COPY patches/ /tmp/patches/
 COPY s390x-linux-gnu.config build-s390x-toolchain.sh /tmp/
 RUN ./build-s390x-toolchain.sh
 
-USER root
-
 COPY build-netbsd-toolchain.sh /tmp/
 RUN ./build-netbsd-toolchain.sh
 
-ENV PATH=$PATH:/x-tools/s390x-ibm-linux-gnu/bin
+USER root
+
+ENV PATH=$PATH:/x-tools/s390x-ibm-linux-gnu/bin:/x-tools/x86_64-unknown-netbsd/bin
 
 ENV \
-    AR_x86_64_unknown_netbsd=x86_64-unknown-netbsd-ar \
-    CC_x86_64_unknown_netbsd=x86_64-unknown-netbsd-gcc \
-    CXX_x86_64_unknown_netbsd=x86_64-unknown-netbsd-g++ \
+    AR_x86_64_unknown_netbsd=x86_64--netbsd-ar \
+    CC_x86_64_unknown_netbsd=x86_64--netbsd-gcc-sysroot \
+    CXX_x86_64_unknown_netbsd=x86_64--netbsd-g++-sysroot \
     CC_s390x_unknown_linux_gnu=s390x-ibm-linux-gnu-gcc \
     AR_s390x_unknown_linux_gnu=s390x-ibm-linux-gnu-ar \
     CXX_s390x_unknown_linux_gnu=s390x-ibm-linux-gnu-g++
diff --git a/src/ci/docker/dist-s390x-linux-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-s390x-linux-netbsd/build-netbsd-toolchain.sh
index 654b458ea40..ea335a24973 100755
--- a/src/ci/docker/dist-s390x-linux-netbsd/build-netbsd-toolchain.sh
+++ b/src/ci/docker/dist-s390x-linux-netbsd/build-netbsd-toolchain.sh
@@ -13,108 +13,71 @@
 
 set -ex
 
-BINUTILS=2.25.1
-GCC=5.3.0
-
-# First up, build binutils
-mkdir binutils
-cd binutils
-curl https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS.tar.bz2 | tar xjf -
-mkdir binutils-build
-cd binutils-build
-../binutils-$BINUTILS/configure \
-  --target=x86_64-unknown-netbsd
-make -j10
-make install
-cd ../..
-rm -rf binutils
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  rm /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  set -x
+}
 
-# Next, download the NetBSD libc and relevant header files
 mkdir netbsd
-# originally from:
-# https://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/amd64/binary/sets/base.tgz
-curl https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-01-16-netbsd-base.tgz | \
-  tar xzf - -C netbsd ./usr/include ./usr/lib ./lib
-# originally from:
-# https://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/amd64/binary/sets/comp.tgz
-curl https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-01-16-netbsd-comp.tgz | \
-  tar xzf - -C netbsd ./usr/include ./usr/lib
-
-dst=/usr/local/x86_64-unknown-netbsd
-cp -r netbsd/usr/include $dst
-cp netbsd/usr/lib/crt0.o $dst/lib
-cp netbsd/usr/lib/crti.o $dst/lib
-cp netbsd/usr/lib/crtn.o $dst/lib
-cp netbsd/usr/lib/crtbeginS.o $dst/lib
-cp netbsd/usr/lib/crtendS.o $dst/lib
-cp netbsd/usr/lib/crtbegin.o $dst/lib
-cp netbsd/usr/lib/crtend.o $dst/lib
-cp netbsd/usr/lib/gcrt0.o $dst/lib
-cp netbsd/usr/lib/libc.a $dst/lib
-cp netbsd/usr/lib/libc_p.a $dst/lib
-cp netbsd/usr/lib/libc_pic.a $dst/lib
-cp netbsd/lib/libc.so.12.193.1 $dst/lib
-cp netbsd/lib/libutil.so.7.21 $dst/lib
-cp netbsd/usr/lib/libm.a $dst/lib
-cp netbsd/usr/lib/libm_p.a $dst/lib
-cp netbsd/usr/lib/libm_pic.a $dst/lib
-cp netbsd/lib/libm.so.0.11 $dst/lib
-cp netbsd/usr/lib/librt.so.1.1 $dst/lib
-cp netbsd/usr/lib/libpthread.a $dst/lib
-cp netbsd/usr/lib/libpthread_p.a $dst/lib
-cp netbsd/usr/lib/libpthread_pic.a $dst/lib
-cp netbsd/usr/lib/libpthread.so.1.2 $dst/lib
-
-ln -s libc.so.12.193.1 $dst/lib/libc.so
-ln -s libc.so.12.193.1 $dst/lib/libc.so.12
-ln -s libm.so.0.11 $dst/lib/libm.so
-ln -s libm.so.0.11 $dst/lib/libm.so.0
-ln -s libutil.so.7.21 $dst/lib/libutil.so
-ln -s libutil.so.7.21 $dst/lib/libutil.so.7
-ln -s libpthread.so.1.2 $dst/lib/libpthread.so
-ln -s libpthread.so.1.2 $dst/lib/libpthread.so.1
-ln -s librt.so.1.1 $dst/lib/librt.so
-
-rm -rf netbsd
-
-# Finally, download and build gcc to target NetBSD
-mkdir gcc
-cd gcc
-curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.bz2 | tar xjf -
-cd gcc-$GCC
-./contrib/download_prerequisites
-
-# Originally from
-# ftp://ftp.netbsd.org/pub/pkgsrc/pkgsrc-2016Q4/pkgsrc/lang/gcc5/patches/patch-libstdc%2B%2B-v3_config_os_bsd_netbsd_ctype__base.h
-PATCHES="https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-01-13-netbsd-patch1.patch"
-
-# Originally from
-# ftp://ftp.netbsd.org/pub/pkgsrc/pkgsrc-2016Q4/pkgsrc/lang/gcc5/patches/patch-libstdc%2B%2B-v3_config_os_bsd_netbsd_ctype__configure__char.cc
-PATCHES="$PATCHES https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-01-13-netbsd-patch2.patch"
-
-for patch in $PATCHES; do
-  curl $patch | patch -Np0
-done
-
-mkdir ../gcc-build
-cd ../gcc-build
-../gcc-$GCC/configure                            \
-  --enable-languages=c,c++                       \
-  --target=x86_64-unknown-netbsd                 \
-  --disable-libcilkrts                           \
-  --disable-multilib                             \
-  --disable-nls                                  \
-  --disable-libgomp                              \
-  --disable-libquadmath                          \
-  --disable-libssp                               \
-  --disable-libvtv                               \
-  --disable-libcilkrt                            \
-  --disable-libada                               \
-  --disable-libsanitizer                         \
-  --disable-libquadmath-support                  \
-  --disable-lto
-make -j10
-make install
+cd netbsd
+
+mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot
+
+URL=https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror
+
+# Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz
+curl $URL/2017-03-17-netbsd-src.tgz | tar xzf -
+curl $URL/2017-03-17-netbsd-gnusrc.tgz | tar xzf -
+curl $URL/2017-03-17-netbsd-sharesrc.tgz | tar xzf -
+curl $URL/2017-03-17-netbsd-syssrc.tgz | tar xzf -
+
+# Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/amd64/binary/sets/*.tgz
+curl $URL/2017-03-17-netbsd-base.tgz | \
+  tar xzf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib ./lib
+curl $URL/2017-03-17-netbsd-comp.tgz | \
+  tar xzf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib
+
+cd usr/src
+
+# The options, in order, do the following
+# * this is an unpriviledged build
+# * output to a predictable location
+# * disable various uneeded stuff
+MKUNPRIVED=yes TOOLDIR=/x-tools/x86_64-unknown-netbsd \
+MKSHARE=no MKDOC=no MKHTML=no MKINFO=no MKKMOD=no MKLINT=no MKMAN=no MKNLS=no MKPROFILE=no \
+hide_output ./build.sh -j10 -m amd64 tools
 
 cd ../..
-rm -rf gcc
+
+rm -rf usr
+
+cat > /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-gcc-sysroot <<'EOF'
+#!/bin/bash
+exec /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-gcc --sysroot=/x-tools/x86_64-unknown-netbsd/sysroot "$@"
+EOF
+
+cat > /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-g++-sysroot <<'EOF'
+#!/bin/bash
+exec /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-g++ --sysroot=/x-tools/x86_64-unknown-netbsd/sysroot "$@"
+EOF
+
+GCC_SHA1=`sha1sum -b /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-gcc | cut -d' ' -f1`
+GPP_SHA1=`sha1sum -b /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-g++ | cut -d' ' -f1`
+
+echo "# $GCC_SHA1" >> /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-gcc-sysroot
+echo "# $GPP_SHA1" >> /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-g++-sysroot
+
+chmod +x /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-gcc-sysroot
+chmod +x /x-tools/x86_64-unknown-netbsd/bin/x86_64--netbsd-g++-sysroot