Auto merge of #47157 - malbarbo:shared-build-musl, r=alexcrichton

ci: use a shared script to build musl

The dist-x86_64-musl, dist-various-1 and dist-i586-gnu-i686-musl builders had different scripts to build musl. This PR creates an unified script, which makes it easier to add new musl targets and update musl and libunwind (used in the musl targets).

The libunwind is update from 3.7 to 3.9 for dist-x86_64-musl and dist-i586-gnu-i686-musl (dist-various-1 already used 3.9 version).

8 files changed, 144 insertions, 272 deletions

diff --git a/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile
index 2fb12196811..c59476fab00 100644
--- a/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile
+++ b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile
@@ -17,8 +17,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   pkg-config
 
 WORKDIR /build/
-COPY dist-i586-gnu-i686-musl/musl-libunwind-patch.patch dist-i586-gnu-i686-musl/build-musl.sh /build/
-RUN sh /build/build-musl.sh && rm -rf /build
+COPY scripts/musl.sh /build/
+RUN CC=gcc CFLAGS="-m32 -fPIC -Wa,-mrelax-relocations=no" \
+    CXX=g++ CXXFLAGS="-m32 -Wa,-mrelax-relocations=no" \
+    bash musl.sh i686 --target=i686 && \
+    rm -rf /build
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
diff --git a/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh b/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh
deleted file mode 100644
index 883859d1fa6..00000000000
--- a/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/sh
-# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution and at
-# http://rust-lang.org/COPYRIGHT.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-
-set -ex
-
-# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well
-export CFLAGS="-fPIC -Wa,-mrelax-relocations=no"
-export CXXFLAGS="-Wa,-mrelax-relocations=no"
-
-MUSL=musl-1.1.17
-curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf -
-cd $MUSL
-CC=gcc \
-  CFLAGS="$CFLAGS -m32" \
-  ./configure --prefix=/musl-i686 --disable-shared \
-    --target=i686
-make AR=ar RANLIB=ranlib -j10
-make install
-cd ..
-
-# To build MUSL we're going to need a libunwind lying around, so acquire that
-# here and build it.
-curl -L https://github.com/llvm-mirror/llvm/archive/release_37.tar.gz | tar xzf -
-curl -L https://github.com/llvm-mirror/libunwind/archive/release_37.tar.gz | tar xzf -
-
-# Whoa what's this mysterious patch we're applying to libunwind! Why are we
-# swapping the values of ESP/EBP in libunwind?!
-#
-# Discovered in #35599 it turns out that the vanilla build of libunwind is not
-# suitable for unwinding 32-bit musl. After some investigation it ended up
-# looking like the register values for ESP/EBP were indeed incorrect (swapped)
-# in the source. Similar commits in libunwind (r280099 and r282589) have noticed
-# this for other platforms, and we just need to realize it for musl linux as
-# well.
-#
-# More technical info can be found at #35599
-cd libunwind-release_37
-patch -Np1 < /build/musl-libunwind-patch.patch
-cd ..
-
-mkdir libunwind-build
-cd libunwind-build
-CFLAGS="$CFLAGS -m32" CXXFLAGS="$CXXFLAGS -m32" cmake ../libunwind-release_37 \
-          -DLLVM_PATH=/build/llvm-release_37 \
-          -DLIBUNWIND_ENABLE_SHARED=0
-make -j10
-cp lib/libunwind.a /musl-i686/lib
diff --git a/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch b/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch
deleted file mode 100644
index 99cd685b72d..00000000000
--- a/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch
+++ /dev/null
@@ -1,15 +0,0 @@
-diff --git a/include/libunwind.h b/include/libunwind.h
-index c5b9633..1360eb2 100644
---- a/include/libunwind.h
-+++ b/include/libunwind.h
-@@ -151,8 +151,8 @@ enum {
-   UNW_X86_ECX = 1,
-   UNW_X86_EDX = 2,
-   UNW_X86_EBX = 3,
--  UNW_X86_EBP = 4,
--  UNW_X86_ESP = 5,
-+  UNW_X86_ESP = 4,
-+  UNW_X86_EBP = 5,
-   UNW_X86_ESI = 6,
-   UNW_X86_EDI = 7
- };
diff --git a/src/ci/docker/dist-various-1/Dockerfile b/src/ci/docker/dist-various-1/Dockerfile
index b58ee7a719c..0f08bcddd38 100644
--- a/src/ci/docker/dist-various-1/Dockerfile
+++ b/src/ci/docker/dist-various-1/Dockerfile
@@ -22,23 +22,39 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libssl-dev \
   pkg-config
 
-WORKDIR /tmp
+WORKDIR /build
 
-COPY dist-various-1/build-rumprun.sh /tmp/
+COPY dist-various-1/build-rumprun.sh /build
 RUN ./build-rumprun.sh
 
-COPY dist-various-1/build-arm-musl.sh /tmp/
-RUN ./build-arm-musl.sh
+COPY dist-various-1/install-x86_64-redox.sh /build
+RUN ./install-x86_64-redox.sh
+
+COPY scripts/musl.sh /build
+RUN env \
+    CC=arm-linux-gnueabi-gcc CFLAGS="-march=armv6 -marm" \
+    CXX=arm-linux-gnueabi-g++ CXXFLAGS="-march=armv6 -marm" \
+    bash musl.sh arm && \
+    env \
+    CC=arm-linux-gnueabihf-gcc CFLAGS="-march=armv6 -marm" \
+    CXX=arm-linux-gnueabihf-g++ CXXFLAGS="-march=armv6 -marm" \
+    bash musl.sh armhf && \
+    env \
+    CC=arm-linux-gnueabihf-gcc CFLAGS="-march=armv7-a" \
+    CXX=arm-linux-gnueabihf-g++ CXXFLAGS="-march=armv7-a" \
+    bash musl.sh armv7 && \
+    env \
+    CC=aarch64-linux-gnu-gcc \
+    CXX=aarch64-linux-gnu-g++ \
+    bash musl.sh aarch64 && \
+    rm -rf /build/*
 
-COPY dist-various-1/install-mips-musl.sh /tmp/
+COPY dist-various-1/install-mips-musl.sh /build
 RUN ./install-mips-musl.sh
 
-COPY dist-various-1/install-mipsel-musl.sh /tmp/
+COPY dist-various-1/install-mipsel-musl.sh /build
 RUN ./install-mipsel-musl.sh
 
-COPY dist-various-1/install-x86_64-redox.sh /tmp/
-RUN ./install-x86_64-redox.sh
-
 ENV TARGETS=asmjs-unknown-emscripten
 ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
 ENV TARGETS=$TARGETS,x86_64-rumprun-netbsd
@@ -67,10 +83,10 @@ ENV STAGING_DIR=/tmp
 ENV RUST_CONFIGURE_ARGS \
       --enable-extended \
       --target=$TARGETS \
-      --musl-root-arm=/usr/local/arm-linux-musleabi \
-      --musl-root-armhf=/usr/local/arm-linux-musleabihf \
-      --musl-root-armv7=/usr/local/armv7-linux-musleabihf \
-      --musl-root-aarch64=/usr/local/aarch64-linux-musl
+      --musl-root-arm=/musl-arm \
+      --musl-root-armhf=/musl-armhf \
+      --musl-root-armv7=/musl-armv7 \
+      --musl-root-aarch64=/musl-aarch64
 ENV SCRIPT python2.7 ../x.py dist --target $TARGETS
 
 # sccache
diff --git a/src/ci/docker/dist-various-1/build-arm-musl.sh b/src/ci/docker/dist-various-1/build-arm-musl.sh
deleted file mode 100755
index f9444a35a8b..00000000000
--- a/src/ci/docker/dist-various-1/build-arm-musl.sh
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution and at
-# http://rust-lang.org/COPYRIGHT.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-
-set -ex
-
-MUSL=1.1.17
-
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  $@ &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  rm /tmp/build.log
-  set -x
-}
-
-curl -O https://www.musl-libc.org/releases/musl-$MUSL.tar.gz
-tar xf musl-$MUSL.tar.gz
-cd musl-$MUSL
-CC=arm-linux-gnueabi-gcc \
-CFLAGS="-march=armv6 -marm" \
-    hide_output ./configure \
-        --prefix=/usr/local/arm-linux-musleabi \
-        --enable-wrapper=gcc
-hide_output make -j$(nproc)
-hide_output make install
-cd ..
-rm -rf musl-$MUSL
-
-tar xf musl-$MUSL.tar.gz
-cd musl-$MUSL
-CC=arm-linux-gnueabihf-gcc \
-CFLAGS="-march=armv6 -marm" \
-    hide_output ./configure \
-        --prefix=/usr/local/arm-linux-musleabihf \
-        --enable-wrapper=gcc
-hide_output make -j$(nproc)
-hide_output make install
-cd ..
-rm -rf musl-$MUSL
-
-tar xf musl-$MUSL.tar.gz
-cd musl-$MUSL
-CC=arm-linux-gnueabihf-gcc \
-CFLAGS="-march=armv7-a" \
-    hide_output ./configure \
-        --prefix=/usr/local/armv7-linux-musleabihf \
-        --enable-wrapper=gcc
-hide_output make -j$(nproc)
-hide_output make install
-cd ..
-rm -rf musl-$MUSL
-
-tar xf musl-$MUSL.tar.gz
-cd musl-$MUSL
-CC=aarch64-linux-gnu-gcc \
-CFLAGS="" \
-    hide_output ./configure \
-        --prefix=/usr/local/aarch64-linux-musl \
-        --enable-wrapper=gcc
-hide_output make -j$(nproc)
-hide_output make install
-cd ..
-rm -rf musl-$MUSL*
-
-ln -nsf ../arm-linux-musleabi/bin/musl-gcc /usr/local/bin/arm-linux-musleabi-gcc
-ln -nsf ../arm-linux-musleabihf/bin/musl-gcc /usr/local/bin/arm-linux-musleabihf-gcc
-ln -nsf ../armv7-linux-musleabihf/bin/musl-gcc /usr/local/bin/armv7-linux-musleabihf-gcc
-ln -nsf ../aarch64-linux-musl/bin/musl-gcc /usr/local/bin/aarch64-unknown-linux-musl-gcc
-
-curl -L https://github.com/llvm-mirror/llvm/archive/release_39.tar.gz | tar xzf -
-curl -L https://github.com/llvm-mirror/libunwind/archive/release_39.tar.gz | tar xzf -
-
-mkdir libunwind-build
-cd libunwind-build
-cmake ../libunwind-release_39 \
-          -DLLVM_PATH=/tmp/llvm-release_39 \
-          -DLIBUNWIND_ENABLE_SHARED=0 \
-          -DCMAKE_C_COMPILER=arm-linux-gnueabi-gcc \
-          -DCMAKE_CXX_COMPILER=arm-linux-gnueabi-g++ \
-          -DCMAKE_C_FLAGS="-march=armv6 -marm" \
-          -DCMAKE_CXX_FLAGS="-march=armv6 -marm"
-make -j$(nproc)
-cp lib/libunwind.a /usr/local/arm-linux-musleabi/lib
-cd ..
-rm -rf libunwind-build
-
-mkdir libunwind-build
-cd libunwind-build
-cmake ../libunwind-release_39 \
-          -DLLVM_PATH=/tmp/llvm-release_39 \
-          -DLIBUNWIND_ENABLE_SHARED=0 \
-          -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc \
-          -DCMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++ \
-          -DCMAKE_C_FLAGS="-march=armv6 -marm" \
-          -DCMAKE_CXX_FLAGS="-march=armv6 -marm"
-make -j$(nproc)
-cp lib/libunwind.a /usr/local/arm-linux-musleabihf/lib
-cd ..
-rm -rf libunwind-build
-
-mkdir libunwind-build
-cd libunwind-build
-cmake ../libunwind-release_39 \
-          -DLLVM_PATH=/tmp/llvm-release_39 \
-          -DLIBUNWIND_ENABLE_SHARED=0 \
-          -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc \
-          -DCMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++ \
-          -DCMAKE_C_FLAGS="-march=armv7-a" \
-          -DCMAKE_CXX_FLAGS="-march=armv7-a"
-make -j$(nproc)
-cp lib/libunwind.a /usr/local/armv7-linux-musleabihf/lib
-cd ..
-rm -rf libunwind-build
-
-mkdir libunwind-build
-cd libunwind-build
-cmake ../libunwind-release_39 \
-          -DLLVM_PATH=/tmp/llvm-release_39 \
-          -DLIBUNWIND_ENABLE_SHARED=0 \
-          -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
-          -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
-          -DCMAKE_C_FLAGS="" \
-          -DCMAKE_CXX_FLAGS=""
-make -j$(nproc)
-cp lib/libunwind.a /usr/local/aarch64-linux-musl/lib
-cd ..
-rm -rf libunwind-build
-
-rm -rf libunwind-release_39
-rm -rf llvm-release_39
diff --git a/src/ci/docker/dist-x86_64-musl/Dockerfile b/src/ci/docker/dist-x86_64-musl/Dockerfile
index 91ed6bfe1f6..77a55b33e41 100644
--- a/src/ci/docker/dist-x86_64-musl/Dockerfile
+++ b/src/ci/docker/dist-x86_64-musl/Dockerfile
@@ -17,8 +17,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   pkg-config
 
 WORKDIR /build/
-COPY dist-x86_64-musl/build-musl.sh /build/
-RUN sh /build/build-musl.sh && rm -rf /build
+
+COPY scripts/musl.sh /build/
+# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well
+RUN CC=gcc \
+    CFLAGS="-fPIC -Wa,-mrelax-relocations=no" \
+    CXX=g++ \
+    CXXFLAGS="-Wa,-mrelax-relocations=no" \
+    bash musl.sh x86_64 && rm -rf /build
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
diff --git a/src/ci/docker/dist-x86_64-musl/build-musl.sh b/src/ci/docker/dist-x86_64-musl/build-musl.sh
deleted file mode 100644
index 9be8d001149..00000000000
--- a/src/ci/docker/dist-x86_64-musl/build-musl.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/sh
-# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution and at
-# http://rust-lang.org/COPYRIGHT.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-
-set -ex
-
-# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well
-export CFLAGS="-fPIC -Wa,-mrelax-relocations=no"
-export CXXFLAGS="-Wa,-mrelax-relocations=no"
-
-MUSL=musl-1.1.17
-curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf -
-cd $MUSL
-./configure --prefix=/musl-x86_64 --disable-shared
-make -j10
-make install
-
-cd ..
-rm -rf $MUSL
-
-# To build MUSL we're going to need a libunwind lying around, so acquire that
-# here and build it.
-curl -L https://github.com/llvm-mirror/llvm/archive/release_37.tar.gz | tar xzf -
-curl -L https://github.com/llvm-mirror/libunwind/archive/release_37.tar.gz | tar xzf -
-
-mkdir libunwind-build
-cd libunwind-build
-cmake ../libunwind-release_37 -DLLVM_PATH=/build/llvm-release_37 \
-          -DLIBUNWIND_ENABLE_SHARED=0
-make -j10
-cp lib/libunwind.a /musl-x86_64/lib
diff --git a/src/ci/docker/scripts/musl.sh b/src/ci/docker/scripts/musl.sh
new file mode 100644
index 00000000000..b704e37d592
--- /dev/null
+++ b/src/ci/docker/scripts/musl.sh
@@ -0,0 +1,102 @@
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  $@ &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  rm /tmp/build.log
+  set -x
+}
+
+TAG=$1
+shift
+
+MUSL=musl-1.1.17
+
+# may have been downloaded in a previous run
+if [ ! -d $MUSL ]; then
+  curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf -
+fi
+
+cd $MUSL
+./configure --disable-shared --prefix=/musl-$TAG $@
+if [ "$TAG" = "i686" ]; then
+  hide_output make -j$(nproc) AR=ar RANLIB=ranlib
+else
+  hide_output make -j$(nproc)
+fi
+hide_output make install
+hide_output make clean
+
+cd ..
+
+LLVM=39
+# may have been downloaded in a previous run
+if [ ! -d libunwind-release_$LLVM ]; then
+  curl -L https://github.com/llvm-mirror/llvm/archive/release_$LLVM.tar.gz | tar xzf -
+  curl -L https://github.com/llvm-mirror/libunwind/archive/release_$LLVM.tar.gz | tar xzf -
+  # Whoa what's this mysterious patch we're applying to libunwind! Why are we
+  # swapping the values of ESP/EBP in libunwind?!
+  #
+  # Discovered in #35599 it turns out that the vanilla build of libunwind is not
+  # suitable for unwinding i686 musl. After some investigation it ended up
+  # looking like the register values for ESP/EBP were indeed incorrect (swapped)
+  # in the source. Similar commits in libunwind (r280099 and r282589) have noticed
+  # this for other platforms, and we just need to realize it for musl linux as
+  # well.
+  #
+  # More technical info can be found at #35599
+  cd libunwind-release_$LLVM
+  patch -Np1 << EOF
+diff --git a/include/libunwind.h b/include/libunwind.h
+index c5b9633..1360eb2 100644
+--- a/include/libunwind.h
++++ b/include/libunwind.h
+@@ -151,8 +151,8 @@ enum {
+   UNW_X86_ECX = 1,
+   UNW_X86_EDX = 2,
+   UNW_X86_EBX = 3,
+-  UNW_X86_EBP = 4,
+-  UNW_X86_ESP = 5,
++  UNW_X86_ESP = 4,
++  UNW_X86_EBP = 5,
+   UNW_X86_ESI = 6,
+   UNW_X86_EDI = 7
+ };
+fi
+EOF
+  cd ..
+fi
+
+mkdir libunwind-build
+cd libunwind-build
+cmake ../libunwind-release_$LLVM \
+          -DLLVM_PATH=/build/llvm-release_$LLVM \
+          -DLIBUNWIND_ENABLE_SHARED=0 \
+          -DCMAKE_C_COMPILER=$CC \
+          -DCMAKE_CXX_COMPILER=$CXX \
+          -DCMAKE_C_FLAGS="$CFLAGS" \
+          -DCMAKE_CXX_FLAGS="$CXXFLAGS"
+
+hide_output make -j$(nproc)
+cp lib/libunwind.a /musl-$TAG/lib
+cd ../ && rm -rf libunwind-build