From f44801c5ee71861c67977992d69d14bfb1a702d3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 Mar 2017 19:46:44 -0800 Subject: travis: Split the linux-tested-targets builder Travis only gives us 30GB disk space and we don't currently have an option to increase that. Each musl target generates "hello world" binaries of about 3.5MB in size, and we're testing two targets in the same image. We have around 3k run-pass tests and 2 musl targets which works out to around 20GB. That's dangerously close to the limit and is causing PRs to bounce. This PR splits up the builder in two, one for x86_64 musl and the other for i686. Hopefully that'll keep us under the disk limit. Closes #40359 --- .travis.yml | 3 +- src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile | 50 ++++++++++++++++ .../docker/dist-i586-gnu-i686-musl/build-musl.sh | 52 +++++++++++++++++ .../musl-libunwind-patch.patch | 15 +++++ src/ci/docker/dist-x86_64-musl/Dockerfile | 45 +++++++++++++++ src/ci/docker/dist-x86_64-musl/build-musl.sh | 38 ++++++++++++ src/ci/docker/linux-tested-targets/Dockerfile | 54 ----------------- src/ci/docker/linux-tested-targets/build-musl.sh | 67 ---------------------- .../musl-libunwind-patch.patch | 15 ----- 9 files changed, 202 insertions(+), 137 deletions(-) create mode 100644 src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile create mode 100644 src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh create mode 100644 src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch create mode 100644 src/ci/docker/dist-x86_64-musl/Dockerfile create mode 100644 src/ci/docker/dist-x86_64-musl/build-musl.sh delete mode 100644 src/ci/docker/linux-tested-targets/Dockerfile delete mode 100644 src/ci/docker/linux-tested-targets/build-musl.sh delete mode 100644 src/ci/docker/linux-tested-targets/musl-libunwind-patch.patch diff --git a/.travis.yml b/.travis.yml index 2f12408242f..fa6c9cf0748 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,17 +15,18 @@ matrix: - env: IMAGE=arm-android - env: IMAGE=armhf-gnu - env: IMAGE=cross DEPLOY=1 - - env: IMAGE=linux-tested-targets DEPLOY=1 - env: IMAGE=dist-android DEPLOY=1 - env: IMAGE=dist-arm-linux DEPLOY=1 - env: IMAGE=dist-armv7-aarch64-linux DEPLOY=1 - env: IMAGE=dist-freebsd DEPLOY=1 + - env: IMAGE=dist-i586-gnu-i686-musl DEPLOY=1 - env: IMAGE=dist-mips-linux DEPLOY=1 - env: IMAGE=dist-mips64-linux DEPLOY=1 - env: IMAGE=dist-powerpc-linux DEPLOY=1 - env: IMAGE=dist-powerpc64-linux DEPLOY=1 - env: IMAGE=dist-s390x-linux-netbsd DEPLOY=1 - env: IMAGE=dist-x86-linux DEPLOY=1 + - env: IMAGE=dist-x86_64-musl DEPLOY=1 - env: IMAGE=emscripten - env: IMAGE=i686-gnu - env: IMAGE=i686-gnu-nopt diff --git a/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile new file mode 100644 index 00000000000..0897be80cc2 --- /dev/null +++ b/src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++-multilib \ + make \ + file \ + curl \ + ca-certificates \ + python2.7 \ + git \ + cmake \ + xz-utils \ + sudo \ + gdb \ + patch \ + libssl-dev \ + pkg-config + +WORKDIR /build/ +COPY musl-libunwind-patch.patch build-musl.sh /build/ +RUN sh /build/build-musl.sh && rm -rf /build + +RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \ + dpkg -i dumb-init_*.deb && \ + rm dumb-init_*.deb +ENTRYPOINT ["/usr/bin/dumb-init", "--"] + +RUN curl -o /usr/local/bin/sccache \ + https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-25-sccache-x86_64-unknown-linux-musl && \ + chmod +x /usr/local/bin/sccache + +ENV RUST_CONFIGURE_ARGS \ + --target=i686-unknown-linux-musl,i586-unknown-linux-gnu \ + --musl-root-i686=/musl-i686 + +# Newer binutils broke things on some vms/distros (i.e., linking against +# unknown relocs disabled by the following flag), so we need to go out of our +# way to produce "super compatible" binaries. +# +# See: https://github.com/rust-lang/rust/issues/34978 +ENV CFLAGS_i686_unknown_linux_musl=-Wa,-mrelax-relocations=no + +ENV SCRIPT \ + python2.7 ../x.py test \ + --target i686-unknown-linux-musl \ + --target i586-unknown-linux-gnu \ + && \ + python2.7 ../x.py dist \ + --target i686-unknown-linux-musl \ + --target i586-unknown-linux-gnu diff --git a/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh b/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh new file mode 100644 index 00000000000..a50a25c7913 --- /dev/null +++ b/src/ci/docker/dist-i586-gnu-i686-musl/build-musl.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well +export CFLAGS="-fPIC -Wa,-mrelax-relocations=no" +export CXXFLAGS="-Wa,-mrelax-relocations=no" + +MUSL=musl-1.1.14 +curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf - +cd $MUSL +CFLAGS="$CFLAGS -m32" ./configure --prefix=/musl-i686 --disable-shared --target=i686 +make -j10 +make install +cd .. + +# To build MUSL we're going to need a libunwind lying around, so acquire that +# here and build it. +curl -L https://github.com/llvm-mirror/llvm/archive/release_37.tar.gz | tar xzf - +curl -L https://github.com/llvm-mirror/libunwind/archive/release_37.tar.gz | tar xzf - + +# Whoa what's this mysterious patch we're applying to libunwind! Why are we +# swapping the values of ESP/EBP in libunwind?! +# +# Discovered in #35599 it turns out that the vanilla build of libunwind is not +# suitable for unwinding 32-bit musl. After some investigation it ended up +# looking like the register values for ESP/EBP were indeed incorrect (swapped) +# in the source. Similar commits in libunwind (r280099 and r282589) have noticed +# this for other platforms, and we just need to realize it for musl linux as +# well. +# +# More technical info can be found at #35599 +cd libunwind-release_37 +patch -Np1 < /build/musl-libunwind-patch.patch +cd .. + +mkdir libunwind-build +cd libunwind-build +CFLAGS="$CFLAGS -m32" CXXFLAGS="$CXXFLAGS -m32" cmake ../libunwind-release_37 \ + -DLLVM_PATH=/build/llvm-release_37 \ + -DLIBUNWIND_ENABLE_SHARED=0 +make -j10 +cp lib/libunwind.a /musl-i686/lib diff --git a/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch b/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch new file mode 100644 index 00000000000..99cd685b72d --- /dev/null +++ b/src/ci/docker/dist-i586-gnu-i686-musl/musl-libunwind-patch.patch @@ -0,0 +1,15 @@ +diff --git a/include/libunwind.h b/include/libunwind.h +index c5b9633..1360eb2 100644 +--- a/include/libunwind.h ++++ b/include/libunwind.h +@@ -151,8 +151,8 @@ enum { + UNW_X86_ECX = 1, + UNW_X86_EDX = 2, + UNW_X86_EBX = 3, +- UNW_X86_EBP = 4, +- UNW_X86_ESP = 5, ++ UNW_X86_ESP = 4, ++ UNW_X86_EBP = 5, + UNW_X86_ESI = 6, + UNW_X86_EDI = 7 + }; diff --git a/src/ci/docker/dist-x86_64-musl/Dockerfile b/src/ci/docker/dist-x86_64-musl/Dockerfile new file mode 100644 index 00000000000..5877404ece1 --- /dev/null +++ b/src/ci/docker/dist-x86_64-musl/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + make \ + file \ + curl \ + ca-certificates \ + python2.7 \ + git \ + cmake \ + xz-utils \ + sudo \ + gdb \ + patch \ + libssl-dev \ + pkg-config + +WORKDIR /build/ +COPY build-musl.sh /build/ +RUN sh /build/build-musl.sh && rm -rf /build + +RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \ + dpkg -i dumb-init_*.deb && \ + rm dumb-init_*.deb +ENTRYPOINT ["/usr/bin/dumb-init", "--"] + +RUN curl -o /usr/local/bin/sccache \ + https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-25-sccache-x86_64-unknown-linux-musl && \ + chmod +x /usr/local/bin/sccache + +ENV RUST_CONFIGURE_ARGS \ + --target=x86_64-unknown-linux-musl \ + --musl-root-x86_64=/musl-x86_64 + +# Newer binutils broke things on some vms/distros (i.e., linking against +# unknown relocs disabled by the following flag), so we need to go out of our +# way to produce "super compatible" binaries. +# +# See: https://github.com/rust-lang/rust/issues/34978 +ENV CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no + +ENV SCRIPT \ + python2.7 ../x.py test --target x86_64-unknown-linux-musl && \ + python2.7 ../x.py dist --target x86_64-unknown-linux-musl diff --git a/src/ci/docker/dist-x86_64-musl/build-musl.sh b/src/ci/docker/dist-x86_64-musl/build-musl.sh new file mode 100644 index 00000000000..86bb259c854 --- /dev/null +++ b/src/ci/docker/dist-x86_64-musl/build-musl.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well +export CFLAGS="-fPIC -Wa,-mrelax-relocations=no" +export CXXFLAGS="-Wa,-mrelax-relocations=no" + +MUSL=musl-1.1.14 +curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf - +cd $MUSL +./configure --prefix=/musl-x86_64 --disable-shared +make -j10 +make install + +cd .. +rm -rf $MUSL + +# To build MUSL we're going to need a libunwind lying around, so acquire that +# here and build it. +curl -L https://github.com/llvm-mirror/llvm/archive/release_37.tar.gz | tar xzf - +curl -L https://github.com/llvm-mirror/libunwind/archive/release_37.tar.gz | tar xzf - + +mkdir libunwind-build +cd libunwind-build +cmake ../libunwind-release_37 -DLLVM_PATH=/build/llvm-release_37 \ + -DLIBUNWIND_ENABLE_SHARED=0 +make -j10 +cp lib/libunwind.a /musl-x86_64/lib diff --git a/src/ci/docker/linux-tested-targets/Dockerfile b/src/ci/docker/linux-tested-targets/Dockerfile deleted file mode 100644 index a7060cd55fe..00000000000 --- a/src/ci/docker/linux-tested-targets/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++-multilib \ - make \ - file \ - curl \ - ca-certificates \ - python2.7 \ - git \ - cmake \ - xz-utils \ - sudo \ - gdb \ - patch \ - libssl-dev \ - pkg-config - -WORKDIR /build/ -COPY musl-libunwind-patch.patch build-musl.sh /build/ -RUN sh /build/build-musl.sh && rm -rf /build - -RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \ - dpkg -i dumb-init_*.deb && \ - rm dumb-init_*.deb -ENTRYPOINT ["/usr/bin/dumb-init", "--"] - -RUN curl -o /usr/local/bin/sccache \ - https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-25-sccache-x86_64-unknown-linux-musl && \ - chmod +x /usr/local/bin/sccache - -ENV RUST_CONFIGURE_ARGS \ - --target=x86_64-unknown-linux-musl,i686-unknown-linux-musl,i586-unknown-linux-gnu \ - --musl-root-x86_64=/musl-x86_64 \ - --musl-root-i686=/musl-i686 - -# Newer binutils broke things on some vms/distros (i.e., linking against -# unknown relocs disabled by the following flag), so we need to go out of our -# way to produce "super compatible" binaries. -# -# See: https://github.com/rust-lang/rust/issues/34978 -ENV CFLAGS_i686_unknown_linux_musl=-Wa,-mrelax-relocations=no \ - CFLAGS_x86_64_unknown_linux_musl=-Wa,-mrelax-relocations=no - -ENV SCRIPT \ - python2.7 ../x.py test \ - --target x86_64-unknown-linux-musl \ - --target i686-unknown-linux-musl \ - --target i586-unknown-linux-gnu \ - && \ - python2.7 ../x.py dist \ - --target x86_64-unknown-linux-musl \ - --target i686-unknown-linux-musl \ - --target i586-unknown-linux-gnu diff --git a/src/ci/docker/linux-tested-targets/build-musl.sh b/src/ci/docker/linux-tested-targets/build-musl.sh deleted file mode 100644 index ce62c392241..00000000000 --- a/src/ci/docker/linux-tested-targets/build-musl.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/sh -# Copyright 2016 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -set -ex - -# We need to mitigate rust-lang/rust#34978 when compiling musl itself as well -export CFLAGS="-fPIC -Wa,-mrelax-relocations=no" -export CXXFLAGS="-Wa,-mrelax-relocations=no" - -MUSL=musl-1.1.14 -curl https://www.musl-libc.org/releases/$MUSL.tar.gz | tar xzf - -cd $MUSL -./configure --prefix=/musl-x86_64 --disable-shared -make -j10 -make install -make clean -# for i686 -CFLAGS="$CFLAGS -m32" ./configure --prefix=/musl-i686 --disable-shared --target=i686 -make -j10 -make install -cd .. - -# To build MUSL we're going to need a libunwind lying around, so acquire that -# here and build it. -curl -L https://github.com/llvm-mirror/llvm/archive/release_37.tar.gz | tar xzf - -curl -L https://github.com/llvm-mirror/libunwind/archive/release_37.tar.gz | tar xzf - - -# Whoa what's this mysterious patch we're applying to libunwind! Why are we -# swapping the values of ESP/EBP in libunwind?! -# -# Discovered in #35599 it turns out that the vanilla build of libunwind is not -# suitable for unwinding 32-bit musl. After some investigation it ended up -# looking like the register values for ESP/EBP were indeed incorrect (swapped) -# in the source. Similar commits in libunwind (r280099 and r282589) have noticed -# this for other platforms, and we just need to realize it for musl linux as -# well. -# -# More technical info can be found at #35599 -cd libunwind-release_37 -patch -Np1 < /build/musl-libunwind-patch.patch -cd .. - -mkdir libunwind-build -cd libunwind-build -cmake ../libunwind-release_37 -DLLVM_PATH=/build/llvm-release_37 \ - -DLIBUNWIND_ENABLE_SHARED=0 -make -j10 -cp lib/libunwind.a /musl-x86_64/lib - -# (Note: the next cmake call doesn't fully override the previous cached one, so remove the cached -# configuration manually. IOW, if don't do this or call make clean we'll end up building libunwind -# for x86_64 again) -rm -rf * -# for i686 -CFLAGS="$CFLAGS -m32" CXXFLAGS="$CXXFLAGS -m32" cmake ../libunwind-release_37 \ - -DLLVM_PATH=/build/llvm-release_37 \ - -DLIBUNWIND_ENABLE_SHARED=0 -make -j10 -cp lib/libunwind.a /musl-i686/lib diff --git a/src/ci/docker/linux-tested-targets/musl-libunwind-patch.patch b/src/ci/docker/linux-tested-targets/musl-libunwind-patch.patch deleted file mode 100644 index 99cd685b72d..00000000000 --- a/src/ci/docker/linux-tested-targets/musl-libunwind-patch.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/include/libunwind.h b/include/libunwind.h -index c5b9633..1360eb2 100644 ---- a/include/libunwind.h -+++ b/include/libunwind.h -@@ -151,8 +151,8 @@ enum { - UNW_X86_ECX = 1, - UNW_X86_EDX = 2, - UNW_X86_EBX = 3, -- UNW_X86_EBP = 4, -- UNW_X86_ESP = 5, -+ UNW_X86_ESP = 4, -+ UNW_X86_EBP = 5, - UNW_X86_ESI = 6, - UNW_X86_EDI = 7 - }; -- cgit 1.4.1-3-g733a5