From eb832b2a3244166c81d8e00d94805525f3bd7526 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 12 Aug 2019 09:41:06 +0200 Subject: ci: move mirrors to their standalone bucket Currently mirrors are stored in the rust-lang-ci2 S3 bucket along with CI toolchains. This is problematic for multiple reasons: - CI IAM credentials are allowed to both edit and delete those files. A malicious user gaining access to those credentials would be able to change our mirrored dependencies, possibly backdooring the compiler. - Contents of the rust-lang-ci2 bucket are disposable except for the mirrors' content. When we implement backups for S3 buckets we'd have to replicate just that part of the bucket, complicating the backup logic and increasing the chance of mistakes. A standalone bucket will be way easier to backup. This commit switches our CI to use the new rust-lang-ci-mirrors bucket. --- src/ci/docker/scripts/android-sdk-manager.py | 8 +++++--- src/ci/docker/scripts/freebsd-toolchain.sh | 2 +- src/ci/docker/scripts/sccache.sh | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) (limited to 'src/ci/docker/scripts') diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py index 7c9a8b82e92..c9e2961f6eb 100755 --- a/src/ci/docker/scripts/android-sdk-manager.py +++ b/src/ci/docker/scripts/android-sdk-manager.py @@ -23,8 +23,9 @@ REPOSITORIES = [ HOST_OS = "linux" # Mirroring options -MIRROR_BUCKET = "rust-lang-ci2" -MIRROR_BASE_DIR = "rust-ci-mirror/android/" +MIRROR_BUCKET = "rust-lang-ci-mirrors" +MIRROR_BUCKET_REGION = "us-west-1" +MIRROR_BASE_DIR = "rustc/android/" import argparse import hashlib @@ -144,7 +145,8 @@ def cli_install(args): lockfile = Lockfile(args.lockfile) for package in lockfile.packages.values(): # Download the file from the mirror into a temp file - url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR + url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \ + ".amazonaws.com/" + MIRROR_BASE_DIR downloaded = package.download(url) # Extract the file in a temporary directory extract_dir = tempfile.mkdtemp() diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh index 8cef69d9c26..70155e770a9 100755 --- a/src/ci/docker/scripts/freebsd-toolchain.sh +++ b/src/ci/docker/scripts/freebsd-toolchain.sh @@ -59,7 +59,7 @@ done # Originally downloaded from: # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz -URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz +URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}" # Fix up absolute symlinks from the system image. This can be removed diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index 194de3c339f..efeb0ed0d72 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,6 +1,6 @@ set -ex curl -fo /usr/local/bin/sccache \ - https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl + https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl chmod +x /usr/local/bin/sccache -- cgit 1.4.1-3-g733a5 From 5941acd80faaf9043cb48737c47d0b94023eddc3 Mon Sep 17 00:00:00 2001 From: Marco A L Barbosa Date: Tue, 30 Jul 2019 16:31:26 -0300 Subject: Use libunwind from llvm-project submodule for musl targets --- src/bootstrap/sanity.rs | 4 ---- src/ci/docker/scripts/musl-toolchain.sh | 26 -------------------------- src/ci/docker/scripts/musl.sh | 26 ++------------------------ src/libunwind/Cargo.toml | 4 ++-- src/libunwind/build.rs | 16 ++++++++++++---- 5 files changed, 16 insertions(+), 60 deletions(-) (limited to 'src/ci/docker/scripts') diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 4e3930c8da7..bffe748f37c 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -202,10 +202,6 @@ pub fn check(build: &mut Build) { panic!("couldn't find libc.a in musl dir: {}", root.join("lib").display()); } - if fs::metadata(root.join("lib/libunwind.a")).is_err() { - panic!("couldn't find libunwind.a in musl dir: {}", - root.join("lib").display()); - } } None => { panic!("when targeting MUSL either the rust.musl-root \ diff --git a/src/ci/docker/scripts/musl-toolchain.sh b/src/ci/docker/scripts/musl-toolchain.sh index 55899fa6c3e..74ba2f0eadb 100644 --- a/src/ci/docker/scripts/musl-toolchain.sh +++ b/src/ci/docker/scripts/musl-toolchain.sh @@ -54,29 +54,3 @@ if [ "$REPLACE_CC" = "1" ]; then ln -s $TARGET-g++ /usr/local/bin/$exec done fi - -export CC=$TARGET-gcc -export CXX=$TARGET-g++ - -LLVM=70 - -# may have been downloaded in a previous run -if [ ! -d libunwind-release_$LLVM ]; then - curl -L https://github.com/llvm-mirror/llvm/archive/release_$LLVM.tar.gz | tar xzf - - curl -L https://github.com/llvm-mirror/libunwind/archive/release_$LLVM.tar.gz | tar xzf - -fi - -# fixme(mati865): Replace it with https://github.com/rust-lang/rust/pull/59089 -mkdir libunwind-build -cd libunwind-build -cmake ../libunwind-release_$LLVM \ - -DLLVM_PATH=/build/llvm-release_$LLVM \ - -DLIBUNWIND_ENABLE_SHARED=0 \ - -DCMAKE_C_COMPILER=$CC \ - -DCMAKE_CXX_COMPILER=$CXX \ - -DCMAKE_C_FLAGS="$CFLAGS" \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS" - -hide_output make -j$(nproc) -cp lib/libunwind.a $OUTPUT/$TARGET/lib -cd - && rm -rf libunwind-build diff --git a/src/ci/docker/scripts/musl.sh b/src/ci/docker/scripts/musl.sh index c2cf77d56cb..d847c407aba 100644 --- a/src/ci/docker/scripts/musl.sh +++ b/src/ci/docker/scripts/musl.sh @@ -20,6 +20,8 @@ exit 1 TAG=$1 shift +# Ancient binutils versions don't understand debug symbols produced by more recent tools. +# Apparently applying `-fPIC` everywhere allows them to link successfully. export CFLAGS="-fPIC $CFLAGS" MUSL=musl-1.1.22 @@ -38,27 +40,3 @@ else fi hide_output make install hide_output make clean - -cd .. - -LLVM=70 - -# may have been downloaded in a previous run -if [ ! -d libunwind-release_$LLVM ]; then - curl -L https://github.com/llvm-mirror/llvm/archive/release_$LLVM.tar.gz | tar xzf - - curl -L https://github.com/llvm-mirror/libunwind/archive/release_$LLVM.tar.gz | tar xzf - -fi - -mkdir libunwind-build -cd libunwind-build -cmake ../libunwind-release_$LLVM \ - -DLLVM_PATH=/build/llvm-release_$LLVM \ - -DLIBUNWIND_ENABLE_SHARED=0 \ - -DCMAKE_C_COMPILER=$CC \ - -DCMAKE_CXX_COMPILER=$CXX \ - -DCMAKE_C_FLAGS="$CFLAGS" \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS" - -hide_output make -j$(nproc) -cp lib/libunwind.a /musl-$TAG/lib -cd ../ && rm -rf libunwind-build diff --git a/src/libunwind/Cargo.toml b/src/libunwind/Cargo.toml index f0f1bab425d..f10df8c85ba 100644 --- a/src/libunwind/Cargo.toml +++ b/src/libunwind/Cargo.toml @@ -22,7 +22,7 @@ compiler_builtins = "0.1.0" cfg-if = "0.1.8" [build-dependencies] -cc = { optional = true, version = "1.0.1" } +cc = { version = "1.0.1" } [features] -llvm-libunwind = ["cc"] +llvm-libunwind = [] diff --git a/src/libunwind/build.rs b/src/libunwind/build.rs index c1b0dbc0881..a266407c602 100644 --- a/src/libunwind/build.rs +++ b/src/libunwind/build.rs @@ -5,14 +5,14 @@ fn main() { let target = env::var("TARGET").expect("TARGET was not set"); if cfg!(feature = "llvm-libunwind") && - (target.contains("linux") || + ((target.contains("linux") && !target.contains("musl")) || target.contains("fuchsia")) { // Build the unwinding from libunwind C/C++ source code. - #[cfg(feature = "llvm-libunwind")] llvm_libunwind::compile(); } else if target.contains("linux") { if target.contains("musl") { - // musl is handled in lib.rs + // linking for musl is handled in lib.rs + llvm_libunwind::compile(); } else if !target.contains("android") { println!("cargo:rustc-link-lib=gcc_s"); } @@ -44,7 +44,6 @@ fn main() { } } -#[cfg(feature = "llvm-libunwind")] mod llvm_libunwind { use std::env; use std::path::Path; @@ -96,6 +95,15 @@ mod llvm_libunwind { cfg.file(root.join("src").join(src)); } + if target_env == "musl" { + // use the same C compiler command to compile C++ code so we do not need to setup the + // C++ compiler env variables on the builders + cfg.cpp(false); + // linking for musl is handled in lib.rs + cfg.cargo_metadata(false); + println!("cargo:rustc-link-search=native={}", env::var("OUT_DIR").unwrap()); + } + cfg.compile("unwind"); } } -- cgit 1.4.1-3-g733a5