diff options
Diffstat (limited to 'src')
19 files changed, 176 insertions, 48 deletions
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 203db2d3876..13e3049f2c8 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -990,42 +990,7 @@ impl Config { config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); - // This is currently all tier 1 targets and tier 2 targets with host tools - // (since others may not have CI artifacts) - // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - // FIXME: this is duplicated in bootstrap.py - let supported_platforms = [ - // tier 1 - "aarch64-unknown-linux-gnu", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - // tier 2 with host tools - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "mips-unknown-linux-gnu", - "mips64-unknown-linux-gnuabi64", - "mips64el-unknown-linux-gnuabi64", - "mipsel-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", - ]; - supported_platforms.contains(&&*config.build.triple) + crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false)) } Some(StringOrBool::Bool(b)) => b, None => false, @@ -1176,6 +1141,7 @@ impl Config { if config.llvm_from_ci { let triple = &config.build.triple; + let ci_llvm_bin = config.ci_llvm_root().join("bin"); let mut build_target = config .target_config .entry(config.build) @@ -1183,7 +1149,6 @@ impl Config { check_ci_llvm!(build_target.llvm_config); check_ci_llvm!(build_target.llvm_filecheck); - let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin"); build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 1bba5e68be4..7ecf74d3068 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -114,23 +114,20 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } -pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { - let config = &builder.config; - if !config.llvm_from_ci { - return; - } +/// This retrieves the LLVM sha we *want* to use, according to git history. +pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String { let mut rev_list = config.git(); rev_list.args(&[ PathBuf::from("rev-list"), - format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(), + format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), "-n1".into(), "--first-parent".into(), "HEAD".into(), "--".into(), - builder.src.join("src/llvm-project"), - builder.src.join("src/bootstrap/download-ci-llvm-stamp"), + config.src.join("src/llvm-project"), + config.src.join("src/bootstrap/download-ci-llvm-stamp"), // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - builder.src.join("src/version"), + config.src.join("src/version"), ]); let llvm_sha = output(&mut rev_list); let llvm_sha = llvm_sha.trim(); @@ -143,8 +140,82 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { panic!(); } + llvm_sha.to_owned() +} + +/// Returns whether the CI-found LLVM is currently usable. +/// +/// This checks both the build triple platform to confirm we're usable at all, +/// and then verifies if the current HEAD matches the detected LLVM SHA head, +/// in which case LLVM is indicated as not available. +pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool { + // This is currently all tier 1 targets and tier 2 targets with host tools + // (since others may not have CI artifacts) + // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 + let supported_platforms = [ + // tier 1 + "aarch64-unknown-linux-gnu", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + // tier 2 with host tools + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "mips-unknown-linux-gnu", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", + ]; + if !supported_platforms.contains(&&*config.build.triple) { + return false; + } + + let triple = &*config.build.triple; + if (triple == "aarch64-unknown-linux-gnu" || triple.contains("i686")) && asserts { + // No alt builder for aarch64-unknown-linux-gnu today. + return false; + } + + if crate::util::CiEnv::is_ci() { + let llvm_sha = detect_llvm_sha(config); + let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); + let head_sha = head_sha.trim(); + if llvm_sha == head_sha { + eprintln!( + "Detected LLVM as non-available: running in CI and modified LLVM in this change" + ); + return false; + } + } + + true +} + +pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { + let config = &builder.config; + if !config.llvm_from_ci { + return; + } let llvm_root = config.ci_llvm_root(); let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&config); let key = format!("{}{}", llvm_sha, config.llvm_assertions); if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { download_ci_llvm(builder, &llvm_sha); diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 1895e290148..3a00e258e00 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -258,6 +258,10 @@ impl CiEnv { } } + pub fn is_ci() -> bool { + Self::current() != CiEnv::None + } + /// If in a CI environment, forces the command to run with colors. pub fn force_coloring_in_ci(self, cmd: &mut Command) { if self != CiEnv::None { diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index a2335687ab3..d0d367b39b4 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools ENV TARGETS=arm-linux-androideabi +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \ --set llvm.allow-old-toolchain diff --git a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile index e2dbc7cfd7c..69f88e49520 100644 --- a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile @@ -78,6 +78,8 @@ RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +COPY static/gitconfig /etc/gitconfig + ENV RUST_CONFIGURE_ARGS --qemu-armhf-rootfs=/tmp/rootfs ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target arm-unknown-linux-gnueabihf diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile index 1f8d0a64ea4..71ea13071bd 100644 --- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile @@ -27,6 +27,10 @@ RUN mkdir -p /config RUN echo "[rust]" > /config/nopt-std-config.toml RUN echo "optimize = false" >> /config/nopt-std-config.toml +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \ --set llvm.allow-old-toolchain ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \ diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile index 7bca0398dea..ebeab3dbdb7 100644 --- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile @@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \ --set llvm.allow-old-toolchain # Exclude some tests that are unlikely to be platform specific, to speed up diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile index 4d554a2852a..b75e2f085cd 100644 --- a/src/ci/docker/host-x86_64/test-various/Dockerfile +++ b/src/ci/docker/host-x86_64/test-various/Dockerfile @@ -30,6 +30,10 @@ WORKDIR / COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# We are disabling CI LLVM since this builder needs to build LLD, which is +# currently unsupported when downloading pre-built LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS \ --musl-root-x86_64=/usr/local/x86_64-linux-musl \ --set build.nodejs=/node-v15.14.0-linux-x64/bin/node \ diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile index 878c4e34158..9e37c2822e8 100644 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ b/src/ci/docker/host-x86_64/wasm32/Dockerfile @@ -48,6 +48,8 @@ ENV TARGETS=wasm32-unknown-emscripten # Use -O1 optimizations in the link step to reduce time spent optimizing. ENV EMCC_CFLAGS=-O1 +COPY static/gitconfig /etc/gitconfig + # Emscripten installation is user-specific ENV NO_CHANGE_USER=1 diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile index bfc6975c19d..321b3d6ace4 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile @@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \ --set llvm.allow-old-toolchain ENV RUST_CHECK_TARGET check-aux diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile index 13d440423b2..739045248fe 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile @@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 +# llvm.use-linker conflicts with downloading CI LLVM +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-debug \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile index 4bb4042cd7e..be4def85f14 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile @@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \ --set llvm.allow-old-toolchain ENV SCRIPT python3 ../x.py --stage 2 test distcheck diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile index 21891c1244d..23f2215c2d9 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile @@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -# using llvm-link-shared due to libffi issues -- see #34486 +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-13 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile index 5f3af2f47ca..8f6831bc54e 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile @@ -37,7 +37,11 @@ RUN apt-get update && \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -# using llvm-link-shared due to libffi issues -- see #34486 +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-13 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index f442a477a7b..e2ec4f27500 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/ # the local version of the package is different than the one used by the CI. RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS \ --set llvm.allow-old-toolchain \ --build=x86_64-unknown-linux-gnu \ diff --git a/src/ci/docker/static/gitconfig b/src/ci/docker/static/gitconfig new file mode 100644 index 00000000000..6bad35f01ce --- /dev/null +++ b/src/ci/docker/static/gitconfig @@ -0,0 +1,2 @@ +[safe] +directory = * diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 91204eb0b05..a6bdd9f3186 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -600,6 +600,9 @@ jobs: --build=i686-pc-windows-gnu --set llvm.allow-old-toolchain SCRIPT: make ci-mingw-subset-1 + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -609,6 +612,9 @@ jobs: --build=i686-pc-windows-gnu --set llvm.allow-old-toolchain SCRIPT: make ci-mingw-subset-2 + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -619,6 +625,9 @@ jobs: --build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -629,6 +638,9 @@ jobs: --build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -678,6 +690,9 @@ jobs: --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 SCRIPT: python x.py dist CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 @@ -691,6 +706,9 @@ jobs: --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl diff --git a/src/ci/run.sh b/src/ci/run.sh index 6545475d996..9a247fb60a8 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -11,6 +11,16 @@ if [ "$NO_CHANGE_USER" = "" ]; then useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user export HOME=/home/user unset LOCAL_USER_ID + + # Ensure that runners are able to execute git commands in the worktree, + # overriding the typical git protections. In our docker container we're running + # as root, while the user owning the checkout is not root. + # This is only necessary when we change the user, otherwise we should + # already be running with the right user. + # + # For NO_CHANGE_USER done in the small number of Dockerfiles affected. + echo -e '[safe]\n\tdirectory = *' > /home/user/gitconfig + exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user fi fi @@ -102,6 +112,18 @@ else fi RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" + + # We enable this for non-dist builders, since those aren't trying to produce + # fresh binaries. We currently don't entirely support distributing a fresh + # copy of the compiler (including llvm tools, etc.) if we haven't actually + # built LLVM, since not everything necessary is copied into the + # local-usage-only LLVM artifacts. If that changes, this could maybe be made + # true for all builds. In practice it's probably a good idea to keep building + # LLVM continuously on at least some builders to ensure it works, though. + # (And PGO is its own can of worms). + if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-available" + fi fi if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 0bc8a0389a8..02b72625d6e 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -61,6 +61,10 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then 7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe" ciCommandSetEnv RUST_CONFIGURE_ARGS \ "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" + + # Disable downloading CI LLVM on this builder; + # setting up clang-cl just above conflicts with the default if-available option. + ciCommandSetEnv NO_DOWNLOAD_CI_LLVM 1 fi if isWindows; then |