Auto merge of #99967 - Mark-Simulacrum:download-llvm-ci, r=jyn514

Download, rather than sccache-cache, LLVM in CI

My hope/expectation is that we can do better than sccache in CI for cached builds -- currently it looks like on macOS those still take upwards of 10-11 minutes, which is a significant amount of time that we could potentially cut.

This enables this mode for all non-dist builders; this should avoid any problems with the artifacts we distribute, while also providing for faster test builders (since they'll make use of PGO'd LLVM on the platforms we do that on, which is hopefully a nice win). It slightly increases the chance of test builders starting to fail only after a PR is merged (if PGO changes runtime behavior), but that should hopefully never happen, so I think this is worthwhile.

Measurements on the PR for apple-1 don't show any noticeable improvement in CI times, but those can be pretty noisy -- I'm inclined to land this since it *should* pretty much always be better and we can reconsider if that ever turns out not to be the case.

16 files changed, 90 insertions, 2 deletions

diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile
index a2335687ab3..d0d367b39b4 100644
--- a/src/ci/docker/host-x86_64/arm-android/Dockerfile
+++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile
@@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools
 
 ENV TARGETS=arm-linux-androideabi
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \
     --set llvm.allow-old-toolchain
 
diff --git a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile
index e2dbc7cfd7c..69f88e49520 100644
--- a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile
+++ b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile
@@ -78,6 +78,8 @@ RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+COPY static/gitconfig /etc/gitconfig
+
 ENV RUST_CONFIGURE_ARGS --qemu-armhf-rootfs=/tmp/rootfs
 ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target arm-unknown-linux-gnueabihf
 
diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
index 1f8d0a64ea4..71ea13071bd 100644
--- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
@@ -27,6 +27,10 @@ RUN mkdir -p /config
 RUN echo "[rust]" > /config/nopt-std-config.toml
 RUN echo "optimize = false" >> /config/nopt-std-config.toml
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \
diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
index 7bca0398dea..ebeab3dbdb7 100644
--- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
@@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 # Exclude some tests that are unlikely to be platform specific, to speed up
diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile
index 4d554a2852a..b75e2f085cd 100644
--- a/src/ci/docker/host-x86_64/test-various/Dockerfile
+++ b/src/ci/docker/host-x86_64/test-various/Dockerfile
@@ -30,6 +30,10 @@ WORKDIR /
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+# We are disabling CI LLVM since this builder needs to build LLD, which is
+# currently unsupported when downloading pre-built LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --musl-root-x86_64=/usr/local/x86_64-linux-musl \
   --set build.nodejs=/node-v15.14.0-linux-x64/bin/node \
diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile
index 878c4e34158..9e37c2822e8 100644
--- a/src/ci/docker/host-x86_64/wasm32/Dockerfile
+++ b/src/ci/docker/host-x86_64/wasm32/Dockerfile
@@ -48,6 +48,8 @@ ENV TARGETS=wasm32-unknown-emscripten
 # Use -O1 optimizations in the link step to reduce time spent optimizing.
 ENV EMCC_CFLAGS=-O1
 
+COPY static/gitconfig /etc/gitconfig
+
 # Emscripten installation is user-specific
 ENV NO_CHANGE_USER=1
 
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
index bfc6975c19d..321b3d6ace4 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
@@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 ENV RUST_CHECK_TARGET check-aux
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
index 13d440423b2..739045248fe 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
@@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh
 ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 
+# llvm.use-linker conflicts with downloading CI LLVM
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --enable-debug \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
index 4bb4042cd7e..be4def85f14 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
@@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py --stage 2 test distcheck
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
index 21891c1244d..23f2215c2d9 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
@@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
     --build=x86_64-unknown-linux-gnu \
     --llvm-root=/usr/lib/llvm-13 \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
index 5f3af2f47ca..8f6831bc54e 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
@@ -37,7 +37,11 @@ RUN apt-get update && \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-13 \
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
index f442a477a7b..e2ec4f27500 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/
 # the local version of the package is different than the one used by the CI.
 RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --set llvm.allow-old-toolchain \
   --build=x86_64-unknown-linux-gnu \
diff --git a/src/ci/docker/static/gitconfig b/src/ci/docker/static/gitconfig
new file mode 100644
index 00000000000..6bad35f01ce
--- /dev/null
+++ b/src/ci/docker/static/gitconfig
@@ -0,0 +1,2 @@
+[safe]
+directory = *
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 91204eb0b05..a6bdd9f3186 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -600,6 +600,9 @@ jobs:
                 --build=i686-pc-windows-gnu
                 --set llvm.allow-old-toolchain
               SCRIPT: make ci-mingw-subset-1
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -609,6 +612,9 @@ jobs:
                 --build=i686-pc-windows-gnu
                 --set llvm.allow-old-toolchain
               SCRIPT: make ci-mingw-subset-2
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -619,6 +625,9 @@ jobs:
                 --build=x86_64-pc-windows-gnu
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -629,6 +638,9 @@ jobs:
                 --build=x86_64-pc-windows-gnu
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -678,6 +690,9 @@ jobs:
                 --enable-full-tools
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               SCRIPT: python x.py dist
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
@@ -691,6 +706,9 @@ jobs:
                 --enable-full-tools
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-xl
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 6545475d996..9a247fb60a8 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -11,6 +11,16 @@ if [ "$NO_CHANGE_USER" = "" ]; then
     useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user
     export HOME=/home/user
     unset LOCAL_USER_ID
+
+    # Ensure that runners are able to execute git commands in the worktree,
+    # overriding the typical git protections. In our docker container we're running
+    # as root, while the user owning the checkout is not root.
+    # This is only necessary when we change the user, otherwise we should
+    # already be running with the right user.
+    #
+    # For NO_CHANGE_USER done in the small number of Dockerfiles affected.
+    echo -e '[safe]\n\tdirectory = *' > /home/user/gitconfig
+
     exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user
   fi
 fi
@@ -102,6 +112,18 @@ else
   fi
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
+
+  # We enable this for non-dist builders, since those aren't trying to produce
+  # fresh binaries. We currently don't entirely support distributing a fresh
+  # copy of the compiler (including llvm tools, etc.) if we haven't actually
+  # built LLVM, since not everything necessary is copied into the
+  # local-usage-only LLVM artifacts. If that changes, this could maybe be made
+  # true for all builds. In practice it's probably a good idea to keep building
+  # LLVM continuously on at least some builders to ensure it works, though.
+  # (And PGO is its own can of worms).
+  if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-available"
+  fi
 fi
 
 if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then
diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh
index 0bc8a0389a8..02b72625d6e 100755
--- a/src/ci/scripts/install-clang.sh
+++ b/src/ci/scripts/install-clang.sh
@@ -61,6 +61,10 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then
     7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe"
     ciCommandSetEnv RUST_CONFIGURE_ARGS \
         "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe"
+
+    # Disable downloading CI LLVM on this builder;
+    # setting up clang-cl just above conflicts with the default if-available option.
+    ciCommandSetEnv NO_DOWNLOAD_CI_LLVM 1
 fi
 
 if isWindows; then