Merge from rustc

17 files changed, 148 insertions, 108 deletions

diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md
index 508b7b40c01..2f52ff5a99a 100644
--- a/src/ci/docker/README.md
+++ b/src/ci/docker/README.md
@@ -1,29 +1,30 @@
 # Docker images for CI
 
 This folder contains a bunch of docker images used by the continuous integration
-(CI) of Rust. An script is accompanied (`run.sh`) with these images to actually
-execute them. To test out an image execute:
+(CI) of Rust. A script is accompanied (`run.sh`) with these images to actually
+execute them.
 
-```
-./src/ci/docker/run.sh $image_name
-```
+Note that a single Docker image can be used by multiple CI jobs, so the job name
+is the important thing that you should know. You can examine the existing CI jobs in
+the [`jobs.yml`](../github-actions/jobs.yml) file.
 
-for example:
+To run a specific CI job locally, you can use the following script:
 
 ```
-./src/ci/docker/run.sh x86_64-gnu
+python3 ./src/ci/github-actions/ci.py run-local <job-name>
 ```
 
-Images will output artifacts in an `obj/$image_name` dir at the root of a repository. Note
-that the script will overwrite the contents of this directory.
-
-To match conditions in rusts CI, also set the environment variable `DEPLOY=1`, e.g.:
+For example, to run the `x86_64-gnu-llvm-18-1` job:
 ```
-DEPLOY=1 ./src/ci/docker/run.sh x86_64-gnu
+python3 ./src/ci/github-actions/ci.py run-local x86_64-gnu-llvm-18-1
 ```
 
+The job will output artifacts in an `obj/<image-name>` dir at the root of a repository. Note
+that the script will overwrite the contents of this directory. `<image-name>` is set based on the
+Docker image executed in the given CI job.
+
 **NOTE**: In CI, the script outputs the artifacts to the `obj` directory,
-while locally, to the `obj/$image_name` directory. This is primarily to prevent
+while locally, to the `obj/<image-name>` directory. This is primarily to prevent
 strange linker errors when using multiple Docker images.
 
 For some Linux workflows (for example `x86_64-gnu-llvm-18-N`), the process is more involved. You will need to see which script is executed for the given workflow inside the [`jobs.yml`](../github-actions/jobs.yml) file and pass it through the `DOCKER_SCRIPT` environment variable. For example, to reproduce the `x86_64-gnu-llvm-18-3` workflow, you can run the following script:
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
new file mode 100644
index 00000000000..8b0fb0710d4
--- /dev/null
+++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
@@ -0,0 +1,100 @@
+# We document platform support for minimum glibc 2.17 and kernel 3.2.
+# CentOS 7 has headers for kernel 3.10, but that's fine as long as we don't
+# actually use newer APIs in rustc or std without a fallback. It's more
+# important that we match glibc for ELF symbol versioning.
+FROM centos:7
+
+WORKDIR /build
+
+# CentOS 7 EOL is June 30, 2024, but the repos remain in the vault.
+RUN sed -i /etc/yum.repos.d/*.repo -e 's!^mirrorlist!#mirrorlist!' \
+  -e 's!^#baseurl=http://mirror.centos.org/!baseurl=https://vault.centos.org/!'
+RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+
+RUN yum upgrade -y && \
+    yum install -y \
+      automake \
+      bzip2 \
+      file \
+      gcc \
+      gcc-c++ \
+      git \
+      glibc-devel \
+      libedit-devel \
+      libstdc++-devel \
+      make \
+      ncurses-devel \
+      openssl-devel \
+      patch \
+      perl \
+      perl-core \
+      pkgconfig \
+      python3 \
+      unzip \
+      wget \
+      xz \
+      zlib-devel \
+      && yum clean all
+
+RUN mkdir -p /rustroot/bin
+
+ENV PATH=/rustroot/bin:$PATH
+ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
+ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
+WORKDIR /tmp
+RUN mkdir /home/user
+COPY scripts/shared.sh /tmp/
+
+# Need at least GCC 5.1 to compile LLVM
+COPY scripts/build-gcc.sh /tmp/
+ENV GCC_VERSION=9.5.0
+ENV GCC_BUILD_TARGET=aarch64-unknown-linux-gnu
+RUN ./build-gcc.sh && yum remove -y gcc gcc-c++
+
+ENV CC=gcc CXX=g++
+
+# LLVM 17 needs cmake 3.20 or higher.
+COPY scripts/cmake.sh /tmp/
+RUN ./cmake.sh
+
+# Build LLVM+Clang
+COPY scripts/build-clang.sh /tmp/
+ENV LLVM_BUILD_TARGETS=AArch64
+RUN ./build-clang.sh
+ENV CC=clang CXX=clang++
+
+# Build zstd to enable `llvm.libzstd`.
+COPY scripts/build-zstd.sh /tmp/
+RUN ./build-zstd.sh
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV PGO_HOST=aarch64-unknown-linux-gnu
+ENV HOSTS=aarch64-unknown-linux-gnu
+
+ENV CPATH=/usr/include/aarch64-linux-gnu/:$CPATH
+
+ENV RUST_CONFIGURE_ARGS \
+      --build=aarch64-unknown-linux-gnu \
+      --enable-full-tools \
+      --enable-profiler \
+      --enable-sanitizers \
+      --enable-compiler-docs \
+      --set target.aarch64-unknown-linux-gnu.linker=clang \
+      --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
+      --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
+      --set llvm.link-shared=true \
+      --set llvm.thin-lto=true \
+      --set llvm.libzstd=true \
+      --set llvm.ninja=false \
+      --set rust.debug-assertions=false \
+      --set rust.jemalloc \
+      --set rust.use-lld=true \
+      --set rust.codegen-units=1
+
+ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS
+
+ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang
+ENV LIBCURL_NO_PKG_CONFIG 1
+ENV DIST_REQUIRE_ALL_TOOLS 1
diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile
deleted file mode 100644
index 18972387e34..00000000000
--- a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile
+++ /dev/null
@@ -1,32 +0,0 @@
-FROM ubuntu:22.04
-
-COPY scripts/cross-apt-packages.sh /scripts/
-RUN sh /scripts/cross-apt-packages.sh
-
-COPY scripts/crosstool-ng.sh /scripts/
-RUN sh /scripts/crosstool-ng.sh
-
-COPY scripts/rustbuild-setup.sh /scripts/
-RUN sh /scripts/rustbuild-setup.sh
-WORKDIR /tmp
-
-COPY scripts/crosstool-ng-build.sh /scripts/
-COPY host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig /tmp/crosstool.defconfig
-RUN /scripts/crosstool-ng-build.sh
-
-COPY scripts/sccache.sh /scripts/
-RUN sh /scripts/sccache.sh
-
-ENV PATH=$PATH:/x-tools/aarch64-unknown-linux-gnu/bin
-
-ENV CC_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-gcc \
-    AR_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-ar \
-    CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-g++
-
-ENV HOSTS=aarch64-unknown-linux-gnu
-
-ENV RUST_CONFIGURE_ARGS \
-      --enable-full-tools \
-      --enable-profiler \
-      --enable-sanitizers
-ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS
diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig b/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig
deleted file mode 100644
index 520b1667c8b..00000000000
--- a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig
+++ /dev/null
@@ -1,10 +0,0 @@
-CT_CONFIG_VERSION="4"
-CT_PREFIX_DIR="/x-tools/${CT_TARGET}"
-CT_USE_MIRROR=y
-CT_MIRROR_BASE_URL="https://ci-mirrors.rust-lang.org/rustc"
-CT_ARCH_ARM=y
-CT_ARCH_64=y
-CT_KERNEL_LINUX=y
-CT_LINUX_V_4_1=y
-CT_GLIBC_V_2_17=y
-CT_CC_LANG_CXX=y
diff --git a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile
index 7e946df6163..ea5b208a7c1 100644
--- a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile
@@ -19,6 +19,7 @@ RUN yum upgrade -y && \
       gcc \
       gcc-c++ \
       git \
+      binutils.i686 \
       glibc-devel.i686 \
       glibc-devel.x86_64 \
       libedit-devel \
@@ -46,11 +47,12 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
 RUN mkdir /home/user
-COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
+COPY scripts/shared.sh /tmp/
 
 # Need at least GCC 5.1 to compile LLVM nowadays
-COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
+COPY scripts/build-gcc.sh /tmp/
 ENV GCC_VERSION=9.5.0
+ENV GCC_BUILD_TARGET=i686-pc-linux-gnu
 RUN ./build-gcc.sh && yum remove -y gcc gcc-c++
 
 COPY scripts/cmake.sh /tmp/
@@ -58,7 +60,8 @@ RUN ./cmake.sh
 
 # Now build LLVM+Clang, afterwards configuring further compilations to use the
 # clang/clang++ compilers.
-COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
+COPY scripts/build-clang.sh /tmp/
+ENV LLVM_BUILD_TARGETS=X86
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile
index 9ef39189249..9d3be51d037 100644
--- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile
@@ -18,7 +18,7 @@ RUN /scripts/crosstool-ng-build.sh
 WORKDIR /build
 
 RUN apt-get install -y --no-install-recommends rpm2cpio cpio
-COPY host-x86_64/dist-powerpc64le-linux/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/
+COPY scripts/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/
 RUN ./build-powerpc64le-toolchain.sh
 
 COPY scripts/sccache.sh /scripts/
diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh
deleted file mode 100644
index dc86dddd464..00000000000
--- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  set -x
-}
diff --git a/src/ci/docker/host-x86_64/dist-riscv64-linux/patches/gcc/8.5.0/0002-hidden-jump-target.patch b/src/ci/docker/host-x86_64/dist-riscv64-linux/patches/gcc/8.5.0/0002-hidden-jump-target.patch
index 7ae4469428b..1ae0ecf6cb5 100644
--- a/src/ci/docker/host-x86_64/dist-riscv64-linux/patches/gcc/8.5.0/0002-hidden-jump-target.patch
+++ b/src/ci/docker/host-x86_64/dist-riscv64-linux/patches/gcc/8.5.0/0002-hidden-jump-target.patch
@@ -10,7 +10,7 @@ https://sourceware.org/bugzilla/show_bug.cgi?id=28509
 And this is the first version of the proposed binutils patch,
 https://sourceware.org/pipermail/binutils/2021-November/118398.html
 
-After applying the binutils patch, I get the the unexpected error when
+After applying the binutils patch, I get the unexpected error when
 building libgcc,
 
 /scratch/nelsonc/riscv-gnu-toolchain/riscv-gcc/libgcc/config/riscv/div.S:42:
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
index bbb4fe216a5..dde6fe7f6d0 100644
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
@@ -47,11 +47,12 @@ ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 # Clang needs to access GCC headers to enable linker plugin LTO
 WORKDIR /tmp
 RUN mkdir /home/user
-COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
+COPY scripts/shared.sh /tmp/
 
 # Need at least GCC 5.1 to compile LLVM nowadays
-COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
+COPY scripts/build-gcc.sh /tmp/
 ENV GCC_VERSION=9.5.0
+ENV GCC_BUILD_TARGET=x86_64-pc-linux-gnu
 RUN ./build-gcc.sh && yum remove -y gcc gcc-c++
 
 # LLVM 17 needs cmake 3.20 or higher.
@@ -60,12 +61,13 @@ RUN ./cmake.sh
 
 # Now build LLVM+Clang, afterwards configuring further compilations to use the
 # clang/clang++ compilers.
-COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
+COPY scripts/build-clang.sh /tmp/
+ENV LLVM_BUILD_TARGETS=X86
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
 # Build zstd to enable `llvm.libzstd`.
-COPY host-x86_64/dist-x86_64-linux/build-zstd.sh /tmp/
+COPY scripts/build-zstd.sh /tmp/
 RUN ./build-zstd.sh
 
 COPY scripts/sccache.sh /scripts/
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh
deleted file mode 100644
index dc86dddd464..00000000000
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  set -x
-}
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile
index e157debfd09..0a58f337d9d 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile
@@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.randomize-layout=true \
       --set rust.thin-lto-import-instr-limit=10
 
-COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
-COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
+COPY scripts/shared.sh /scripts/
+COPY scripts/build-gccjit.sh /scripts/
 
 RUN /scripts/build-gccjit.sh /scripts
 
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile
index e7016e7d3c0..092847cdfe0 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile
@@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.randomize-layout=true \
       --set rust.thin-lto-import-instr-limit=10
 
-COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
-COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
+COPY scripts/shared.sh /scripts/
+COPY scripts/build-gccjit.sh /scripts/
 
 RUN /scripts/build-gccjit.sh /scripts
 
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
index 2a09cd54b13..ab749b3fdd5 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@@ -89,8 +89,8 @@ ENV HOST_TARGET x86_64-unknown-linux-gnu
 # assertions enabled! Therefore, we cannot force download CI rustc.
 #ENV FORCE_CI_RUSTC 1
 
-COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
-COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
+COPY scripts/shared.sh /scripts/
+COPY scripts/build-gccjit.sh /scripts/
 
 RUN /scripts/build-gccjit.sh /scripts
 
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/scripts/build-clang.sh
index 3c8123d90de..841a0adb2ab 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh
+++ b/src/ci/docker/scripts/build-clang.sh
@@ -21,6 +21,12 @@ cd clang-build
 # include path, /rustroot/include, to clang's default include path.
 INC="/rustroot/include:/usr/include"
 
+GCC_PLUGIN_TARGET=$GCC_BUILD_TARGET
+# We build gcc for the i686 job on x86_64 so the plugin will end up under an x86_64 path
+if [[ $GCC_PLUGIN_TARGET == "i686-pc-linux-gnu" ]]; then
+  GCC_PLUGIN_TARGET=x86_64-pc-linux-gnu
+fi
+
 # We need compiler-rt for the profile runtime (used later to PGO the LLVM build)
 # but sanitizers aren't currently building. Since we don't need those, just
 # disable them. BOLT is used for optimizing LLVM.
@@ -34,12 +40,12 @@ hide_output \
       -DCOMPILER_RT_BUILD_XRAY=OFF \
       -DCOMPILER_RT_BUILD_MEMPROF=OFF \
       -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \
-      -DLLVM_TARGETS_TO_BUILD=X86 \
+      -DLLVM_TARGETS_TO_BUILD=$LLVM_BUILD_TARGETS \
       -DLLVM_INCLUDE_BENCHMARKS=OFF \
       -DLLVM_INCLUDE_TESTS=OFF \
       -DLLVM_INCLUDE_EXAMPLES=OFF \
       -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \
-      -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/plugin/include/" \
+      -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/$GCC_PLUGIN_TARGET/$GCC_VERSION/plugin/include/" \
       -DC_INCLUDE_DIRS="$INC"
 
 hide_output make -j$(nproc)
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/scripts/build-gcc.sh
index 57d4d338a50..11db5aa811c 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh
+++ b/src/ci/docker/scripts/build-gcc.sh
@@ -51,7 +51,9 @@ cd ..
 rm -rf gcc-build
 rm -rf gcc-$GCC
 
-# FIXME: clang doesn't find 32-bit libraries in /rustroot/lib,
-# but it does look all the way under /rustroot/lib/[...]/32,
-# so we can link stuff there to help it out.
-ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/
+if [[ $GCC_BUILD_TARGET == "i686-pc-linux-gnu" ]]; then
+    # FIXME: clang doesn't find 32-bit libraries in /rustroot/lib,
+    # but it does look all the way under /rustroot/lib/[...]/32,
+    # so we can link stuff there to help it out.
+    ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/
+fi
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh b/src/ci/docker/scripts/build-gccjit.sh
index c565922dcd1..c565922dcd1 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh
+++ b/src/ci/docker/scripts/build-gccjit.sh
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh
index a3d37ccc311..a3d37ccc311 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh
+++ b/src/ci/docker/scripts/build-zstd.sh