ci: Organise shared helper scripts

Move shared helper scripts used by Docker builds under docker/scripts.

17 files changed, 41 insertions, 200 deletions

diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
index c7778dbf6bb..8b0fb0710d4 100644
--- a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
+++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
@@ -43,10 +43,12 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
 RUN mkdir /home/user
-COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/
+COPY scripts/shared.sh /tmp/
 
 # Need at least GCC 5.1 to compile LLVM
-COPY host-aarch64/dist-aarch64-linux/build-gcc.sh /tmp/
+COPY scripts/build-gcc.sh /tmp/
+ENV GCC_VERSION=9.5.0
+ENV GCC_BUILD_TARGET=aarch64-unknown-linux-gnu
 RUN ./build-gcc.sh && yum remove -y gcc gcc-c++
 
 ENV CC=gcc CXX=g++
@@ -56,12 +58,13 @@ COPY scripts/cmake.sh /tmp/
 RUN ./cmake.sh
 
 # Build LLVM+Clang
-COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/
+COPY scripts/build-clang.sh /tmp/
+ENV LLVM_BUILD_TARGETS=AArch64
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
 # Build zstd to enable `llvm.libzstd`.
-COPY host-aarch64/dist-aarch64-linux/build-zstd.sh /tmp/
+COPY scripts/build-zstd.sh /tmp/
 RUN ./build-zstd.sh
 
 COPY scripts/sccache.sh /scripts/
@@ -82,7 +85,7 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
       --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
       --set llvm.link-shared=true \
-      --set llvm.thin-lto=false \
+      --set llvm.thin-lto=true \
       --set llvm.libzstd=true \
       --set llvm.ninja=false \
       --set rust.debug-assertions=false \
@@ -93,6 +96,5 @@ ENV RUST_CONFIGURE_ARGS \
 ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS
 
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang
-ENV DIST_SRC 1
 ENV LIBCURL_NO_PKG_CONFIG 1
 ENV DIST_REQUIRE_ALL_TOOLS 1
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh
deleted file mode 100755
index fb5f4d8c4f7..00000000000
--- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-source shared.sh
-
-# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh
-LLVM=llvmorg-19.1.5
-
-mkdir llvm-project
-cd llvm-project
-
-curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \
-  tar xzf - --strip-components=1
-
-mkdir clang-build
-cd clang-build
-
-# For whatever reason the default set of include paths for clang is different
-# than that of gcc. As a result we need to manually include our sysroot's
-# include path, /rustroot/include, to clang's default include path.
-INC="/rustroot/include:/usr/include"
-
-# We need compiler-rt for the profile runtime (used later to PGO the LLVM build)
-# but sanitizers aren't currently building. Since we don't need those, just
-# disable them. BOLT is used for optimizing LLVM.
-hide_output \
-    cmake ../llvm \
-      -DCMAKE_BUILD_TYPE=Release \
-      -DCMAKE_INSTALL_PREFIX=/rustroot \
-      -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
-      -DCOMPILER_RT_BUILD_XRAY=OFF \
-      -DCOMPILER_RT_BUILD_MEMPROF=OFF \
-      -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \
-      -DLLVM_TARGETS_TO_BUILD=AArch64 \
-      -DLLVM_INCLUDE_BENCHMARKS=OFF \
-      -DLLVM_INCLUDE_TESTS=OFF \
-      -DLLVM_INCLUDE_EXAMPLES=OFF \
-      -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \
-      -DC_INCLUDE_DIRS="$INC"
-
-hide_output make -j$(nproc)
-hide_output make install
-
-cd ../..
-rm -rf llvm-project
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh
deleted file mode 100755
index ad33b21b9b5..00000000000
--- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-set -ex
-
-source shared.sh
-
-# Note: in the future when bumping to version 10.1.0, also take care of the sed block below.
-GCC=9.5.0
-
-curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf -
-cd gcc-$GCC
-
-# FIXME(#49246): Remove the `sed` below.
-#
-# On 2018 March 21st, two Travis builders' cache for Docker are suddenly invalidated. Normally this
-# is fine, because we just need to rebuild the Docker image. However, it reveals a network issue:
-# downloading from `ftp://gcc.gnu.org/` from Travis (using passive mode) often leads to "Connection
-# timed out" error, and even when the download completed, the file is usually corrupted. This causes
-# nothing to be landed that day.
-#
-# We observed that the `gcc-4.8.5.tar.bz2` above can be downloaded successfully, so as a stability
-# improvement we try to download from the HTTPS mirror instead. Turns out this uncovered the third
-# bug: the host `gcc.gnu.org` and `cygwin.com` share the same IP, and the TLS certificate of the
-# latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server
-# instead here.
-#
-# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to
-# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or
-# the expression will need to be updated. That new URL is available at:
-# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35
-#
-sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites
-
-./contrib/download_prerequisites
-mkdir ../gcc-build
-cd ../gcc-build
-
-# '-fno-reorder-blocks-and-partition' is required to
-# enable BOLT optimization of the C++ standard library,
-# which is included in librustc_driver.so
-hide_output ../gcc-$GCC/configure \
-    --prefix=/rustroot \
-    --enable-languages=c,c++ \
-    --disable-gnu-unique-object \
-    --enable-cxx-flags='-fno-reorder-blocks-and-partition'
-hide_output make -j$(nproc)
-hide_output make install
-ln -s gcc /rustroot/bin/cc
-
-cd ..
-rm -rf gcc-build
-rm -rf gcc-$GCC
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh
deleted file mode 100644
index dc86dddd464..00000000000
--- a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  set -x
-}
diff --git a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile
index 7e946df6163..ea5b208a7c1 100644
--- a/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile
@@ -19,6 +19,7 @@ RUN yum upgrade -y && \
       gcc \
       gcc-c++ \
       git \
+      binutils.i686 \
       glibc-devel.i686 \
       glibc-devel.x86_64 \
       libedit-devel \
@@ -46,11 +47,12 @@ ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
 RUN mkdir /home/user
-COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
+COPY scripts/shared.sh /tmp/
 
 # Need at least GCC 5.1 to compile LLVM nowadays
-COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
+COPY scripts/build-gcc.sh /tmp/
 ENV GCC_VERSION=9.5.0
+ENV GCC_BUILD_TARGET=i686-pc-linux-gnu
 RUN ./build-gcc.sh && yum remove -y gcc gcc-c++
 
 COPY scripts/cmake.sh /tmp/
@@ -58,7 +60,8 @@ RUN ./cmake.sh
 
 # Now build LLVM+Clang, afterwards configuring further compilations to use the
 # clang/clang++ compilers.
-COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
+COPY scripts/build-clang.sh /tmp/
+ENV LLVM_BUILD_TARGETS=X86
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile
index 9ef39189249..9d3be51d037 100644
--- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/Dockerfile
@@ -18,7 +18,7 @@ RUN /scripts/crosstool-ng-build.sh
 WORKDIR /build
 
 RUN apt-get install -y --no-install-recommends rpm2cpio cpio
-COPY host-x86_64/dist-powerpc64le-linux/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/
+COPY scripts/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /build/
 RUN ./build-powerpc64le-toolchain.sh
 
 COPY scripts/sccache.sh /scripts/
diff --git a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh b/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh
deleted file mode 100644
index dc86dddd464..00000000000
--- a/src/ci/docker/host-x86_64/dist-powerpc64le-linux/shared.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  set -x
-}
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
index bbb4fe216a5..dde6fe7f6d0 100644
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
@@ -47,11 +47,12 @@ ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 # Clang needs to access GCC headers to enable linker plugin LTO
 WORKDIR /tmp
 RUN mkdir /home/user
-COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
+COPY scripts/shared.sh /tmp/
 
 # Need at least GCC 5.1 to compile LLVM nowadays
-COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
+COPY scripts/build-gcc.sh /tmp/
 ENV GCC_VERSION=9.5.0
+ENV GCC_BUILD_TARGET=x86_64-pc-linux-gnu
 RUN ./build-gcc.sh && yum remove -y gcc gcc-c++
 
 # LLVM 17 needs cmake 3.20 or higher.
@@ -60,12 +61,13 @@ RUN ./cmake.sh
 
 # Now build LLVM+Clang, afterwards configuring further compilations to use the
 # clang/clang++ compilers.
-COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
+COPY scripts/build-clang.sh /tmp/
+ENV LLVM_BUILD_TARGETS=X86
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
 # Build zstd to enable `llvm.libzstd`.
-COPY host-x86_64/dist-x86_64-linux/build-zstd.sh /tmp/
+COPY scripts/build-zstd.sh /tmp/
 RUN ./build-zstd.sh
 
 COPY scripts/sccache.sh /scripts/
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh
deleted file mode 100755
index a3d37ccc311..00000000000
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-zstd.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-set -ex
-
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/zstd_build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/zstd_build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  rm /tmp/zstd_build.log
-  set -x
-}
-
-ZSTD=1.5.6
-curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf -
-
-cd zstd-$ZSTD
-CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1
-hide_output make install
-
-cd ..
-rm -rf zstd-$ZSTD
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh
deleted file mode 100644
index dc86dddd464..00000000000
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/shared.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-hide_output() {
-  set +x
-  on_err="
-echo ERROR: An error was encountered with the build.
-cat /tmp/build.log
-exit 1
-"
-  trap "$on_err" ERR
-  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
-  PING_LOOP_PID=$!
-  "$@" &> /tmp/build.log
-  trap - ERR
-  kill $PING_LOOP_PID
-  set -x
-}
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile
index e157debfd09..0a58f337d9d 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile
@@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.randomize-layout=true \
       --set rust.thin-lto-import-instr-limit=10
 
-COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
-COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
+COPY scripts/shared.sh /scripts/
+COPY scripts/build-gccjit.sh /scripts/
 
 RUN /scripts/build-gccjit.sh /scripts
 
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile
index e7016e7d3c0..092847cdfe0 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile
@@ -54,8 +54,8 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.randomize-layout=true \
       --set rust.thin-lto-import-instr-limit=10
 
-COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
-COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
+COPY scripts/shared.sh /scripts/
+COPY scripts/build-gccjit.sh /scripts/
 
 RUN /scripts/build-gccjit.sh /scripts
 
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
index 2a09cd54b13..ab749b3fdd5 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@@ -89,8 +89,8 @@ ENV HOST_TARGET x86_64-unknown-linux-gnu
 # assertions enabled! Therefore, we cannot force download CI rustc.
 #ENV FORCE_CI_RUSTC 1
 
-COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
-COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/
+COPY scripts/shared.sh /scripts/
+COPY scripts/build-gccjit.sh /scripts/
 
 RUN /scripts/build-gccjit.sh /scripts
 
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/scripts/build-clang.sh
index 3c8123d90de..841a0adb2ab 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh
+++ b/src/ci/docker/scripts/build-clang.sh
@@ -21,6 +21,12 @@ cd clang-build
 # include path, /rustroot/include, to clang's default include path.
 INC="/rustroot/include:/usr/include"
 
+GCC_PLUGIN_TARGET=$GCC_BUILD_TARGET
+# We build gcc for the i686 job on x86_64 so the plugin will end up under an x86_64 path
+if [[ $GCC_PLUGIN_TARGET == "i686-pc-linux-gnu" ]]; then
+  GCC_PLUGIN_TARGET=x86_64-pc-linux-gnu
+fi
+
 # We need compiler-rt for the profile runtime (used later to PGO the LLVM build)
 # but sanitizers aren't currently building. Since we don't need those, just
 # disable them. BOLT is used for optimizing LLVM.
@@ -34,12 +40,12 @@ hide_output \
       -DCOMPILER_RT_BUILD_XRAY=OFF \
       -DCOMPILER_RT_BUILD_MEMPROF=OFF \
       -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \
-      -DLLVM_TARGETS_TO_BUILD=X86 \
+      -DLLVM_TARGETS_TO_BUILD=$LLVM_BUILD_TARGETS \
       -DLLVM_INCLUDE_BENCHMARKS=OFF \
       -DLLVM_INCLUDE_TESTS=OFF \
       -DLLVM_INCLUDE_EXAMPLES=OFF \
       -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \
-      -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC_VERSION/plugin/include/" \
+      -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/$GCC_PLUGIN_TARGET/$GCC_VERSION/plugin/include/" \
       -DC_INCLUDE_DIRS="$INC"
 
 hide_output make -j$(nproc)
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/scripts/build-gcc.sh
index 57d4d338a50..11db5aa811c 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh
+++ b/src/ci/docker/scripts/build-gcc.sh
@@ -51,7 +51,9 @@ cd ..
 rm -rf gcc-build
 rm -rf gcc-$GCC
 
-# FIXME: clang doesn't find 32-bit libraries in /rustroot/lib,
-# but it does look all the way under /rustroot/lib/[...]/32,
-# so we can link stuff there to help it out.
-ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/
+if [[ $GCC_BUILD_TARGET == "i686-pc-linux-gnu" ]]; then
+    # FIXME: clang doesn't find 32-bit libraries in /rustroot/lib,
+    # but it does look all the way under /rustroot/lib/[...]/32,
+    # so we can link stuff there to help it out.
+    ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/
+fi
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh b/src/ci/docker/scripts/build-gccjit.sh
index c565922dcd1..c565922dcd1 100755
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh
+++ b/src/ci/docker/scripts/build-gccjit.sh
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh
index a3d37ccc311..a3d37ccc311 100755
--- a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh
+++ b/src/ci/docker/scripts/build-zstd.sh