Auto merge of #142245 - marcoieni:split-gnu-tools, r=Kobzol

ci: split x86_64-gnu-tools job

try-job: x86_64-gnu-tools
try-job: x86_64-gnu-miri
try-job: aarch64-gnu

4 files changed, 125 insertions, 56 deletions

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile
new file mode 100644
index 00000000000..b937bc3e678
--- /dev/null
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-miri/Dockerfile
@@ -0,0 +1,57 @@
+FROM ghcr.io/rust-lang/ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  ninja-build \
+  file \
+  curl \
+  ca-certificates \
+  python3 \
+  git \
+  cmake \
+  libssl-dev \
+  sudo \
+  xz-utils \
+  tidy \
+  \
+  libc6 \
+  wget \
+  # libgccjit dependencies
+  flex \
+  libmpfr-dev \
+  libgmp-dev \
+  libmpc3 \
+  libmpc-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+# Fix rustc_codegen_gcc lto issues.
+ENV GCC_EXEC_PREFIX="/usr/lib/gcc/"
+
+COPY host-x86_64/x86_64-gnu-miri/check-miri.sh /tmp/
+
+ENV RUST_CONFIGURE_ARGS \
+  --build=x86_64-unknown-linux-gnu \
+  --enable-new-symbol-mangling
+
+ENV HOST_TARGET x86_64-unknown-linux-gnu
+
+# FIXME(#133381): currently rustc alt builds do *not* have rustc debug
+# assertions enabled! Therefore, we cannot force download CI rustc.
+#ENV FORCE_CI_RUSTC 1
+
+COPY scripts/shared.sh /scripts/
+
+# For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries
+# to create a new folder. For reference:
+# https://github.com/puppeteer/puppeteer/issues/375
+#
+# We also specify the version in case we need to update it to go around cache limitations.
+#
+# The `browser-ui-test.version` file is also used by bootstrap to emit warnings in case
+# the local version of the package is different than the one used by the CI.
+ENV SCRIPT /tmp/check-miri.sh ../x.py
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh b/src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh
new file mode 100755
index 00000000000..c2a5b308b32
--- /dev/null
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-miri/check-miri.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+# ignore-tidy-linelength
+
+set -eu
+set -x # so one can see where we are in the script
+
+X_PY="$1"
+
+# Testing Miri is a bit complicated.
+# We set the GC interval to the shortest possible value (0 would be off) to increase the chance
+# that bugs which only surface when the GC runs at a specific time are more likely to cause CI to fail.
+# This significantly increases the runtime of our test suite, or we'd do this in PR CI too.
+if [ -z "${PR_CI_JOB:-}" ]; then
+    MIRIFLAGS=-Zmiri-provenance-gc=1 python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri
+else
+    python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri
+fi
+# We re-run the test suite for a chance to find bugs in the intrinsic fallback bodies and in MIR
+# optimizations. This can miss UB, so we only run the "pass" tests. We need to enable debug
+# assertions as `-O` disables them but some tests rely on them. We also set a cfg flag so tests can
+# adjust their expectations if needed. This can change the output of the tests so we ignore that,
+# we only ensure that all assertions still pass.
+MIRIFLAGS="-Zmiri-force-intrinsic-fallback --cfg force_intrinsic_fallback -O -Zmir-opt-level=4 -Cdebug-assertions=yes" \
+  MIRI_SKIP_UI_CHECKS=1 \
+  python3 "$X_PY" test --stage 2 src/tools/miri -- tests/pass tests/panic
+# We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc.
+# Also cover some other targets via cross-testing, in particular all tier 1 targets.
+case $HOST_TARGET in
+  x86_64-unknown-linux-gnu)
+    # Only this branch runs in PR CI.
+    # Fully test all main OSes, and all main architectures.
+    python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target aarch64-apple-darwin
+    python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target i686-pc-windows-msvc
+    # Only run "pass" tests for the remaining targets, which is quite a bit faster.
+    python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass
+    ;;
+  x86_64-pc-windows-msvc)
+    # Strangely, Linux targets do not work here. cargo always says
+    # "error: cannot produce cdylib for ... as the target ... does not support these crate types".
+    # Only run "pass" tests, which is quite a bit faster.
+    #FIXME: Re-enable this once CI issues are fixed
+    # See <https://github.com/rust-lang/rust/issues/127883>
+    # For now, these tests are moved to `x86_64-msvc-ext2` in `src/ci/github-actions/jobs.yml`.
+    #python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin --test-args pass
+    ;;
+  *)
+    echo "FATAL: unexpected host $HOST_TARGET"
+    exit 1
+    ;;
+esac
+# Also smoke-test `x.py miri`. This doesn't run any actual tests (that would take too long),
+# but it ensures that the crates build properly when tested with Miri.
+
+#FIXME: Re-enable this for msvc once CI issues are fixed
+if [ "$HOST_TARGET" != "x86_64-pc-windows-msvc" ]; then
+  python3 "$X_PY" miri --stage 2 library/core --test-args notest
+  python3 "$X_PY" miri --stage 2 library/alloc --test-args notest
+  python3 "$X_PY" miri --stage 2 library/std --test-args notest
+fi
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh
index 62e0451814b..ff9fedad656 100755
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh
@@ -30,58 +30,3 @@ cat /tmp/toolstate/toolstates.json
 python3 "$X_PY" test --stage 2 check-tools
 python3 "$X_PY" test --stage 2 src/tools/clippy
 python3 "$X_PY" test --stage 2 src/tools/rustfmt
-
-# Testing Miri is a bit more complicated.
-# We set the GC interval to the shortest possible value (0 would be off) to increase the chance
-# that bugs which only surface when the GC runs at a specific time are more likely to cause CI to fail.
-# This significantly increases the runtime of our test suite, or we'd do this in PR CI too.
-if [ -z "${PR_CI_JOB:-}" ]; then
-    MIRIFLAGS=-Zmiri-provenance-gc=1 python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri
-else
-    python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri
-fi
-# We re-run the test suite for a chance to find bugs in the intrinsic fallback bodies and in MIR
-# optimizations. This can miss UB, so we only run the "pass" tests. We need to enable debug
-# assertions as `-O` disables them but some tests rely on them. We also set a cfg flag so tests can
-# adjust their expectations if needed. This can change the output of the tests so we ignore that,
-# we only ensure that all assertions still pass.
-MIRIFLAGS="-Zmiri-force-intrinsic-fallback --cfg force_intrinsic_fallback -O -Zmir-opt-level=4 -Cdebug-assertions=yes" \
-  MIRI_SKIP_UI_CHECKS=1 \
-  python3 "$X_PY" test --stage 2 src/tools/miri -- tests/pass tests/panic
-# We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc.
-# Also cover some other targets via cross-testing, in particular all tier 1 targets.
-case $HOST_TARGET in
-  x86_64-unknown-linux-gnu)
-    # Only this branch runs in PR CI.
-    # Fully test all main OSes, and all main architectures.
-    python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target aarch64-apple-darwin
-    python3 "$X_PY" test --stage 2 src/tools/miri src/tools/miri/cargo-miri --target i686-pc-windows-msvc
-    # Only run "pass" tests for the remaining targets, which is quite a bit faster.
-    python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass
-    python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass
-    python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass
-    python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass
-    ;;
-  x86_64-pc-windows-msvc)
-    # Strangely, Linux targets do not work here. cargo always says
-    # "error: cannot produce cdylib for ... as the target ... does not support these crate types".
-    # Only run "pass" tests, which is quite a bit faster.
-    #FIXME: Re-enable this once CI issues are fixed
-    # See <https://github.com/rust-lang/rust/issues/127883>
-    # For now, these tests are moved to `x86_64-msvc-ext2` in `src/ci/github-actions/jobs.yml`.
-    #python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin --test-args pass
-    ;;
-  *)
-    echo "FATAL: unexpected host $HOST_TARGET"
-    exit 1
-    ;;
-esac
-# Also smoke-test `x.py miri`. This doesn't run any actual tests (that would take too long),
-# but it ensures that the crates build properly when tested with Miri.
-
-#FIXME: Re-enable this for msvc once CI issues are fixed
-if [ "$HOST_TARGET" != "x86_64-pc-windows-msvc" ]; then
-  python3 "$X_PY" miri --stage 2 library/core --test-args notest
-  python3 "$X_PY" miri --stage 2 library/alloc --test-args notest
-  python3 "$X_PY" miri --stage 2 library/std --test-args notest
-fi
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index e996a8f7139..3aa435003d3 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -150,7 +150,9 @@ pr:
       DOCKER_SCRIPT: stage_2_test_set2.sh
     <<: *job-aarch64-linux
   - name: x86_64-gnu-tools
-    <<: *job-linux-36c-codebuild
+    <<: *job-linux-4c
+  - name: x86_64-gnu-miri
+    <<: *job-linux-4c
 
 # Jobs that run when you perform a try build (@bors try)
 # These jobs automatically inherit envs.try, to avoid repeating
@@ -425,6 +427,9 @@ auto:
       DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
     <<: *job-linux-4c
 
+  - name: x86_64-gnu-miri
+    <<: *job-linux-4c
+
   ####################
   #  macOS Builders  #
   ####################