Auto merge of #124360 - matthiaskrgr:rollup-k6bffhd, r=matthiaskrgr

Rollup of 3 pull requests

Successful merges:

 - #124257 (Rewrite the `no-input-file.stderr` test in Rust and support diff)
 - #124324 (Minor AST cleanups)
 - #124327 (CI: implement job skipping in Python matrix calculation)

r? `@ghost`
`@rustbot` modify labels: rollup

10 files changed, 159 insertions, 70 deletions

diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
index c24cefa8d89..124c22bd979 100755
--- a/src/ci/github-actions/calculate-job-matrix.py
+++ b/src/ci/github-actions/calculate-job-matrix.py
@@ -17,10 +17,13 @@ from typing import List, Dict, Any, Optional
 
 import yaml
 
+CI_DIR = Path(__file__).absolute().parent.parent
 JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
 
+Job = Dict[str, Any]
 
-def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+
+def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]:
     """
     Add a `name` attribute to each job, based on its image and the given `prefix`.
     """
@@ -29,7 +32,7 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
     return jobs
 
 
-def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]:
+def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]:
     """
     Prepends `environment` to the `env` attribute of each job.
     The `env` of each job has higher precedence than `environment`.
@@ -77,7 +80,7 @@ def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
     return None
 
 
-def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]:
     if job_type == JobType.PR:
         return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
     elif job_type == JobType.Try:
@@ -88,6 +91,13 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str
     return []
 
 
+def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]:
+    """
+    Skip CI jobs that are not supposed to be executed on the given `channel`.
+    """
+    return [j for j in jobs if j.get("only_on_channel", channel) == channel]
+
+
 def get_github_ctx() -> GitHubCtx:
     return GitHubCtx(
         event_name=os.environ["GITHUB_EVENT_NAME"],
@@ -107,9 +117,13 @@ if __name__ == "__main__":
     job_type = find_job_type(github_ctx)
     logging.info(f"Job type: {job_type}")
 
+    with open(CI_DIR / "channel") as f:
+        channel = f.read().strip()
+
     jobs = []
     if job_type is not None:
         jobs = calculate_jobs(job_type, data)
+    jobs = skip_jobs(jobs, channel)
 
     logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}")
     print(f"jobs={json.dumps(jobs)}")
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 19d6b517552..bc4b1b815cf 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -107,9 +107,6 @@ x--expand-yaml-anchors--remove:
   - &job-aarch64-linux
     os: [self-hosted, ARM64, linux]
 
-  - &step
-    if: success() && !env.SKIP_JOB
-
   - &base-ci-job
     defaults:
       run:
@@ -151,7 +148,7 @@ x--expand-yaml-anchors--remove:
         run: echo "[CI_PR_NUMBER=$num]"
         env:
           num: ${{ github.event.number }}
-        if: success() && !env.SKIP_JOB && github.event_name == 'pull_request'
+        if: success() && github.event_name == 'pull_request'
 
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
@@ -161,71 +158,51 @@ x--expand-yaml-anchors--remove:
           # are passed to the `setup-environment.sh` script encoded in JSON,
           # which then uses log commands to actually set them.
           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
-        <<: *step
-
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        <<: *step
 
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
-        <<: *step
 
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
-        <<: *step
 
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
-        <<: *step
 
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
-        <<: *step
 
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
-        <<: *step
 
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
-        <<: *step
 
       - name: install clang
         run: src/ci/scripts/install-clang.sh
-        <<: *step
 
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
-        <<: *step
 
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
-        <<: *step
 
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        <<: *step
 
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
-        <<: *step
 
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
-        <<: *step
 
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
-        <<: *step
 
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
-        <<: *step
 
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
-        <<: *step
 
       # Disable automatic line ending conversion (again). On Windows, when we're
       # installing dependencies, something switches the git configuration directory or
@@ -234,19 +211,15 @@ x--expand-yaml-anchors--remove:
       # appropriate line endings.
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        <<: *step
 
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
-        <<: *step
 
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
-        <<: *step
 
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
-        <<: *step
 
       - name: run the build
         # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
@@ -255,11 +228,9 @@ x--expand-yaml-anchors--remove:
           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-        <<: *step
 
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
-        <<: *step
 
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
@@ -269,7 +240,6 @@ x--expand-yaml-anchors--remove:
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
-        <<: *step
 
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
@@ -281,8 +251,7 @@ x--expand-yaml-anchors--remove:
         # adding the condition is helpful as this way CI will not silently skip
         # deploying artifacts from a dist builder if the variables are misconfigured,
         # erroring about invalid credentials instead.
-        if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
-        <<: *step
+        if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
 
   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
   # Check out their documentation for more information on why they're needed.
@@ -399,7 +368,6 @@ jobs:
         shell: bash
         env:
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-        <<: *step
 
   # These jobs don't actually test anything, but they're used to tell bors the
   # build completed, as there is no practical way to detect when a workflow is
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index ec58bd0924e..df6c5d6079d 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -79,7 +79,7 @@ pr:
     <<: *job-linux-16c
 
 # Jobs that run when you perform a try build (@bors try)
-# These jobs automatically inherit envs.production, to avoid repeating
+# These jobs automatically inherit envs.try, to avoid repeating
 # it in each job definition.
 try:
   - image: dist-x86_64-linux
@@ -88,7 +88,7 @@ try:
     <<: *job-linux-16c
 
 # Main CI jobs that have to be green to merge a commit into master
-# These jobs automatically inherit envs.production, to avoid repeating
+# These jobs automatically inherit envs.auto, to avoid repeating
 # it in each job definition.
 auto:
   #############################
@@ -200,24 +200,23 @@ auto:
   # channel name on the output), and this builder prevents landing
   # changes that would result in broken builds after a promotion.
   - image: x86_64-gnu-stable
+    # Only run this job on the nightly channel. Running this on beta
+    # could cause failures when `dev: 1` in `stage0.txt`, and running
+    # this on stable is useless.
+    only_on_channel: nightly
     env:
       IMAGE: x86_64-gnu
       RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
-      # Only run this job on the nightly channel. Running this on beta
-      # could cause failures when `dev: 1` in `stage0.txt`, and running
-      # this on stable is useless.
-      CI_ONLY_WHEN_CHANNEL: nightly
     <<: *job-linux-4c
 
   - image: x86_64-gnu-aux
     <<: *job-linux-4c
 
   - image: x86_64-gnu-integration
-    env:
-      # Only run this job on the nightly channel. Fuchsia requires
-      # nightly features to compile, and this job would fail if
-      # executed on beta and stable.
-      CI_ONLY_WHEN_CHANNEL: nightly
+    # Only run this job on the nightly channel. Fuchsia requires
+    # nightly features to compile, and this job would fail if
+    # executed on beta and stable.
+    only_on_channel: nightly
     <<: *job-linux-8c
 
   - image: x86_64-gnu-debug
diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh
deleted file mode 100755
index 48127166ad0..00000000000
--- a/src/ci/scripts/should-skip-this.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-# Set the SKIP_JOB environment variable if this job is not supposed to run on the current builder.
-
-set -euo pipefail
-IFS=$'\n\t'
-
-source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
-
-if [[ -n "${CI_ONLY_WHEN_CHANNEL-}" ]]; then
-    if [[ "${CI_ONLY_WHEN_CHANNEL}" = "$(cat src/ci/channel)" ]]; then
-        echo "The channel is the expected one"
-    else
-        echo "Not executing this job as the channel is not the expected one"
-        ciCommandSetEnv SKIP_JOB 1
-        exit 0
-    fi
-fi
-
-
-echo "Executing the job since there is no skip rule preventing the execution"
-exit 0
diff --git a/src/tools/run-make-support/Cargo.toml b/src/tools/run-make-support/Cargo.toml
index 3ea35c7940c..61a24c97e77 100644
--- a/src/tools/run-make-support/Cargo.toml
+++ b/src/tools/run-make-support/Cargo.toml
@@ -5,5 +5,6 @@ edition = "2021"
 
 [dependencies]
 object = "0.34.0"
+similar = "2.5.0"
 wasmparser = "0.118.2"
 regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace
diff --git a/src/tools/run-make-support/src/diff/mod.rs b/src/tools/run-make-support/src/diff/mod.rs
new file mode 100644
index 00000000000..54532c6e35b
--- /dev/null
+++ b/src/tools/run-make-support/src/diff/mod.rs
@@ -0,0 +1,81 @@
+use similar::TextDiff;
+use std::path::Path;
+
+#[cfg(test)]
+mod tests;
+
+pub fn diff() -> Diff {
+    Diff::new()
+}
+
+#[derive(Debug)]
+pub struct Diff {
+    expected: Option<String>,
+    expected_name: Option<String>,
+    actual: Option<String>,
+    actual_name: Option<String>,
+}
+
+impl Diff {
+    /// Construct a bare `diff` invocation.
+    pub fn new() -> Self {
+        Self { expected: None, expected_name: None, actual: None, actual_name: None }
+    }
+
+    /// Specify the expected output for the diff from a file.
+    pub fn expected_file<P: AsRef<Path>>(&mut self, path: P) -> &mut Self {
+        let path = path.as_ref();
+        let content = std::fs::read_to_string(path).expect("failed to read file");
+        let name = path.to_string_lossy().to_string();
+
+        self.expected = Some(content);
+        self.expected_name = Some(name);
+        self
+    }
+
+    /// Specify the expected output for the diff from a given text string.
+    pub fn expected_text<T: AsRef<[u8]>>(&mut self, name: &str, text: T) -> &mut Self {
+        self.expected = Some(String::from_utf8_lossy(text.as_ref()).to_string());
+        self.expected_name = Some(name.to_string());
+        self
+    }
+
+    /// Specify the actual output for the diff from a file.
+    pub fn actual_file<P: AsRef<Path>>(&mut self, path: P) -> &mut Self {
+        let path = path.as_ref();
+        let content = std::fs::read_to_string(path).expect("failed to read file");
+        let name = path.to_string_lossy().to_string();
+
+        self.actual = Some(content);
+        self.actual_name = Some(name);
+        self
+    }
+
+    /// Specify the actual output for the diff from a given text string.
+    pub fn actual_text<T: AsRef<[u8]>>(&mut self, name: &str, text: T) -> &mut Self {
+        self.actual = Some(String::from_utf8_lossy(text.as_ref()).to_string());
+        self.actual_name = Some(name.to_string());
+        self
+    }
+
+    /// Executes the diff process, prints any differences to the standard error.
+    #[track_caller]
+    pub fn run(&self) {
+        let expected = self.expected.as_ref().expect("expected text not set");
+        let actual = self.actual.as_ref().expect("actual text not set");
+        let expected_name = self.expected_name.as_ref().unwrap();
+        let actual_name = self.actual_name.as_ref().unwrap();
+
+        let output = TextDiff::from_lines(expected, actual)
+            .unified_diff()
+            .header(expected_name, actual_name)
+            .to_string();
+
+        if !output.is_empty() {
+            panic!(
+                "test failed: `{}` is different from `{}`\n\n{}",
+                expected_name, actual_name, output
+            )
+        }
+    }
+}
diff --git a/src/tools/run-make-support/src/diff/tests.rs b/src/tools/run-make-support/src/diff/tests.rs
new file mode 100644
index 00000000000..e6d72544b7e
--- /dev/null
+++ b/src/tools/run-make-support/src/diff/tests.rs
@@ -0,0 +1,39 @@
+#[cfg(test)]
+mod tests {
+    use crate::*;
+
+    #[test]
+    fn test_diff() {
+        let expected = "foo\nbar\nbaz\n";
+        let actual = "foo\nbar\nbaz\n";
+        diff().expected_text("EXPECTED_TEXT", expected).actual_text("ACTUAL_TEXT", actual).run();
+    }
+
+    #[test]
+    fn test_should_panic() {
+        let expected = "foo\nbar\nbaz\n";
+        let actual = "foo\nbaz\nbar\n";
+
+        let output = std::panic::catch_unwind(|| {
+            diff()
+                .expected_text("EXPECTED_TEXT", expected)
+                .actual_text("ACTUAL_TEXT", actual)
+                .run();
+        })
+        .unwrap_err();
+
+        let expected_output = "\
+test failed: `EXPECTED_TEXT` is different from `ACTUAL_TEXT`
+
+--- EXPECTED_TEXT
++++ ACTUAL_TEXT
+@@ -1,3 +1,3 @@
+ foo
++baz
+ bar
+-baz
+";
+
+        assert_eq!(output.downcast_ref::<String>().unwrap(), expected_output);
+    }
+}
diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs
index e723e824ed6..76e8838d27c 100644
--- a/src/tools/run-make-support/src/lib.rs
+++ b/src/tools/run-make-support/src/lib.rs
@@ -5,6 +5,7 @@
 
 pub mod cc;
 pub mod clang;
+pub mod diff;
 pub mod llvm_readobj;
 pub mod run;
 pub mod rustc;
@@ -20,6 +21,7 @@ pub use wasmparser;
 
 pub use cc::{cc, extra_c_flags, extra_cxx_flags, Cc};
 pub use clang::{clang, Clang};
+pub use diff::{diff, Diff};
 pub use llvm_readobj::{llvm_readobj, LlvmReadobj};
 pub use run::{run, run_fail};
 pub use rustc::{aux_build, rustc, Rustc};
diff --git a/src/tools/run-make-support/src/rustc.rs b/src/tools/run-make-support/src/rustc.rs
index 9bf41c6e2e9..ddaae3236c2 100644
--- a/src/tools/run-make-support/src/rustc.rs
+++ b/src/tools/run-make-support/src/rustc.rs
@@ -148,6 +148,13 @@ impl Rustc {
         self
     }
 
+    /// Specify the print request.
+    pub fn print(&mut self, request: &str) -> &mut Self {
+        self.cmd.arg("--print");
+        self.cmd.arg(request);
+        self
+    }
+
     /// Add an extra argument to the linker invocation, via `-Clink-arg`.
     pub fn link_arg(&mut self, link_arg: &str) -> &mut Self {
         self.cmd.arg(format!("-Clink-arg={link_arg}"));
diff --git a/src/tools/tidy/src/allowed_run_make_makefiles.txt b/src/tools/tidy/src/allowed_run_make_makefiles.txt
index 93188b4fbae..9b3c0d0f1a5 100644
--- a/src/tools/tidy/src/allowed_run_make_makefiles.txt
+++ b/src/tools/tidy/src/allowed_run_make_makefiles.txt
@@ -189,7 +189,6 @@ run-make/no-builtins-attribute/Makefile
 run-make/no-builtins-lto/Makefile
 run-make/no-cdylib-as-rdylib/Makefile
 run-make/no-duplicate-libs/Makefile
-run-make/no-input-file/Makefile
 run-make/no-intermediate-extras/Makefile
 run-make/obey-crate-type-flag/Makefile
 run-make/optimization-remarks-dir-pgo/Makefile