Rollup merge of #131831 - onur-ozkan:improve-rustc-if-unchanged-logic, r=Mark-Simulacrum

extend the "if-unchanged" logic for compiler builds

Implements the first item from [this tracking issue](https://github.com/rust-lang/rust/issues/131744).

In short, we want to make "if-unchanged" logic to check for changes outside of certain allowed directories, and this PR implements that.

See https://github.com/rust-lang/rust/issues/131658 for more context.

26 files changed, 70 insertions, 37 deletions

diff --git a/Cargo.toml b/Cargo.toml
index e1d667bf015..b773030b4ca 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,12 +2,12 @@
 resolver = "2"
 members = [
   "compiler/rustc",
+  "src/build_helper",
   "src/etc/test-float-parse",
   "src/rustc-std-workspace/rustc-std-workspace-core",
   "src/rustc-std-workspace/rustc-std-workspace-alloc",
   "src/rustc-std-workspace/rustc-std-workspace-std",
   "src/rustdoc-json-types",
-  "src/tools/build_helper",
   "src/tools/cargotest",
   "src/tools/clippy",
   "src/tools/clippy/clippy_dev",
diff --git a/config.example.toml b/config.example.toml
index cd7ec6a05bc..62c41fee31b 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -500,8 +500,9 @@
 # This is useful if you are working on tools, doc-comments, or library (you will be able to build
 # the standard library without needing to build the compiler).
 #
-# Set this to "if-unchanged" to only download if the compiler (and library if running on CI) have
-# not been modified.
+# Set this to "if-unchanged" if you are working on `src/tools`, `tests` or `library` (on CI, `library`
+# changes triggers in-tree compiler build) to speed up the build process.
+#
 # Set this to `true` to download unconditionally.
 #download-rustc = false
 
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
index ba505089a00..7950f1004a2 100644
--- a/src/bootstrap/Cargo.toml
+++ b/src/bootstrap/Cargo.toml
@@ -40,7 +40,7 @@ test = false
 cc = "=1.1.22"
 cmake = "=0.1.48"
 
-build_helper = { path = "../tools/build_helper" }
+build_helper = { path = "../build_helper" }
 clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] }
 clap_complete = "4.4"
 fd-lock = "4.0"
diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs
index 3d4b89a363e..6fb37e8cfc4 100644
--- a/src/bootstrap/src/core/build_steps/clippy.rs
+++ b/src/bootstrap/src/core/build_steps/clippy.rs
@@ -295,7 +295,7 @@ macro_rules! lint_any {
 
 lint_any!(
     Bootstrap, "src/bootstrap", "bootstrap";
-    BuildHelper, "src/tools/build_helper", "build_helper";
+    BuildHelper, "src/build_helper", "build_helper";
     BuildManifest, "src/tools/build-manifest", "build-manifest";
     CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri";
     Clippy, "src/tools/clippy", "clippy";
diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
index 69ec832a44a..8a9321f8e79 100644
--- a/src/bootstrap/src/core/build_steps/doc.rs
+++ b/src/bootstrap/src/core/build_steps/doc.rs
@@ -1030,7 +1030,7 @@ macro_rules! tool_doc {
 // NOTE: make sure to register these in `Builder::get_step_description`.
 tool_doc!(
     BuildHelper,
-    "src/tools/build_helper",
+    "src/build_helper",
     rustc_tool = false,
     is_library = true,
     crates = ["build_helper"]
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index 2c36d8bab82..532c8f767eb 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -1354,7 +1354,7 @@ impl Step for CrateBuildHelper {
     const ONLY_HOSTS: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/build_helper")
+        run.path("src/build_helper")
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -1372,7 +1372,7 @@ impl Step for CrateBuildHelper {
             Mode::ToolBootstrap,
             host,
             Kind::Test,
-            "src/tools/build_helper",
+            "src/build_helper",
             SourceType::InTree,
             &[],
         );
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index f977c285a74..8afabda1403 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -28,6 +28,24 @@ use crate::utils::cache::{INTERNER, Interned};
 use crate::utils::channel::{self, GitInfo};
 use crate::utils::helpers::{self, exe, output, t};
 
+/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic.
+/// This means they can be modified and changes to these paths should never trigger a compiler build
+/// when "if-unchanged" is set.
+///
+/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during
+/// the diff check.
+///
+/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build
+/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results.
+/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the
+/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources.
+#[rustfmt::skip] // We don't want rustfmt to oneline this list
+pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[
+    ":!src/tools",
+    ":!tests",
+    ":!triagebot.toml",
+];
+
 macro_rules! check_ci_llvm {
     ($name:expr) => {
         assert!(
@@ -2768,32 +2786,33 @@ impl Config {
             }
         };
 
-        let mut files_to_track = vec!["compiler", "src/version", "src/stage0", "src/ci/channel"];
+        // RUSTC_IF_UNCHANGED_ALLOWED_PATHS
+        let mut allowed_paths = RUSTC_IF_UNCHANGED_ALLOWED_PATHS.to_vec();
 
-        // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, ignore
+        // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, allow
         // these changes to speed up the build process for library developers. This provides consistent
         // functionality for library developers between `download-rustc=true` and `download-rustc="if-unchanged"`
         // options.
-        if CiEnv::is_ci() {
-            files_to_track.push("library");
+        if !CiEnv::is_ci() {
+            allowed_paths.push(":!library");
         }
 
         // Look for a version to compare to based on the current commit.
         // Only commits merged by bors will have CI artifacts.
-        let commit =
-            match self.last_modified_commit(&files_to_track, "download-rustc", if_unchanged) {
-                Some(commit) => commit,
-                None => {
-                    if if_unchanged {
-                        return None;
-                    }
-                    println!("ERROR: could not find commit hash for downloading rustc");
-                    println!("HELP: maybe your repository history is too shallow?");
-                    println!("HELP: consider disabling `download-rustc`");
-                    println!("HELP: or fetch enough history to include one upstream commit");
-                    crate::exit!(1);
+        let commit = match self.last_modified_commit(&allowed_paths, "download-rustc", if_unchanged)
+        {
+            Some(commit) => commit,
+            None => {
+                if if_unchanged {
+                    return None;
                 }
-            };
+                println!("ERROR: could not find commit hash for downloading rustc");
+                println!("HELP: maybe your repository history is too shallow?");
+                println!("HELP: consider setting `rust.download-rustc=false` in config.toml");
+                println!("HELP: or fetch enough history to include one upstream commit");
+                crate::exit!(1);
+            }
+        };
 
         if CiEnv::is_ci() && {
             let head_sha =
diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs
index e4ce64e2bc1..014555296d0 100644
--- a/src/bootstrap/src/core/config/tests.rs
+++ b/src/bootstrap/src/core/config/tests.rs
@@ -8,7 +8,7 @@ use clap::CommandFactory;
 use serde::Deserialize;
 
 use super::flags::Flags;
-use super::{ChangeIdWrapper, Config};
+use super::{ChangeIdWrapper, Config, RUSTC_IF_UNCHANGED_ALLOWED_PATHS};
 use crate::core::build_steps::clippy::get_clippy_rules_in_order;
 use crate::core::build_steps::llvm;
 use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig};
@@ -427,3 +427,18 @@ fn jobs_precedence() {
     );
     assert_eq!(config.jobs, Some(123));
 }
+
+#[test]
+fn check_rustc_if_unchanged_paths() {
+    let config = parse("");
+    let normalised_allowed_paths: Vec<_> = RUSTC_IF_UNCHANGED_ALLOWED_PATHS
+        .iter()
+        .map(|t| {
+            t.strip_prefix(":!").expect(&format!("{t} doesn't have ':!' prefix, but it should."))
+        })
+        .collect();
+
+    for p in normalised_allowed_paths {
+        assert!(config.src.join(p).exists(), "{p} doesn't exist.");
+    }
+}
diff --git a/src/tools/build_helper/Cargo.toml b/src/build_helper/Cargo.toml
index 66894e1abc4..66894e1abc4 100644
--- a/src/tools/build_helper/Cargo.toml
+++ b/src/build_helper/Cargo.toml
diff --git a/src/tools/build_helper/README.md b/src/build_helper/README.md
index f81b631c3fd..f81b631c3fd 100644
--- a/src/tools/build_helper/README.md
+++ b/src/build_helper/README.md
diff --git a/src/tools/build_helper/src/ci.rs b/src/build_helper/src/ci.rs
index 60f319129a0..60f319129a0 100644
--- a/src/tools/build_helper/src/ci.rs
+++ b/src/build_helper/src/ci.rs
diff --git a/src/tools/build_helper/src/drop_bomb/mod.rs b/src/build_helper/src/drop_bomb/mod.rs
index 0a5bb04b55b..0a5bb04b55b 100644
--- a/src/tools/build_helper/src/drop_bomb/mod.rs
+++ b/src/build_helper/src/drop_bomb/mod.rs
diff --git a/src/tools/build_helper/src/drop_bomb/tests.rs b/src/build_helper/src/drop_bomb/tests.rs
index 4a488c0f670..4a488c0f670 100644
--- a/src/tools/build_helper/src/drop_bomb/tests.rs
+++ b/src/build_helper/src/drop_bomb/tests.rs
diff --git a/src/tools/build_helper/src/git.rs b/src/build_helper/src/git.rs
index 2aad5650fa8..2aad5650fa8 100644
--- a/src/tools/build_helper/src/git.rs
+++ b/src/build_helper/src/git.rs
diff --git a/src/tools/build_helper/src/lib.rs b/src/build_helper/src/lib.rs
index 4a4f0ca2a9d..4a4f0ca2a9d 100644
--- a/src/tools/build_helper/src/lib.rs
+++ b/src/build_helper/src/lib.rs
diff --git a/src/tools/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs
index 2d0c66a8f33..2d0c66a8f33 100644
--- a/src/tools/build_helper/src/metrics.rs
+++ b/src/build_helper/src/metrics.rs
diff --git a/src/tools/build_helper/src/stage0_parser.rs b/src/build_helper/src/stage0_parser.rs
index ff05b116989..2a0c12a1c91 100644
--- a/src/tools/build_helper/src/stage0_parser.rs
+++ b/src/build_helper/src/stage0_parser.rs
@@ -25,7 +25,7 @@ pub struct Stage0Config {
 }
 
 pub fn parse_stage0_file() -> Stage0 {
-    let stage0_content = include_str!("../../../stage0");
+    let stage0_content = include_str!("../../stage0");
 
     let mut stage0 = Stage0::default();
     for line in stage0_content.lines() {
diff --git a/src/tools/build_helper/src/util.rs b/src/build_helper/src/util.rs
index 72c05c4c48a..72c05c4c48a 100644
--- a/src/tools/build_helper/src/util.rs
+++ b/src/build_helper/src/util.rs
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 9f39ad9c55c..8e2f525db68 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -179,9 +179,7 @@ else
   fi
 
   if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then
-    # disabled for now, see https://github.com/rust-lang/rust/issues/131658
-    #RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
-    true
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
   fi
 fi
 
diff --git a/src/tools/bump-stage0/Cargo.toml b/src/tools/bump-stage0/Cargo.toml
index de5d821133d..6ee7a831839 100644
--- a/src/tools/bump-stage0/Cargo.toml
+++ b/src/tools/bump-stage0/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.34"
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 curl = "0.4.38"
 indexmap = { version = "2.0.0", features = ["serde"] }
 serde = { version = "1.0.125", features = ["derive"] }
diff --git a/src/tools/compiletest/Cargo.toml b/src/tools/compiletest/Cargo.toml
index cef6b525a7b..b784bdb7139 100644
--- a/src/tools/compiletest/Cargo.toml
+++ b/src/tools/compiletest/Cargo.toml
@@ -14,7 +14,7 @@ unified-diff = "0.2.1"
 getopts = "0.2"
 indexmap = "2.0.0"
 miropt-test-tools = { path = "../miropt-test-tools" }
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", default-features = false, features = ["fmt", "env-filter", "smallvec", "parking_lot", "ansi"] }
 regex = "1.0"
diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml
index d34f8ad0520..d0413911014 100644
--- a/src/tools/opt-dist/Cargo.toml
+++ b/src/tools/opt-dist/Cargo.toml
@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 env_logger = "0.11"
 log = "0.4"
 anyhow = { version = "1", features = ["backtrace"] }
diff --git a/src/tools/run-make-support/Cargo.toml b/src/tools/run-make-support/Cargo.toml
index 3affa199fa5..3c172b2d956 100644
--- a/src/tools/run-make-support/Cargo.toml
+++ b/src/tools/run-make-support/Cargo.toml
@@ -10,7 +10,7 @@ similar = "2.5.0"
 wasmparser = { version = "0.216", default-features = false, features = ["std"] }
 regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace
 gimli = "0.31.0"
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 serde_json = "1.0"
 libc = "0.2"
 
diff --git a/src/tools/rustdoc-gui-test/Cargo.toml b/src/tools/rustdoc-gui-test/Cargo.toml
index 4cb200ebc7c..f7384a98f85 100644
--- a/src/tools/rustdoc-gui-test/Cargo.toml
+++ b/src/tools/rustdoc-gui-test/Cargo.toml
@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 compiletest = { path = "../compiletest" }
 getopts = "0.2"
 walkdir = "2"
diff --git a/src/tools/suggest-tests/Cargo.toml b/src/tools/suggest-tests/Cargo.toml
index 7c048d53a50..d6f86078d7e 100644
--- a/src/tools/suggest-tests/Cargo.toml
+++ b/src/tools/suggest-tests/Cargo.toml
@@ -5,4 +5,4 @@ edition = "2021"
 
 [dependencies]
 glob = "0.3.0"
-build_helper = { version = "0.1.0", path = "../build_helper" }
+build_helper = { version = "0.1.0", path = "../../build_helper" }
diff --git a/src/tools/tidy/Cargo.toml b/src/tools/tidy/Cargo.toml
index 42e608ff5ce..bc75787fb1a 100644
--- a/src/tools/tidy/Cargo.toml
+++ b/src/tools/tidy/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2021"
 autobins = false
 
 [dependencies]
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 cargo_metadata = "0.18"
 regex = "1"
 miropt-test-tools = { path = "../miropt-test-tools" }