Auto merge of #124456 - matthiaskrgr:rollup-8bm3m6v, r=matthiaskrgr

Rollup of 7 pull requests

Successful merges:

 - #123248 (1.78 release notes)
 - #123942 (`x vendor`)
 - #124165 (add test for incremental ICE: slice-pattern-const.rs #83085)
 - #124242 (bootstrap: Describe build_steps modules)
 - #124406 (Remove unused `[patch]` for clippy_lints)
 - #124429 (bootstrap: Document `struct Builder` and its fields)
 - #124447 (Unconditionally call `really_init` on GNU/Linux)

r? `@ghost`
`@rustbot` modify labels: rollup

12 files changed, 149 insertions, 17 deletions

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 39add60e705..e464e444fea 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -1035,14 +1035,8 @@ class RustBuild(object):
         if self.use_vendored_sources:
             vendor_dir = os.path.join(self.rust_root, 'vendor')
             if not os.path.exists(vendor_dir):
-                sync_dirs = "--sync ./src/tools/cargo/Cargo.toml " \
-                            "--sync ./src/tools/rust-analyzer/Cargo.toml " \
-                            "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \
-                            "--sync ./compiler/rustc_codegen_gcc/Cargo.toml " \
-                            "--sync ./src/bootstrap/Cargo.toml "
                 eprint('ERROR: vendoring required, but vendor directory does not exist.')
-                eprint('       Run `cargo vendor {}` to initialize the '
-                      'vendor directory.'.format(sync_dirs))
+                eprint('       Run `x.py vendor` to initialize the vendor directory.')
                 eprint('       Alternatively, use the pre-vendored `rustc-src` dist component.')
                 eprint('       To get a stable/beta/nightly version, download it from: ')
                 eprint('       '
diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs
index 5bcaeed7faa..a81d6403013 100644
--- a/src/bootstrap/src/core/build_steps/clean.rs
+++ b/src/bootstrap/src/core/build_steps/clean.rs
@@ -1,4 +1,4 @@
-//! Implementation of `make clean` in rustbuild.
+//! `./x.py clean`
 //!
 //! Responsible for cleaning out a build directory of all old and stale
 //! artifacts to prepare for a fresh build. Currently doesn't remove the
diff --git a/src/bootstrap/src/core/build_steps/mod.rs b/src/bootstrap/src/core/build_steps/mod.rs
index 9b7378165de..381ee7ef53b 100644
--- a/src/bootstrap/src/core/build_steps/mod.rs
+++ b/src/bootstrap/src/core/build_steps/mod.rs
@@ -14,3 +14,4 @@ pub(crate) mod synthetic_targets;
 pub(crate) mod test;
 pub(crate) mod tool;
 pub(crate) mod toolstate;
+pub(crate) mod vendor;
diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs
index 7028bffea54..0a428ec5ca0 100644
--- a/src/bootstrap/src/core/build_steps/run.rs
+++ b/src/bootstrap/src/core/build_steps/run.rs
@@ -1,3 +1,8 @@
+//! Build-and-run steps for in-repo tools
+//!
+//! A bit of a hodge-podge as e.g. if a tool's a test fixture it should be in `build_steps::test`.
+//! If it can be reached from `./x.py run` it can go here.
+
 use std::path::PathBuf;
 use std::process::Command;
 
diff --git a/src/bootstrap/src/core/build_steps/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs
index c0683cdda1e..df38d6166eb 100644
--- a/src/bootstrap/src/core/build_steps/setup.rs
+++ b/src/bootstrap/src/core/build_steps/setup.rs
@@ -1,3 +1,10 @@
+//! First time setup of a dev environment
+//!
+//! These are build-and-run steps for `./x.py setup`, which allows quickly setting up the directory
+//! for modifying, building, and running the compiler and library. Running arbitrary configuration
+//! allows setting up things that cannot be simply captured inside the config.toml, in addition to
+//! leading people away from manually editing most of the config.toml values.
+
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::t;
 use crate::utils::change_tracker::CONFIG_CHANGE_HISTORY;
@@ -25,6 +32,8 @@ pub enum Profile {
     None,
 }
 
+static PROFILE_DIR: &str = "src/bootstrap/defaults";
+
 /// A list of historical hashes of `src/etc/rust_analyzer_settings.json`.
 /// New entries should be appended whenever this is updated so we can detect
 /// outdated vs. user-modified settings files.
@@ -41,7 +50,7 @@ static RUST_ANALYZER_SETTINGS: &str = include_str!("../../../../etc/rust_analyze
 
 impl Profile {
     fn include_path(&self, src_path: &Path) -> PathBuf {
-        PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self))
+        PathBuf::from(format!("{}/{PROFILE_DIR}/config.{}.toml", src_path.display(), self))
     }
 
     pub fn all() -> impl Iterator<Item = Self> {
@@ -220,7 +229,7 @@ fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) {
 
     let latest_change_id = CONFIG_CHANGE_HISTORY.last().unwrap().change_id;
     let settings = format!(
-        "# Includes one of the default files in src/bootstrap/defaults\n\
+        "# Includes one of the default files in {PROFILE_DIR}\n\
     profile = \"{profile}\"\n\
     change-id = {latest_change_id}\n"
     );
diff --git a/src/bootstrap/src/core/build_steps/suggest.rs b/src/bootstrap/src/core/build_steps/suggest.rs
index c057fa9a566..754d1e61da8 100644
--- a/src/bootstrap/src/core/build_steps/suggest.rs
+++ b/src/bootstrap/src/core/build_steps/suggest.rs
@@ -1,3 +1,5 @@
+//! Attempt to magically identify good tests to run
+
 #![cfg_attr(feature = "build-metrics", allow(unused))]
 
 use clap::Parser;
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index d581987c29e..411ed99532f 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -1,7 +1,7 @@
-//! Implementation of the test-related targets of the build system.
+//! Build-and-run steps for `./x.py test` test fixtures
 //!
-//! This file implements the various regression test suites that we execute on
-//! our CI.
+//! `./x.py test` (aka [`Kind::Test`]) is currently allowed to reach build steps in other modules.
+//! However, this contains ~all test parts we expect people to be able to build and run locally.
 
 use std::env;
 use std::ffi::OsStr;
diff --git a/src/bootstrap/src/core/build_steps/toolstate.rs b/src/bootstrap/src/core/build_steps/toolstate.rs
index f88c1b3ee82..ca3756df4d7 100644
--- a/src/bootstrap/src/core/build_steps/toolstate.rs
+++ b/src/bootstrap/src/core/build_steps/toolstate.rs
@@ -1,3 +1,9 @@
+//! [Toolstate] checks to keep tools building
+//!
+//! Reachable via `./x.py test` but mostly relevant for CI, since it isn't run locally by default.
+//!
+//! [Toolstate]: https://forge.rust-lang.org/infra/toolstate.html
+
 use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::utils::helpers::t;
 use serde_derive::{Deserialize, Serialize};
diff --git a/src/bootstrap/src/core/build_steps/vendor.rs b/src/bootstrap/src/core/build_steps/vendor.rs
new file mode 100644
index 00000000000..68f1b1bef3f
--- /dev/null
+++ b/src/bootstrap/src/core/build_steps/vendor.rs
@@ -0,0 +1,61 @@
+use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
+use std::path::PathBuf;
+use std::process::Command;
+
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub(crate) struct Vendor {
+    sync_args: Vec<PathBuf>,
+    versioned_dirs: bool,
+    root_dir: PathBuf,
+}
+
+impl Step for Vendor {
+    type Output = ();
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("placeholder").default_condition(true)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Vendor {
+            sync_args: run.builder.config.cmd.vendor_sync_args(),
+            versioned_dirs: run.builder.config.cmd.vendor_versioned_dirs(),
+            root_dir: run.builder.src.clone(),
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let mut cmd = Command::new(&builder.initial_cargo);
+        cmd.arg("vendor");
+
+        if self.versioned_dirs {
+            cmd.arg("--versioned-dirs");
+        }
+
+        // Sync these paths by default.
+        for p in [
+            "src/tools/cargo/Cargo.toml",
+            "src/tools/rust-analyzer/Cargo.toml",
+            "compiler/rustc_codegen_cranelift/Cargo.toml",
+            "compiler/rustc_codegen_gcc/Cargo.toml",
+            "src/bootstrap/Cargo.toml",
+        ] {
+            cmd.arg("--sync").arg(builder.src.join(p));
+        }
+
+        // Also sync explicitly requested paths.
+        for sync_arg in self.sync_args {
+            cmd.arg("--sync").arg(sync_arg);
+        }
+
+        // Will read the libstd Cargo.toml
+        // which uses the unstable `public-dependency` feature.
+        cmd.env("RUSTC_BOOTSTRAP", "1");
+
+        cmd.current_dir(self.root_dir);
+
+        builder.run(&mut cmd);
+    }
+}
diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs
index 70125aa429d..ae5440de572 100644
--- a/src/bootstrap/src/core/builder.rs
+++ b/src/bootstrap/src/core/builder.rs
@@ -14,8 +14,9 @@ use std::sync::OnceLock;
 use std::time::{Duration, Instant};
 
 use crate::core::build_steps::tool::{self, SourceType};
-use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test};
-use crate::core::build_steps::{clippy, llvm};
+use crate::core::build_steps::{
+    check, clean, clippy, compile, dist, doc, install, llvm, run, setup, test, vendor,
+};
 use crate::core::config::flags::{Color, Subcommand};
 use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection};
 use crate::prepare_behaviour_dump_dir;
@@ -34,13 +35,34 @@ use once_cell::sync::Lazy;
 #[cfg(test)]
 mod tests;
 
+/// Builds and performs different [`Self::kind`]s of stuff and actions, taking
+/// into account build configuration from e.g. config.toml.
 pub struct Builder<'a> {
+    /// Build configuration from e.g. config.toml.
     pub build: &'a Build,
+
+    /// The stage to use. Either implicitly determined based on subcommand, or
+    /// explicitly specified with `--stage N`. Normally this is the stage we
+    /// use, but sometimes we want to run steps with a lower stage than this.
     pub top_stage: u32,
+
+    /// What to build or what action to perform.
     pub kind: Kind,
+
+    /// A cache of outputs of [`Step`]s so we can avoid running steps we already
+    /// ran.
     cache: Cache,
+
+    /// A stack of [`Step`]s to run before we can run this builder. The output
+    /// of steps is cached in [`Self::cache`].
     stack: RefCell<Vec<Box<dyn Any>>>,
+
+    /// The total amount of time we spent running [`Step`]s in [`Self::stack`].
     time_spent_on_dependencies: Cell<Duration>,
+
+    /// The paths passed on the command line. Used by steps to figure out what
+    /// to do. For example: with `./x check foo bar` we get `paths=["foo",
+    /// "bar"]`.
     pub paths: Vec<PathBuf>,
 }
 
@@ -638,6 +660,7 @@ pub enum Kind {
     Run,
     Setup,
     Suggest,
+    Vendor,
 }
 
 impl Kind {
@@ -658,6 +681,7 @@ impl Kind {
             Kind::Run => "run",
             Kind::Setup => "setup",
             Kind::Suggest => "suggest",
+            Kind::Vendor => "vendor",
         }
     }
 
@@ -921,6 +945,7 @@ impl<'a> Builder<'a> {
             ),
             Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode),
             Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std),
+            Kind::Vendor => describe!(vendor::Vendor),
             // special-cased in Build::build()
             Kind::Format | Kind::Suggest => vec![],
         }
@@ -993,6 +1018,7 @@ impl<'a> Builder<'a> {
                 Kind::Setup,
                 path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)),
             ),
+            Subcommand::Vendor { .. } => (Kind::Vendor, &paths[..]),
         };
 
         Self::new_internal(build, kind, paths.to_owned())
diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 2acce627359..149a62f88c9 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -345,6 +345,8 @@ pub struct Config {
     #[cfg(test)]
     pub initial_rustfmt: RefCell<RustfmtState>,
 
+    /// The paths to work with. For example: with `./x check foo bar` we get
+    /// `paths=["foo", "bar"]`.
     pub paths: Vec<PathBuf>,
 }
 
@@ -2027,7 +2029,8 @@ impl Config {
             | Subcommand::Run { .. }
             | Subcommand::Setup { .. }
             | Subcommand::Format { .. }
-            | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0),
+            | Subcommand::Suggest { .. }
+            | Subcommand::Vendor { .. } => flags.stage.unwrap_or(0),
         };
 
         // CI should always run stage 2 builds, unless it specifically states otherwise
@@ -2054,7 +2057,8 @@ impl Config {
                 | Subcommand::Run { .. }
                 | Subcommand::Setup { .. }
                 | Subcommand::Format { .. }
-                | Subcommand::Suggest { .. } => {}
+                | Subcommand::Suggest { .. }
+                | Subcommand::Vendor { .. } => {}
             }
         }
 
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
index b2124c65c22..a5af87ad5c7 100644
--- a/src/bootstrap/src/core/config/flags.rs
+++ b/src/bootstrap/src/core/config/flags.rs
@@ -456,6 +456,15 @@ Arguments:
         #[arg(long)]
         run: bool,
     },
+    /// Vendor dependencies
+    Vendor {
+        /// Additional `Cargo.toml` to sync and vendor
+        #[arg(long)]
+        sync: Vec<PathBuf>,
+        /// Always include version in subdir name
+        #[arg(long)]
+        versioned_dirs: bool,
+    },
 }
 
 impl Subcommand {
@@ -476,6 +485,7 @@ impl Subcommand {
             Subcommand::Run { .. } => Kind::Run,
             Subcommand::Setup { .. } => Kind::Setup,
             Subcommand::Suggest { .. } => Kind::Suggest,
+            Subcommand::Vendor { .. } => Kind::Vendor,
         }
     }
 
@@ -581,6 +591,20 @@ impl Subcommand {
             _ => false,
         }
     }
+
+    pub fn vendor_versioned_dirs(&self) -> bool {
+        match *self {
+            Subcommand::Vendor { versioned_dirs, .. } => versioned_dirs,
+            _ => false,
+        }
+    }
+
+    pub fn vendor_sync_args(&self) -> Vec<PathBuf> {
+        match self {
+            Subcommand::Vendor { sync, .. } => sync.clone(),
+            _ => vec![],
+        }
+    }
 }
 
 /// Returns the shell completion for a given shell, if the result differs from the current