Rollup merge of #135391 - jieyouxu:conditional-tracing, r=onur-ozkan

bootstrap: Implement conditional `tracing` infra

Add a conditional `tracing` setup that is gated behind `BOOTSTRAP_TRACING` env var. This `tracing` infra is implemented by:

- Introducing an optional `tracing` cargo feature in bootstrap.
- Added optional `tracing*` dependencies which are gated behind the `tracing` cargo feature.
- When `BOOTSTRAP_TRACING` is set, `bootstrap.py` will build bootstrap with `--features=tracing`.

There is a small trick here to share `BOOTSTRAP_TRACING` env var without having to add a separate env var:

- `BOOTSTRAP_TRACING=1` is not a registered `tracing` filter target, so that can be used to enable the `tracing` cargo feature yet not actually enable any tracing logs (useful for editor r-a setups without actually outputting any tracing logs).
- `BOOTSTRAP_TRACING=TRACE` and such are actually valid `tracing` filters, but that sets `BOOTSTRAP_TRACING` anyway.

Example usage: https://github.com/rust-lang/rust/pull/135299 (that experimental PR is not conditionally gated)
This PR is intentionally kept minimal to focus on the infra itself. To get actual mileage, instrumentations will need to be added to individual `Step`s and such.

r? `@onur-ozkan` (or reroll)

8 files changed, 341 insertions, 6 deletions

diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock
index c9697e670b7..c46aae2ded1 100644
--- a/src/bootstrap/Cargo.lock
+++ b/src/bootstrap/Cargo.lock
@@ -58,6 +58,9 @@ dependencies = [
  "tar",
  "termcolor",
  "toml",
+ "tracing",
+ "tracing-subscriber",
+ "tracing-tree",
  "walkdir",
  "windows 0.52.0",
  "xz2",
@@ -70,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
 dependencies = [
  "memchr",
- "regex-automata",
+ "regex-automata 0.4.9",
  "serde",
 ]
 
@@ -271,8 +274,8 @@ dependencies = [
  "aho-corasick",
  "bstr",
  "log",
- "regex-automata",
- "regex-syntax",
+ "regex-automata 0.4.9",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -300,7 +303,7 @@ dependencies = [
  "globset",
  "log",
  "memchr",
- "regex-automata",
+ "regex-automata 0.4.9",
  "same-file",
  "walkdir",
  "winapi-util",
@@ -323,6 +326,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
 name = "libc"
 version = "0.2.167"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -363,6 +372,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "matchers"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
+dependencies = [
+ "regex-automata 0.1.10",
+]
+
+[[package]]
 name = "memchr"
 version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -378,6 +396,25 @@ dependencies = [
 ]
 
 [[package]]
+name = "nu-ansi-term"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
+dependencies = [
+ "overload",
+ "winapi",
+]
+
+[[package]]
+name = "nu-ansi-term"
+version = "0.50.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
 name = "object"
 version = "0.36.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -387,6 +424,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "once_cell"
+version = "1.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+
+[[package]]
 name = "opener"
 version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -397,6 +440,18 @@ dependencies = [
 ]
 
 [[package]]
+name = "overload"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
+
+[[package]]
 name = "pkg-config"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -440,6 +495,27 @@ dependencies = [
 ]
 
 [[package]]
+name = "regex"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata 0.4.9",
+ "regex-syntax 0.8.5",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+dependencies = [
+ "regex-syntax 0.6.29",
+]
+
+[[package]]
 name = "regex-automata"
 version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -447,11 +523,17 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
 name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
+[[package]]
+name = "regex-syntax"
 version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
@@ -540,12 +622,27 @@ dependencies = [
 ]
 
 [[package]]
+name = "sharded-slab"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
 name = "shlex"
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
+[[package]]
 name = "syn"
 version = "2.0.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -590,6 +687,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "thread_local"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+]
+
+[[package]]
 name = "toml"
 version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -599,6 +706,79 @@ dependencies = [
 ]
 
 [[package]]
+name = "tracing"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
+dependencies = [
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
+dependencies = [
+ "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
+dependencies = [
+ "log",
+ "once_cell",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
+dependencies = [
+ "matchers",
+ "nu-ansi-term 0.46.0",
+ "once_cell",
+ "regex",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+]
+
+[[package]]
+name = "tracing-tree"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f459ca79f1b0d5f71c54ddfde6debfc59c8b6eeb46808ae492077f739dc7b49c"
+dependencies = [
+ "nu-ansi-term 0.50.1",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+]
+
+[[package]]
 name = "typenum"
 version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -611,6 +791,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
 
 [[package]]
+name = "valuable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+
+[[package]]
 name = "version_check"
 version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
index d8775a67e19..5322f079214 100644
--- a/src/bootstrap/Cargo.toml
+++ b/src/bootstrap/Cargo.toml
@@ -7,6 +7,7 @@ default-run = "bootstrap"
 
 [features]
 build-metrics = ["sysinfo"]
+tracing = ["dep:tracing", "dep:tracing-subscriber", "dep:tracing-tree"]
 
 [lib]
 path = "src/lib.rs"
@@ -64,6 +65,11 @@ xz2 = "0.1"
 # Dependencies needed by the build-metrics feature
 sysinfo = { version = "0.33.0", default-features = false, optional = true, features = ["system"] }
 
+# Dependencies needed by the `logging` feature
+tracing = { version = "0.1", optional = true, features = ["attributes"] }
+tracing-subscriber = { version = "0.3", optional = true, features = ["env-filter", "fmt", "registry", "std"] }
+tracing-tree = { version = "0.4.0", optional = true }
+
 [target.'cfg(windows)'.dependencies.junction]
 version = "1.0.0"
 
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 89415afbe3b..76ee40c6f45 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -1129,6 +1129,10 @@ class RustBuild(object):
             "-Zroot-dir=" + self.rust_root,
         ]
         args.extend("--verbose" for _ in range(self.verbose))
+
+        if "BOOTSTRAP_TRACING" in env:
+            args.append("--features=tracing")
+
         if self.use_locked_deps:
             args.append("--locked")
         if self.use_vendored_sources:
diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs
index ee813de1c9e..b6552774195 100644
--- a/src/bootstrap/src/bin/main.rs
+++ b/src/bootstrap/src/bin/main.rs
@@ -15,8 +15,18 @@ use bootstrap::{
     human_readable_changes, t,
 };
 use build_helper::ci::CiEnv;
-
+#[cfg(feature = "tracing")]
+use tracing::*;
+#[cfg(feature = "tracing")]
+use tracing_subscriber::EnvFilter;
+#[cfg(feature = "tracing")]
+use tracing_subscriber::prelude::*;
+
+#[cfg_attr(feature = "tracing", instrument(level = "trace", name = "main"))]
 fn main() {
+    #[cfg(feature = "tracing")]
+    setup_tracing();
+
     let args = env::args().skip(1).collect::<Vec<_>>();
 
     if Flags::try_parse_verbose_help(&args) {
@@ -187,3 +197,28 @@ fn check_version(config: &Config) -> Option<String> {
 
     Some(msg)
 }
+
+// # Note on `tracing` usage in bootstrap
+//
+// Due to the conditional compilation via the `tracing` cargo feature, this means that `tracing`
+// usages in bootstrap need to be also gated behind the `tracing` feature:
+//
+// - `tracing` macros (like `trace!`) and anything from `tracing`, `tracing_subscriber` and
+//   `tracing-tree` will need to be gated by `#[cfg(feature = "tracing")]`.
+// - `tracing`'s `#[instrument(..)]` macro will need to be gated like `#![cfg_attr(feature =
+//   "tracing", instrument(..))]`.
+#[cfg(feature = "tracing")]
+fn setup_tracing() {
+    let filter = EnvFilter::from_env("BOOTSTRAP_TRACING");
+    let layer = tracing_tree::HierarchicalLayer::default()
+        .with_writer(std::io::stderr)
+        .with_ansi(true)
+        .with_targets(true)
+        .with_bracketed_fields(true)
+        .with_indent_amount(2)
+        .with_indent_lines(true);
+    let subscriber = tracing_subscriber::registry().with(filter).with(layer);
+
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+    trace!("tracing subscriber setup");
+}
diff --git a/src/doc/rustc-dev-guide/src/SUMMARY.md b/src/doc/rustc-dev-guide/src/SUMMARY.md
index a8fddbc7562..91c4aeacbd7 100644
--- a/src/doc/rustc-dev-guide/src/SUMMARY.md
+++ b/src/doc/rustc-dev-guide/src/SUMMARY.md
@@ -75,6 +75,7 @@
 - [Prologue](./building/bootstrapping/intro.md)
 - [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md)
 - [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md)
+- [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md)
 
 # High-level Compiler Architecture
 
diff --git a/src/doc/rustc-dev-guide/src/building/bootstrapping/debugging-bootstrap.md b/src/doc/rustc-dev-guide/src/building/bootstrapping/debugging-bootstrap.md
new file mode 100644
index 00000000000..972b4a8fb0e
--- /dev/null
+++ b/src/doc/rustc-dev-guide/src/building/bootstrapping/debugging-bootstrap.md
@@ -0,0 +1,100 @@
+# Debugging bootstrap
+
+> FIXME: this page could be expanded 
+
+## `tracing` in bootstrap
+
+Bootstrap has conditional [`tracing`][tracing] setup to provide structured logging.
+
+[tracing]: https://docs.rs/tracing/0.1.41/tracing/index.html
+
+### Enabling `tracing` output
+
+Bootstrap will conditionally enable `tracing` output if the `BOOTSTRAP_TRACING` env var is set.
+
+Example usage:
+
+```bash
+$ BOOTSTRAP_TRACING=TRACE ./x build library --stage 1
+```
+
+Example output[^experimental]:
+
+![Example bootstrap tracing output](./debugging-bootstrap/tracing-output-example.png)
+
+[^experimental]: This shows what's *possible* with the infra in an experimental implementation.
+
+The env var `BOOTSTRAP_TRACING` accepts a [`tracing` env-filter][tracing-env-filter]. The `TRACE` filter will enable *all* `trace` level or less verbose level tracing output.
+
+[tracing-env-filter]: https://docs.rs/tracing-subscriber/0.3.19/tracing_subscriber/filter/struct.EnvFilter.html
+
+### Using `tracing` in bootstrap
+
+Both `tracing::*` macros and the `tracing::instrument` proc-macro attribute need to be gated behind `tracing` feature. Examples:
+
+```rs
+#[cfg(feature = "tracing")]
+use tracing::{instrument, trace};
+
+struct Foo;
+
+impl Step for Foo {
+    type Output = ();
+
+    #[cfg_attr(feature = "tracing", instrument(level = "trace", name = "Foo::should_run", skip_all))]
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        #[cfg(feature = "tracing")]
+        trace!(?run, "entered Foo::should_run");
+
+        todo!()
+    }
+
+    #[cfg_attr(
+        feature = "tracing",
+        instrument(
+            level = "trace",
+            name = "Foo::run",
+            skip_all,
+            fields(compiler = ?builder.compiler),
+        ),
+    )]
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        #[cfg(feature = "tracing")]
+        trace!(?run, "entered Foo::run");
+
+        todo!()
+    }    
+}
+```
+
+For `#[instrument]`, it's recommended to:
+
+- Gate it behind `trace` level for fine-granularity, possibly `debug` level for core functions.
+- Explicitly pick an instrumentation name via `name = ".."` to distinguish between e.g. `run` of different steps.
+- Take care to not cause diverging behavior via tracing, e.g. building extra things only when tracing infra is enabled.
+
+### Enabling `tracing` bootstrap feature in rust-analyzer
+
+You can adjust your `settings.json`'s `rust-analyzer.check.overrideCommand` and `rust-analyzer.cargo.buildScripts.overrideCommand` if you want to also enable `logging` cargo feature by default in your editor. This is mostly useful if you want proper r-a completions and such when working on bootstrap itself.
+
+```json
+"rust-analyzer.check.overrideCommand": [
+    "BOOTSTRAP_TRACING=1", // <- BOOTSTRAP_TRACING=1 won't enable tracing filter, but it will activate bootstrap's `tracing` feature
+    "python3",
+    "x.py",
+    "check",
+    "--json-output",
+    "--build-dir=build-rust-analyzer"
+],
+```
+
+```json
+"rust-analyzer.cargo.buildScripts.overrideCommand": [
+    "BOOTSTRAP_TRACING=1", // <- note this
+    "python3",
+    "x.py",
+    "check",
+    "--json-output",
+    "--build-dir=build-rust-analyzer"
+],
+```
diff --git a/src/doc/rustc-dev-guide/src/building/bootstrapping/debugging-bootstrap/tracing-output-example.png b/src/doc/rustc-dev-guide/src/building/bootstrapping/debugging-bootstrap/tracing-output-example.png
new file mode 100644
index 00000000000..745aec50d4a
--- /dev/null
+++ b/src/doc/rustc-dev-guide/src/building/bootstrapping/debugging-bootstrap/tracing-output-example.png
Binary files differdiff --git a/src/doc/rustc-dev-guide/src/building/bootstrapping/intro.md b/src/doc/rustc-dev-guide/src/building/bootstrapping/intro.md
index f829884fb93..f72918c8377 100644
--- a/src/doc/rustc-dev-guide/src/building/bootstrapping/intro.md
+++ b/src/doc/rustc-dev-guide/src/building/bootstrapping/intro.md
@@ -17,5 +17,8 @@ In this section, we give a high-level overview of
 [what Bootstrap does](./what-bootstrapping-does.md), followed by a high-level
 introduction to [how Bootstrap does it](./how-bootstrap-does-it.md).
 
+Additionally, see [debugging bootstrap](./debugging-bootstrap.md) to learn
+about debugging methods.
+
 [boot]: https://en.wikipedia.org/wiki/Bootstrapping_(compilers)
 [ocaml-compiler]: https://github.com/rust-lang/rust/tree/ef75860a0a72f79f97216f8aaa5b388d98da6480/src/boot