Rollup merge of #144027 - RalfJung:clippy, r=Mark-Simulacrum

clippy: make tests work in stage 1

This finally fixes https://github.com/rust-lang/rust/issues/78717 :)

Similar to what Miri already does, the clippy test step needs to carefully consider  which compiler is used to build clippy and which compiler is linked into clippy (and thus must be used to build the test dependencies). On top of that we have some extra complications that Miri avoided by using `cargo-miri` for building its test dependencies: we need cargo to use the right rustc and the right sysroot, but https://github.com/rust-lang/cargo/issues/4423 makes this quite hard to do. See the long comment in `src/tools/clippy/tests/compile-test.rs` for details.

Some clippy tests tried to import rustc crates; that fundamentally requires a full bootstrap loop so it cannot work in stage 1. I had to kind of guess what those tests were doing so I don't know if my changes there make any sense.

Cc ```@flip1995``` ```@Kobzol```

14 files changed, 84 insertions, 63 deletions

diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index 7652ea1a488..8344b0e03b4 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -739,7 +739,6 @@ impl Step for CompiletestTest {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Clippy {
-    stage: u32,
     host: TargetSelection,
 }
 
@@ -753,33 +752,23 @@ impl Step for Clippy {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        // If stage is explicitly set or not lower than 2, keep it. Otherwise, make sure it's at least 2
-        // as tests for this step don't work with a lower stage.
-        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {
-            run.builder.top_stage
-        } else {
-            2
-        };
-
-        run.builder.ensure(Clippy { stage, host: run.target });
+        run.builder.ensure(Clippy { host: run.target });
     }
 
     /// Runs `cargo test` for clippy.
     fn run(self, builder: &Builder<'_>) {
-        let stage = self.stage;
+        let stage = builder.top_stage;
         let host = self.host;
-        let compiler = builder.compiler(stage, host);
-
-        if stage < 2 {
-            eprintln!("WARNING: clippy tests on stage {stage} may not behave well.");
-            eprintln!("HELP: consider using stage 2");
-        }
+        // We need to carefully distinguish the compiler that builds clippy, and the compiler
+        // that is linked into the clippy being tested. `target_compiler` is the latter,
+        // and it must also be used by clippy's test runner to build tests and their dependencies.
+        let target_compiler = builder.compiler(stage, host);
 
-        let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host });
-        let compiler = tool_result.build_compiler;
+        let tool_result = builder.ensure(tool::Clippy { compiler: target_compiler, target: host });
+        let tool_compiler = tool_result.build_compiler;
         let mut cargo = tool::prepare_tool_cargo(
             builder,
-            compiler,
+            tool_compiler,
             Mode::ToolRustc,
             host,
             Kind::Test,
@@ -788,11 +777,17 @@ impl Step for Clippy {
             &[],
         );
 
-        cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
-        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
-        let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir());
+        cargo.env("RUSTC_TEST_SUITE", builder.rustc(tool_compiler));
+        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(tool_compiler));
+        let host_libs = builder.stage_out(tool_compiler, Mode::ToolRustc).join(builder.cargo_dir());
         cargo.env("HOST_LIBS", host_libs);
 
+        // Build the standard library that the tests can use.
+        builder.std(target_compiler, host);
+        cargo.env("TEST_SYSROOT", builder.sysroot(target_compiler));
+        cargo.env("TEST_RUSTC", builder.rustc(target_compiler));
+        cargo.env("TEST_RUSTC_LIB", builder.rustc_libdir(target_compiler));
+
         // Collect paths of tests to run
         'partially_test: {
             let paths = &builder.config.paths[..];
@@ -813,7 +808,8 @@ impl Step for Clippy {
         cargo.add_rustc_lib_path(builder);
         let cargo = prepare_cargo_test(cargo, &[], &[], host, builder);
 
-        let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host);
+        let _guard =
+            builder.msg_sysroot_tool(Kind::Test, tool_compiler.stage, "clippy", host, host);
 
         // Clippy reports errors if it blessed the outputs
         if cargo.allow_failure().run(builder) {
diff --git a/src/tools/clippy/clippy_test_deps/Cargo.lock b/src/tools/clippy/clippy_test_deps/Cargo.lock
index a591dae3a1a..5be404f24e6 100644
--- a/src/tools/clippy/clippy_test_deps/Cargo.lock
+++ b/src/tools/clippy/clippy_test_deps/Cargo.lock
@@ -72,6 +72,7 @@ dependencies = [
  "futures",
  "if_chain",
  "itertools",
+ "libc",
  "parking_lot",
  "quote",
  "regex",
diff --git a/src/tools/clippy/clippy_test_deps/Cargo.toml b/src/tools/clippy/clippy_test_deps/Cargo.toml
index a23ffcaf2f9..fcedc5d4843 100644
--- a/src/tools/clippy/clippy_test_deps/Cargo.toml
+++ b/src/tools/clippy/clippy_test_deps/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2021"
 # Add dependencies here to make them available in ui tests.
 
 [dependencies]
+libc = "0.2"
 regex = "1.5.5"
 serde = { version = "1.0.145", features = ["derive"] }
 if_chain = "1.0"
diff --git a/src/tools/clippy/tests/compile-test.rs b/src/tools/clippy/tests/compile-test.rs
index 57d623b2cfc..83f91ccaa7b 100644
--- a/src/tools/clippy/tests/compile-test.rs
+++ b/src/tools/clippy/tests/compile-test.rs
@@ -151,7 +151,31 @@ impl TestContext {
         defaults.set_custom(
             "dependencies",
             DependencyBuilder {
-                program: CommandBuilder::cargo(),
+                program: {
+                    let mut p = CommandBuilder::cargo();
+                    // If we run in bootstrap, we need to use the right compiler for building the
+                    // tests -- not the compiler that built clippy, but the compiler that got linked
+                    // into clippy. Just invoking TEST_RUSTC does not work because LD_LIBRARY_PATH
+                    // is set in a way that makes it pick the wrong sysroot. Sadly due to
+                    // <https://github.com/rust-lang/cargo/issues/4423> we cannot use RUSTFLAGS to
+                    // set `--sysroot`, so we need to use bootstrap's rustc wrapper. That wrapper
+                    // however has some staging logic that is hurting us here, so to work around
+                    // that we set both the "real" and "staging" rustc to TEST_RUSTC, including the
+                    // associated library paths.
+                    if let Some(rustc) = option_env!("TEST_RUSTC") {
+                        let libdir = option_env!("TEST_RUSTC_LIB").unwrap();
+                        let sysroot = option_env!("TEST_SYSROOT").unwrap();
+                        p.envs.push(("RUSTC_REAL".into(), Some(rustc.into())));
+                        p.envs.push(("RUSTC_REAL_LIBDIR".into(), Some(libdir.into())));
+                        p.envs.push(("RUSTC_SNAPSHOT".into(), Some(rustc.into())));
+                        p.envs.push(("RUSTC_SNAPSHOT_LIBDIR".into(), Some(libdir.into())));
+                        p.envs.push((
+                            "RUSTC_SYSROOT".into(),
+                            Some(sysroot.into()),
+                        ));
+                    }
+                    p
+                },
                 crate_manifest_path: Path::new("clippy_test_deps").join("Cargo.toml"),
                 build_std: None,
                 bless_lockfile: self.args.bless,
@@ -192,6 +216,9 @@ impl TestContext {
             let dep = format!("-Ldependency={}", Path::new(host_libs).join("deps").display());
             config.program.args.push(dep.into());
         }
+        if let Some(sysroot) = option_env!("TEST_SYSROOT") {
+            config.program.args.push(format!("--sysroot={sysroot}").into());
+        }
 
         config.program.program = profile_path.join(if cfg!(windows) {
             "clippy-driver.exe"
diff --git a/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs b/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs
index 5992d15935d..54650922871 100644
--- a/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs
+++ b/src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs
@@ -16,7 +16,7 @@ pub fn derive(_: TokenStream) -> TokenStream {
     let output = quote! {
         // Should not trigger `useless_attribute`
         #[allow(dead_code)]
-        extern crate rustc_middle;
+        extern crate core;
     };
     output
 }
diff --git a/src/tools/clippy/tests/ui/cast_alignment.rs b/src/tools/clippy/tests/ui/cast_alignment.rs
index 5773ffddb91..ef667f5598a 100644
--- a/src/tools/clippy/tests/ui/cast_alignment.rs
+++ b/src/tools/clippy/tests/ui/cast_alignment.rs
@@ -1,6 +1,5 @@
 //! Test casts for alignment issues
 
-#![feature(rustc_private)]
 #![feature(core_intrinsics)]
 #![warn(clippy::cast_ptr_alignment)]
 #![allow(
@@ -10,8 +9,6 @@
     clippy::borrow_as_ptr
 )]
 
-extern crate libc;
-
 fn main() {
     /* These should be warned against */
 
diff --git a/src/tools/clippy/tests/ui/cast_alignment.stderr b/src/tools/clippy/tests/ui/cast_alignment.stderr
index 6d9a81f0ecf..ee4c3e9a77e 100644
--- a/src/tools/clippy/tests/ui/cast_alignment.stderr
+++ b/src/tools/clippy/tests/ui/cast_alignment.stderr
@@ -1,5 +1,5 @@
 error: casting from `*const u8` to a more-strictly-aligned pointer (`*const u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:19:5
+  --> tests/ui/cast_alignment.rs:16:5
    |
 LL |     (&1u8 as *const u8) as *const u16;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -8,19 +8,19 @@ LL |     (&1u8 as *const u8) as *const u16;
    = help: to override `-D warnings` add `#[allow(clippy::cast_ptr_alignment)]`
 
 error: casting from `*mut u8` to a more-strictly-aligned pointer (`*mut u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:22:5
+  --> tests/ui/cast_alignment.rs:19:5
    |
 LL |     (&mut 1u8 as *mut u8) as *mut u16;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: casting from `*const u8` to a more-strictly-aligned pointer (`*const u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:26:5
+  --> tests/ui/cast_alignment.rs:23:5
    |
 LL |     (&1u8 as *const u8).cast::<u16>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: casting from `*mut u8` to a more-strictly-aligned pointer (`*mut u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:29:5
+  --> tests/ui/cast_alignment.rs:26:5
    |
 LL |     (&mut 1u8 as *mut u8).cast::<u16>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/src/tools/clippy/tests/ui/iter_over_hash_type.rs b/src/tools/clippy/tests/ui/iter_over_hash_type.rs
index 914cc9df0de..9a3e7033cd8 100644
--- a/src/tools/clippy/tests/ui/iter_over_hash_type.rs
+++ b/src/tools/clippy/tests/ui/iter_over_hash_type.rs
@@ -3,15 +3,18 @@
 #![warn(clippy::iter_over_hash_type)]
 use std::collections::{HashMap, HashSet};
 
-extern crate rustc_data_structures;
-
 extern crate proc_macros;
 
+// Ensure it also works via type aliases (this isn't really the Fx hasher but that does not matter).
+type FxBuildHasher = std::collections::hash_map::RandomState;
+type FxHashMap<K, V> = HashMap<K, V, FxBuildHasher>;
+type FxHashSet<K> = HashSet<K, FxBuildHasher>;
+
 fn main() {
     let mut hash_set = HashSet::<i32>::new();
     let mut hash_map = HashMap::<i32, i32>::new();
-    let mut fx_hash_map = rustc_data_structures::fx::FxHashMap::<i32, i32>::default();
-    let mut fx_hash_set = rustc_data_structures::fx::FxHashMap::<i32, i32>::default();
+    let mut fx_hash_map = FxHashMap::<i32, i32>::default();
+    let mut fx_hash_set = FxHashSet::<i32>::default();
     let vec = Vec::<i32>::new();
 
     // test hashset
diff --git a/src/tools/clippy/tests/ui/iter_over_hash_type.stderr b/src/tools/clippy/tests/ui/iter_over_hash_type.stderr
index 1bc6f4588d4..3356186547d 100644
--- a/src/tools/clippy/tests/ui/iter_over_hash_type.stderr
+++ b/src/tools/clippy/tests/ui/iter_over_hash_type.stderr
@@ -1,5 +1,5 @@
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:18:5
+  --> tests/ui/iter_over_hash_type.rs:21:5
    |
 LL | /     for x in &hash_set {
 LL | |
@@ -11,7 +11,7 @@ LL | |     }
    = help: to override `-D warnings` add `#[allow(clippy::iter_over_hash_type)]`
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:22:5
+  --> tests/ui/iter_over_hash_type.rs:25:5
    |
 LL | /     for x in hash_set.iter() {
 LL | |
@@ -20,7 +20,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:26:5
+  --> tests/ui/iter_over_hash_type.rs:29:5
    |
 LL | /     for x in hash_set.clone() {
 LL | |
@@ -29,7 +29,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:30:5
+  --> tests/ui/iter_over_hash_type.rs:33:5
    |
 LL | /     for x in hash_set.drain() {
 LL | |
@@ -38,7 +38,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:36:5
+  --> tests/ui/iter_over_hash_type.rs:39:5
    |
 LL | /     for (x, y) in &hash_map {
 LL | |
@@ -47,7 +47,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:40:5
+  --> tests/ui/iter_over_hash_type.rs:43:5
    |
 LL | /     for x in hash_map.keys() {
 LL | |
@@ -56,7 +56,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:44:5
+  --> tests/ui/iter_over_hash_type.rs:47:5
    |
 LL | /     for x in hash_map.values() {
 LL | |
@@ -65,7 +65,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:48:5
+  --> tests/ui/iter_over_hash_type.rs:51:5
    |
 LL | /     for x in hash_map.values_mut() {
 LL | |
@@ -74,7 +74,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:52:5
+  --> tests/ui/iter_over_hash_type.rs:55:5
    |
 LL | /     for x in hash_map.iter() {
 LL | |
@@ -83,7 +83,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:56:5
+  --> tests/ui/iter_over_hash_type.rs:59:5
    |
 LL | /     for x in hash_map.clone() {
 LL | |
@@ -92,7 +92,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:60:5
+  --> tests/ui/iter_over_hash_type.rs:63:5
    |
 LL | /     for x in hash_map.drain() {
 LL | |
@@ -101,7 +101,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:66:5
+  --> tests/ui/iter_over_hash_type.rs:69:5
    |
 LL | /     for x in fx_hash_set {
 LL | |
@@ -110,7 +110,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:70:5
+  --> tests/ui/iter_over_hash_type.rs:73:5
    |
 LL | /     for x in fx_hash_map {
 LL | |
diff --git a/src/tools/clippy/tests/ui/strlen_on_c_strings.fixed b/src/tools/clippy/tests/ui/strlen_on_c_strings.fixed
index 31ed1cf03a2..17c1b541f77 100644
--- a/src/tools/clippy/tests/ui/strlen_on_c_strings.fixed
+++ b/src/tools/clippy/tests/ui/strlen_on_c_strings.fixed
@@ -1,7 +1,5 @@
 #![warn(clippy::strlen_on_c_strings)]
 #![allow(dead_code, clippy::manual_c_str_literals)]
-#![feature(rustc_private)]
-extern crate libc;
 
 #[allow(unused)]
 use libc::strlen;
diff --git a/src/tools/clippy/tests/ui/strlen_on_c_strings.rs b/src/tools/clippy/tests/ui/strlen_on_c_strings.rs
index 0f3798c9fd8..c641422f5df 100644
--- a/src/tools/clippy/tests/ui/strlen_on_c_strings.rs
+++ b/src/tools/clippy/tests/ui/strlen_on_c_strings.rs
@@ -1,7 +1,5 @@
 #![warn(clippy::strlen_on_c_strings)]
 #![allow(dead_code, clippy::manual_c_str_literals)]
-#![feature(rustc_private)]
-extern crate libc;
 
 #[allow(unused)]
 use libc::strlen;
diff --git a/src/tools/clippy/tests/ui/strlen_on_c_strings.stderr b/src/tools/clippy/tests/ui/strlen_on_c_strings.stderr
index b8619fa2df3..84a93b99ee3 100644
--- a/src/tools/clippy/tests/ui/strlen_on_c_strings.stderr
+++ b/src/tools/clippy/tests/ui/strlen_on_c_strings.stderr
@@ -1,5 +1,5 @@
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:13:13
+  --> tests/ui/strlen_on_c_strings.rs:11:13
    |
 LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `cstring.as_bytes().len()`
@@ -8,37 +8,37 @@ LL |     let _ = unsafe { libc::strlen(cstring.as_ptr()) };
    = help: to override `-D warnings` add `#[allow(clippy::strlen_on_c_strings)]`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:18:13
+  --> tests/ui/strlen_on_c_strings.rs:16:13
    |
 LL |     let _ = unsafe { libc::strlen(cstr.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `cstr.to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:21:13
+  --> tests/ui/strlen_on_c_strings.rs:19:13
    |
 LL |     let _ = unsafe { strlen(cstr.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `cstr.to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:25:22
+  --> tests/ui/strlen_on_c_strings.rs:23:22
    |
 LL |     let _ = unsafe { strlen((*pcstr).as_ptr()) };
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `(*pcstr).to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:31:22
+  --> tests/ui/strlen_on_c_strings.rs:29:22
    |
 LL |     let _ = unsafe { strlen(unsafe_identity(cstr).as_ptr()) };
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `unsafe_identity(cstr).to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:33:13
+  --> tests/ui/strlen_on_c_strings.rs:31:13
    |
 LL |     let _ = unsafe { strlen(unsafe { unsafe_identity(cstr) }.as_ptr()) };
    |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `unsafe { unsafe_identity(cstr) }.to_bytes().len()`
 
 error: using `libc::strlen` on a `CString` or `CStr` value
-  --> tests/ui/strlen_on_c_strings.rs:37:22
+  --> tests/ui/strlen_on_c_strings.rs:35:22
    |
 LL |     let _ = unsafe { strlen(f(cstr).as_ptr()) };
    |                      ^^^^^^^^^^^^^^^^^^^^^^^^ help: try: `f(cstr).to_bytes().len()`
diff --git a/src/tools/clippy/tests/ui/useless_attribute.fixed b/src/tools/clippy/tests/ui/useless_attribute.fixed
index a96c8f46f55..930bc1eaecf 100644
--- a/src/tools/clippy/tests/ui/useless_attribute.fixed
+++ b/src/tools/clippy/tests/ui/useless_attribute.fixed
@@ -13,7 +13,7 @@
 #[allow(unused_imports)]
 #[allow(unused_extern_crates)]
 #[macro_use]
-extern crate rustc_middle;
+extern crate regex as regex_crate;
 
 #[macro_use]
 extern crate proc_macro_derive;
diff --git a/src/tools/clippy/tests/ui/useless_attribute.rs b/src/tools/clippy/tests/ui/useless_attribute.rs
index b26410134bb..50fafd478e5 100644
--- a/src/tools/clippy/tests/ui/useless_attribute.rs
+++ b/src/tools/clippy/tests/ui/useless_attribute.rs
@@ -13,7 +13,7 @@
 #[allow(unused_imports)]
 #[allow(unused_extern_crates)]
 #[macro_use]
-extern crate rustc_middle;
+extern crate regex as regex_crate;
 
 #[macro_use]
 extern crate proc_macro_derive;