Auto merge of #64328 - Mark-Simulacrum:rustdoc-find-rustc, r=GuillaumeGomez

rustdoc: change doctests locating rustc binary

We previously used the "naive" approach of replacing the `current_exe()`'s file name with rustc, but now load from the sysroot by default (`$sysroot/bin/rustc`). The functionality of locating the sysroot overlaps/is the same as the functionality used by codegen backend loading; this ensures that any failure cases we've introduced are not exceeding those, and that improvements to finding the sysroot for loading codegen backends likewise enhance rustdoc.

The second commit adds an unstable `--test-builder` flag to rustdoc, and is largely separate (I can split into separate PR, but it's a simple and related change). This is largely intended for "advanced" users at this point (I'm not sure if we'll ever stabilize it); it permits use of a different rustc binary for rustdoc compilation of doctests than the rustdoc binary used when loading. Note, that this may not be what you want as the parsers and such differ (and rustdoc uses its own libsyntax, etc.). However, I've been told that running doctests in miri may be assisted by this change, so I've implemented it; I'll file a tracking issue for it if there's interest in it (and we land this PR).

10 files changed, 143 insertions, 79 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 27ee3814609..f2eaf470658 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2138,6 +2138,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "once_cell"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6a04cb71e910d0034815600180f62a95bf6e67942d7ab52a166a68c7d7e9cd0"
+
+[[package]]
 name = "open"
 version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3425,6 +3431,7 @@ name = "rustc_interface"
 version = "0.0.0"
 dependencies = [
  "log",
+ "once_cell",
  "rustc",
  "rustc-rayon",
  "rustc_ast_borrowck",
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 4f5de1ecd2b..b7873fd1d35 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -1167,6 +1167,8 @@ impl<'a> Builder<'a> {
             cargo.arg("--frozen");
         }
 
+        cargo.env("RUSTC_INSTALL_BINDIR", &self.config.bindir);
+
         self.ci_env.force_coloring_in_ci(&mut cargo);
 
         cargo
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 43d9264eaca..52b5cd888df 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -137,7 +137,7 @@ pub struct Config {
     pub sysconfdir: Option<PathBuf>,
     pub datadir: Option<PathBuf>,
     pub docdir: Option<PathBuf>,
-    pub bindir: Option<PathBuf>,
+    pub bindir: PathBuf,
     pub libdir: Option<PathBuf>,
     pub mandir: Option<PathBuf>,
     pub codegen_tests: bool,
@@ -400,6 +400,7 @@ impl Config {
         config.incremental = flags.incremental;
         config.dry_run = flags.dry_run;
         config.keep_stage = flags.keep_stage;
+        config.bindir = "bin".into(); // default
         if let Some(value) = flags.deny_warnings {
             config.deny_warnings = value;
         }
@@ -482,7 +483,7 @@ impl Config {
             config.sysconfdir = install.sysconfdir.clone().map(PathBuf::from);
             config.datadir = install.datadir.clone().map(PathBuf::from);
             config.docdir = install.docdir.clone().map(PathBuf::from);
-            config.bindir = install.bindir.clone().map(PathBuf::from);
+            set(&mut config.bindir, install.bindir.clone().map(PathBuf::from));
             config.libdir = install.libdir.clone().map(PathBuf::from);
             config.mandir = install.mandir.clone().map(PathBuf::from);
         }
diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs
index 557586709c6..384219c38fd 100644
--- a/src/bootstrap/install.rs
+++ b/src/bootstrap/install.rs
@@ -67,7 +67,6 @@ fn install_sh(
     let sysconfdir_default = PathBuf::from("/etc");
     let datadir_default = PathBuf::from("share");
     let docdir_default = datadir_default.join("doc/rust");
-    let bindir_default = PathBuf::from("bin");
     let libdir_default = PathBuf::from("lib");
     let mandir_default = datadir_default.join("man");
     let prefix = builder.config.prefix.as_ref().map_or(prefix_default, |p| {
@@ -76,7 +75,7 @@ fn install_sh(
     let sysconfdir = builder.config.sysconfdir.as_ref().unwrap_or(&sysconfdir_default);
     let datadir = builder.config.datadir.as_ref().unwrap_or(&datadir_default);
     let docdir = builder.config.docdir.as_ref().unwrap_or(&docdir_default);
-    let bindir = builder.config.bindir.as_ref().unwrap_or(&bindir_default);
+    let bindir = &builder.config.bindir;
     let libdir = builder.config.libdir.as_ref().unwrap_or(&libdir_default);
     let mandir = builder.config.mandir.as_ref().unwrap_or(&mandir_default);
 
diff --git a/src/librustc_interface/Cargo.toml b/src/librustc_interface/Cargo.toml
index 16b377d5bcc..f6293107a94 100644
--- a/src/librustc_interface/Cargo.toml
+++ b/src/librustc_interface/Cargo.toml
@@ -34,3 +34,4 @@ rustc_plugin = { path = "../librustc_plugin", package = "rustc_plugin_impl" }
 rustc_privacy = { path = "../librustc_privacy" }
 rustc_resolve = { path = "../librustc_resolve" }
 tempfile = "3.0.5"
+once_cell = "1"
diff --git a/src/librustc_interface/build.rs b/src/librustc_interface/build.rs
new file mode 100644
index 00000000000..79a343e0fee
--- /dev/null
+++ b/src/librustc_interface/build.rs
@@ -0,0 +1,4 @@
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=RUSTC_INSTALL_BINDIR");
+}
diff --git a/src/librustc_interface/util.rs b/src/librustc_interface/util.rs
index 9eaf7b77716..b81f814de0f 100644
--- a/src/librustc_interface/util.rs
+++ b/src/librustc_interface/util.rs
@@ -289,20 +289,39 @@ pub fn get_codegen_backend(sess: &Session) -> Box<dyn CodegenBackend> {
     backend
 }
 
-pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
-    // For now we only allow this function to be called once as it'll dlopen a
-    // few things, which seems to work best if we only do that once. In
-    // general this assertion never trips due to the once guard in `get_codegen_backend`,
-    // but there's a few manual calls to this function in this file we protect
-    // against.
-    static LOADED: AtomicBool = AtomicBool::new(false);
-    assert!(!LOADED.fetch_or(true, Ordering::SeqCst),
-            "cannot load the default codegen backend twice");
+// This is used for rustdoc, but it uses similar machinery to codegen backend
+// loading, so we leave the code here. It is potentially useful for other tools
+// that want to invoke the rustc binary while linking to rustc as well.
+pub fn rustc_path<'a>() -> Option<&'a Path> {
+    static RUSTC_PATH: once_cell::sync::OnceCell<Option<PathBuf>> =
+        once_cell::sync::OnceCell::new();
 
+    const BIN_PATH: &str = env!("RUSTC_INSTALL_BINDIR");
+
+    RUSTC_PATH.get_or_init(|| get_rustc_path_inner(BIN_PATH)).as_ref().map(|v| &**v)
+}
+
+fn get_rustc_path_inner(bin_path: &str) -> Option<PathBuf> {
+    sysroot_candidates().iter()
+        .filter_map(|sysroot| {
+            let candidate = sysroot.join(bin_path).join(if cfg!(target_os = "windows") {
+                "rustc.exe"
+            } else {
+                "rustc"
+            });
+            if candidate.exists() {
+                Some(candidate)
+            } else {
+                None
+            }
+        })
+        .next()
+}
+
+fn sysroot_candidates() -> Vec<PathBuf> {
     let target = session::config::host_triple();
     let mut sysroot_candidates = vec![filesearch::get_or_default_sysroot()];
-    let path = current_dll_path()
-        .and_then(|s| s.canonicalize().ok());
+    let path = current_dll_path().and_then(|s| s.canonicalize().ok());
     if let Some(dll) = path {
         // use `parent` twice to chop off the file name and then also the
         // directory containing the dll which should be either `lib` or `bin`.
@@ -327,69 +346,7 @@ pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend
         }
     }
 
-    let sysroot = sysroot_candidates.iter()
-        .map(|sysroot| {
-            let libdir = filesearch::relative_target_lib_path(&sysroot, &target);
-            sysroot.join(libdir).with_file_name(
-                option_env!("CFG_CODEGEN_BACKENDS_DIR").unwrap_or("codegen-backends"))
-        })
-        .filter(|f| {
-            info!("codegen backend candidate: {}", f.display());
-            f.exists()
-        })
-        .next();
-    let sysroot = sysroot.unwrap_or_else(|| {
-        let candidates = sysroot_candidates.iter()
-            .map(|p| p.display().to_string())
-            .collect::<Vec<_>>()
-            .join("\n* ");
-        let err = format!("failed to find a `codegen-backends` folder \
-                           in the sysroot candidates:\n* {}", candidates);
-        early_error(ErrorOutputType::default(), &err);
-    });
-    info!("probing {} for a codegen backend", sysroot.display());
-
-    let d = sysroot.read_dir().unwrap_or_else(|e| {
-        let err = format!("failed to load default codegen backend, couldn't \
-                           read `{}`: {}", sysroot.display(), e);
-        early_error(ErrorOutputType::default(), &err);
-    });
-
-    let mut file: Option<PathBuf> = None;
-
-    let expected_name = format!("rustc_codegen_llvm-{}", backend_name);
-    for entry in d.filter_map(|e| e.ok()) {
-        let path = entry.path();
-        let filename = match path.file_name().and_then(|s| s.to_str()) {
-            Some(s) => s,
-            None => continue,
-        };
-        if !(filename.starts_with(DLL_PREFIX) && filename.ends_with(DLL_SUFFIX)) {
-            continue
-        }
-        let name = &filename[DLL_PREFIX.len() .. filename.len() - DLL_SUFFIX.len()];
-        if name != expected_name {
-            continue
-        }
-        if let Some(ref prev) = file {
-            let err = format!("duplicate codegen backends found\n\
-                               first:  {}\n\
-                               second: {}\n\
-            ", prev.display(), path.display());
-            early_error(ErrorOutputType::default(), &err);
-        }
-        file = Some(path.clone());
-    }
-
-    match file {
-        Some(ref s) => return load_backend_from_dylib(s),
-        None => {
-            let err = format!("failed to load default codegen backend for `{}`, \
-                               no appropriate codegen dylib found in `{}`",
-                              backend_name, sysroot.display());
-            early_error(ErrorOutputType::default(), &err);
-        }
-    }
+    return sysroot_candidates;
 
     #[cfg(unix)]
     fn current_dll_path() -> Option<PathBuf> {
@@ -459,6 +416,85 @@ pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend
     }
 }
 
+pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> {
+    // For now we only allow this function to be called once as it'll dlopen a
+    // few things, which seems to work best if we only do that once. In
+    // general this assertion never trips due to the once guard in `get_codegen_backend`,
+    // but there's a few manual calls to this function in this file we protect
+    // against.
+    static LOADED: AtomicBool = AtomicBool::new(false);
+    assert!(!LOADED.fetch_or(true, Ordering::SeqCst),
+            "cannot load the default codegen backend twice");
+
+    let target = session::config::host_triple();
+    let sysroot_candidates = sysroot_candidates();
+
+    let sysroot = sysroot_candidates.iter()
+        .map(|sysroot| {
+            let libdir = filesearch::relative_target_lib_path(&sysroot, &target);
+            sysroot.join(libdir).with_file_name(
+                option_env!("CFG_CODEGEN_BACKENDS_DIR").unwrap_or("codegen-backends"))
+        })
+        .filter(|f| {
+            info!("codegen backend candidate: {}", f.display());
+            f.exists()
+        })
+        .next();
+    let sysroot = sysroot.unwrap_or_else(|| {
+        let candidates = sysroot_candidates.iter()
+            .map(|p| p.display().to_string())
+            .collect::<Vec<_>>()
+            .join("\n* ");
+        let err = format!("failed to find a `codegen-backends` folder \
+                           in the sysroot candidates:\n* {}", candidates);
+        early_error(ErrorOutputType::default(), &err);
+    });
+    info!("probing {} for a codegen backend", sysroot.display());
+
+    let d = sysroot.read_dir().unwrap_or_else(|e| {
+        let err = format!("failed to load default codegen backend, couldn't \
+                           read `{}`: {}", sysroot.display(), e);
+        early_error(ErrorOutputType::default(), &err);
+    });
+
+    let mut file: Option<PathBuf> = None;
+
+    let expected_name = format!("rustc_codegen_llvm-{}", backend_name);
+    for entry in d.filter_map(|e| e.ok()) {
+        let path = entry.path();
+        let filename = match path.file_name().and_then(|s| s.to_str()) {
+            Some(s) => s,
+            None => continue,
+        };
+        if !(filename.starts_with(DLL_PREFIX) && filename.ends_with(DLL_SUFFIX)) {
+            continue
+        }
+        let name = &filename[DLL_PREFIX.len() .. filename.len() - DLL_SUFFIX.len()];
+        if name != expected_name {
+            continue
+        }
+        if let Some(ref prev) = file {
+            let err = format!("duplicate codegen backends found\n\
+                               first:  {}\n\
+                               second: {}\n\
+            ", prev.display(), path.display());
+            early_error(ErrorOutputType::default(), &err);
+        }
+        file = Some(path.clone());
+    }
+
+    match file {
+        Some(ref s) => return load_backend_from_dylib(s),
+        None => {
+            let err = format!("failed to load default codegen backend for `{}`, \
+                               no appropriate codegen dylib found in `{}`",
+                              backend_name, sysroot.display());
+            early_error(ErrorOutputType::default(), &err);
+        }
+    }
+
+}
+
 pub(crate) fn compute_crate_disambiguator(session: &Session) -> CrateDisambiguator {
     use std::hash::Hasher;
 
diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs
index 995a340143f..19ea7814300 100644
--- a/src/librustdoc/config.rs
+++ b/src/librustdoc/config.rs
@@ -86,6 +86,10 @@ pub struct Options {
     /// contains "foo" as a substring
     pub enable_per_target_ignores: bool,
 
+    /// The path to a rustc-like binary to build tests with. If not set, we
+    /// default to loading from $sysroot/bin/rustc.
+    pub test_builder: Option<PathBuf>,
+
     // Options that affect the documentation process
 
     /// The selected default set of passes to use.
@@ -476,6 +480,7 @@ impl Options {
         let generate_search_filter = !matches.opt_present("disable-per-crate-search");
         let persist_doctests = matches.opt_str("persist-doctests").map(PathBuf::from);
         let generate_redirect_pages = matches.opt_present("generate-redirect-pages");
+        let test_builder = matches.opt_str("test-builder").map(PathBuf::from);
         let codegen_options_strs = matches.opt_strs("C");
         let lib_strs = matches.opt_strs("L");
         let extern_strs = matches.opt_strs("extern");
@@ -515,6 +520,7 @@ impl Options {
             runtool,
             runtool_args,
             enable_per_target_ignores,
+            test_builder,
             render_options: RenderOptions {
                 output,
                 external_html,
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index 88da1b16686..d77e790d4a4 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -373,6 +373,11 @@ fn opts() -> Vec<RustcOptGroup> {
                        "",
                        "One (of possibly many) arguments to pass to the runtool")
         }),
+        unstable("test-builder", |o| {
+            o.optflag("",
+                      "test-builder",
+                      "specified the rustc-like binary to use as the test builder")
+        }),
     ]
 }
 
diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs
index acc41b7c13b..482c69c1ab5 100644
--- a/src/librustdoc/test.rs
+++ b/src/librustdoc/test.rs
@@ -248,7 +248,10 @@ fn run_test(
     };
     let output_file = outdir.path().join("rust_out");
 
-    let mut compiler = Command::new(std::env::current_exe().unwrap().with_file_name("rustc"));
+    let rustc_binary = options.test_builder.as_ref().map(|v| &**v).unwrap_or_else(|| {
+        rustc_interface::util::rustc_path().expect("found rustc")
+    });
+    let mut compiler = Command::new(&rustc_binary);
     compiler.arg("--crate-type").arg("bin");
     for cfg in &options.cfgs {
         compiler.arg("--cfg").arg(&cfg);