Auto merge of #146376 - durin42:dwo-specify-path, r=davidtwco

debuginfo: add an unstable flag to write split DWARF to an explicit directory

Bazel requires knowledge of outputs from actions at analysis time, including file or directory name. In order to work around the lack of predictable output name for dwo files, we group the dwo files in a subdirectory of --out-dir as a post-processing step before returning control to bazel. Unfortunately some debugging workflows rely on directly opening the dwo file rather than loading the merged dwp file, and our trick of moving the files breaks those users. We can't just hardlink the file or copy it, because with remote build execution we wouldn't end up with the un-moved file copied back to the developer's workstation. As a fix, we add this unstable flag that causes dwo files to be written to a build-system-controllable location, which then lets bazel hoover up the dwo files, but the objects also have the correct path for the dwo files.

r? `@davidtwco`

6 files changed, 167 insertions, 13 deletions

diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 76ccd12797e..58ec72b5b45 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -542,6 +542,7 @@ pub fn build_output_filenames(attrs: &[ast::Attribute], sess: &Session) -> Outpu
                 stem,
                 None,
                 sess.io.temps_dir.clone(),
+                sess.opts.unstable_opts.split_dwarf_out_dir.clone(),
                 sess.opts.cg.extra_filename.clone(),
                 sess.opts.output_types.clone(),
             )
@@ -571,6 +572,7 @@ pub fn build_output_filenames(attrs: &[ast::Attribute], sess: &Session) -> Outpu
                 out_filestem,
                 ofile,
                 sess.io.temps_dir.clone(),
+                sess.opts.unstable_opts.split_dwarf_out_dir.clone(),
                 sess.opts.cg.extra_filename.clone(),
                 sess.opts.output_types.clone(),
             )
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 811f2890fe3..d1426ff55fb 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1195,6 +1195,7 @@ pub struct OutputFilenames {
     filestem: String,
     pub single_output_file: Option<OutFileName>,
     temps_directory: Option<PathBuf>,
+    explicit_dwo_out_directory: Option<PathBuf>,
     pub outputs: OutputTypes,
 }
 
@@ -1227,6 +1228,7 @@ impl OutputFilenames {
         out_filestem: String,
         single_output_file: Option<OutFileName>,
         temps_directory: Option<PathBuf>,
+        explicit_dwo_out_directory: Option<PathBuf>,
         extra: String,
         outputs: OutputTypes,
     ) -> Self {
@@ -1234,6 +1236,7 @@ impl OutputFilenames {
             out_directory,
             single_output_file,
             temps_directory,
+            explicit_dwo_out_directory,
             outputs,
             crate_stem: format!("{out_crate_name}{extra}"),
             filestem: format!("{out_filestem}{extra}"),
@@ -1283,7 +1286,14 @@ impl OutputFilenames {
         codegen_unit_name: &str,
         invocation_temp: Option<&str>,
     ) -> PathBuf {
-        self.temp_path_ext_for_cgu(DWARF_OBJECT_EXT, codegen_unit_name, invocation_temp)
+        let p = self.temp_path_ext_for_cgu(DWARF_OBJECT_EXT, codegen_unit_name, invocation_temp);
+        if let Some(dwo_out) = &self.explicit_dwo_out_directory {
+            let mut o = dwo_out.clone();
+            o.push(p.file_name().unwrap());
+            o
+        } else {
+            p
+        }
     }
 
     /// Like `temp_path`, but also supports things where there is no corresponding
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 732cf61effc..6dd90546de1 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -2634,6 +2634,8 @@ written to standard error output)"),
                  file which is ignored by the linker
         `single`: sections which do not require relocation are written into object file but ignored
                   by the linker"),
+    split_dwarf_out_dir : Option<PathBuf> = (None, parse_opt_pathbuf, [TRACKED],
+        "location for writing split DWARF objects (`.dwo`) if enabled"),
     split_lto_unit: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "enable LTO unit splitting (default: no)"),
     src_hash_algorithm: Option<SourceFileHashAlgorithm> = (None, parse_src_file_hash, [TRACKED],
diff --git a/src/doc/unstable-book/src/compiler-flags/split-dwarf-out-dir.md b/src/doc/unstable-book/src/compiler-flags/split-dwarf-out-dir.md
new file mode 100644
index 00000000000..a2070730b42
--- /dev/null
+++ b/src/doc/unstable-book/src/compiler-flags/split-dwarf-out-dir.md
@@ -0,0 +1,7 @@
+# `split-dwarf-out-dir`
+
+On systems which use DWARF debug info this flag causes `.dwo` files produced
+by `-C split-debuginfo` to be written to the specified directory rather than
+placed next to the object files. This is mostly useful if you have a build
+system which needs to control where to find compile outputs without running the
+compiler and have to put your `.dwo` files in a separate directory.
diff --git a/src/tools/run-make-support/src/external_deps/rustc.rs b/src/tools/run-make-support/src/external_deps/rustc.rs
index b74b1d5e166..b461a24a061 100644
--- a/src/tools/run-make-support/src/external_deps/rustc.rs
+++ b/src/tools/run-make-support/src/external_deps/rustc.rs
@@ -366,6 +366,13 @@ impl Rustc {
         self
     }
 
+    pub fn split_dwarf_out_dir(&mut self, out_dir: Option<&str>) -> &mut Self {
+        if let Some(out_dir) = out_dir {
+            self.cmd.arg(format!("-Zsplit-dwarf-out-dir={out_dir}"));
+        }
+        self
+    }
+
     /// Pass the `--verbose` flag.
     pub fn verbose(&mut self) -> &mut Self {
         self.cmd.arg("--verbose");
diff --git a/tests/run-make/split-debuginfo/rmake.rs b/tests/run-make/split-debuginfo/rmake.rs
index e8de5aed172..e53b7101078 100644
--- a/tests/run-make/split-debuginfo/rmake.rs
+++ b/tests/run-make/split-debuginfo/rmake.rs
@@ -188,6 +188,25 @@ enum UnstableOptions {
 }
 
 #[track_caller]
+fn dwo_out_filenames(dwo_out: Option<&str>) -> BTreeSet<String> {
+    let dwo_out = if let Some(d) = dwo_out {
+        d
+    } else {
+        return BTreeSet::new();
+    };
+    let files = shallow_find_files(dwo_out, |path| {
+        // Fiilter out source files
+        !has_extension(path, "rs")
+    });
+    files
+        .iter()
+        .map(|p| {
+            format!("{}/{}", dwo_out, p.file_name().unwrap().to_os_string().into_string().unwrap())
+        })
+        .collect()
+}
+
+#[track_caller]
 fn cwd_filenames() -> BTreeSet<String> {
     let files = shallow_find_files(cwd(), |path| {
         // Fiilter out source files
@@ -197,6 +216,17 @@ fn cwd_filenames() -> BTreeSet<String> {
 }
 
 #[track_caller]
+fn dwo_out_dwo_filenames(dwo_out: &str) -> BTreeSet<String> {
+    let files = shallow_find_files(dwo_out, |p| has_extension(p, "dwo"));
+    files
+        .iter()
+        .map(|p| {
+            format!("{}/{}", dwo_out, p.file_name().unwrap().to_os_string().into_string().unwrap())
+        })
+        .collect()
+}
+
+#[track_caller]
 fn cwd_dwo_filenames() -> BTreeSet<String> {
     let files = shallow_find_files(cwd(), |path| has_extension(path, "dwo"));
     files.iter().map(|p| p.file_name().unwrap().to_os_string().into_string().unwrap()).collect()
@@ -376,17 +406,19 @@ mod shared_linux_other_tests {
         lto: LinkerPluginLto,
         remap_path_prefix: RemapPathPrefix,
         remap_path_scope: RemapPathScope,
+        split_dwarf_output_directory: Option<&str>,
     ) {
         run_in_tmpdir(|| {
             println!(
-                "checking: unstable_options={:?} + split_kind={:?} + level={:?} + split_dwarf_kind={:?} + lto={:?} + remap_path_prefix={:?} + remap_path_scope={:?}",
+                "checking: unstable_options={:?} + split_kind={:?} + level={:?} + split_dwarf_kind={:?} + lto={:?} + remap_path_prefix={:?} + remap_path_scope={:?} + split_dwarf_out_dir={:?}",
                 unstable_options,
                 split_kind,
                 level,
                 split_dwarf_kind,
                 lto,
                 remap_path_prefix,
-                remap_path_scope
+                remap_path_scope,
+                split_dwarf_output_directory,
             );
 
             match cross_crate_test {
@@ -398,6 +430,7 @@ mod shared_linux_other_tests {
                     lto,
                     remap_path_prefix,
                     remap_path_scope,
+                    split_dwarf_output_directory,
                 ),
                 CrossCrateTest::No => simple_split_debuginfo(
                     unstable_options,
@@ -407,6 +440,7 @@ mod shared_linux_other_tests {
                     lto,
                     remap_path_prefix,
                     remap_path_scope,
+                    split_dwarf_output_directory,
                 ),
             }
         });
@@ -420,7 +454,11 @@ mod shared_linux_other_tests {
         lto: LinkerPluginLto,
         remap_path_prefix: RemapPathPrefix,
         remap_path_scope: RemapPathScope,
+        split_dwarf_output_directory: Option<&str>,
     ) {
+        if let Some(dwo_out) = split_dwarf_output_directory {
+            run_make_support::rfs::create_dir(dwo_out);
+        }
         match (split_kind, level, split_dwarf_kind, lto, remap_path_prefix, remap_path_scope) {
             // packed-crosscrate-split
             // - Debuginfo in `.dwo` files
@@ -531,13 +569,19 @@ mod shared_linux_other_tests {
                     .input("bar.rs")
                     .crate_type("lib")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .run();
 
-                let bar_found_files = cwd_filenames();
+                let mut bar_found_files = cwd_filenames();
+                bar_found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let bar_dwo_files = cwd_dwo_filenames();
+                let bar_dwo_files = if let Some(dwo_out) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_out)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(bar_dwo_files.len(), 1);
 
                 let mut bar_expected_files = BTreeSet::new();
@@ -553,13 +597,19 @@ mod shared_linux_other_tests {
                     .extern_("bar", "libbar.rlib")
                     .input("main.rs")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .run();
 
-                let overall_found_files = cwd_filenames();
+                let mut overall_found_files = cwd_filenames();
+                overall_found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let overall_dwo_files = cwd_dwo_filenames();
+                let overall_dwo_files = if let Some(dwo_out) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_out)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(overall_dwo_files.len(), 2);
 
                 let mut overall_expected_files = BTreeSet::new();
@@ -648,7 +698,11 @@ mod shared_linux_other_tests {
         lto: LinkerPluginLto,
         remap_path_prefix: RemapPathPrefix,
         remap_path_scope: RemapPathScope,
+        split_dwarf_output_directory: Option<&str>,
     ) {
+        if let Some(dwo_out) = split_dwarf_output_directory {
+            run_make_support::rfs::create_dir(dwo_out);
+        }
         match (split_kind, level, split_dwarf_kind, lto, remap_path_prefix, remap_path_scope) {
             // off (unspecified):
             // - Debuginfo in `.o` files
@@ -921,14 +975,19 @@ mod shared_linux_other_tests {
                 rustc(unstable_options)
                     .input("foo.rs")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .run();
-                let found_files = cwd_filenames();
-
-                let dwo_files = cwd_dwo_filenames();
+                let mut found_files = cwd_filenames();
+                found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
+
+                let dwo_files = if let Some(dwo_dir) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_dir)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(dwo_files.len(), 1);
-
                 let mut expected_files = BTreeSet::new();
                 expected_files.extend(dwo_files);
                 expected_files.insert("foo".to_string());
@@ -1056,14 +1115,20 @@ mod shared_linux_other_tests {
                 rustc(unstable_options)
                     .input("foo.rs")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .remap_path_prefix(cwd(), remapped_prefix)
                     .run();
 
-                let found_files = cwd_filenames();
+                let mut found_files = cwd_filenames();
+                found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let dwo_files = cwd_dwo_filenames();
+                let dwo_files = if let Some(dwo_out) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_out)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(dwo_files.len(), 1);
 
                 let mut expected_files = BTreeSet::new();
@@ -1358,6 +1423,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // off
@@ -1370,6 +1436,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-split
@@ -1382,6 +1449,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-single
@@ -1394,6 +1462,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-lto-split
@@ -1406,6 +1475,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-lto-single
@@ -1418,6 +1488,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // FIXME: the remapping tests probably need to be reworked, see
@@ -1433,6 +1504,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-remapped-single
@@ -1445,6 +1517,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-remapped-scope
@@ -1457,6 +1530,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("debuginfo"),
+            None,
         );
 
         // packed-remapped-wrong-scope
@@ -1469,6 +1543,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("macro"),
+            None,
         );
 
         // packed-crosscrate-split
@@ -1481,6 +1556,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-crosscrate-single
@@ -1493,6 +1569,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-split
@@ -1505,6 +1582,20 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
+        );
+
+        // unpacked-split with split-dwarf-out-dir
+        shared_linux_other_tests::split_debuginfo(
+            CrossCrateTest::No,
+            UnstableOptions::Yes,
+            SplitDebuginfo::Unpacked,
+            DebuginfoLevel::Full,
+            SplitDwarfKind::Split,
+            LinkerPluginLto::Unspecified,
+            RemapPathPrefix::Unspecified,
+            RemapPathScope::Unspecified,
+            Some("other-dir"),
         );
 
         // unpacked-single
@@ -1517,6 +1608,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-lto-split
@@ -1529,6 +1621,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-lto-single
@@ -1541,6 +1634,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-remapped-split
@@ -1553,6 +1647,20 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
+        );
+
+        // unpacked-remapped-split with split-dwarf-out-dir
+        shared_linux_other_tests::split_debuginfo(
+            CrossCrateTest::No,
+            UnstableOptions::Yes,
+            SplitDebuginfo::Unpacked,
+            DebuginfoLevel::Full,
+            SplitDwarfKind::Split,
+            LinkerPluginLto::Unspecified,
+            RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
+            RemapPathScope::Unspecified,
+            Some("other-dir"),
         );
 
         // unpacked-remapped-single
@@ -1565,6 +1673,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-remapped-scope
@@ -1577,6 +1686,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("debuginfo"),
+            None,
         );
 
         // unpacked-remapped-wrong-scope
@@ -1589,6 +1699,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("macro"),
+            None,
         );
 
         // unpacked-crosscrate-split
@@ -1601,6 +1712,20 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
+        );
+
+        // unpacked-crosscrate-split with split-dwarf-out-dir
+        shared_linux_other_tests::split_debuginfo(
+            CrossCrateTest::Yes,
+            UnstableOptions::Yes,
+            SplitDebuginfo::Unpacked,
+            DebuginfoLevel::Full,
+            SplitDwarfKind::Split,
+            LinkerPluginLto::Unspecified,
+            RemapPathPrefix::Unspecified,
+            RemapPathScope::Unspecified,
+            Some("other-dir"),
         );
 
         // unpacked-crosscrate-single
@@ -1613,6 +1738,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
     }
 }