Rollup merge of #127050 - Kobzol:reproducibility-git, r=onur-ozkan

Make mtime of reproducible tarballs dependent on git commit

Since https://github.com/rust-lang/rust/pull/123246, our tarballs should be fully reproducible. That means that the mtime of all files and directories in the tarballs is set to the date of the first Rust commit (from 2006). However, this is causing some mtime invalidation issues (https://github.com/rust-lang/rust/issues/125578#issuecomment-2141068906).

Ideally, we would like to keep the mtime reproducible, but still update it with new versions of Rust. That's what this PR does. It modifies the tarball installer bootstrap invocation so that if the current rustc directory is managed by git, we will set the UTC timestamp of the latest commit as the mtime for all files in the archive. This means that the archive should be still fully reproducible from a given commit SHA, but it will also be changed with new beta bumps and `download-rustc` versions.

Note that only files are set to this mtime, directories are still set to the year 2006, because the `tar` library used by `rust-installer` doesn't allow us to selectively override mtime for directories (or at least I haven't found it). We could work around that by doing all the mtime modifications in bootstrap, but that would require more changes. I think/hope that just modifying the file mtimes should be enough. It should at least fix cargo `rustc` mtime invalidation.

Fixes: https://github.com/rust-lang/rust/issues/125578

r? ``@onur-ozkan``

try-job: x86_64-gnu-distcheck

3 files changed, 32 insertions, 4 deletions

diff --git a/src/tools/rust-installer/src/combiner.rs b/src/tools/rust-installer/src/combiner.rs
index 565d19d863f..c211b34850a 100644
--- a/src/tools/rust-installer/src/combiner.rs
+++ b/src/tools/rust-installer/src/combiner.rs
@@ -55,6 +55,12 @@ actor! {
         /// The formats used to compress the tarball
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -145,7 +151,8 @@ impl Combiner {
             .input(self.package_name)
             .output(path_to_str(&output)?.into())
             .compression_profile(self.compression_profile)
-            .compression_formats(self.compression_formats);
+            .compression_formats(self.compression_formats)
+            .override_file_mtime(self.override_file_mtime);
         tarballer.run()?;
 
         Ok(())
diff --git a/src/tools/rust-installer/src/generator.rs b/src/tools/rust-installer/src/generator.rs
index b101e67d8df..035a394deeb 100644
--- a/src/tools/rust-installer/src/generator.rs
+++ b/src/tools/rust-installer/src/generator.rs
@@ -61,6 +61,12 @@ actor! {
         /// The formats used to compress the tarball
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -114,7 +120,8 @@ impl Generator {
             .input(self.package_name)
             .output(path_to_str(&output)?.into())
             .compression_profile(self.compression_profile)
-            .compression_formats(self.compression_formats);
+            .compression_formats(self.compression_formats)
+            .override_file_mtime(self.override_file_mtime);
         tarballer.run()?;
 
         Ok(())
diff --git a/src/tools/rust-installer/src/tarballer.rs b/src/tools/rust-installer/src/tarballer.rs
index 2f093e7ad17..b5e87a66ffc 100644
--- a/src/tools/rust-installer/src/tarballer.rs
+++ b/src/tools/rust-installer/src/tarballer.rs
@@ -32,6 +32,12 @@ actor! {
         /// The formats used to compress the tarball.
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -65,6 +71,8 @@ impl Tarballer {
         let buf = BufWriter::with_capacity(1024 * 1024, encoder);
         let mut builder = Builder::new(buf);
         // Make uid, gid and mtime deterministic to improve reproducibility
+        // The modification time of directories will be set to the date of the first Rust commit.
+        // The modification time of files will be set to `override_file_mtime` (see `append_path`).
         builder.mode(HeaderMode::Deterministic);
 
         let pool = rayon::ThreadPoolBuilder::new().num_threads(2).build().unwrap();
@@ -77,7 +85,7 @@ impl Tarballer {
             }
             for path in files {
                 let src = Path::new(&self.work_dir).join(&path);
-                append_path(&mut builder, &src, &path)
+                append_path(&mut builder, &src, &path, self.override_file_mtime)
                     .with_context(|| format!("failed to tar file '{}'", src.display()))?;
             }
             builder
@@ -93,10 +101,16 @@ impl Tarballer {
     }
 }
 
-fn append_path<W: Write>(builder: &mut Builder<W>, src: &Path, path: &String) -> Result<()> {
+fn append_path<W: Write>(
+    builder: &mut Builder<W>,
+    src: &Path,
+    path: &String,
+    override_file_mtime: u64,
+) -> Result<()> {
     let stat = symlink_metadata(src)?;
     let mut header = Header::new_gnu();
     header.set_metadata_in_mode(&stat, HeaderMode::Deterministic);
+    header.set_mtime(override_file_mtime);
 
     if stat.file_type().is_symlink() {
         let link = read_link(src)?;