Rollup merge of #127050 - Kobzol:reproducibility-git, r=onur-ozkan

Make mtime of reproducible tarballs dependent on git commit

Since https://github.com/rust-lang/rust/pull/123246, our tarballs should be fully reproducible. That means that the mtime of all files and directories in the tarballs is set to the date of the first Rust commit (from 2006). However, this is causing some mtime invalidation issues (https://github.com/rust-lang/rust/issues/125578#issuecomment-2141068906).

Ideally, we would like to keep the mtime reproducible, but still update it with new versions of Rust. That's what this PR does. It modifies the tarball installer bootstrap invocation so that if the current rustc directory is managed by git, we will set the UTC timestamp of the latest commit as the mtime for all files in the archive. This means that the archive should be still fully reproducible from a given commit SHA, but it will also be changed with new beta bumps and `download-rustc` versions.

Note that only files are set to this mtime, directories are still set to the year 2006, because the `tar` library used by `rust-installer` doesn't allow us to selectively override mtime for directories (or at least I haven't found it). We could work around that by doing all the mtime modifications in bootstrap, but that would require more changes. I think/hope that just modifying the file mtimes should be enough. It should at least fix cargo `rustc` mtime invalidation.

Fixes: https://github.com/rust-lang/rust/issues/125578

r? ``@onur-ozkan``

try-job: x86_64-gnu-distcheck

4 files changed, 57 insertions, 5 deletions

diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs
index 5cc319826db..3c15bb296f3 100644
--- a/src/bootstrap/src/utils/tarball.rs
+++ b/src/bootstrap/src/utils/tarball.rs
@@ -9,9 +9,9 @@ use std::path::{Path, PathBuf};
 
 use crate::core::builder::Builder;
 use crate::core::{build_steps::dist::distdir, builder::Kind};
-use crate::utils::channel;
 use crate::utils::exec::BootstrapCommand;
 use crate::utils::helpers::{move_file, t};
+use crate::utils::{channel, helpers};
 
 #[derive(Copy, Clone)]
 pub(crate) enum OverlayKind {
@@ -351,6 +351,30 @@ impl<'a> Tarball<'a> {
         };
 
         cmd.args(["--compression-profile", compression_profile]);
+
+        // We want to use a pinned modification time for files in the archive
+        // to achieve better reproducibility. However, using the same mtime for all
+        // releases is not ideal, because it can break e.g. Cargo mtime checking
+        // (https://github.com/rust-lang/rust/issues/125578).
+        // Therefore, we set mtime to the date of the latest commit (if we're managed
+        // by git). In this way, the archive will still be always the same for a given commit
+        // (achieving reproducibility), but it will also change between different commits and
+        // Rust versions, so that it won't break mtime-based caches.
+        //
+        // Note that this only overrides the mtime of files, not directories, due to the
+        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
+
+        // Get the UTC timestamp of the last git commit, if we're under git.
+        // We need to use UTC, so that anyone who tries to rebuild from the same commit
+        // gets the same timestamp.
+        if self.builder.rust_info().is_managed_git_subrepository() {
+            // %ct means committer date
+            let timestamp = helpers::output(
+                helpers::git(Some(&self.builder.src)).arg("log").arg("-1").arg("--format=%ct"),
+            );
+            cmd.args(["--override-file-mtime", timestamp.trim()]);
+        }
+
         self.builder.run(cmd);
 
         // Ensure there are no symbolic links in the tarball. In particular,
diff --git a/src/tools/rust-installer/src/combiner.rs b/src/tools/rust-installer/src/combiner.rs
index 565d19d863f..c211b34850a 100644
--- a/src/tools/rust-installer/src/combiner.rs
+++ b/src/tools/rust-installer/src/combiner.rs
@@ -55,6 +55,12 @@ actor! {
         /// The formats used to compress the tarball
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -145,7 +151,8 @@ impl Combiner {
             .input(self.package_name)
             .output(path_to_str(&output)?.into())
             .compression_profile(self.compression_profile)
-            .compression_formats(self.compression_formats);
+            .compression_formats(self.compression_formats)
+            .override_file_mtime(self.override_file_mtime);
         tarballer.run()?;
 
         Ok(())
diff --git a/src/tools/rust-installer/src/generator.rs b/src/tools/rust-installer/src/generator.rs
index b101e67d8df..035a394deeb 100644
--- a/src/tools/rust-installer/src/generator.rs
+++ b/src/tools/rust-installer/src/generator.rs
@@ -61,6 +61,12 @@ actor! {
         /// The formats used to compress the tarball
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -114,7 +120,8 @@ impl Generator {
             .input(self.package_name)
             .output(path_to_str(&output)?.into())
             .compression_profile(self.compression_profile)
-            .compression_formats(self.compression_formats);
+            .compression_formats(self.compression_formats)
+            .override_file_mtime(self.override_file_mtime);
         tarballer.run()?;
 
         Ok(())
diff --git a/src/tools/rust-installer/src/tarballer.rs b/src/tools/rust-installer/src/tarballer.rs
index 2f093e7ad17..b5e87a66ffc 100644
--- a/src/tools/rust-installer/src/tarballer.rs
+++ b/src/tools/rust-installer/src/tarballer.rs
@@ -32,6 +32,12 @@ actor! {
         /// The formats used to compress the tarball.
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -65,6 +71,8 @@ impl Tarballer {
         let buf = BufWriter::with_capacity(1024 * 1024, encoder);
         let mut builder = Builder::new(buf);
         // Make uid, gid and mtime deterministic to improve reproducibility
+        // The modification time of directories will be set to the date of the first Rust commit.
+        // The modification time of files will be set to `override_file_mtime` (see `append_path`).
         builder.mode(HeaderMode::Deterministic);
 
         let pool = rayon::ThreadPoolBuilder::new().num_threads(2).build().unwrap();
@@ -77,7 +85,7 @@ impl Tarballer {
             }
             for path in files {
                 let src = Path::new(&self.work_dir).join(&path);
-                append_path(&mut builder, &src, &path)
+                append_path(&mut builder, &src, &path, self.override_file_mtime)
                     .with_context(|| format!("failed to tar file '{}'", src.display()))?;
             }
             builder
@@ -93,10 +101,16 @@ impl Tarballer {
     }
 }
 
-fn append_path<W: Write>(builder: &mut Builder<W>, src: &Path, path: &String) -> Result<()> {
+fn append_path<W: Write>(
+    builder: &mut Builder<W>,
+    src: &Path,
+    path: &String,
+    override_file_mtime: u64,
+) -> Result<()> {
     let stat = symlink_metadata(src)?;
     let mut header = Header::new_gnu();
     header.set_metadata_in_mode(&stat, HeaderMode::Deterministic);
+    header.set_mtime(override_file_mtime);
 
     if stat.file_type().is_symlink() {
         let link = read_link(src)?;