Make mtime of reproducible tarball dependent on git commit

Kobzol · Kobzol · commit c54a2a53f83d · 2024-06-29T15:08:17.000+02:00
diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs
@@ -9,9 +9,9 @@ use std::path::{Path, PathBuf};
 
 use crate::core::builder::Builder;
 use crate::core::{build_steps::dist::distdir, builder::Kind};
-use crate::utils::channel;
 use crate::utils::exec::BootstrapCommand;
 use crate::utils::helpers::{move_file, t};
+use crate::utils::{channel, helpers};
 
 #[derive(Copy, Clone)]
 pub(crate) enum OverlayKind {
@@ -351,6 +351,30 @@ impl<'a> Tarball<'a> {
         };
 
         cmd.args(["--compression-profile", compression_profile]);
+
+        // We want to use a pinned modification time for files in the archive
+        // to achieve better reproducibility. However, using the same mtime for all
+        // releases is not ideal, because it can break e.g. Cargo mtime checking
+        // (https://github.com/rust-lang/rust/issues/125578).
+        // Therefore, we set mtime to the date of the latest commit (if we're managed
+        // by git). In this way, the archive will still be always the same for a given commit
+        // (achieving reproducibility), but it will also change between different commits and
+        // Rust versions, so that it won't break mtime-based caches.
+        //
+        // Note that this only overrides the mtime of files, not directories, due to the
+        // limitations of the tarballer tool. Directories will have their mtime set to 2006.
+
+        // Get the UTC timestamp of the last git commit, if we're under git.
+        // We need to use UTC, so that anyone who tries to rebuild from the same commit
+        // gets the same timestamp.
+        if self.builder.rust_info().is_managed_git_subrepository() {
+            // %ct means committer date
+            let timestamp = helpers::output(
+                helpers::git(Some(&self.builder.src)).arg("log").arg("-1").arg("--format=%ct"),
+            );
+            cmd.args(["--override-file-mtime", timestamp.trim()]);
+        }
+
         self.builder.run(cmd);
 
         // Ensure there are no symbolic links in the tarball. In particular,
diff --git a/src/tools/rust-installer/src/tarballer.rs b/src/tools/rust-installer/src/tarballer.rs
@@ -32,6 +32,12 @@ actor! {
         /// The formats used to compress the tarball.
         #[arg(value_name = "FORMAT", default_value_t)]
         compression_formats: CompressionFormats,
+
+        /// Modification time that will be set for all files added to the archive.
+        /// The default is the date of the first Rust commit from 2006.
+        /// This serves for better reproducibility of the archives.
+        #[arg(value_name = "FILE_MTIME", default_value_t = 1153704088)]
+        override_file_mtime: u64,
     }
 }
 
@@ -65,6 +71,8 @@ impl Tarballer {
         let buf = BufWriter::with_capacity(1024 * 1024, encoder);
         let mut builder = Builder::new(buf);
         // Make uid, gid and mtime deterministic to improve reproducibility
+        // The modification time of directories will be set to the date of the first Rust commit.
+        // The modification time of files will be set to `override_file_mtime` (see `append_path`).
         builder.mode(HeaderMode::Deterministic);
 
         let pool = rayon::ThreadPoolBuilder::new().num_threads(2).build().unwrap();
@@ -77,7 +85,7 @@ impl Tarballer {
             }
             for path in files {
                 let src = Path::new(&self.work_dir).join(&path);
-                append_path(&mut builder, &src, &path)
+                append_path(&mut builder, &src, &path, self.override_file_mtime)
                     .with_context(|| format!("failed to tar file '{}'", src.display()))?;
             }
             builder
@@ -93,10 +101,16 @@ impl Tarballer {
     }
 }
 
-fn append_path<W: Write>(builder: &mut Builder<W>, src: &Path, path: &String) -> Result<()> {
+fn append_path<W: Write>(
+    builder: &mut Builder<W>,
+    src: &Path,
+    path: &String,
+    override_file_mtime: u64,
+) -> Result<()> {
     let stat = symlink_metadata(src)?;
     let mut header = Header::new_gnu();
     header.set_metadata_in_mode(&stat, HeaderMode::Deterministic);
+    header.set_mtime(override_file_mtime);
 
     if stat.file_type().is_symlink() {
         let link = read_link(src)?;
