Auto merge of #77557 - Dylan-DPC:rollup-aib9ptp, r=Dylan-DPC

Rollup of 11 pull requests

Successful merges:

 - #75853 (Use more intra-doc-links in `core::fmt`)
 - #75928 (Remove trait_selection error message in specific case)
 - #76329 (Add check for doc alias attribute at crate level)
 - #77219 (core::global_allocator docs link to std::alloc::GlobalAlloc)
 - #77395 (BTreeMap: admit the existence of leaf edges in comments)
 - #77407 (Improve build-manifest to work with the improved promote-release)
 - #77426 (Include scope id in SocketAddrV6::Display)
 - #77439 (Fix missing diagnostic span for `impl Trait` with const generics, and add various tests for `min_const_generics` and `const_generics`)
 - #77471 (BTreeMap: refactoring around edges, missed spots)
 - #77512 (Allow `Abort` terminators in all const-contexts)
 - #77514 (Replace some once(x).chain(once(y)) with [x, y] IntoIter)

Failed merges:

r? `@ghost`

4 files changed, 225 insertions, 120 deletions

diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
index 4f89c31936d..4ae4dbfc06e 100644
--- a/src/tools/build-manifest/Cargo.toml
+++ b/src/tools/build-manifest/Cargo.toml
@@ -11,3 +11,6 @@ serde_json = "1.0"
 anyhow = "1.0.32"
 flate2 = "1.0.16"
 tar = "0.4.29"
+sha2 = "0.9.1"
+rayon = "1.3.1"
+hex = "0.4.2"
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
index 4d7d9f7da18..26e96c9fd8f 100644
--- a/src/tools/build-manifest/README.md
+++ b/src/tools/build-manifest/README.md
@@ -20,8 +20,7 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
 `path/to/output` with:
 
 ```
-$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
-    path/to/dist path/to/output 1970-01-01 http://example.com \
+$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
     CHANNEL path/to/rust/repo
 ```
 
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index e1dc9111bf3..c7e7d88c68f 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -4,17 +4,22 @@
 //! via `x.py dist hash-and-sign`; the cmdline arguments are set up
 //! by rustbuild (in `src/bootstrap/dist.rs`).
 
+mod manifest;
 mod versions;
 
+use crate::manifest::{Component, FileHash, Manifest, Package, Rename, Target};
 use crate::versions::{PkgType, Versions};
-use serde::Serialize;
-use std::collections::BTreeMap;
-use std::collections::HashMap;
+use rayon::prelude::*;
+use sha2::Digest;
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::env;
+use std::error::Error;
 use std::fs::{self, File};
-use std::io::{self, Read, Write};
+use std::io::{self, BufReader, Read, Write};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
+use std::sync::Mutex;
+use std::time::Instant;
 
 static HOSTS: &[&str] = &[
     "aarch64-unknown-linux-gnu",
@@ -167,57 +172,6 @@ static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 
 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
 
-#[derive(Serialize)]
-#[serde(rename_all = "kebab-case")]
-struct Manifest {
-    manifest_version: String,
-    date: String,
-    pkg: BTreeMap<String, Package>,
-    renames: BTreeMap<String, Rename>,
-    profiles: BTreeMap<String, Vec<String>>,
-}
-
-#[derive(Serialize)]
-struct Package {
-    version: String,
-    git_commit_hash: Option<String>,
-    target: BTreeMap<String, Target>,
-}
-
-#[derive(Serialize)]
-struct Rename {
-    to: String,
-}
-
-#[derive(Serialize, Default)]
-struct Target {
-    available: bool,
-    url: Option<String>,
-    hash: Option<String>,
-    xz_url: Option<String>,
-    xz_hash: Option<String>,
-    components: Option<Vec<Component>>,
-    extensions: Option<Vec<Component>>,
-}
-
-impl Target {
-    fn unavailable() -> Self {
-        Self::default()
-    }
-}
-
-#[derive(Serialize)]
-struct Component {
-    pkg: String,
-    target: String,
-}
-
-impl Component {
-    fn from_str(pkg: &str, target: &str) -> Self {
-        Self { pkg: pkg.to_string(), target: target.to_string() }
-    }
-}
-
 macro_rules! t {
     ($e:expr) => {
         match $e {
@@ -232,25 +186,33 @@ struct Builder {
 
     input: PathBuf,
     output: PathBuf,
-    gpg_passphrase: String,
-    digests: BTreeMap<String, String>,
     s3_address: String,
     date: String,
 
-    should_sign: bool,
+    legacy: bool,
+    legacy_gpg_passphrase: String,
 }
 
 fn main() {
-    // Avoid signing packages while manually testing
-    // Do NOT set this envvar in CI
-    let should_sign = env::var("BUILD_MANIFEST_DISABLE_SIGNING").is_err();
-
-    // Safety check to ensure signing is always enabled on CI
-    // The CI environment variable is set by both Travis and AppVeyor
-    if !should_sign && env::var("CI").is_ok() {
-        println!("The 'BUILD_MANIFEST_DISABLE_SIGNING' env var can't be enabled on CI.");
-        println!("If you're not running this on CI, unset the 'CI' env var.");
-        panic!();
+    // Up until Rust 1.48 the release process relied on build-manifest to create the SHA256
+    // checksums of released files and to sign the tarballs. That was moved over to promote-release
+    // in time for the branching of Rust 1.48, but the old release process still had to work the
+    // old way.
+    //
+    // When running build-manifest through the old ./x.py dist hash-and-sign the environment
+    // variable will be set, enabling the legacy behavior of generating the .sha256 files and
+    // signing the tarballs.
+    //
+    // Once the old release process is fully decommissioned, the environment variable, all the
+    // related code in this tool and ./x.py dist hash-and-sign can be removed.
+    let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok();
+
+    // Avoid overloading the old server in legacy mode.
+    if legacy {
+        rayon::ThreadPoolBuilder::new()
+            .num_threads(1)
+            .build_global()
+            .expect("failed to initialize Rayon");
     }
 
     let mut args = env::args().skip(1);
@@ -263,7 +225,7 @@ fn main() {
 
     // Do not ask for a passphrase while manually testing
     let mut passphrase = String::new();
-    if should_sign {
+    if legacy {
         // `x.py` passes the passphrase via stdin.
         t!(io::stdin().read_to_string(&mut passphrase));
     }
@@ -273,12 +235,11 @@ fn main() {
 
         input,
         output,
-        gpg_passphrase: passphrase,
-        digests: BTreeMap::new(),
         s3_address,
         date,
 
-        should_sign,
+        legacy,
+        legacy_gpg_passphrase: passphrase,
     }
     .build();
 }
@@ -286,7 +247,9 @@ fn main() {
 impl Builder {
     fn build(&mut self) {
         self.check_toolstate();
-        self.digest_and_sign();
+        if self.legacy {
+            self.digest_and_sign();
+        }
         let manifest = self.build_manifest();
 
         let rust_version = self.versions.package_version(&PkgType::Rust).unwrap();
@@ -324,10 +287,9 @@ impl Builder {
     /// Hash all files, compute their signatures, and collect the hashes in `self.digests`.
     fn digest_and_sign(&mut self) {
         for file in t!(self.input.read_dir()).map(|e| t!(e).path()) {
-            let filename = file.file_name().unwrap().to_str().unwrap();
-            let digest = self.hash(&file);
+            file.file_name().unwrap().to_str().unwrap();
+            self.hash(&file);
             self.sign(&file);
-            assert!(self.digests.insert(filename.to_string(), digest).is_none());
         }
     }
 
@@ -343,6 +305,9 @@ impl Builder {
         self.add_profiles_to(&mut manifest);
         self.add_renames_to(&mut manifest);
         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
+
+        self.fill_missing_hashes(&mut manifest);
+
         manifest
     }
 
@@ -438,9 +403,12 @@ impl Builder {
 
     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
         let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
-        let digest = self.digests.remove(&filename)?;
-        let xz_filename = filename.replace(".tar.gz", ".tar.xz");
-        let xz_digest = self.digests.remove(&xz_filename);
+
+        let mut target = Target::from_compressed_tar(self, &filename);
+        if !target.available {
+            return None;
+        }
+
         let mut components = Vec::new();
         let mut extensions = Vec::new();
 
@@ -496,15 +464,9 @@ impl Builder {
         extensions.retain(&has_component);
         components.retain(&has_component);
 
-        Some(Target {
-            available: true,
-            url: Some(self.url(&filename)),
-            hash: Some(digest),
-            xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
-            xz_hash: xz_digest,
-            components: Some(components),
-            extensions: Some(extensions),
-        })
+        target.components = Some(components);
+        target.extensions = Some(extensions);
+        Some(target)
     }
 
     fn profile(
@@ -542,37 +504,19 @@ impl Builder {
         let targets = targets
             .iter()
             .map(|name| {
-                if is_present {
-                    // The component generally exists, but it might still be missing for this target.
+                let target = if is_present {
                     let filename = self
                         .versions
                         .tarball_name(&PkgType::from_component(pkgname), name)
                         .unwrap();
-                    let digest = match self.digests.remove(&filename) {
-                        Some(digest) => digest,
-                        // This component does not exist for this target -- skip it.
-                        None => return (name.to_string(), Target::unavailable()),
-                    };
-                    let xz_filename = filename.replace(".tar.gz", ".tar.xz");
-                    let xz_digest = self.digests.remove(&xz_filename);
-
-                    (
-                        name.to_string(),
-                        Target {
-                            available: true,
-                            url: Some(self.url(&filename)),
-                            hash: Some(digest),
-                            xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
-                            xz_hash: xz_digest,
-                            components: None,
-                            extensions: None,
-                        },
-                    )
+
+                    Target::from_compressed_tar(self, &filename)
                 } else {
                     // If the component is not present for this build add it anyway but mark it as
                     // unavailable -- this way rustup won't allow upgrades without --force
-                    (name.to_string(), Target::unavailable())
-                }
+                    Target::unavailable()
+                };
+                (name.to_string(), target)
             })
             .collect();
 
@@ -586,8 +530,9 @@ impl Builder {
         );
     }
 
-    fn url(&self, filename: &str) -> String {
-        format!("{}/{}/{}", self.s3_address, self.date, filename)
+    fn url(&self, path: &Path) -> String {
+        let file_name = path.file_name().unwrap().to_str().unwrap();
+        format!("{}/{}/{}", self.s3_address, self.date, file_name)
     }
 
     fn hash(&self, path: &Path) -> String {
@@ -608,7 +553,7 @@ impl Builder {
     }
 
     fn sign(&self, path: &Path) {
-        if !self.should_sign {
+        if !self.legacy {
             return;
         }
 
@@ -631,10 +576,45 @@ impl Builder {
             .arg(path)
             .stdin(Stdio::piped());
         let mut child = t!(cmd.spawn());
-        t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes()));
+        t!(child.stdin.take().unwrap().write_all(self.legacy_gpg_passphrase.as_bytes()));
         assert!(t!(child.wait()).success());
     }
 
+    fn fill_missing_hashes(&self, manifest: &mut Manifest) {
+        // First collect all files that need hashes
+        let mut need_hashes = HashSet::new();
+        crate::manifest::visit_file_hashes(manifest, |file_hash| {
+            if let FileHash::Missing(path) = file_hash {
+                need_hashes.insert(path.clone());
+            }
+        });
+
+        let collected = Mutex::new(HashMap::new());
+        let collection_start = Instant::now();
+        println!(
+            "collecting hashes for {} tarballs across {} threads",
+            need_hashes.len(),
+            rayon::current_num_threads().min(need_hashes.len()),
+        );
+        need_hashes.par_iter().for_each(|path| match fetch_hash(path) {
+            Ok(hash) => {
+                collected.lock().unwrap().insert(path, hash);
+            }
+            Err(err) => eprintln!("error while fetching the hash for {}: {}", path.display(), err),
+        });
+        let collected = collected.into_inner().unwrap();
+        println!("collected {} hashes in {:.2?}", collected.len(), collection_start.elapsed());
+
+        crate::manifest::visit_file_hashes(manifest, |file_hash| {
+            if let FileHash::Missing(path) = file_hash {
+                match collected.get(path) {
+                    Some(hash) => *file_hash = FileHash::Present(hash.clone()),
+                    None => panic!("missing hash for file {}", path.display()),
+                }
+            }
+        })
+    }
+
     fn write_channel_files(&self, channel_name: &str, manifest: &Manifest) {
         self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
         self.write(&manifest.date, channel_name, "-date.txt");
@@ -648,7 +628,16 @@ impl Builder {
     fn write(&self, contents: &str, channel_name: &str, suffix: &str) {
         let dst = self.output.join(format!("channel-rust-{}{}", channel_name, suffix));
         t!(fs::write(&dst, contents));
-        self.hash(&dst);
-        self.sign(&dst);
+        if self.legacy {
+            self.hash(&dst);
+            self.sign(&dst);
+        }
     }
 }
+
+fn fetch_hash(path: &Path) -> Result<String, Box<dyn Error>> {
+    let mut file = BufReader::new(File::open(path)?);
+    let mut sha256 = sha2::Sha256::default();
+    std::io::copy(&mut file, &mut sha256)?;
+    Ok(hex::encode(sha256.finalize()))
+}
diff --git a/src/tools/build-manifest/src/manifest.rs b/src/tools/build-manifest/src/manifest.rs
new file mode 100644
index 00000000000..20e62abb54c
--- /dev/null
+++ b/src/tools/build-manifest/src/manifest.rs
@@ -0,0 +1,114 @@
+use crate::Builder;
+use serde::{Serialize, Serializer};
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+#[derive(Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct Manifest {
+    pub(crate) manifest_version: String,
+    pub(crate) date: String,
+    pub(crate) pkg: BTreeMap<String, Package>,
+    pub(crate) renames: BTreeMap<String, Rename>,
+    pub(crate) profiles: BTreeMap<String, Vec<String>>,
+}
+
+#[derive(Serialize)]
+pub(crate) struct Package {
+    pub(crate) version: String,
+    pub(crate) git_commit_hash: Option<String>,
+    pub(crate) target: BTreeMap<String, Target>,
+}
+
+#[derive(Serialize)]
+pub(crate) struct Rename {
+    pub(crate) to: String,
+}
+
+#[derive(Serialize, Default)]
+pub(crate) struct Target {
+    pub(crate) available: bool,
+    pub(crate) url: Option<String>,
+    pub(crate) hash: Option<FileHash>,
+    pub(crate) xz_url: Option<String>,
+    pub(crate) xz_hash: Option<FileHash>,
+    pub(crate) components: Option<Vec<Component>>,
+    pub(crate) extensions: Option<Vec<Component>>,
+}
+
+impl Target {
+    pub(crate) fn from_compressed_tar(builder: &Builder, base_path: &str) -> Self {
+        let base_path = builder.input.join(base_path);
+        let gz = Self::tarball_variant(&base_path, "gz");
+        let xz = Self::tarball_variant(&base_path, "xz");
+
+        if gz.is_none() {
+            return Self::unavailable();
+        }
+
+        Self {
+            available: true,
+            components: None,
+            extensions: None,
+            // .gz
+            url: gz.as_ref().map(|path| builder.url(path)),
+            hash: gz.map(FileHash::Missing),
+            // .xz
+            xz_url: xz.as_ref().map(|path| builder.url(path)),
+            xz_hash: xz.map(FileHash::Missing),
+        }
+    }
+
+    fn tarball_variant(base: &Path, ext: &str) -> Option<PathBuf> {
+        let mut path = base.to_path_buf();
+        path.set_extension(ext);
+        if path.is_file() { Some(path) } else { None }
+    }
+
+    pub(crate) fn unavailable() -> Self {
+        Self::default()
+    }
+}
+
+#[derive(Serialize)]
+pub(crate) struct Component {
+    pub(crate) pkg: String,
+    pub(crate) target: String,
+}
+
+impl Component {
+    pub(crate) fn from_str(pkg: &str, target: &str) -> Self {
+        Self { pkg: pkg.to_string(), target: target.to_string() }
+    }
+}
+
+#[allow(unused)]
+pub(crate) enum FileHash {
+    Missing(PathBuf),
+    Present(String),
+}
+
+impl Serialize for FileHash {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        match self {
+            FileHash::Missing(path) => Err(serde::ser::Error::custom(format!(
+                "can't serialize a missing hash for file {}",
+                path.display()
+            ))),
+            FileHash::Present(inner) => inner.serialize(serializer),
+        }
+    }
+}
+
+pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut FileHash)) {
+    for pkg in manifest.pkg.values_mut() {
+        for target in pkg.target.values_mut() {
+            if let Some(hash) = &mut target.hash {
+                f(hash);
+            }
+            if let Some(hash) = &mut target.xz_hash {
+                f(hash);
+            }
+        }
+    }
+}