diff options
| -rw-r--r-- | src/tools/build-manifest/README.md | 32 | ||||
| -rw-r--r-- | src/tools/build-manifest/src/main.rs | 212 | ||||
| -rw-r--r-- | src/tools/build-manifest/src/manifest.rs | 5 | ||||
| -rw-r--r-- | src/tools/build-manifest/src/versions.rs | 140 |
4 files changed, 208 insertions, 181 deletions
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index 44c96f31d0b..9d30c554186 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -1,7 +1,16 @@ # build-manifest -This tool generates the manifests uploaded to static.rust-lang.org and used by -rustup. The tool is invoked by the bootstrap tool. +This tool generates the manifests uploaded to static.rust-lang.org and used by rustup. +You can see a full list of all manifests at <https://static.rust-lang.org/manifests.txt>. +This listing is updated by <https://github.com/rust-lang/generate-manifest-list> every 7 days. + +This gets called by `promote-release` <https://github.com/rust-lang/promote-release> via `x.py dist hash-and-sign`. + +## Adding a new component + +1. Add a new `Step` to `dist.rs`. This should usually be named after the filename of the uploaded tarball. See https://github.com/rust-lang/rust/pull/101799/files#diff-2c56335faa24486df09ba392d8900c57e2fac4633e1f7038469bcf9ed3feb871 for an example. + a. If appropriate, call `tarball.is_preview(true)` for the component. +2. Add a new `PkgType` to build-manifest. Fix all the compile errors as appropriate. ## Testing changes locally @@ -9,19 +18,16 @@ In order to test the changes locally you need to have a valid dist directory available locally. If you don't want to build all the compiler, you can easily create one from the nightly artifacts with: -``` -#!/bin/bash -for cmpn in rust rustc rust-std rust-docs cargo; do - wget https://static.rust-lang.org/dist/${cmpn}-nightly-x86_64-unknown-linux-gnu.tar.gz +```sh +for component in rust rustc rust-std rust-docs cargo; do + wget -P build/dist https://static.rust-lang.org/dist/${component}-nightly-x86_64-unknown-linux-gnu.tar.gz done ``` -Then, you can generate the manifest and all the packages from `path/to/dist` to -`path/to/output` with: +Then, you can generate the manifest and all the packages from `build/dist` to +`build/manifest` with: +```sh +mkdir -p build/manifest +cargo +nightly run --release -p build-manifest build/dist build/manifest 1970-01-01 http://example.com nightly ``` -$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com CHANNEL -``` - -Remember to replace `CHANNEL` with the channel you produced dist artifacts of -and `VERSION` with the current Rust version. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index b0006cb90bd..371e60f969e 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -1,8 +1,4 @@ -//! Build a dist manifest, hash and sign everything. -//! This gets called by `promote-release` -//! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release) -//! via `x.py dist hash-and-sign`; the cmdline arguments are set up -//! by rustbuild (in `src/bootstrap/dist.rs`). +#![doc = include_str!("../README.md")] mod checksum; mod manifest; @@ -184,7 +180,7 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin" static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; -static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-docs-json-preview"]; +static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs]; macro_rules! t { ($e:expr) => { @@ -287,28 +283,9 @@ impl Builder { } fn add_packages_to(&mut self, manifest: &mut Manifest) { - macro_rules! package { - ($name:expr, $targets:expr) => { - self.package($name, &mut manifest.pkg, $targets, &[]) - }; + for pkg in PkgType::all() { + self.package(pkg, &mut manifest.pkg); } - package!("rustc", HOSTS); - package!("rustc-dev", HOSTS); - package!("reproducible-artifacts", HOSTS); - package!("rustc-docs", HOSTS); - package!("cargo", HOSTS); - package!("rust-mingw", MINGW); - package!("rust-std", TARGETS); - self.package("rust-docs", &mut manifest.pkg, HOSTS, DOCS_FALLBACK); - self.package("rust-docs-json-preview", &mut manifest.pkg, HOSTS, DOCS_FALLBACK); - package!("rust-src", &["*"]); - package!("rls-preview", HOSTS); - package!("rust-analyzer-preview", HOSTS); - package!("clippy-preview", HOSTS); - package!("miri-preview", HOSTS); - package!("rustfmt-preview", HOSTS); - package!("rust-analysis", TARGETS); - package!("llvm-tools-preview", TARGETS); } fn add_artifacts_to(&mut self, manifest: &mut Manifest) { @@ -333,44 +310,28 @@ impl Builder { } fn add_profiles_to(&mut self, manifest: &mut Manifest) { - let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs); - profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]); - profile( - "default", - &[ - "rustc", - "cargo", - "rust-std", - "rust-mingw", - "rust-docs", - "rustfmt-preview", - "clippy-preview", - ], - ); - profile( - "complete", - &[ - "rustc", - "cargo", - "rust-std", - "rust-mingw", - "rust-docs", - "rustfmt-preview", - "clippy-preview", - "rls-preview", - "rust-analyzer-preview", - "rust-src", - "llvm-tools-preview", - "rust-analysis", - "miri-preview", - ], - ); + use PkgType::*; + + let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs); + + // Use a Vec here to make sure we don't exclude any components in an earlier profile. + let minimal = vec![Rustc, Cargo, RustStd, RustMingw]; + profile("minimal", &minimal); + + let mut default = minimal; + default.extend([HtmlDocs, Rustfmt, Clippy]); + profile("default", &default); + + // NOTE: this profile is effectively deprecated; do not add new components to it. + let mut complete = default; + complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]); + profile("complete", &complete); // The compiler libraries are not stable for end users, and they're also huge, so we only // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible // for users to install the additional component manually, if needed. if self.versions.channel() == "nightly" { - self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); + self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]); // Do not include the rustc-docs component for now, as it causes // conflicts with the rust-docs component when installed. See // #75833. @@ -382,12 +343,11 @@ impl Builder { let mut rename = |from: &str, to: &str| { manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() }) }; - rename("rls", "rls-preview"); - rename("rustfmt", "rustfmt-preview"); - rename("clippy", "clippy-preview"); - rename("miri", "miri-preview"); - rename("rust-docs-json", "rust-docs-json-preview"); - rename("rust-analyzer", "rust-analyzer-preview"); + for pkg in PkgType::all() { + if pkg.is_preview() { + rename(pkg.tarball_component_name(), &pkg.manifest_component_name()); + } + } } fn rust_package(&mut self, manifest: &Manifest) -> Package { @@ -419,42 +379,52 @@ impl Builder { let mut components = Vec::new(); let mut extensions = Vec::new(); - let host_component = |pkg| Component::from_str(pkg, host); - - // rustc/rust-std/cargo/docs are all required, - // and so is rust-mingw if it's available for the target. - components.extend(vec![ - host_component("rustc"), - host_component("rust-std"), - host_component("cargo"), - host_component("rust-docs"), - ]); - if host.contains("pc-windows-gnu") { - components.push(host_component("rust-mingw")); - } + let host_component = |pkg: &_| Component::from_pkg(pkg, host); - // Tools are always present in the manifest, - // but might be marked as unavailable if they weren't built. - extensions.extend(vec![ - host_component("clippy-preview"), - host_component("miri-preview"), - host_component("rls-preview"), - host_component("rust-analyzer-preview"), - host_component("rustfmt-preview"), - host_component("llvm-tools-preview"), - host_component("rust-analysis"), - host_component("rust-docs-json-preview"), - ]); - - extensions.extend( - TARGETS - .iter() - .filter(|&&target| target != host) - .map(|target| Component::from_str("rust-std", target)), - ); - extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target))); - extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target))); - extensions.push(Component::from_str("rust-src", "*")); + for pkg in PkgType::all() { + match pkg { + // rustc/rust-std/cargo/docs are all required + PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => { + components.push(host_component(pkg)); + } + PkgType::RustStd => { + components.push(host_component(pkg)); + extensions.extend( + TARGETS + .iter() + .filter(|&&target| target != host) + .map(|target| Component::from_pkg(pkg, target)), + ); + } + // so is rust-mingw if it's available for the target + PkgType::RustMingw => { + if host.contains("pc-windows-gnu") { + components.push(host_component(pkg)); + } + } + // Tools are always present in the manifest, + // but might be marked as unavailable if they weren't built. + PkgType::Clippy + | PkgType::Miri + | PkgType::Rls + | PkgType::RustAnalyzer + | PkgType::Rustfmt + | PkgType::LlvmTools + | PkgType::RustAnalysis + | PkgType::JsonDocs => { + extensions.push(host_component(pkg)); + } + PkgType::RustcDev | PkgType::RustcDocs => { + extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target))); + } + PkgType::RustSrc => { + extensions.push(Component::from_pkg(pkg, "*")); + } + PkgType::Rust => {} + // NOTE: this is intentional, these artifacts aren't intended to be used with rustup + PkgType::ReproducibleArtifacts => {} + } + } // If the components/extensions don't actually exist for this // particular host/target combination then nix it entirely from our @@ -481,43 +451,44 @@ impl Builder { &mut self, profile_name: &str, dst: &mut BTreeMap<String, Vec<String>>, - pkgs: &[&str], + pkgs: &[PkgType], ) { - dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect()); + dst.insert( + profile_name.to_owned(), + pkgs.iter().map(|s| s.manifest_component_name()).collect(), + ); } fn extend_profile( &mut self, profile_name: &str, dst: &mut BTreeMap<String, Vec<String>>, - pkgs: &[&str], + pkgs: &[PkgType], ) { dst.get_mut(profile_name) .expect("existing profile") - .extend(pkgs.iter().map(|s| (*s).to_owned())); + .extend(pkgs.iter().map(|s| s.manifest_component_name())); } - fn package( - &mut self, - pkgname: &str, - dst: &mut BTreeMap<String, Package>, - targets: &[&str], - fallback: &[(&str, &str)], - ) { - let version_info = self - .versions - .version(&PkgType::from_component(pkgname)) - .expect("failed to load package version"); + fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap<String, Package>) { + if *pkg == PkgType::Rust { + // This is handled specially by `rust_package` later. + // Order is important, so don't call `rust_package` here. + return; + } + + let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; + let version_info = self.versions.version(&pkg).expect("failed to load package version"); let mut is_present = version_info.present; // Never ship nightly-only components for other trains. - if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { + if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) { is_present = false; // Pretend the component is entirely missing. } macro_rules! tarball_name { ($target_name:expr) => { - self.versions.tarball_name(&PkgType::from_component(pkgname), $target_name).unwrap() + self.versions.tarball_name(pkg, $target_name).unwrap() }; } let mut target_from_compressed_tar = |target_name| { @@ -546,7 +517,8 @@ impl Builder { Target::unavailable() }; - let targets = targets + let targets = pkg + .targets() .iter() .map(|name| { let target = if is_present { @@ -561,7 +533,7 @@ impl Builder { .collect(); dst.insert( - pkgname.to_string(), + pkg.manifest_component_name(), Package { version: version_info.version.unwrap_or_default(), git_commit_hash: version_info.git_commit, diff --git a/src/tools/build-manifest/src/manifest.rs b/src/tools/build-manifest/src/manifest.rs index 547c270d89a..a9f19d8e565 100644 --- a/src/tools/build-manifest/src/manifest.rs +++ b/src/tools/build-manifest/src/manifest.rs @@ -1,3 +1,4 @@ +use crate::versions::PkgType; use crate::Builder; use serde::{Serialize, Serializer}; use std::collections::BTreeMap; @@ -116,8 +117,8 @@ pub(crate) struct Component { } impl Component { - pub(crate) fn from_str(pkg: &str, target: &str) -> Self { - Self { pkg: pkg.to_string(), target: target.to_string() } + pub(crate) fn from_pkg(pkg: &PkgType, target: &str) -> Self { + Self { pkg: pkg.manifest_component_name(), target: target.to_string() } } } diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 0186194a41f..dde9745afb7 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -8,55 +8,63 @@ use tar::Archive; const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; -#[derive(Debug, Hash, Eq, PartialEq, Clone)] -pub(crate) enum PkgType { - Rust, - RustSrc, - Rustc, - Cargo, - Rls, - RustAnalyzer, - Clippy, - Rustfmt, - LlvmTools, - Miri, - JsonDocs, - Other(String), -} +macro_rules! pkg_type { + ( $($variant:ident = $component:literal $(; preview = true $(@$is_preview:tt)? )? ),+ $(,)? ) => { + #[derive(Debug, Hash, Eq, PartialEq, Clone)] + pub(crate) enum PkgType { + $($variant,)+ + } -impl PkgType { - pub(crate) fn from_component(component: &str) -> Self { - match component { - "rust" => PkgType::Rust, - "rust-src" => PkgType::RustSrc, - "rustc" => PkgType::Rustc, - "cargo" => PkgType::Cargo, - "rls" | "rls-preview" => PkgType::Rls, - "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer, - "clippy" | "clippy-preview" => PkgType::Clippy, - "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt, - "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools, - "miri" | "miri-preview" => PkgType::Miri, - "rust-docs-json" | "rust-docs-json-preview" => PkgType::JsonDocs, - other => PkgType::Other(other.into()), + impl PkgType { + pub(crate) fn is_preview(&self) -> bool { + match self { + $( $( $($is_preview)? PkgType::$variant => true, )? )+ + _ => false, + } + } + + /// First part of the tarball name. + pub(crate) fn tarball_component_name(&self) -> &str { + match self { + $( PkgType::$variant => $component,)+ + } + } + + pub(crate) fn all() -> &'static [PkgType] { + &[ $(PkgType::$variant),+ ] + } } } +} - /// First part of the tarball name. - fn tarball_component_name(&self) -> &str { - match self { - PkgType::Rust => "rust", - PkgType::RustSrc => "rust-src", - PkgType::Rustc => "rustc", - PkgType::Cargo => "cargo", - PkgType::Rls => "rls", - PkgType::RustAnalyzer => "rust-analyzer", - PkgType::Clippy => "clippy", - PkgType::Rustfmt => "rustfmt", - PkgType::LlvmTools => "llvm-tools", - PkgType::Miri => "miri", - PkgType::JsonDocs => "rust-docs-json", - PkgType::Other(component) => component, +pkg_type! { + Rust = "rust", + RustSrc = "rust-src", + Rustc = "rustc", + RustcDev = "rustc-dev", + RustcDocs = "rustc-docs", + ReproducibleArtifacts = "reproducible-artifacts", + RustMingw = "rust-mingw", + RustStd = "rust-std", + Cargo = "cargo", + HtmlDocs = "rust-docs", + RustAnalysis = "rust-analysis", + Rls = "rls"; preview = true, + RustAnalyzer = "rust-analyzer"; preview = true, + Clippy = "clippy"; preview = true, + Rustfmt = "rustfmt"; preview = true, + LlvmTools = "llvm-tools"; preview = true, + Miri = "miri"; preview = true, + JsonDocs = "rust-docs-json"; preview = true, +} + +impl PkgType { + /// Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. + pub(crate) fn manifest_component_name(&self) -> String { + if self.is_preview() { + format!("{}-preview", self.tarball_component_name()) + } else { + self.tarball_component_name().to_string() } } @@ -73,10 +81,42 @@ impl PkgType { PkgType::Miri => false, PkgType::Rust => true, + PkgType::RustStd => true, PkgType::RustSrc => true, PkgType::Rustc => true, PkgType::JsonDocs => true, - PkgType::Other(_) => true, + PkgType::HtmlDocs => true, + PkgType::RustcDev => true, + PkgType::RustcDocs => true, + PkgType::ReproducibleArtifacts => true, + PkgType::RustMingw => true, + PkgType::RustAnalysis => true, + } + } + + pub(crate) fn targets(&self) -> &[&str] { + use crate::{HOSTS, MINGW, TARGETS}; + use PkgType::*; + + match self { + Rust => HOSTS, // doesn't matter in practice, but return something to avoid panicking + Rustc => HOSTS, + RustcDev => HOSTS, + ReproducibleArtifacts => HOSTS, + RustcDocs => HOSTS, + Cargo => HOSTS, + RustMingw => MINGW, + RustStd => TARGETS, + HtmlDocs => HOSTS, + JsonDocs => HOSTS, + RustSrc => &["*"], + Rls => HOSTS, + RustAnalyzer => HOSTS, + Clippy => HOSTS, + Miri => HOSTS, + Rustfmt => HOSTS, + RustAnalysis => TARGETS, + LlvmTools => TARGETS, } } @@ -84,6 +124,14 @@ impl PkgType { fn target_independent(&self) -> bool { *self == PkgType::RustSrc } + + /// Whether to package these target-specific docs for another similar target. + pub(crate) fn use_docs_fallback(&self) -> bool { + match self { + PkgType::JsonDocs | PkgType::HtmlDocs => true, + _ => false, + } + } } #[derive(Debug, Default, Clone)] |