From 9e8785f01757a50e321e85adeb659a9aff6fc00e Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 24 Jan 2017 14:37:04 -0800 Subject: rustbuild: Add manifest generation in-tree This commit adds a new tool, `build-manifest`, which is used to generate a distribution manifest of all produced artifacts. This tool is intended to replace the `build-rust-manifest.py` script that's currently located on the buildmaster. The intention is that we'll have a builder which periodically: * Downloads all artifacts for a commit * Runs `./x.py dist hash-and-sign`. This will generate `sha256` and `asc` files as well as TOML manifests. * Upload all generated hashes and manifests to the directory the artifacts came from. * Upload *all* artifacts (tarballs and hashes and manifests) to an archived location. * If necessary, upload all artifacts to the main location. This script is intended to just be the second step here where orchestrating uploads and such will all happen externally from the build system itself. --- src/bootstrap/config.rs | 19 +++++++++++++++++++ src/bootstrap/config.toml.example | 30 ++++++++++++++++++++++++++++++ src/bootstrap/dist.rs | 33 ++++++++++++++++++++++++++++++++- src/bootstrap/step.rs | 10 ++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 7d1abcfa6f6..e035f8157ff 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -78,6 +78,11 @@ pub struct Config { pub cargo: Option, pub local_rebuild: bool, + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_gpg_password_file: Option, + // libstd features pub debug_jemalloc: bool, pub use_jemalloc: bool, @@ -123,6 +128,7 @@ struct TomlConfig { llvm: Option, rust: Option, target: Option>, + dist: Option, } /// TOML representation of various global build decisions. @@ -166,6 +172,13 @@ struct Llvm { targets: Option, } +#[derive(RustcDecodable, Default, Clone)] +struct Dist { + sign_folder: Option, + gpg_password_file: Option, + upload_addr: Option, +} + #[derive(RustcDecodable)] enum StringOrBool { String(String), @@ -352,6 +365,12 @@ impl Config { } } + if let Some(ref t) = toml.dist { + config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from); + config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from); + config.dist_upload_addr = t.upload_addr.clone(); + } + return config } diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 4b859482562..a53419ad7fd 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -242,3 +242,33 @@ # that this option only makes sense for MUSL targets that produce statically # linked binaries #musl-root = "..." + +# ============================================================================= +# Distribution options +# +# These options are related to distribution, mostly for the Rust project itself. +# You probably won't need to concern yourself with any of these options +# ============================================================================= +[dist] + +# This is the folder of artifacts that the build system will sign. All files in +# this directory will be signed with the default gpg key using the system `gpg` +# binary. The `asc` and `sha256` files will all be output into the standard dist +# output folder (currently `build/dist`) +# +# This folder should be populated ahead of time before the build system is +# invoked. +#sign-folder = "path/to/folder/to/sign" + +# This is a file which contains the password of the default gpg key. This will +# be passed to `gpg` down the road when signing all files in `sign-folder` +# above. This should be stored in plaintext. +#gpg-password-file = "path/to/gpg/password" + +# The remote address that all artifacts will eventually be uploaded to. The +# build system generates manifests which will point to these urls, and for the +# manifests to be correct they'll have to have the right URLs encoded. +# +# Note that this address should not contain a trailing slash as file names will +# be appended to it. +#upload-addr = "https://example.com/folder" diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e5f05059523..71a5f313bbd 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -22,7 +22,7 @@ use std::env; use std::fs::{self, File}; use std::io::{Read, Write}; use std::path::{PathBuf, Path}; -use std::process::Command; +use std::process::{Command, Stdio}; use build_helper::output; @@ -876,3 +876,34 @@ fn add_env(build: &Build, cmd: &mut Command, target: &str) { cmd.env("CFG_PLATFORM", "x86"); } } + +pub fn hash_and_sign(build: &Build) { + let compiler = Compiler::new(0, &build.config.build); + let mut cmd = build.tool_cmd(&compiler, "build-manifest"); + let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n") + }); + let mut pass = String::new(); + t!(t!(File::open(&file)).read_to_string(&mut pass)); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(build)); + cmd.arg(today.trim()); + cmd.arg(package_vers(build)); + cmd.arg(addr); + + t!(fs::create_dir_all(distdir(build))); + + let mut child = t!(cmd.stdin(Stdio::piped()).spawn()); + t!(child.stdin.take().unwrap().write_all(pass.as_bytes())); + let status = t!(child.wait()); + assert!(status.success()); +} diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index 697b14c6050..3932a7cf8c5 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -513,6 +513,9 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.build("tool-compiletest", "src/tools/compiletest") .dep(|s| s.name("libtest")) .run(move |s| compile::tool(build, s.stage, s.target, "compiletest")); + rules.build("tool-build-manifest", "src/tools/build-manifest") + .dep(|s| s.name("libstd")) + .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest")); // ======================================================================== // Documentation targets @@ -633,6 +636,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|d| d.name("dist-cargo")) .run(move |s| dist::extended(build, s.stage, s.target)); + rules.dist("dist-sign", "hash-and-sign") + .host(true) + .only_build(true) + .only_host_build(true) + .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0)) + .run(move |_| dist::hash_and_sign(build)); + rules.verify(); return rules; } -- cgit 1.4.1-3-g733a5