use std::net::{SocketAddr, TcpStream}; use std::process::{Command, Stdio, exit}; use std::time::Duration; use std::{env, fs, process, thread}; const JOSH_PORT: u16 = 42042; const DEFAULT_PR_BRANCH: &str = "update-builtins"; pub struct GitSync { upstream_repo: String, upstream_ref: String, upstream_url: String, josh_filter: String, josh_url_base: String, } /// This code was adapted from the miri repository, via the rustc-dev-guide /// () impl GitSync { pub fn from_current_dir() -> Self { let upstream_repo = env::var("UPSTREAM_ORG").unwrap_or_else(|_| "rust-lang".to_owned()) + "/rust"; Self { upstream_url: format!("https://github.com/{upstream_repo}"), upstream_repo, upstream_ref: env::var("UPSTREAM_REF").unwrap_or_else(|_| "HEAD".to_owned()), josh_filter: ":/library/compiler-builtins".to_owned(), josh_url_base: format!("http://localhost:{JOSH_PORT}"), } } /// Pull from rust-lang/rust to compiler-builtins. pub fn rustc_pull(&self, commit: Option) { let Self { upstream_ref, upstream_url, upstream_repo, .. } = self; let new_upstream_base = commit.unwrap_or_else(|| { let out = check_output(["git", "ls-remote", upstream_url, upstream_ref]); out.split_whitespace() .next() .unwrap_or_else(|| panic!("could not split output: '{out}'")) .to_owned() }); ensure_clean(); // Make sure josh is running. let _josh = Josh::start(); let josh_url_filtered = self.josh_url( &self.upstream_repo, Some(&new_upstream_base), Some(&self.josh_filter), ); let previous_upstream_base = fs::read_to_string("rust-version") .expect("failed to read `rust-version`") .trim() .to_string(); assert_ne!(previous_upstream_base, new_upstream_base, "nothing to pull"); let orig_head = check_output(["git", "rev-parse", "HEAD"]); println!("original upstream base: {previous_upstream_base}"); println!("new upstream base: {new_upstream_base}"); println!("original HEAD: {orig_head}"); // Fetch the latest upstream HEAD so we can get a summary. Use the Josh URL for caching. run([ "git", "fetch", &self.josh_url(&self.upstream_repo, Some(&new_upstream_base), Some(":/")), &new_upstream_base, "--depth=1", ]); let new_summary = check_output(["git", "log", "-1", "--format=%h %s", &new_upstream_base]); // Update rust-version file. As a separate commit, since making it part of // the merge has confused the heck out of josh in the past. // We pass `--no-verify` to avoid running git hooks. // We do this before the merge so that if there are merge conflicts, we have // the right rust-version file while resolving them. fs::write("rust-version", format!("{new_upstream_base}\n")) .expect("failed to write rust-version"); let prep_message = format!( "Update the upstream Rust version\n\n\ To prepare for merging from {upstream_repo}, set the version file to:\n\n \ {new_summary}\n\ ", ); run([ "git", "commit", "rust-version", "--no-verify", "-m", &prep_message, ]); // Fetch given rustc commit. run(["git", "fetch", &josh_url_filtered]); let incoming_ref = check_output(["git", "rev-parse", "FETCH_HEAD"]); println!("incoming ref: {incoming_ref}"); let merge_message = format!( "Merge ref '{upstream_head_short}{filter}' from {upstream_url}\n\n\ Pull recent changes from {upstream_repo} via Josh.\n\n\ Upstream ref: {new_upstream_base}\n\ Filtered ref: {incoming_ref}\n\ ", upstream_head_short = &new_upstream_base[..12], filter = self.josh_filter ); // This should not add any new root commits. So count those before and after merging. let num_roots = || -> u32 { let out = check_output(["git", "rev-list", "HEAD", "--max-parents=0", "--count"]); out.trim() .parse::() .unwrap_or_else(|e| panic!("failed to parse `{out}`: {e}")) }; let num_roots_before = num_roots(); let pre_merge_sha = check_output(["git", "rev-parse", "HEAD"]); println!("pre-merge HEAD: {pre_merge_sha}"); // Merge the fetched commit. run([ "git", "merge", "FETCH_HEAD", "--no-verify", "--no-ff", "-m", &merge_message, ]); let current_sha = check_output(["git", "rev-parse", "HEAD"]); if current_sha == pre_merge_sha { run(["git", "reset", "--hard", &orig_head]); eprintln!( "No merge was performed, no changes to pull were found. \ Rolled back the preparation commit." ); exit(1); } // Check that the number of roots did not increase. assert_eq!( num_roots(), num_roots_before, "Josh created a new root commit. This is probably not the history you want." ); } /// Construct an update to rust-lang/rust from compiler-builtins. pub fn rustc_push(&self, github_user: &str, branch: Option<&str>) { let Self { josh_filter, upstream_url, .. } = self; let branch = branch.unwrap_or(DEFAULT_PR_BRANCH); let josh_url = self.josh_url(&format!("{github_user}/rust"), None, Some(josh_filter)); let user_upstream_url = format!("git@github.com:{github_user}/rust.git"); let Ok(rustc_git) = env::var("RUSTC_GIT") else { panic!("the RUSTC_GIT environment variable must be set to a rust-lang/rust checkout") }; ensure_clean(); let base = fs::read_to_string("rust-version") .expect("failed to read `rust-version`") .trim() .to_string(); // Make sure josh is running. let _josh = Josh::start(); // Prepare the branch. Pushing works much better if we use as base exactly // the commit that we pulled from last time, so we use the `rust-version` // file to find out which commit that would be. println!("Preparing {github_user}/rust (base: {base})..."); if Command::new("git") .args(["-C", &rustc_git, "fetch", &user_upstream_url, branch]) .output() // capture output .expect("could not run fetch") .status .success() { panic!( "The branch '{branch}' seems to already exist in '{user_upstream_url}'. \ Please delete it and try again." ); } run(["git", "-C", &rustc_git, "fetch", upstream_url, &base]); run_cfg("git", |c| { c.args([ "-C", &rustc_git, "push", &user_upstream_url, &format!("{base}:refs/heads/{branch}"), ]) .stdout(Stdio::null()) .stderr(Stdio::null()) // silence the "create GitHub PR" message }); println!("pushed PR branch"); // Do the actual push. println!("Pushing changes..."); run(["git", "push", &josh_url, &format!("HEAD:{branch}")]); println!(); // Do a round-trip check to make sure the push worked as expected. run(["git", "fetch", &josh_url, branch]); let head = check_output(["git", "rev-parse", "HEAD"]); let fetch_head = check_output(["git", "rev-parse", "FETCH_HEAD"]); assert_eq!( head, fetch_head, "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ Expected {head}, got {fetch_head}." ); println!( "Confirmed that the push round-trips back to compiler-builtins properly. Please \ create a rustc PR:" ); // Open PR with `subtree update` title to silence the `no-merges` triagebot check println!( " {upstream_url}/compare/{github_user}:{branch}?quick_pull=1\ &title=Update%20the%20%60compiler-builtins%60%20subtree\ &body=Update%20the%20Josh%20subtree%20to%20https%3A%2F%2Fgithub.com%2Frust-lang%2F\ compiler-builtins%2Fcommit%2F{head_short}.%0A%0Ar%3F%20%40ghost", head_short = &head[..12], ); } /// Construct a url to the local Josh server with (optionally) fn josh_url(&self, repo: &str, rev: Option<&str>, filter: Option<&str>) -> String { format!( "{base}/{repo}.git{at}{rev}{filter}{filt_git}", base = self.josh_url_base, at = if rev.is_some() { "@" } else { "" }, rev = rev.unwrap_or_default(), filter = filter.unwrap_or_default(), filt_git = if filter.is_some() { ".git" } else { "" } ) } } /// Fail if there are files that need to be checked in. fn ensure_clean() { let read = check_output(["git", "status", "--untracked-files=no", "--porcelain"]); assert!( read.is_empty(), "working directory must be clean before performing rustc pull" ); } /* Helpers for running commands with logged invocations */ /// Run a command from an array, passing its output through. fn run<'a, Args: AsRef<[&'a str]>>(l: Args) { let l = l.as_ref(); run_cfg(l[0], |c| c.args(&l[1..])); } /// Run a command from an array, collecting its output. fn check_output<'a, Args: AsRef<[&'a str]>>(l: Args) -> String { let l = l.as_ref(); check_output_cfg(l[0], |c| c.args(&l[1..])) } /// [`run`] with configuration. fn run_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) { // self.read(l.as_ref()); check_output_cfg(prog, |c| f(c.stdout(Stdio::inherit()))); } /// [`read`] with configuration. All shell helpers print the command and pass stderr. fn check_output_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) -> String { let mut cmd = Command::new(prog); cmd.stderr(Stdio::inherit()); f(&mut cmd); eprintln!("+ {cmd:?}"); let out = cmd.output().expect("command failed"); assert!(out.status.success()); String::from_utf8(out.stdout.trim_ascii().to_vec()).expect("non-UTF8 output") } /// Create a wrapper that stops Josh on drop. pub struct Josh(process::Child); impl Josh { pub fn start() -> Self { // Determine cache directory. let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "rustc-compiler-builtins-josh") .unwrap(); let local_dir = user_dirs.cache_dir().to_owned(); // Start josh, silencing its output. #[expect(clippy::zombie_processes, reason = "clippy can't handle the loop")] let josh = process::Command::new("josh-proxy") .arg("--local") .arg(local_dir) .args([ "--remote=https://github.com", &format!("--port={JOSH_PORT}"), "--no-background", ]) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() .expect("failed to start josh-proxy, make sure it is installed"); // Wait until the port is open. We try every 10ms until 1s passed. for _ in 0..100 { // This will generally fail immediately when the port is still closed. let addr = SocketAddr::from(([127, 0, 0, 1], JOSH_PORT)); let josh_ready = TcpStream::connect_timeout(&addr, Duration::from_millis(1)); if josh_ready.is_ok() { println!("josh up and running"); return Josh(josh); } // Not ready yet. thread::sleep(Duration::from_millis(10)); } panic!("Even after waiting for 1s, josh-proxy is still not available.") } } impl Drop for Josh { fn drop(&mut self) { if cfg!(unix) { // Try to gracefully shut it down. Command::new("kill") .args(["-s", "INT", &self.0.id().to_string()]) .output() .expect("failed to SIGINT josh-proxy"); // Sadly there is no "wait with timeout"... so we just give it some time to finish. thread::sleep(Duration::from_millis(100)); // Now hopefully it is gone. if self .0 .try_wait() .expect("failed to wait for josh-proxy") .is_some() { return; } } // If that didn't work (or we're not on Unix), kill it hard. eprintln!( "I have to kill josh-proxy the hard way, let's hope this does not \ break anything." ); self.0.kill().expect("failed to SIGKILL josh-proxy"); } }