diff options
| -rw-r--r-- | src/bootstrap/src/core/builder.rs | 9 | ||||
| -rw-r--r-- | src/bootstrap/src/core/builder/tests.rs | 48 | ||||
| -rw-r--r-- | src/bootstrap/src/core/config/config.rs | 51 | ||||
| -rw-r--r-- | src/bootstrap/src/core/config/tests.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/src/core/download.rs | 40 | ||||
| -rw-r--r-- | src/ci/docker/host-x86_64/mingw-check/Dockerfile | 15 | ||||
| -rw-r--r-- | src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile | 4 | ||||
| -rw-r--r-- | src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 1 | ||||
| -rwxr-xr-x | src/ci/docker/run.sh | 1 | ||||
| -rw-r--r-- | src/ci/github-actions/jobs.yml | 5 | ||||
| -rwxr-xr-x | src/ci/run.sh | 13 |
11 files changed, 176 insertions, 13 deletions
diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 155c6515db8..d32830c0a96 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -414,6 +414,15 @@ impl StepDescription { .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) .collect::<Vec<_>>(); + if builder.download_rustc() && (builder.kind == Kind::Dist || builder.kind == Kind::Install) + { + eprintln!( + "ERROR: '{}' subcommand is incompatible with `rust.download-rustc`.", + builder.kind.as_str() + ); + crate::exit!(1); + } + // sanity checks on rules for (desc, should_run) in v.iter().zip(&should_runs) { assert!( diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index bd81dc930be..4a96ecf1421 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -1,5 +1,7 @@ use std::thread; +use build_helper::git::get_closest_merge_commit; + use super::*; use crate::Flags; use crate::core::build_steps::doc::DocumentationFormat; @@ -212,6 +214,52 @@ fn alias_and_path_for_library() { assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]); } +#[test] +fn ci_rustc_if_unchanged_logic() { + let config = Config::parse_inner( + Flags::parse(&[ + "build".to_owned(), + "--dry-run".to_owned(), + "--set=rust.download-rustc='if-unchanged'".to_owned(), + ]), + |&_| Ok(Default::default()), + ); + + if config.rust_info.is_from_tarball() { + return; + } + + let build = Build::new(config.clone()); + let builder = Builder::new(&build); + + if config.out.exists() { + fs::remove_dir_all(&config.out).unwrap(); + } + + builder.run_step_descriptions(&Builder::get_step_descriptions(config.cmd.kind()), &[]); + + let compiler_path = build.src.join("compiler"); + let library_path = build.src.join("compiler"); + + let commit = + get_closest_merge_commit(Some(&builder.config.src), &builder.config.git_config(), &[ + compiler_path.clone(), + library_path.clone(), + ]) + .unwrap(); + + let has_changes = !helpers::git(Some(&builder.src)) + .args(["diff-index", "--quiet", &commit]) + .arg("--") + .args([compiler_path, library_path]) + .as_command_mut() + .status() + .unwrap() + .success(); + + assert!(has_changes == config.download_rustc_commit.is_none()); +} + mod defaults { use pretty_assertions::assert_eq; diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 7dc3b7b081a..9f84b492b80 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -13,6 +13,7 @@ use std::str::FromStr; use std::sync::OnceLock; use std::{cmp, env, fs}; +use build_helper::ci::CiEnv; use build_helper::exit; use build_helper::git::{GitConfig, get_closest_merge_commit, output_result}; use serde::{Deserialize, Deserializer}; @@ -22,6 +23,7 @@ use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; use crate::core::build_steps::llvm; pub use crate::core::config::flags::Subcommand; use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::core::download::is_download_ci_available; use crate::utils::cache::{INTERNER, Interned}; use crate::utils::channel::{self, GitInfo}; use crate::utils::helpers::{self, exe, output, t}; @@ -1627,9 +1629,11 @@ impl Config { config.mandir = mandir.map(PathBuf::from); } + config.llvm_assertions = + toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false)); + // Store off these values as options because if they're not provided // we'll infer default values for them later - let mut llvm_assertions = None; let mut llvm_tests = None; let mut llvm_enzyme = None; let mut llvm_plugins = None; @@ -1712,7 +1716,8 @@ impl Config { is_user_configured_rust_channel = channel.is_some(); set(&mut config.channel, channel.clone()); - config.download_rustc_commit = config.download_ci_rustc_commit(download_rustc); + config.download_rustc_commit = + config.download_ci_rustc_commit(download_rustc, config.llvm_assertions); debug = debug_toml; debug_assertions = debug_assertions_toml; @@ -1848,7 +1853,7 @@ impl Config { optimize: optimize_toml, thin_lto, release_debuginfo, - assertions, + assertions: _, tests, enzyme, plugins, @@ -1882,7 +1887,6 @@ impl Config { Some(StringOrBool::Bool(false)) | None => {} } set(&mut config.ninja_in_file, ninja); - llvm_assertions = assertions; llvm_tests = tests; llvm_enzyme = enzyme; llvm_plugins = plugins; @@ -1911,8 +1915,8 @@ impl Config { config.llvm_enable_warnings = enable_warnings.unwrap_or(false); config.llvm_build_config = build_config.clone().unwrap_or(Default::default()); - let asserts = llvm_assertions.unwrap_or(false); - config.llvm_from_ci = config.parse_download_ci_llvm(download_ci_llvm, asserts); + config.llvm_from_ci = + config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions); if config.llvm_from_ci { let warn = |option: &str| { @@ -2080,7 +2084,6 @@ impl Config { // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. - config.llvm_assertions = llvm_assertions.unwrap_or(false); config.llvm_tests = llvm_tests.unwrap_or(false); config.llvm_enzyme = llvm_enzyme.unwrap_or(false); config.llvm_plugins = llvm_plugins.unwrap_or(false); @@ -2419,8 +2422,9 @@ impl Config { ci_config_toml, ); - let disable_ci_rustc_if_incompatible = - env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE") + // Primarily used by CI runners to avoid handling download-rustc incompatible + // options one by one on shell scripts. + let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE") .is_some_and(|s| s == "1" || s == "true"); if disable_ci_rustc_if_incompatible && res.is_err() { @@ -2711,7 +2715,15 @@ impl Config { } /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> { + fn download_ci_rustc_commit( + &self, + download_rustc: Option<StringOrBool>, + llvm_assertions: bool, + ) -> Option<String> { + if !is_download_ci_available(&self.build.triple, llvm_assertions) { + return None; + } + // If `download-rustc` is not set, default to rebuilding. let if_unchanged = match download_rustc { None | Some(StringOrBool::Bool(false)) => return None, @@ -2724,7 +2736,11 @@ impl Config { // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. - let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap(); + let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[ + self.src.join("compiler"), + self.src.join("library"), + ]) + .unwrap(); if commit.is_empty() { println!("ERROR: could not find commit hash for downloading rustc"); println!("HELP: maybe your repository history is too shallow?"); @@ -2733,6 +2749,19 @@ impl Config { crate::exit!(1); } + if CiEnv::is_ci() && { + let head_sha = + output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut()); + let head_sha = head_sha.trim(); + commit == head_sha + } { + eprintln!("CI rustc commit matches with HEAD and we are in CI."); + eprintln!( + "`rustc.download-ci` functionality will be skipped as artifacts are not available." + ); + return None; + } + // Warn if there were changes to the compiler or standard library since the ancestor commit. let has_changes = !t!(helpers::git(Some(&self.src)) .args(["diff-index", "--quiet", &commit]) diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs index 278becdcbc7..ed89112de90 100644 --- a/src/bootstrap/src/core/config/tests.rs +++ b/src/bootstrap/src/core/config/tests.rs @@ -12,7 +12,7 @@ use super::{ChangeIdWrapper, Config}; use crate::core::build_steps::clippy::get_clippy_rules_in_order; use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig}; -fn parse(config: &str) -> Config { +pub(crate) fn parse(config: &str) -> Config { Config::parse_inner( Flags::parse(&["check".to_string(), "--config=/does/not/exist".to_string()]), |&_| toml::from_str(&config), diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index 444b75876f2..db1f5b08338 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -832,3 +832,43 @@ fn path_is_dylib(path: &Path) -> bool { // The .so is not necessarily the extension, it might be libLLVM.so.18.1 path.to_str().map_or(false, |path| path.contains(".so")) } + +/// Checks whether the CI rustc is available for the given target triple. +pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool { + // All tier 1 targets and tier 2 targets with host tools. + const SUPPORTED_PLATFORMS: &[&str] = &[ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "loongarch64-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", + ]; + + const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] = + &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"]; + + if llvm_assertions { + SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple) + } else { + SUPPORTED_PLATFORMS.contains(&target_triple) + } +} diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 571378774be..467ca1dac67 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -46,7 +46,20 @@ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 # Check library crates on all tier 1 targets. # We disable optimized compiler built-ins because that requires a C toolchain for the target. # We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs. -ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \ +ENV SCRIPT \ + # `core::builder::tests::ci_rustc_if_unchanged_logic` bootstrap test covers the `rust.download-rustc=if-unchanged` logic. + # Here we are adding a dummy commit on compiler and running that test to ensure when there is a change on the compiler, + # we never download ci rustc with `rust.download-rustc=if-unchanged` option. + echo \"\" >> ../compiler/rustc/src/main.rs && \ + git config --global user.email \"dummy@dummy.com\" && \ + git config --global user.name \"dummy\" && \ + git add ../compiler/rustc/src/main.rs && \ + git commit -m \"test commit for rust.download-rustc=if-unchanged logic\" && \ + DISABLE_CI_RUSTC_IF_INCOMPATIBLE=0 python3 ../x.py test bootstrap -- core::builder::tests::ci_rustc_if_unchanged_logic && \ + # Revert the dummy commit + git reset --hard HEAD~1 && \ + + python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \ /scripts/check-default-config-profiles.sh && \ python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python3 ../x.py clippy bootstrap -Dwarnings && \ diff --git a/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile b/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile index ba3e8bdb687..0cae83a85b3 100644 --- a/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile @@ -58,6 +58,9 @@ RUN mkdir -p $RUST_INSTALL_DIR/etc # Fuchsia only supports LLVM. ENV CODEGEN_BACKENDS llvm +# download-rustc is not allowed for `x install` +ENV NO_DOWNLOAD_CI_RUSTC 1 + ENV RUST_CONFIGURE_ARGS \ --prefix=$RUST_INSTALL_DIR \ --sysconfdir=etc \ @@ -70,6 +73,7 @@ ENV RUST_CONFIGURE_ARGS \ --set target.x86_64-unknown-fuchsia.ar=/usr/local/bin/llvm-ar \ --set target.x86_64-unknown-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \ --set target.x86_64-unknown-fuchsia.linker=/usr/local/bin/ld.lld + ENV SCRIPT \ python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \ bash ../src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 145f41f21e1..17fc1a57492 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -84,6 +84,7 @@ ENV RUST_CONFIGURE_ARGS \ --enable-new-symbol-mangling ENV HOST_TARGET x86_64-unknown-linux-gnu +ENV FORCE_CI_RUSTC 1 COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/ COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/ diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index fad4b5af095..28487bce482 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -343,6 +343,7 @@ docker \ --env PR_CI_JOB \ --env OBJDIR_ON_HOST="$objdir" \ --env CODEGEN_BACKENDS \ + --env DISABLE_CI_RUSTC_IF_INCOMPATIBLE="$DISABLE_CI_RUSTC_IF_INCOMPATIBLE" \ --init \ --rm \ rust-ci \ diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 6379f1ade1c..4bbebbc4697 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -85,6 +85,9 @@ envs: # it in each job definition. pr: - image: mingw-check + env: + # We are adding (temporarily) a dummy commit on the compiler + READ_ONLY_SRC: "0" <<: *job-linux-4c - image: mingw-check-tidy continue_on_error: true @@ -207,6 +210,8 @@ auto: <<: *job-linux-8c - image: mingw-check + env: + READ_ONLY_SRC: 0 <<: *job-linux-4c - image: test-various diff --git a/src/ci/run.sh b/src/ci/run.sh index c8201d9bcfd..1ce54f9ecb3 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -52,6 +52,13 @@ if [ "$CI" != "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set change-id=99999999" fi +# If runner uses an incompatible option and `FORCE_CI_RUSTC` is not defined, +# switch to in-tree rustc. +if [ "$FORCE_CI_RUSTC" == "" ]; then + echo "debug: `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` configured." + DISABLE_CI_RUSTC_IF_INCOMPATIBLE=1 +fi + if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \ isCiBranch automation/bors/try; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" @@ -169,10 +176,16 @@ else if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-unchanged" else + # CI rustc requires CI LLVM to be enabled (see https://github.com/rust-lang/rust/issues/123586). + NO_DOWNLOAD_CI_RUSTC=1 # When building for CI we want to use the static C++ Standard library # included with LLVM, since a dynamic libstdcpp may not be available. RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.static-libstdcpp" fi + + if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged" + fi fi if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then |