diff options
| -rw-r--r-- | src/bootstrap/lib.rs | 9 | ||||
| -rw-r--r-- | src/bootstrap/test.rs | 10 | ||||
| -rw-r--r-- | src/ci/stage-build.py | 115 |
3 files changed, 127 insertions, 7 deletions
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index a1aaee68c62..eb121b110d6 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -222,6 +222,7 @@ pub struct Build { initial_cargo: PathBuf, initial_lld: PathBuf, initial_libdir: PathBuf, + initial_sysroot: PathBuf, // Runtime state filled in later on // C/C++ compilers and archiver for all targets @@ -389,13 +390,16 @@ impl Build { "/dummy".to_string() } else { output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) - }; + } + .trim() + .to_string(); + let initial_libdir = initial_target_dir .parent() .unwrap() .parent() .unwrap() - .strip_prefix(initial_sysroot.trim()) + .strip_prefix(&initial_sysroot) .unwrap() .to_path_buf(); @@ -425,6 +429,7 @@ impl Build { initial_cargo: config.initial_cargo.clone(), initial_lld, initial_libdir, + initial_sysroot: initial_sysroot.into(), local_rebuild: config.local_rebuild, fail_fast: config.cmd.fail_fast(), doc_tests: config.cmd.doc_tests(), diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index eec8c4ad69f..35ba3dab764 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1424,7 +1424,15 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); - cmd.arg("--sysroot-base").arg(builder.sysroot(compiler)); + + // When top stage is 0, that means that we're testing an externally provided compiler. + // In that case we need to use its specific sysroot for tests to pass. + let sysroot = if builder.top_stage == 0 { + builder.initial_sysroot.clone() + } else { + builder.sysroot(compiler).to_path_buf() + }; + cmd.arg("--sysroot-base").arg(sysroot); cmd.arg("--stage-id").arg(stage_id); cmd.arg("--suite").arg(suite); cmd.arg("--mode").arg(mode); diff --git a/src/ci/stage-build.py b/src/ci/stage-build.py index 4141296bd42..91bd137085e 100644 --- a/src/ci/stage-build.py +++ b/src/ci/stage-build.py @@ -124,6 +124,12 @@ class Pipeline: def metrics_path(self) -> Path: return self.build_root() / "build" / "metrics.json" + def executable_extension(self) -> str: + raise NotImplementedError + + def skipped_tests(self) -> Iterable[str]: + return () + class LinuxPipeline(Pipeline): def checkout_path(self) -> Path: @@ -152,6 +158,13 @@ class LinuxPipeline(Pipeline): def supports_bolt(self) -> bool: return True + def executable_extension(self) -> str: + return "" + + def skipped_tests(self) -> Iterable[str]: + # This test fails because of linker errors, as of June 2023. + yield "tests/ui/process/nofile-limit.rs" + class WindowsPipeline(Pipeline): def __init__(self): @@ -211,6 +224,13 @@ class WindowsPipeline(Pipeline): def supports_bolt(self) -> bool: return False + def executable_extension(self) -> str: + return ".exe" + + def skipped_tests(self) -> Iterable[str]: + # This test fails as of June 2023 + yield "tests\\codegen\\vec-shrink-panik.rs" + def get_timestamp() -> float: return time.time() @@ -403,9 +423,9 @@ def delete_directory(path: Path): shutil.rmtree(path) -def unpack_archive(archive: Path): +def unpack_archive(archive: Path, target_dir: Optional[Path] = None): LOGGER.info(f"Unpacking archive `{archive}`") - shutil.unpack_archive(archive) + shutil.unpack_archive(str(archive), extract_dir=str(target_dir) if target_dir is not None else None) def download_file(src: str, target: Path): @@ -455,6 +475,7 @@ def cmd( ) return subprocess.run(args, env=environment, check=True) + class BenchmarkRunner: def run_rustc(self, pipeline: Pipeline): raise NotImplementedError @@ -465,6 +486,7 @@ class BenchmarkRunner: def run_bolt(self, pipeline: Pipeline): raise NotImplementedError + class DefaultBenchmarkRunner(BenchmarkRunner): def run_rustc(self, pipeline: Pipeline): # Here we're profiling the `rustc` frontend, so we also include `Check`. @@ -478,6 +500,7 @@ class DefaultBenchmarkRunner(BenchmarkRunner): LLVM_PROFILE_FILE=str(pipeline.rustc_profile_template_path()) ) ) + def run_llvm(self, pipeline: Pipeline): run_compiler_benchmarks( pipeline, @@ -494,6 +517,7 @@ class DefaultBenchmarkRunner(BenchmarkRunner): crates=LLVM_BOLT_CRATES ) + def run_compiler_benchmarks( pipeline: Pipeline, profiles: List[str], @@ -650,10 +674,8 @@ def gather_llvm_profiles(pipeline: Pipeline, runner: BenchmarkRunner): def gather_rustc_profiles(pipeline: Pipeline, runner: BenchmarkRunner): LOGGER.info("Running benchmarks with PGO instrumented rustc") - runner.run_rustc(pipeline) - profile_path = pipeline.rustc_profile_merged_file() LOGGER.info(f"Merging Rustc PGO profiles to {profile_path}") cmd([ @@ -770,6 +792,86 @@ def record_metrics(pipeline: Pipeline, timer: Timer): log_metrics(metrics) +def run_tests(pipeline: Pipeline): + """ + After `dist` is executed, we extract its archived components into a sysroot directory, + and then use that extracted rustc as a stage0 compiler. + Then we run a subset of tests using that compiler, to have a basic smoke test which checks + whether the optimization pipeline hasn't broken something. + """ + build_dir = pipeline.build_root() / "build" + dist_dir = build_dir / "dist" + + def extract_dist_dir(name: str) -> Path: + target_dir = build_dir / "optimized-dist" + target_dir.mkdir(parents=True, exist_ok=True) + unpack_archive(dist_dir / f"{name}.tar.xz", target_dir=target_dir) + extracted_path = target_dir / name + assert extracted_path.is_dir() + return extracted_path + + # Extract rustc, libstd, cargo and src archives to create the optimized sysroot + rustc_dir = extract_dist_dir(f"rustc-nightly-{PGO_HOST}") / "rustc" + libstd_dir = extract_dist_dir(f"rust-std-nightly-{PGO_HOST}") / f"rust-std-{PGO_HOST}" + cargo_dir = extract_dist_dir(f"cargo-nightly-{PGO_HOST}") / f"cargo" + extracted_src_dir = extract_dist_dir("rust-src-nightly") / "rust-src" + + # We need to manually copy libstd to the extracted rustc sysroot + shutil.copytree( + libstd_dir / "lib" / "rustlib" / PGO_HOST / "lib", + rustc_dir / "lib" / "rustlib" / PGO_HOST / "lib" + ) + + # Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd + # sources to the extracted sysroot. We need sources available so that `-Zsimulate-remapped-rust-src-base` + # works correctly. + shutil.copytree( + extracted_src_dir / "lib" / "rustlib" / "src", + rustc_dir / "lib" / "rustlib" / "src" + ) + + rustc_path = rustc_dir / "bin" / f"rustc{pipeline.executable_extension()}" + assert rustc_path.is_file() + cargo_path = cargo_dir / "bin" / f"cargo{pipeline.executable_extension()}" + assert cargo_path.is_file() + + config_content = f"""profile = "user" +changelog-seen = 2 + +[build] +rustc = "{rustc_path.as_posix()}" +cargo = "{cargo_path.as_posix()}" + +[llvm] +download-ci-llvm = true +""" + logging.info(f"Using following `config.toml` for running tests:\n{config_content}") + + # Simulate a stage 0 compiler with the extracted optimized dist artifacts. + with open("config.toml", "w") as f: + f.write(config_content) + + args = [ + sys.executable, + pipeline.checkout_path() / "x.py", + "test", + "--stage", "0", + "tests/assembly", + "tests/codegen", + "tests/codegen-units", + "tests/incremental", + "tests/mir-opt", + "tests/pretty", + "tests/run-pass-valgrind", + "tests/ui", + ] + for test_path in pipeline.skipped_tests(): + args.extend(["--exclude", test_path]) + cmd(args=args, env=dict( + COMPILETEST_FORCE_STAGE0="1" + )) + + def execute_build_pipeline(timer: Timer, pipeline: Pipeline, runner: BenchmarkRunner, final_build_args: List[str]): # Clear and prepare tmp directory shutil.rmtree(pipeline.opt_artifacts(), ignore_errors=True) @@ -844,6 +946,11 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, runner: BenchmarkRu cmd(final_build_args) record_metrics(pipeline, stage4) + # Try builds can be in various broken states, so we don't want to gatekeep them with tests + if not is_try_build(): + with timer.section("Run tests"): + run_tests(pipeline) + def run(runner: BenchmarkRunner): logging.basicConfig( |