Merge from rustc

5 files changed, 151 insertions, 120 deletions

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
deleted file mode 100644
index bcbf58253b1..00000000000
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
+++ /dev/null
@@ -1,49 +0,0 @@
-FROM ubuntu:22.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    g++ \
-    gcc-multilib \
-    make \
-    ninja-build \
-    file \
-    curl \
-    ca-certificates \
-    python2.7 \
-    git \
-    cmake \
-    sudo \
-    gdb \
-    llvm-13-tools \
-    llvm-13-dev \
-    libedit-dev \
-    libssl-dev \
-    pkg-config \
-    zlib1g-dev \
-    xz-utils \
-    nodejs \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY scripts/sccache.sh /scripts/
-RUN sh /scripts/sccache.sh
-
-# We are disabling CI LLVM since this builder is intentionally using a host
-# LLVM, rather than the typical src/llvm-project LLVM.
-ENV NO_DOWNLOAD_CI_LLVM 1
-
-# Using llvm-link-shared due to libffi issues -- see #34486
-ENV RUST_CONFIGURE_ARGS \
-    --build=x86_64-unknown-linux-gnu \
-    --llvm-root=/usr/lib/llvm-13 \
-    --enable-llvm-link-shared \
-    --set rust.thin-lto-import-instr-limit=10
-
-ENV SCRIPT python2.7 ../x.py --stage 1 test --exclude src/tools/tidy && \
-    # Run the `mir-opt` tests again but this time for a 32-bit target.
-    # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
-    # both 32-bit and 64-bit outputs updated by the PR author, before
-    # the PR is approved and tested for merging.
-    # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`,
-    # despite having different output on 32-bit vs 64-bit targets.
-    python2.7 ../x.py --stage 1 test tests/mir-opt \
-    --host='' --target=i686-unknown-linux-gnu
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile
index 9fc9e9cbffb..b99a0886b4d 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14-stage1/Dockerfile
@@ -1,8 +1,6 @@
 FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
-
-# NOTE: intentionally installs both python2 and python3 so we can test support for both.
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
   gcc-multilib \
@@ -11,28 +9,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   file \
   curl \
   ca-certificates \
-  python2.7 \
-  python3.9 \
+  python3 \
   git \
   cmake \
   sudo \
   gdb \
-  llvm-13-tools \
-  llvm-13-dev \
+  llvm-14-tools \
+  llvm-14-dev \
   libedit-dev \
   libssl-dev \
   pkg-config \
   zlib1g-dev \
   xz-utils \
   nodejs \
-  \
-# Install powershell so we can test x.ps1 on Linux
-    apt-transport-https software-properties-common && \
-    curl -s "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb" > packages-microsoft-prod.deb && \
-    dpkg -i packages-microsoft-prod.deb && \
-    apt-get update && \
-    apt-get install -y powershell \
-    && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/*
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
@@ -44,26 +34,16 @@ ENV NO_DOWNLOAD_CI_LLVM 1
 # Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
-      --llvm-root=/usr/lib/llvm-13 \
+      --llvm-root=/usr/lib/llvm-14 \
       --enable-llvm-link-shared \
       --set rust.thin-lto-import-instr-limit=10
 
-# NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux.
-ENV SCRIPT ../x.py --stage 2 test --exclude src/tools/tidy && \
+ENV SCRIPT ../x.py --stage 1 test --exclude src/tools/tidy && \
            # Run the `mir-opt` tests again but this time for a 32-bit target.
            # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have
            # both 32-bit and 64-bit outputs updated by the PR author, before
            # the PR is approved and tested for merging.
            # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`,
            # despite having different output on 32-bit vs 64-bit targets.
-           ../x --stage 2 test tests/mir-opt \
-                             --host='' --target=i686-unknown-linux-gnu && \
-           # Run the UI test suite again, but in `--pass=check` mode
-           #
-           # This is intended to make sure that both `--pass=check` continues to
-           # work.
-           #
-           ../x.ps1 --stage 2 test tests/ui --pass=check \
-                             --host='' --target=i686-unknown-linux-gnu && \
-           # Run tidy at the very end, after all the other tests.
-           python2.7 ../x.py --stage 2 test src/tools/tidy
+           ../x.py --stage 1 test tests/mir-opt \
+                             --host='' --target=i686-unknown-linux-gnu
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version
index c39e9c5fbc9..94ec2401292 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version
@@ -1 +1 @@
-0.14.1
\ No newline at end of file
+0.14.3
\ No newline at end of file
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 3c128c0ca25..ad9c308ad85 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -300,7 +300,7 @@ jobs:
             <<: *job-linux-xl
             tidy: true
 
-          - name: x86_64-gnu-llvm-13
+          - name: x86_64-gnu-llvm-14
             <<: *job-linux-xl
             tidy: false
 
@@ -459,12 +459,7 @@ jobs:
               RUST_BACKTRACE: 1
             <<: *job-linux-xl
 
-          - name: x86_64-gnu-llvm-13
-            env:
-              RUST_BACKTRACE: 1
-            <<: *job-linux-xl
-
-          - name: x86_64-gnu-llvm-13-stage1
+          - name: x86_64-gnu-llvm-14-stage1
             env:
               RUST_BACKTRACE: 1
             <<: *job-linux-xl
diff --git a/src/ci/stage-build.py b/src/ci/stage-build.py
index 662c9e36694..bd8fd524a26 100644
--- a/src/ci/stage-build.py
+++ b/src/ci/stage-build.py
@@ -6,6 +6,7 @@
 import contextlib
 import getpass
 import glob
+import json
 import logging
 import os
 import pprint
@@ -17,7 +18,8 @@ import traceback
 import urllib.request
 from io import StringIO
 from pathlib import Path
-from typing import Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
+from typing import Callable, ContextManager, Dict, Iterable, Iterator, List, Optional, \
+    Tuple, Union
 
 PGO_HOST = os.environ["PGO_HOST"]
 
@@ -115,6 +117,9 @@ class Pipeline:
     def llvm_bolt_profile_merged_file(self) -> Path:
         return self.opt_artifacts() / "bolt.profdata"
 
+    def metrics_path(self) -> Path:
+        return self.build_root() / "build" / "metrics.json"
+
 
 class LinuxPipeline(Pipeline):
     def checkout_path(self) -> Path:
@@ -208,38 +213,34 @@ def get_timestamp() -> float:
 
 
 Duration = float
-TimerSection = Union[Duration, "Timer"]
 
 
-def iterate_sections(section: TimerSection, name: str, level: int = 0) -> Iterator[Tuple[int, str, Duration]]:
+def iterate_timers(timer: "Timer", name: str, level: int = 0) -> Iterator[
+    Tuple[int, str, Duration]]:
     """
-    Hierarchically iterate the sections of a timer, in a depth-first order.
+    Hierarchically iterate the children of a timer, in a depth-first order.
     """
-    if isinstance(section, Duration):
-        yield (level, name, section)
-    elif isinstance(section, Timer):
-        yield (level, name, section.total_duration())
-        for (child_name, child_section) in section.sections:
-            yield from iterate_sections(child_section, child_name, level=level + 1)
-    else:
-        assert False
+    yield (level, name, timer.total_duration())
+    for (child_name, child_timer) in timer.children:
+        yield from iterate_timers(child_timer, child_name, level=level + 1)
 
 
 class Timer:
     def __init__(self, parent_names: Tuple[str, ...] = ()):
-        self.sections: List[Tuple[str, TimerSection]] = []
+        self.children: List[Tuple[str, Timer]] = []
         self.section_active = False
         self.parent_names = parent_names
+        self.duration_excluding_children: Duration = 0
 
     @contextlib.contextmanager
-    def section(self, name: str) -> "Timer":
+    def section(self, name: str) -> ContextManager["Timer"]:
         assert not self.section_active
         self.section_active = True
 
         start = get_timestamp()
         exc = None
 
-        child_timer = Timer(parent_names=self.parent_names + (name, ))
+        child_timer = Timer(parent_names=self.parent_names + (name,))
         full_name = " > ".join(child_timer.parent_names)
         try:
             LOGGER.info(f"Section `{full_name}` starts")
@@ -251,10 +252,8 @@ class Timer:
             end = get_timestamp()
             duration = end - start
 
-            if child_timer.has_children():
-                self.sections.append((name, child_timer))
-            else:
-                self.sections.append((name, duration))
+            child_timer.duration_excluding_children = duration - child_timer.total_duration()
+            self.add_child(name, child_timer)
             if exc is None:
                 LOGGER.info(f"Section `{full_name}` ended: OK ({duration:.2f}s)")
             else:
@@ -262,22 +261,17 @@ class Timer:
             self.section_active = False
 
     def total_duration(self) -> Duration:
-        duration = 0
-        for (_, section) in self.sections:
-            if isinstance(section, Duration):
-                duration += section
-            else:
-                duration += section.total_duration()
-        return duration
+        return self.duration_excluding_children + sum(
+            c.total_duration() for (_, c) in self.children)
 
     def has_children(self) -> bool:
-        return len(self.sections) > 0
+        return len(self.children) > 0
 
     def print_stats(self):
         rows = []
-        for (child_name, child_section) in self.sections:
-            for (level, name, duration) in iterate_sections(child_section, child_name, level=0):
-                label = f"{' ' * level}{name}:"
+        for (child_name, child_timer) in self.children:
+            for (level, name, duration) in iterate_timers(child_timer, child_name, level=0):
+                label = f"{'  ' * level}{name}:"
                 rows.append((label, duration))
 
         # Empty row
@@ -305,6 +299,60 @@ class Timer:
             print(divider, file=output, end="")
             LOGGER.info(f"Timer results\n{output.getvalue()}")
 
+    def add_child(self, name: str, timer: "Timer"):
+        self.children.append((name, timer))
+
+    def add_duration(self, name: str, duration: Duration):
+        timer = Timer(parent_names=self.parent_names + (name,))
+        timer.duration_excluding_children = duration
+        self.add_child(name, timer)
+
+
+class BuildStep:
+    def __init__(self, type: str, children: List["BuildStep"], duration: float):
+        self.type = type
+        self.children = children
+        self.duration = duration
+
+    def find_all_by_type(self, type: str) -> Iterator["BuildStep"]:
+        if type == self.type:
+            yield self
+        for child in self.children:
+            yield from child.find_all_by_type(type)
+
+    def __repr__(self):
+        return f"BuildStep(type={self.type}, duration={self.duration}, children={len(self.children)})"
+
+
+def load_last_metrics(path: Path) -> BuildStep:
+    """
+    Loads the metrics of the most recent bootstrap execution from a metrics.json file.
+    """
+    with open(path, "r") as f:
+        metrics = json.load(f)
+    invocation = metrics["invocations"][-1]
+
+    def parse(entry) -> Optional[BuildStep]:
+        if "kind" not in entry or entry["kind"] != "rustbuild_step":
+            return None
+        type = entry.get("type", "")
+        duration = entry.get("duration_excluding_children_sec", 0)
+        children = []
+
+        for child in entry.get("children", ()):
+            step = parse(child)
+            if step is not None:
+                children.append(step)
+                duration += step.duration
+        return BuildStep(type=type, children=children, duration=duration)
+
+    children = [parse(child) for child in invocation.get("children", ())]
+    return BuildStep(
+        type="root",
+        children=children,
+        duration=invocation.get("duration_including_children_sec", 0)
+    )
+
 
 @contextlib.contextmanager
 def change_cwd(dir: Path):
@@ -644,10 +692,58 @@ def print_binary_sizes(pipeline: Pipeline):
     with StringIO() as output:
         for path in paths:
             path_str = f"{path.name}:"
-            print(f"{path_str:<30}{format_bytes(path.stat().st_size):>14}", file=output)
+            print(f"{path_str:<50}{format_bytes(path.stat().st_size):>14}", file=output)
         LOGGER.info(f"Rustc binary size\n{output.getvalue()}")
 
 
+def print_free_disk_space(pipeline: Pipeline):
+    usage = shutil.disk_usage(pipeline.opt_artifacts())
+    total = usage.total
+    used = usage.used
+    free = usage.free
+
+    logging.info(
+        f"Free disk space: {format_bytes(free)} out of total {format_bytes(total)} ({(used / total) * 100:.2f}% used)")
+
+
+def log_metrics(step: BuildStep):
+    substeps: List[Tuple[int, BuildStep]] = []
+
+    def visit(step: BuildStep, level: int):
+        substeps.append((level, step))
+        for child in step.children:
+            visit(child, level=level + 1)
+
+    visit(step, 0)
+
+    output = StringIO()
+    for (level, step) in substeps:
+        label = f"{'.' * level}{step.type}"
+        print(f"{label:<65}{step.duration:>8.2f}s", file=output)
+    logging.info(f"Build step durations\n{output.getvalue()}")
+
+
+def record_metrics(pipeline: Pipeline, timer: Timer):
+    metrics = load_last_metrics(pipeline.metrics_path())
+    if metrics is None:
+        return
+    llvm_steps = tuple(metrics.find_all_by_type("bootstrap::native::Llvm"))
+    assert len(llvm_steps) > 0
+    llvm_duration = sum(step.duration for step in llvm_steps)
+
+    rustc_steps = tuple(metrics.find_all_by_type("bootstrap::compile::Rustc"))
+    assert len(rustc_steps) > 0
+    rustc_duration = sum(step.duration for step in rustc_steps)
+
+    # The LLVM step is part of the Rustc step
+    rustc_duration -= llvm_duration
+
+    timer.add_duration("LLVM", llvm_duration)
+    timer.add_duration("Rustc", rustc_duration)
+
+    log_metrics(metrics)
+
+
 def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: List[str]):
     # Clear and prepare tmp directory
     shutil.rmtree(pipeline.opt_artifacts(), ignore_errors=True)
@@ -657,15 +753,17 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: L
 
     # Stage 1: Build rustc + PGO instrumented LLVM
     with timer.section("Stage 1 (LLVM PGO)") as stage1:
-        with stage1.section("Build rustc and LLVM"):
+        with stage1.section("Build rustc and LLVM") as rustc_build:
             build_rustc(pipeline, args=[
                 "--llvm-profile-generate"
             ], env=dict(
                 LLVM_PROFILE_DIR=str(pipeline.llvm_profile_dir_root() / "prof-%p")
             ))
+            record_metrics(pipeline, rustc_build)
 
         with stage1.section("Gather profiles"):
             gather_llvm_profiles(pipeline)
+        print_free_disk_space(pipeline)
 
     clear_llvm_files(pipeline)
     final_build_args += [
@@ -675,14 +773,16 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: L
 
     # Stage 2: Build PGO instrumented rustc + LLVM
     with timer.section("Stage 2 (rustc PGO)") as stage2:
-        with stage2.section("Build rustc and LLVM"):
+        with stage2.section("Build rustc and LLVM") as rustc_build:
             build_rustc(pipeline, args=[
                 "--rust-profile-generate",
                 pipeline.rustc_profile_dir_root()
             ])
+            record_metrics(pipeline, rustc_build)
 
         with stage2.section("Gather profiles"):
             gather_rustc_profiles(pipeline)
+        print_free_disk_space(pipeline)
 
     clear_llvm_files(pipeline)
     final_build_args += [
@@ -693,15 +793,18 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: L
     # Stage 3: Build rustc + BOLT instrumented LLVM
     if pipeline.supports_bolt():
         with timer.section("Stage 3 (LLVM BOLT)") as stage3:
-            with stage3.section("Build rustc and LLVM"):
+            with stage3.section("Build rustc and LLVM") as rustc_build:
                 build_rustc(pipeline, args=[
                     "--llvm-profile-use",
                     pipeline.llvm_profile_merged_file(),
                     "--llvm-bolt-profile-generate",
                 ])
+                record_metrics(pipeline, rustc_build)
+
             with stage3.section("Gather profiles"):
                 gather_llvm_bolt_profiles(pipeline)
 
+        print_free_disk_space(pipeline)
         clear_llvm_files(pipeline)
         final_build_args += [
             "--llvm-bolt-profile-use",
@@ -709,8 +812,9 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, final_build_args: L
         ]
 
     # Stage 4: Build PGO optimized rustc + PGO/BOLT optimized LLVM
-    with timer.section("Stage 4 (final build)"):
+    with timer.section("Stage 4 (final build)") as stage4:
         cmd(final_build_args)
+        record_metrics(pipeline, stage4)
 
 
 if __name__ == "__main__":
@@ -733,5 +837,6 @@ if __name__ == "__main__":
         raise e
     finally:
         timer.print_stats()
+        print_free_disk_space(pipeline)
 
     print_binary_sizes(pipeline)