From 8ecd033654667f283cd7295ac532de92898bf301 Mon Sep 17 00:00:00 2001 From: Jakub Beránek Date: Sun, 29 Dec 2024 18:42:33 +0100 Subject: Rename CI script from `calculate-job-matrix` to `ci.py` --- src/ci/github-actions/calculate-job-matrix.py | 210 -------------------------- src/ci/github-actions/ci.py | 210 ++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 210 deletions(-) delete mode 100755 src/ci/github-actions/calculate-job-matrix.py create mode 100755 src/ci/github-actions/ci.py (limited to 'src/ci/github-actions') diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py deleted file mode 100755 index 1f994f0ffd2..00000000000 --- a/src/ci/github-actions/calculate-job-matrix.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env python3 - -""" -This script serves for generating a matrix of jobs that should -be executed on CI. - -It reads job definitions from `src/ci/github-actions/jobs.yml` -and filters them based on the event that happened on CI. -""" - -import dataclasses -import json -import logging -import os -import re -import typing -from pathlib import Path -from typing import List, Dict, Any, Optional - -import yaml - -CI_DIR = Path(__file__).absolute().parent.parent -JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" - -Job = Dict[str, Any] - - -def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]: - """ - Add a `name` attribute to each job, based on its image and the given `prefix`. - """ - for job in jobs: - job["name"] = f"{prefix} - {job['image']}" - return jobs - - -def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: - """ - Prepends `environment` to the `env` attribute of each job. - The `env` of each job has higher precedence than `environment`. - """ - for job in jobs: - env = environment.copy() - env.update(job.get("env", {})) - job["env"] = env - return jobs - - -@dataclasses.dataclass -class PRRunType: - pass - - -@dataclasses.dataclass -class TryRunType: - custom_jobs: List[str] - - -@dataclasses.dataclass -class AutoRunType: - pass - - -WorkflowRunType = typing.Union[PRRunType, TryRunType, AutoRunType] - - -@dataclasses.dataclass -class GitHubCtx: - event_name: str - ref: str - repository: str - commit_message: Optional[str] - - -def get_custom_jobs(ctx: GitHubCtx) -> List[str]: - """ - Tries to parse names of specific CI jobs that should be executed in the form of - try-job: - from the commit message of the passed GitHub context. - """ - if ctx.commit_message is None: - return [] - - regex = re.compile(r"^try-job: (.*)", re.MULTILINE) - jobs = [] - for match in regex.finditer(ctx.commit_message): - jobs.append(match.group(1)) - return jobs - - -def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: - if ctx.event_name == "pull_request": - return PRRunType() - elif ctx.event_name == "push": - try_build = ctx.ref in ( - "refs/heads/try", - "refs/heads/try-perf", - "refs/heads/automation/bors/try", - ) - - # Unrolled branch from a rollup for testing perf - # This should **not** allow custom try jobs - is_unrolled_perf_build = ctx.ref == "refs/heads/try-perf" - - if try_build: - custom_jobs = [] - if not is_unrolled_perf_build: - custom_jobs = get_custom_jobs(ctx) - return TryRunType(custom_jobs=custom_jobs) - - if ctx.ref == "refs/heads/auto": - return AutoRunType() - - return None - - -def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: - if isinstance(run_type, PRRunType): - return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) - elif isinstance(run_type, TryRunType): - jobs = job_data["try"] - custom_jobs = run_type.custom_jobs - if custom_jobs: - if len(custom_jobs) > 10: - raise Exception( - f"It is only possible to schedule up to 10 custom jobs, " - f"received {len(custom_jobs)} jobs" - ) - - jobs = [] - unknown_jobs = [] - for custom_job in custom_jobs: - job = [j for j in job_data["auto"] if j["image"] == custom_job] - if not job: - unknown_jobs.append(custom_job) - continue - jobs.append(job[0]) - if unknown_jobs: - raise Exception( - f"Custom job(s) `{unknown_jobs}` not found in auto jobs" - ) - - return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) - elif isinstance(run_type, AutoRunType): - return add_base_env( - name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"] - ) - - return [] - - -def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: - """ - Skip CI jobs that are not supposed to be executed on the given `channel`. - """ - return [j for j in jobs if j.get("only_on_channel", channel) == channel] - - -def get_github_ctx() -> GitHubCtx: - event_name = os.environ["GITHUB_EVENT_NAME"] - - commit_message = None - if event_name == "push": - commit_message = os.environ["COMMIT_MESSAGE"] - return GitHubCtx( - event_name=event_name, - ref=os.environ["GITHUB_REF"], - repository=os.environ["GITHUB_REPOSITORY"], - commit_message=commit_message, - ) - - -def format_run_type(run_type: WorkflowRunType) -> str: - if isinstance(run_type, PRRunType): - return "pr" - elif isinstance(run_type, AutoRunType): - return "auto" - elif isinstance(run_type, TryRunType): - return "try" - else: - raise AssertionError() - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - with open(JOBS_YAML_PATH) as f: - data = yaml.safe_load(f) - - github_ctx = get_github_ctx() - - run_type = find_run_type(github_ctx) - logging.info(f"Job type: {run_type}") - - with open(CI_DIR / "channel") as f: - channel = f.read().strip() - - jobs = [] - if run_type is not None: - jobs = calculate_jobs(run_type, data) - jobs = skip_jobs(jobs, channel) - - if not jobs: - raise Exception("Scheduled job list is empty, this is an error") - - run_type = format_run_type(run_type) - - logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") - print(f"jobs={json.dumps(jobs)}") - print(f"run_type={run_type}") diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py new file mode 100755 index 00000000000..1f994f0ffd2 --- /dev/null +++ b/src/ci/github-actions/ci.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 + +""" +This script serves for generating a matrix of jobs that should +be executed on CI. + +It reads job definitions from `src/ci/github-actions/jobs.yml` +and filters them based on the event that happened on CI. +""" + +import dataclasses +import json +import logging +import os +import re +import typing +from pathlib import Path +from typing import List, Dict, Any, Optional + +import yaml + +CI_DIR = Path(__file__).absolute().parent.parent +JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" + +Job = Dict[str, Any] + + +def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]: + """ + Add a `name` attribute to each job, based on its image and the given `prefix`. + """ + for job in jobs: + job["name"] = f"{prefix} - {job['image']}" + return jobs + + +def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: + """ + Prepends `environment` to the `env` attribute of each job. + The `env` of each job has higher precedence than `environment`. + """ + for job in jobs: + env = environment.copy() + env.update(job.get("env", {})) + job["env"] = env + return jobs + + +@dataclasses.dataclass +class PRRunType: + pass + + +@dataclasses.dataclass +class TryRunType: + custom_jobs: List[str] + + +@dataclasses.dataclass +class AutoRunType: + pass + + +WorkflowRunType = typing.Union[PRRunType, TryRunType, AutoRunType] + + +@dataclasses.dataclass +class GitHubCtx: + event_name: str + ref: str + repository: str + commit_message: Optional[str] + + +def get_custom_jobs(ctx: GitHubCtx) -> List[str]: + """ + Tries to parse names of specific CI jobs that should be executed in the form of + try-job: + from the commit message of the passed GitHub context. + """ + if ctx.commit_message is None: + return [] + + regex = re.compile(r"^try-job: (.*)", re.MULTILINE) + jobs = [] + for match in regex.finditer(ctx.commit_message): + jobs.append(match.group(1)) + return jobs + + +def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: + if ctx.event_name == "pull_request": + return PRRunType() + elif ctx.event_name == "push": + try_build = ctx.ref in ( + "refs/heads/try", + "refs/heads/try-perf", + "refs/heads/automation/bors/try", + ) + + # Unrolled branch from a rollup for testing perf + # This should **not** allow custom try jobs + is_unrolled_perf_build = ctx.ref == "refs/heads/try-perf" + + if try_build: + custom_jobs = [] + if not is_unrolled_perf_build: + custom_jobs = get_custom_jobs(ctx) + return TryRunType(custom_jobs=custom_jobs) + + if ctx.ref == "refs/heads/auto": + return AutoRunType() + + return None + + +def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: + if isinstance(run_type, PRRunType): + return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) + elif isinstance(run_type, TryRunType): + jobs = job_data["try"] + custom_jobs = run_type.custom_jobs + if custom_jobs: + if len(custom_jobs) > 10: + raise Exception( + f"It is only possible to schedule up to 10 custom jobs, " + f"received {len(custom_jobs)} jobs" + ) + + jobs = [] + unknown_jobs = [] + for custom_job in custom_jobs: + job = [j for j in job_data["auto"] if j["image"] == custom_job] + if not job: + unknown_jobs.append(custom_job) + continue + jobs.append(job[0]) + if unknown_jobs: + raise Exception( + f"Custom job(s) `{unknown_jobs}` not found in auto jobs" + ) + + return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) + elif isinstance(run_type, AutoRunType): + return add_base_env( + name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"] + ) + + return [] + + +def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: + """ + Skip CI jobs that are not supposed to be executed on the given `channel`. + """ + return [j for j in jobs if j.get("only_on_channel", channel) == channel] + + +def get_github_ctx() -> GitHubCtx: + event_name = os.environ["GITHUB_EVENT_NAME"] + + commit_message = None + if event_name == "push": + commit_message = os.environ["COMMIT_MESSAGE"] + return GitHubCtx( + event_name=event_name, + ref=os.environ["GITHUB_REF"], + repository=os.environ["GITHUB_REPOSITORY"], + commit_message=commit_message, + ) + + +def format_run_type(run_type: WorkflowRunType) -> str: + if isinstance(run_type, PRRunType): + return "pr" + elif isinstance(run_type, AutoRunType): + return "auto" + elif isinstance(run_type, TryRunType): + return "try" + else: + raise AssertionError() + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + with open(JOBS_YAML_PATH) as f: + data = yaml.safe_load(f) + + github_ctx = get_github_ctx() + + run_type = find_run_type(github_ctx) + logging.info(f"Job type: {run_type}") + + with open(CI_DIR / "channel") as f: + channel = f.read().strip() + + jobs = [] + if run_type is not None: + jobs = calculate_jobs(run_type, data) + jobs = skip_jobs(jobs, channel) + + if not jobs: + raise Exception("Scheduled job list is empty, this is an error") + + run_type = format_run_type(run_type) + + logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") + print(f"jobs={json.dumps(jobs)}") + print(f"run_type={run_type}") -- cgit 1.4.1-3-g733a5 From 0cc11f8c480cce40859ecb82fd8db54771b6aba7 Mon Sep 17 00:00:00 2001 From: Jakub Beránek Date: Sun, 29 Dec 2024 19:15:36 +0100 Subject: Add a command to run a given Linux CI job locally --- .github/workflows/ci.yml | 2 +- src/ci/github-actions/ci.py | 86 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 20 deletions(-) (limited to 'src/ci/github-actions') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bedfe72137c..d8d7560ffcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - name: Calculate the CI job matrix env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: python3 src/ci/github-actions/ci.py >> $GITHUB_OUTPUT + run: python3 src/ci/github-actions/ci.py calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: name: ${{ matrix.name }} diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py index 1f994f0ffd2..354c0f4844a 100755 --- a/src/ci/github-actions/ci.py +++ b/src/ci/github-actions/ci.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 """ -This script serves for generating a matrix of jobs that should -be executed on CI. +This script contains CI functionality. +It can be used to generate a matrix of jobs that should +be executed on CI, or run a specific CI job locally. -It reads job definitions from `src/ci/github-actions/jobs.yml` -and filters them based on the event that happened on CI. +It reads job definitions from `src/ci/github-actions/jobs.yml`. """ +import argparse import dataclasses import json import logging import os import re +import subprocess import typing from pathlib import Path from typing import List, Dict, Any, Optional @@ -181,30 +183,76 @@ def format_run_type(run_type: WorkflowRunType) -> str: raise AssertionError() +def run_workflow_locally(job_data: Dict[str, Any], job_name: str): + DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" + + jobs = list(job_data["auto"]) + jobs.extend(job_data["pr"]) + + jobs = [job for job in jobs if job.get("image") == job_name] + if len(jobs) == 0: + raise Exception(f"Job `{job_name}` not found") + job = jobs[0] + if "ubuntu" not in job["os"]: + raise Exception("Only Linux jobs can be executed locally") + + image = job.get("env", {}).get("IMAGE", job["image"]) + custom_env = {} + custom_env["DEPLOY"] = "1" + custom_env.update({k: str(v) for (k, v) in job.get("env", {}).items()}) + + args = [ + str(DOCKER_DIR / "run.sh"), + image + ] + env_formatted = [f"{k}={v}" for (k, v) in sorted(custom_env.items())] + print(f"Executing `{' '.join(env_formatted)} {' '.join(args)}`") + + env = os.environ.copy() + env.update(custom_env) + subprocess.run(args, env=env) + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) with open(JOBS_YAML_PATH) as f: data = yaml.safe_load(f) - github_ctx = get_github_ctx() + parser = argparse.ArgumentParser( + prog="ci.py", + description="Generate or run CI workflows" + ) + generate_matrix = argparse.ArgumentParser() + subparsers = parser.add_subparsers(help="Command to execute", dest="command", required=True) + subparsers.add_parser("calculate-job-matrix") + run_parser = subparsers.add_parser("run-local") + run_parser.add_argument("job_name", help="CI job that should be executed") + args = parser.parse_args() - run_type = find_run_type(github_ctx) - logging.info(f"Job type: {run_type}") + if args.command == "calculate-job-matrix": + github_ctx = get_github_ctx() - with open(CI_DIR / "channel") as f: - channel = f.read().strip() + run_type = find_run_type(github_ctx) + logging.info(f"Job type: {run_type}") - jobs = [] - if run_type is not None: - jobs = calculate_jobs(run_type, data) - jobs = skip_jobs(jobs, channel) + with open(CI_DIR / "channel") as f: + channel = f.read().strip() - if not jobs: - raise Exception("Scheduled job list is empty, this is an error") + jobs = [] + if run_type is not None: + jobs = calculate_jobs(run_type, data) + jobs = skip_jobs(jobs, channel) - run_type = format_run_type(run_type) + if not jobs: + raise Exception("Scheduled job list is empty, this is an error") - logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") - print(f"jobs={json.dumps(jobs)}") - print(f"run_type={run_type}") + run_type = format_run_type(run_type) + + logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") + print(f"jobs={json.dumps(jobs)}") + print(f"run_type={run_type}") + elif args.command == "run-local": + run_workflow_locally(data, args.job_name) + else: + raise Exception(f"Unknown command {args.command}") -- cgit 1.4.1-3-g733a5 From e62d1e46bbc4660ae5ec64c7e9f18244bf3f93b2 Mon Sep 17 00:00:00 2001 From: Jakub Beránek Date: Sun, 29 Dec 2024 22:52:45 +0100 Subject: Rename `image` property of CI jobs to `name` The `image` part didn't really make sense, especially since we started splitting CI jobs. --- src/ci/github-actions/ci.py | 34 ++++++---- src/ci/github-actions/jobs.yml | 146 ++++++++++++++++++++--------------------- 2 files changed, 96 insertions(+), 84 deletions(-) (limited to 'src/ci/github-actions') diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py index 354c0f4844a..4ec4cf11b5b 100755 --- a/src/ci/github-actions/ci.py +++ b/src/ci/github-actions/ci.py @@ -27,13 +27,18 @@ JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" Job = Dict[str, Any] -def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]: +def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: """ - Add a `name` attribute to each job, based on its image and the given `prefix`. + Modify the `name` attribute of each job, based on its base name and the given `prefix`. + Add an `image` attribute to each job, base don its image. """ + modified_jobs = [] for job in jobs: - job["name"] = f"{prefix} - {job['image']}" - return jobs + job = dict(job) + job["image"] = get_job_image(job) + job["name"] = f"{prefix} - {job['name']}" + modified_jobs.append(job) + return modified_jobs def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: @@ -118,7 +123,7 @@ def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: if isinstance(run_type, PRRunType): - return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) + return add_base_env(add_job_properties(job_data["pr"], "PR"), job_data["envs"]["pr"]) elif isinstance(run_type, TryRunType): jobs = job_data["try"] custom_jobs = run_type.custom_jobs @@ -132,7 +137,7 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ jobs = [] unknown_jobs = [] for custom_job in custom_jobs: - job = [j for j in job_data["auto"] if j["image"] == custom_job] + job = [j for j in job_data["auto"] if j["name"] == custom_job] if not job: unknown_jobs.append(custom_job) continue @@ -142,10 +147,10 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ f"Custom job(s) `{unknown_jobs}` not found in auto jobs" ) - return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) + return add_base_env(add_job_properties(jobs, "try"), job_data["envs"]["try"]) elif isinstance(run_type, AutoRunType): return add_base_env( - name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"] + add_job_properties(job_data["auto"], "auto"), job_data["envs"]["auto"] ) return [] @@ -183,27 +188,34 @@ def format_run_type(run_type: WorkflowRunType) -> str: raise AssertionError() +def get_job_image(job) -> str: + """ + By default, the Docker image of a job is based on its name. + However, it can be overridden by its IMAGE environment variable. + """ + return job.get("env", {}).get("IMAGE", job["name"]) + + def run_workflow_locally(job_data: Dict[str, Any], job_name: str): DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" jobs = list(job_data["auto"]) jobs.extend(job_data["pr"]) - jobs = [job for job in jobs if job.get("image") == job_name] + jobs = [job for job in jobs if job.get("name") == job_name] if len(jobs) == 0: raise Exception(f"Job `{job_name}` not found") job = jobs[0] if "ubuntu" not in job["os"]: raise Exception("Only Linux jobs can be executed locally") - image = job.get("env", {}).get("IMAGE", job["image"]) custom_env = {} custom_env["DEPLOY"] = "1" custom_env.update({k: str(v) for (k, v) in job.get("env", {}).items()}) args = [ str(DOCKER_DIR / "run.sh"), - image + get_job_image(job) ] env_formatted = [f"{k}={v}" for (k, v) in sorted(custom_env.items())] print(f"Executing `{' '.join(env_formatted)} {' '.join(args)}`") diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 876a7793592..d88be6543a3 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -91,26 +91,26 @@ envs: # These jobs automatically inherit envs.pr, to avoid repeating # it in each job definition. pr: - - image: mingw-check + - name: mingw-check <<: *job-linux-4c - - image: mingw-check-tidy + - name: mingw-check-tidy continue_on_error: true <<: *job-linux-4c - - image: x86_64-gnu-llvm-18 + - name: x86_64-gnu-llvm-18 env: ENABLE_GCC_CODEGEN: "1" # We are adding (temporarily) a dummy commit on the compiler READ_ONLY_SRC: "0" DOCKER_SCRIPT: x86_64-gnu-llvm.sh <<: *job-linux-16c - - image: x86_64-gnu-tools + - name: x86_64-gnu-tools <<: *job-linux-16c # Jobs that run when you perform a try build (@bors try) # These jobs automatically inherit envs.try, to avoid repeating # it in each job definition. try: - - image: dist-x86_64-linux + - name: dist-x86_64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c @@ -123,106 +123,106 @@ auto: # Linux/Docker builders # ############################# - - image: aarch64-gnu + - name: aarch64-gnu <<: *job-aarch64-linux - - image: aarch64-gnu-debug + - name: aarch64-gnu-debug <<: *job-aarch64-linux - - image: arm-android + - name: arm-android <<: *job-linux-4c - - image: armhf-gnu + - name: armhf-gnu <<: *job-linux-4c - - image: dist-aarch64-linux + - name: dist-aarch64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-4c - - image: dist-android + - name: dist-android <<: *job-linux-4c - - image: dist-arm-linux + - name: dist-arm-linux <<: *job-linux-8c - - image: dist-armhf-linux + - name: dist-armhf-linux <<: *job-linux-4c - - image: dist-armv7-linux + - name: dist-armv7-linux <<: *job-linux-4c - - image: dist-i586-gnu-i586-i686-musl + - name: dist-i586-gnu-i586-i686-musl <<: *job-linux-4c - - image: dist-i686-linux + - name: dist-i686-linux <<: *job-linux-4c - - image: dist-loongarch64-linux + - name: dist-loongarch64-linux <<: *job-linux-4c - - image: dist-loongarch64-musl + - name: dist-loongarch64-musl <<: *job-linux-4c - - image: dist-ohos + - name: dist-ohos <<: *job-linux-4c - - image: dist-powerpc-linux + - name: dist-powerpc-linux <<: *job-linux-4c - - image: dist-powerpc64-linux + - name: dist-powerpc64-linux <<: *job-linux-4c - - image: dist-powerpc64le-linux + - name: dist-powerpc64le-linux <<: *job-linux-4c-largedisk - - image: dist-riscv64-linux + - name: dist-riscv64-linux <<: *job-linux-4c - - image: dist-s390x-linux + - name: dist-s390x-linux <<: *job-linux-4c - - image: dist-various-1 + - name: dist-various-1 <<: *job-linux-4c - - image: dist-various-2 + - name: dist-various-2 <<: *job-linux-4c - - image: dist-x86_64-freebsd + - name: dist-x86_64-freebsd <<: *job-linux-4c - - image: dist-x86_64-illumos + - name: dist-x86_64-illumos <<: *job-linux-4c - - image: dist-x86_64-linux + - name: dist-x86_64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - - image: dist-x86_64-linux-alt + - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - - image: dist-x86_64-musl + - name: dist-x86_64-musl env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-4c - - image: dist-x86_64-netbsd + - name: dist-x86_64-netbsd <<: *job-linux-4c # The i686-gnu job is split into multiple jobs to run tests in parallel. # i686-gnu-1 skips tests that run in i686-gnu-2. - - image: i686-gnu-1 + - name: i686-gnu-1 env: IMAGE: i686-gnu DOCKER_SCRIPT: stage_2_test_set1.sh <<: *job-linux-4c # Skip tests that run in i686-gnu-1 - - image: i686-gnu-2 + - name: i686-gnu-2 env: IMAGE: i686-gnu DOCKER_SCRIPT: stage_2_test_set2.sh @@ -230,14 +230,14 @@ auto: # The i686-gnu-nopt job is split into multiple jobs to run tests in parallel. # i686-gnu-nopt-1 skips tests that run in i686-gnu-nopt-2 - - image: i686-gnu-nopt-1 + - name: i686-gnu-nopt-1 env: IMAGE: i686-gnu-nopt DOCKER_SCRIPT: /scripts/stage_2_test_set1.sh <<: *job-linux-4c # Skip tests that run in i686-gnu-nopt-1 - - image: i686-gnu-nopt-2 + - name: i686-gnu-nopt-2 env: IMAGE: i686-gnu-nopt DOCKER_SCRIPT: >- @@ -245,13 +245,13 @@ auto: /scripts/stage_2_test_set2.sh <<: *job-linux-4c - - image: mingw-check + - name: mingw-check <<: *job-linux-4c - - image: test-various + - name: test-various <<: *job-linux-4c - - image: x86_64-fuchsia + - name: x86_64-fuchsia # Only run this job on the nightly channel. Fuchsia requires # nightly features to compile, and this job would fail if # executed on beta and stable. @@ -260,10 +260,10 @@ auto: # Tests integration with Rust for Linux. # Builds stage 1 compiler and tries to compile a few RfL examples with it. - - image: x86_64-rust-for-linux + - name: x86_64-rust-for-linux <<: *job-linux-4c - - image: x86_64-gnu + - name: x86_64-gnu <<: *job-linux-4c # This job ensures commits landing on nightly still pass the full @@ -271,7 +271,7 @@ auto: # depend on the channel being built (for example if they include the # channel name on the output), and this builder prevents landing # changes that would result in broken builds after a promotion. - - image: x86_64-gnu-stable + - name: x86_64-gnu-stable # Only run this job on the nightly channel. Running this on beta # could cause failures when `dev: 1` in `stage0.txt`, and running # this on stable is useless. @@ -281,20 +281,20 @@ auto: RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable <<: *job-linux-4c - - image: x86_64-gnu-aux + - name: x86_64-gnu-aux <<: *job-linux-4c - - image: x86_64-gnu-debug + - name: x86_64-gnu-debug # This seems to be needed because a full stage 2 build + run-make tests # overwhelms the storage capacity of the standard 4c runner. <<: *job-linux-4c-largedisk - - image: x86_64-gnu-distcheck + - name: x86_64-gnu-distcheck <<: *job-linux-8c # The x86_64-gnu-llvm-19 job is split into multiple jobs to run tests in parallel. # x86_64-gnu-llvm-19-1 skips tests that run in x86_64-gnu-llvm-19-{2,3}. - - image: x86_64-gnu-llvm-19-1 + - name: x86_64-gnu-llvm-19-1 env: RUST_BACKTRACE: 1 IMAGE: x86_64-gnu-llvm-19 @@ -302,7 +302,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-19-{1,3} - - image: x86_64-gnu-llvm-19-2 + - name: x86_64-gnu-llvm-19-2 env: RUST_BACKTRACE: 1 IMAGE: x86_64-gnu-llvm-19 @@ -310,7 +310,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-19-{1,2} - - image: x86_64-gnu-llvm-19-3 + - name: x86_64-gnu-llvm-19-3 env: RUST_BACKTRACE: 1 IMAGE: x86_64-gnu-llvm-19 @@ -319,7 +319,7 @@ auto: # The x86_64-gnu-llvm-18 job is split into multiple jobs to run tests in parallel. # x86_64-gnu-llvm-18-1 skips tests that run in x86_64-gnu-llvm-18-{2,3}. - - image: x86_64-gnu-llvm-18-1 + - name: x86_64-gnu-llvm-18-1 env: RUST_BACKTRACE: 1 READ_ONLY_SRC: "0" @@ -328,7 +328,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-18-{1,3} - - image: x86_64-gnu-llvm-18-2 + - name: x86_64-gnu-llvm-18-2 env: RUST_BACKTRACE: 1 READ_ONLY_SRC: "0" @@ -337,7 +337,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-18-{1,2} - - image: x86_64-gnu-llvm-18-3 + - name: x86_64-gnu-llvm-18-3 env: RUST_BACKTRACE: 1 READ_ONLY_SRC: "0" @@ -345,10 +345,10 @@ auto: DOCKER_SCRIPT: x86_64-gnu-llvm3.sh <<: *job-linux-4c - - image: x86_64-gnu-nopt + - name: x86_64-gnu-nopt <<: *job-linux-4c - - image: x86_64-gnu-tools + - name: x86_64-gnu-tools env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json <<: *job-linux-4c @@ -357,7 +357,7 @@ auto: # macOS Builders # #################### - - image: dist-x86_64-apple + - name: dist-x86_64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1 @@ -371,7 +371,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-macos-xl - - image: dist-apple-various + - name: dist-apple-various env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim,aarch64-apple-ios-macabi,x86_64-apple-ios-macabi # Mac Catalyst cannot currently compile the sanitizer: @@ -385,19 +385,19 @@ auto: NO_OVERFLOW_CHECKS: 1 <<: *job-macos-xl - - image: x86_64-apple-1 + - name: x86_64-apple-1 env: <<: *env-x86_64-apple-tests <<: *job-macos-xl - - image: x86_64-apple-2 + - name: x86_64-apple-2 env: SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc <<: *env-x86_64-apple-tests <<: *job-macos-xl # This target only needs to support 11.0 and up as nothing else supports the hardware - - image: dist-aarch64-apple + - name: dist-aarch64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- @@ -421,7 +421,7 @@ auto: <<: *job-macos-m1 # This target only needs to support 11.0 and up as nothing else supports the hardware - - image: aarch64-apple + - name: aarch64-apple env: SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- @@ -442,20 +442,20 @@ auto: # Windows Builders # ###################### - - image: x86_64-msvc + - name: x86_64-msvc env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-msvc <<: *job-windows-8c - - image: i686-msvc + - name: i686-msvc env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-msvc <<: *job-windows-8c # x86_64-msvc-ext is split into multiple jobs to run tests in parallel. - - image: x86_64-msvc-ext1 + - name: x86_64-msvc-ext1 env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld @@ -464,7 +464,7 @@ auto: # Temporary builder to workaround CI issues # See #FIXME: Remove this, and re-enable the same tests in `checktools.sh`, once CI issues are fixed. - - image: x86_64-msvc-ext2 + - name: x86_64-msvc-ext2 env: SCRIPT: > python x.py test --stage 2 src/tools/miri --target aarch64-apple-darwin --test-args pass && @@ -476,7 +476,7 @@ auto: <<: *job-windows # Run `checktools.sh` and upload the toolstate file. - - image: x86_64-msvc-ext3 + - name: x86_64-msvc-ext3 env: SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows HOST_TARGET: x86_64-pc-windows-msvc @@ -500,7 +500,7 @@ auto: # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - - image: i686-mingw + - name: i686-mingw env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw @@ -510,7 +510,7 @@ auto: <<: *job-windows-8c # x86_64-mingw is split into two jobs to run tests in parallel. - - image: x86_64-mingw-1 + - name: x86_64-mingw-1 env: SCRIPT: make ci-mingw-x RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu @@ -519,7 +519,7 @@ auto: NO_DOWNLOAD_CI_LLVM: 1 <<: *job-windows - - image: x86_64-mingw-2 + - name: x86_64-mingw-2 env: SCRIPT: make ci-mingw-bootstrap RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu @@ -528,7 +528,7 @@ auto: NO_DOWNLOAD_CI_LLVM: 1 <<: *job-windows - - image: dist-x86_64-msvc + - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc @@ -542,7 +542,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows-8c - - image: dist-i686-msvc + - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc @@ -555,7 +555,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows - - image: dist-aarch64-msvc + - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc @@ -567,7 +567,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows - - image: dist-i686-mingw + - name: dist-i686-mingw env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-gnu @@ -580,7 +580,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows - - image: dist-x86_64-mingw + - name: dist-x86_64-mingw env: SCRIPT: python x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: >- @@ -593,7 +593,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows - - image: dist-x86_64-msvc-alt + - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths -- cgit 1.4.1-3-g733a5 From 0fc2f14c0cd15cda261ca43b74b2495656df6dab Mon Sep 17 00:00:00 2001 From: Jakub Beránek Date: Tue, 31 Dec 2024 10:18:02 +0100 Subject: Make it possible to select PR vs auto jobs --- src/ci/github-actions/ci.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) (limited to 'src/ci/github-actions') diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py index 4ec4cf11b5b..d1c665503b1 100755 --- a/src/ci/github-actions/ci.py +++ b/src/ci/github-actions/ci.py @@ -30,7 +30,7 @@ Job = Dict[str, Any] def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: """ Modify the `name` attribute of each job, based on its base name and the given `prefix`. - Add an `image` attribute to each job, base don its image. + Add an `image` attribute to each job, based on its image. """ modified_jobs = [] for job in jobs: @@ -196,15 +196,14 @@ def get_job_image(job) -> str: return job.get("env", {}).get("IMAGE", job["name"]) -def run_workflow_locally(job_data: Dict[str, Any], job_name: str): +def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool): DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" - jobs = list(job_data["auto"]) - jobs.extend(job_data["pr"]) - + jobs = job_data["pr"] if pr_jobs else job_data["auto"] jobs = [job for job in jobs if job.get("name") == job_name] if len(jobs) == 0: - raise Exception(f"Job `{job_name}` not found") + raise Exception(f"Job `{job_name}` not found in {'pr' if pr_jobs else 'auto'} jobs") + assert len(jobs) == 1 job = jobs[0] if "ubuntu" not in job["os"]: raise Exception("Only Linux jobs can be executed locally") @@ -222,7 +221,12 @@ def run_workflow_locally(job_data: Dict[str, Any], job_name: str): env = os.environ.copy() env.update(custom_env) - subprocess.run(args, env=env) + + process = subprocess.Popen(args, env=env) + try: + process.wait() + except KeyboardInterrupt: + process.kill() if __name__ == "__main__": @@ -239,7 +243,16 @@ if __name__ == "__main__": subparsers = parser.add_subparsers(help="Command to execute", dest="command", required=True) subparsers.add_parser("calculate-job-matrix") run_parser = subparsers.add_parser("run-local") - run_parser.add_argument("job_name", help="CI job that should be executed") + run_parser.add_argument( + "job_name", + help="CI job that should be executed. By default, a merge (auto) " + "job with the given name will be executed" + ) + run_parser.add_argument( + "--pr", + action="store_true", + help="Run a PR job instead of an auto job" + ) args = parser.parse_args() if args.command == "calculate-job-matrix": @@ -265,6 +278,6 @@ if __name__ == "__main__": print(f"jobs={json.dumps(jobs)}") print(f"run_type={run_type}") elif args.command == "run-local": - run_workflow_locally(data, args.job_name) + run_workflow_locally(data, args.job_name, args.pr) else: raise Exception(f"Unknown command {args.command}") -- cgit 1.4.1-3-g733a5 From 5a95dda44ab42d4d2cdb199a2fa9ac8ba1d7a3f0 Mon Sep 17 00:00:00 2001 From: Jakub Beránek Date: Tue, 7 Jan 2025 18:40:08 +0100 Subject: Apply review comments --- src/ci/github-actions/ci.py | 151 +++++++++++++++++++++++++++----------------- 1 file changed, 93 insertions(+), 58 deletions(-) (limited to 'src/ci/github-actions') diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py index d1c665503b1..97539cf1cdb 100755 --- a/src/ci/github-actions/ci.py +++ b/src/ci/github-actions/ci.py @@ -34,10 +34,11 @@ def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: """ modified_jobs = [] for job in jobs: - job = dict(job) - job["image"] = get_job_image(job) - job["name"] = f"{prefix} - {job['name']}" - modified_jobs.append(job) + # Create a copy of the `job` dictionary to avoid modifying `jobs` + new_job = dict(job) + new_job["image"] = get_job_image(new_job) + new_job["name"] = f"{prefix} - {new_job['name']}" + modified_jobs.append(new_job) return modified_jobs @@ -46,11 +47,15 @@ def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: Prepends `environment` to the `env` attribute of each job. The `env` of each job has higher precedence than `environment`. """ + modified_jobs = [] for job in jobs: env = environment.copy() env.update(job.get("env", {})) - job["env"] = env - return jobs + + new_job = dict(job) + new_job["env"] = env + modified_jobs.append(new_job) + return modified_jobs @dataclasses.dataclass @@ -123,7 +128,9 @@ def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: if isinstance(run_type, PRRunType): - return add_base_env(add_job_properties(job_data["pr"], "PR"), job_data["envs"]["pr"]) + return add_base_env( + add_job_properties(job_data["pr"], "PR"), job_data["envs"]["pr"] + ) elif isinstance(run_type, TryRunType): jobs = job_data["try"] custom_jobs = run_type.custom_jobs @@ -188,95 +195,123 @@ def format_run_type(run_type: WorkflowRunType) -> str: raise AssertionError() -def get_job_image(job) -> str: +def get_job_image(job: Job) -> str: """ By default, the Docker image of a job is based on its name. However, it can be overridden by its IMAGE environment variable. """ - return job.get("env", {}).get("IMAGE", job["name"]) + env = job.get("env", {}) + # Return the IMAGE environment variable if it exists, otherwise return the job name + return env.get("IMAGE", job["name"]) -def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool): - DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" +def is_linux_job(job: Job) -> bool: + return "ubuntu" in job["os"] - jobs = job_data["pr"] if pr_jobs else job_data["auto"] - jobs = [job for job in jobs if job.get("name") == job_name] + +def find_linux_job(job_data: Dict[str, Any], job_name: str, pr_jobs: bool) -> Job: + candidates = job_data["pr"] if pr_jobs else job_data["auto"] + jobs = [job for job in candidates if job.get("name") == job_name] if len(jobs) == 0: - raise Exception(f"Job `{job_name}` not found in {'pr' if pr_jobs else 'auto'} jobs") + available_jobs = "\n".join( + sorted(job["name"] for job in candidates if is_linux_job(job)) + ) + raise Exception(f"""Job `{job_name}` not found in {'pr' if pr_jobs else 'auto'} jobs. +The following jobs are available: +{available_jobs}""") assert len(jobs) == 1 + job = jobs[0] - if "ubuntu" not in job["os"]: + if not is_linux_job(job): raise Exception("Only Linux jobs can be executed locally") + return job + + +def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool): + DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" + + job = find_linux_job(job_data, job_name=job_name, pr_jobs=pr_jobs) custom_env = {} - custom_env["DEPLOY"] = "1" + # Replicate src/ci/scripts/setup-environment.sh + # Adds custom environment variables to the job + if job_name.startswith("dist-"): + if job_name.endswith("-alt"): + custom_env["DEPLOY_ALT"] = "1" + else: + custom_env["DEPLOY"] = "1" custom_env.update({k: str(v) for (k, v) in job.get("env", {}).items()}) - args = [ - str(DOCKER_DIR / "run.sh"), - get_job_image(job) - ] + args = [str(DOCKER_DIR / "run.sh"), get_job_image(job)] env_formatted = [f"{k}={v}" for (k, v) in sorted(custom_env.items())] print(f"Executing `{' '.join(env_formatted)} {' '.join(args)}`") env = os.environ.copy() env.update(custom_env) - process = subprocess.Popen(args, env=env) - try: - process.wait() - except KeyboardInterrupt: - process.kill() + subprocess.run(args, env=env) -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) +def calculate_job_matrix(job_data: Dict[str, Any]): + github_ctx = get_github_ctx() - with open(JOBS_YAML_PATH) as f: - data = yaml.safe_load(f) + run_type = find_run_type(github_ctx) + logging.info(f"Job type: {run_type}") + with open(CI_DIR / "channel") as f: + channel = f.read().strip() + + jobs = [] + if run_type is not None: + jobs = calculate_jobs(run_type, job_data) + jobs = skip_jobs(jobs, channel) + + if not jobs: + raise Exception("Scheduled job list is empty, this is an error") + + run_type = format_run_type(run_type) + + logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") + print(f"jobs={json.dumps(jobs)}") + print(f"run_type={run_type}") + + +def create_cli_parser(): parser = argparse.ArgumentParser( - prog="ci.py", - description="Generate or run CI workflows" + prog="ci.py", description="Generate or run CI workflows" + ) + subparsers = parser.add_subparsers( + help="Command to execute", dest="command", required=True + ) + subparsers.add_parser( + "calculate-job-matrix", + help="Generate a matrix of jobs that should be executed in CI", + ) + run_parser = subparsers.add_parser( + "run-local", help="Run a CI jobs locally (on Linux)" ) - generate_matrix = argparse.ArgumentParser() - subparsers = parser.add_subparsers(help="Command to execute", dest="command", required=True) - subparsers.add_parser("calculate-job-matrix") - run_parser = subparsers.add_parser("run-local") run_parser.add_argument( "job_name", help="CI job that should be executed. By default, a merge (auto) " - "job with the given name will be executed" + "job with the given name will be executed", ) run_parser.add_argument( - "--pr", - action="store_true", - help="Run a PR job instead of an auto job" + "--pr", action="store_true", help="Run a PR job instead of an auto job" ) - args = parser.parse_args() - - if args.command == "calculate-job-matrix": - github_ctx = get_github_ctx() + return parser - run_type = find_run_type(github_ctx) - logging.info(f"Job type: {run_type}") - with open(CI_DIR / "channel") as f: - channel = f.read().strip() - - jobs = [] - if run_type is not None: - jobs = calculate_jobs(run_type, data) - jobs = skip_jobs(jobs, channel) +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) - if not jobs: - raise Exception("Scheduled job list is empty, this is an error") + with open(JOBS_YAML_PATH) as f: + data = yaml.safe_load(f) - run_type = format_run_type(run_type) + parser = create_cli_parser() + args = parser.parse_args() - logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") - print(f"jobs={json.dumps(jobs)}") - print(f"run_type={run_type}") + if args.command == "calculate-job-matrix": + calculate_job_matrix(data) elif args.command == "run-local": run_workflow_locally(data, args.job_name, args.pr) else: -- cgit 1.4.1-3-g733a5 From c8b29d6f6f9a7953e41fc71a6a7c6762cf489c39 Mon Sep 17 00:00:00 2001 From: Jakub Beránek Date: Thu, 9 Jan 2025 21:40:29 +0100 Subject: CI: fix name of jobs There is a difference between the `image` (the Dockerfile), the `name` of the job (which determines also its properties) and the `full_name`, which includes the `auto/try/pr` prefix. --- .github/workflows/ci.yml | 6 +++--- src/ci/github-actions/ci.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/ci/github-actions') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8d7560ffcf..c650df6a0ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: run: python3 src/ci/github-actions/ci.py calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: - name: ${{ matrix.name }} + name: ${{ matrix.full_name }} needs: [ calculate_matrix ] runs-on: "${{ matrix.os }}" defaults: @@ -67,7 +67,7 @@ jobs: shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} timeout-minutes: 360 env: - CI_JOB_NAME: ${{ matrix.image }} + CI_JOB_NAME: ${{ matrix.name }} CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -233,7 +233,7 @@ jobs: env: DATADOG_SITE: datadoghq.com DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_GITHUB_JOB_NAME: ${{ matrix.name }} + DD_GITHUB_JOB_NAME: ${{ matrix.full_name }} run: | cd src/ci npm ci diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py index 97539cf1cdb..b7dac412dbe 100755 --- a/src/ci/github-actions/ci.py +++ b/src/ci/github-actions/ci.py @@ -37,7 +37,7 @@ def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: # Create a copy of the `job` dictionary to avoid modifying `jobs` new_job = dict(job) new_job["image"] = get_job_image(new_job) - new_job["name"] = f"{prefix} - {new_job['name']}" + new_job["full_name"] = f"{prefix} - {new_job['name']}" modified_jobs.append(new_job) return modified_jobs -- cgit 1.4.1-3-g733a5 From dce8c06de30e41fe3de36ae739829756d0f841c7 Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Fri, 10 Jan 2025 14:07:27 +0000 Subject: ci: Move dist-aarch64-linux to an aarch64 runner Move the dist-aarch64-linux CI job to an aarch64 runner instead of cross-compiling it from an x86 one. This will make it possible to perform optimisations such as LTO, PGO and BOLT later on. --- .../host-aarch64/dist-aarch64-linux/Dockerfile | 98 ++++++++++++++++++++++ .../host-aarch64/dist-aarch64-linux/build-clang.sh | 46 ++++++++++ .../host-aarch64/dist-aarch64-linux/build-gcc.sh | 51 +++++++++++ .../host-aarch64/dist-aarch64-linux/build-zstd.sh | 29 +++++++ .../host-aarch64/dist-aarch64-linux/shared.sh | 16 ++++ .../host-x86_64/dist-aarch64-linux/Dockerfile | 32 ------- .../dist-aarch64-linux/aarch64-linux-gnu.defconfig | 10 --- src/ci/github-actions/jobs.yml | 2 +- 8 files changed, 241 insertions(+), 43 deletions(-) create mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh create mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh delete mode 100644 src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile delete mode 100644 src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig (limited to 'src/ci/github-actions') diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile new file mode 100644 index 00000000000..c7778dbf6bb --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile @@ -0,0 +1,98 @@ +# We document platform support for minimum glibc 2.17 and kernel 3.2. +# CentOS 7 has headers for kernel 3.10, but that's fine as long as we don't +# actually use newer APIs in rustc or std without a fallback. It's more +# important that we match glibc for ELF symbol versioning. +FROM centos:7 + +WORKDIR /build + +# CentOS 7 EOL is June 30, 2024, but the repos remain in the vault. +RUN sed -i /etc/yum.repos.d/*.repo -e 's!^mirrorlist!#mirrorlist!' \ + -e 's!^#baseurl=http://mirror.centos.org/!baseurl=https://vault.centos.org/!' +RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf + +RUN yum upgrade -y && \ + yum install -y \ + automake \ + bzip2 \ + file \ + gcc \ + gcc-c++ \ + git \ + glibc-devel \ + libedit-devel \ + libstdc++-devel \ + make \ + ncurses-devel \ + openssl-devel \ + patch \ + perl \ + perl-core \ + pkgconfig \ + python3 \ + unzip \ + wget \ + xz \ + zlib-devel \ + && yum clean all + +RUN mkdir -p /rustroot/bin + +ENV PATH=/rustroot/bin:$PATH +ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib +ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig +WORKDIR /tmp +RUN mkdir /home/user +COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/ + +# Need at least GCC 5.1 to compile LLVM +COPY host-aarch64/dist-aarch64-linux/build-gcc.sh /tmp/ +RUN ./build-gcc.sh && yum remove -y gcc gcc-c++ + +ENV CC=gcc CXX=g++ + +# LLVM 17 needs cmake 3.20 or higher. +COPY scripts/cmake.sh /tmp/ +RUN ./cmake.sh + +# Build LLVM+Clang +COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/ +RUN ./build-clang.sh +ENV CC=clang CXX=clang++ + +# Build zstd to enable `llvm.libzstd`. +COPY host-aarch64/dist-aarch64-linux/build-zstd.sh /tmp/ +RUN ./build-zstd.sh + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV PGO_HOST=aarch64-unknown-linux-gnu +ENV HOSTS=aarch64-unknown-linux-gnu + +ENV CPATH=/usr/include/aarch64-linux-gnu/:$CPATH + +ENV RUST_CONFIGURE_ARGS \ + --build=aarch64-unknown-linux-gnu \ + --enable-full-tools \ + --enable-profiler \ + --enable-sanitizers \ + --enable-compiler-docs \ + --set target.aarch64-unknown-linux-gnu.linker=clang \ + --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ + --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ + --set llvm.link-shared=true \ + --set llvm.thin-lto=false \ + --set llvm.libzstd=true \ + --set llvm.ninja=false \ + --set rust.debug-assertions=false \ + --set rust.jemalloc \ + --set rust.use-lld=true \ + --set rust.codegen-units=1 + +ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS + +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang +ENV DIST_SRC 1 +ENV LIBCURL_NO_PKG_CONFIG 1 +ENV DIST_REQUIRE_ALL_TOOLS 1 diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh new file mode 100755 index 00000000000..fb5f4d8c4f7 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -ex + +source shared.sh + +# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh +LLVM=llvmorg-19.1.5 + +mkdir llvm-project +cd llvm-project + +curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \ + tar xzf - --strip-components=1 + +mkdir clang-build +cd clang-build + +# For whatever reason the default set of include paths for clang is different +# than that of gcc. As a result we need to manually include our sysroot's +# include path, /rustroot/include, to clang's default include path. +INC="/rustroot/include:/usr/include" + +# We need compiler-rt for the profile runtime (used later to PGO the LLVM build) +# but sanitizers aren't currently building. Since we don't need those, just +# disable them. BOLT is used for optimizing LLVM. +hide_output \ + cmake ../llvm \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/rustroot \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_BUILD_MEMPROF=OFF \ + -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ + -DLLVM_TARGETS_TO_BUILD=AArch64 \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ + -DC_INCLUDE_DIRS="$INC" + +hide_output make -j$(nproc) +hide_output make install + +cd ../.. +rm -rf llvm-project diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh new file mode 100755 index 00000000000..ad33b21b9b5 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-gcc.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -ex + +source shared.sh + +# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. +GCC=9.5.0 + +curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - +cd gcc-$GCC + +# FIXME(#49246): Remove the `sed` below. +# +# On 2018 March 21st, two Travis builders' cache for Docker are suddenly invalidated. Normally this +# is fine, because we just need to rebuild the Docker image. However, it reveals a network issue: +# downloading from `ftp://gcc.gnu.org/` from Travis (using passive mode) often leads to "Connection +# timed out" error, and even when the download completed, the file is usually corrupted. This causes +# nothing to be landed that day. +# +# We observed that the `gcc-4.8.5.tar.bz2` above can be downloaded successfully, so as a stability +# improvement we try to download from the HTTPS mirror instead. Turns out this uncovered the third +# bug: the host `gcc.gnu.org` and `cygwin.com` share the same IP, and the TLS certificate of the +# latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server +# instead here. +# +# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to +# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or +# the expression will need to be updated. That new URL is available at: +# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 +# +sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites + +./contrib/download_prerequisites +mkdir ../gcc-build +cd ../gcc-build + +# '-fno-reorder-blocks-and-partition' is required to +# enable BOLT optimization of the C++ standard library, +# which is included in librustc_driver.so +hide_output ../gcc-$GCC/configure \ + --prefix=/rustroot \ + --enable-languages=c,c++ \ + --disable-gnu-unique-object \ + --enable-cxx-flags='-fno-reorder-blocks-and-partition' +hide_output make -j$(nproc) +hide_output make install +ln -s gcc /rustroot/bin/cc + +cd .. +rm -rf gcc-build +rm -rf gcc-$GCC diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh new file mode 100755 index 00000000000..a3d37ccc311 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-zstd.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -ex + +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/zstd_build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/zstd_build.log + trap - ERR + kill $PING_LOOP_PID + rm /tmp/zstd_build.log + set -x +} + +ZSTD=1.5.6 +curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf - + +cd zstd-$ZSTD +CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1 +hide_output make install + +cd .. +rm -rf zstd-$ZSTD diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh new file mode 100644 index 00000000000..dc86dddd464 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh @@ -0,0 +1,16 @@ +#!/bin/sh +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + set -x +} diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile deleted file mode 100644 index 18972387e34..00000000000 --- a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:22.04 - -COPY scripts/cross-apt-packages.sh /scripts/ -RUN sh /scripts/cross-apt-packages.sh - -COPY scripts/crosstool-ng.sh /scripts/ -RUN sh /scripts/crosstool-ng.sh - -COPY scripts/rustbuild-setup.sh /scripts/ -RUN sh /scripts/rustbuild-setup.sh -WORKDIR /tmp - -COPY scripts/crosstool-ng-build.sh /scripts/ -COPY host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig /tmp/crosstool.defconfig -RUN /scripts/crosstool-ng-build.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -ENV PATH=$PATH:/x-tools/aarch64-unknown-linux-gnu/bin - -ENV CC_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-gcc \ - AR_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-ar \ - CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-g++ - -ENV HOSTS=aarch64-unknown-linux-gnu - -ENV RUST_CONFIGURE_ARGS \ - --enable-full-tools \ - --enable-profiler \ - --enable-sanitizers -ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig b/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig deleted file mode 100644 index 520b1667c8b..00000000000 --- a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig +++ /dev/null @@ -1,10 +0,0 @@ -CT_CONFIG_VERSION="4" -CT_PREFIX_DIR="/x-tools/${CT_TARGET}" -CT_USE_MIRROR=y -CT_MIRROR_BASE_URL="https://ci-mirrors.rust-lang.org/rustc" -CT_ARCH_ARM=y -CT_ARCH_64=y -CT_KERNEL_LINUX=y -CT_LINUX_V_4_1=y -CT_GLIBC_V_2_17=y -CT_CC_LANG_CXX=y diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index d88be6543a3..6bf4a75d080 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -138,7 +138,7 @@ auto: - name: dist-aarch64-linux env: CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-4c + <<: *job-aarch64-linux - name: dist-android <<: *job-linux-4c -- cgit 1.4.1-3-g733a5