From a1b9a599584e9044da30bfd8fbc0d16ca72c009e Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Tue, 7 Nov 2023 18:11:07 -0800 Subject: Build Fuchsia in CI --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- .../docker/host-x86_64/dist-various-2/Dockerfile | 4 +- .../dist-various-2/build-fuchsia-toolchain.sh | 60 ------------------- src/ci/docker/host-x86_64/dist-various-2/shared.sh | 35 ----------- .../host-x86_64/x86_64-gnu-integration/Dockerfile | 68 ++++++++++++++++++++++ .../x86_64-gnu-integration/build-fuchsia.sh | 51 ++++++++++++++++ src/ci/docker/scripts/build-fuchsia-toolchain.sh | 60 +++++++++++++++++++ src/ci/docker/scripts/shared.sh | 42 +++++++++++++ src/ci/github-actions/ci.yml | 4 ++ src/ci/shared.sh | 2 +- 10 files changed, 229 insertions(+), 99 deletions(-) delete mode 100755 src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh delete mode 100644 src/ci/docker/host-x86_64/dist-various-2/shared.sh create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile create mode 100755 src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh create mode 100755 src/ci/docker/scripts/build-fuchsia-toolchain.sh create mode 100644 src/ci/docker/scripts/shared.sh (limited to 'src') diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index c253dd2c6aa..1ab488ef656 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1722,7 +1722,7 @@ impl Step for Assemble { let dst_exe = exe("rust-lld", target_compiler.host); builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&self_contained_lld_dir)); + t!(fs::create_dir_all(&self_contained_lld_dir)); let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { compiler: build_compiler, target: target_compiler.host, diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 3f4e025111a..5f1fec74bed 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -76,8 +76,8 @@ RUN env \ rm -rf /build/* WORKDIR /tmp -COPY host-x86_64/dist-various-2/shared.sh /tmp/ -COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/ +COPY scripts/shared.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/ RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 pc diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh b/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh deleted file mode 100755 index d762b4672c6..00000000000 --- a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -ex -source shared.sh - -FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64 -FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC -FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179 -FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk -CLANG_DOWNLOAD_URL=\ -https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64 -CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC -CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b - -install_clang() { - mkdir -p clang_download - pushd clang_download > /dev/null - - # Download clang+llvm - curl -LO "${CLANG_DOWNLOAD_URL}/+/${CLANG_DOWNLOAD_ID}" - echo "$(echo ${CLANG_DOWNLOAD_SHA256}) ${CLANG_DOWNLOAD_ID}" | sha256sum --check --status - unzip -qq ${CLANG_DOWNLOAD_ID} -d clang-linux-amd64 - - # Other dists currently depend on our Clang... moving into /usr/local for other - # dist usage instead of a Fuchsia /usr/local directory - chmod -R 777 clang-linux-amd64/. - cp -a clang-linux-amd64/. /usr/local - - # CFLAGS and CXXFLAGS env variables in main Dockerfile handle sysroot linking - for arch in x86_64 aarch64; do - for tool in clang clang++; do - ln -s /usr/local/bin/${tool} /usr/local/bin/${arch}-unknown-fuchsia-${tool} - done - ln -s /usr/local/bin/llvm-ar /usr/local/bin/${arch}-unknown-fuchsia-ar - done - - popd > /dev/null - rm -rf clang_download -} - -install_zircon_libs() { - mkdir -p zircon - pushd zircon > /dev/null - - # Download Fuchsia SDK (with Zircon libs) - curl -LO "${FUCHSIA_SDK_URL}/+/${FUCHSIA_SDK_ID}" - echo "$(echo ${FUCHSIA_SDK_SHA256}) ${FUCHSIA_SDK_ID}" | sha256sum --check --status - unzip -qq ${FUCHSIA_SDK_ID} -d core-linux-amd64 - - # Moving SDK into Docker's user-space - mkdir -p ${FUCHSIA_SDK_USR_DIR} - chmod -R 777 core-linux-amd64/. - cp -r core-linux-amd64/* ${FUCHSIA_SDK_USR_DIR} - - popd > /dev/null - rm -rf zircon -} - -hide_output install_clang -hide_output install_zircon_libs diff --git a/src/ci/docker/host-x86_64/dist-various-2/shared.sh b/src/ci/docker/host-x86_64/dist-various-2/shared.sh deleted file mode 100644 index 291f26bdaeb..00000000000 --- a/src/ci/docker/host-x86_64/dist-various-2/shared.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -hide_output() { - { set +x; } 2>/dev/null - on_err=" -echo ERROR: An error was encountered with the build. -cat /tmp/build.log -exit 1 -" - trap "$on_err" ERR - bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & - PING_LOOP_PID=$! - "$@" &> /tmp/build.log - trap - ERR - kill $PING_LOOP_PID - set -x -} - -# Copied from ../../shared.sh -function retry { - echo "Attempting with retry:" "$@" - local n=1 - local max=5 - while true; do - "$@" && break || { - if [[ $n -lt $max ]]; then - sleep $n # don't retry immediately - ((n++)) - echo "Command failed. Attempt $n/$max:" - else - echo "The command has failed after $n attempts." - return 1 - fi - } - done -} diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile new file mode 100644 index 00000000000..ba65ba9bed4 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile @@ -0,0 +1,68 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + g++ \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + libssl-dev \ + sudo \ + xz-utils \ + pkg-config \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# Duplicated in dist-various-2 Dockerfile. +# FIXME: Move to canonical triple +ENV \ + AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \ + CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \ + CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \ + CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" + +WORKDIR /tmp +COPY scripts/shared.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ +RUN /tmp/build-fuchsia-toolchain.sh + +ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \ + -C panic=abort \ + -C force-unwind-tables=yes \ + -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ + -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \ + -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib + +ENV TARGETS=x86_64-fuchsia +ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV RUST_INSTALL_DIR /checkout/obj/install +RUN mkdir -p $RUST_INSTALL_DIR/etc + +ENV RUST_CONFIGURE_ARGS \ + --prefix=$RUST_INSTALL_DIR \ + --sysconfdir=etc \ + --enable-lld \ + --llvm-libunwind=in-tree \ + --enable-extended \ + --disable-docs \ + --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \ + --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \ + --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \ + --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \ + --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld +ENV SCRIPT \ + python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \ + bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh new file mode 100755 index 00000000000..4a246f591d7 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Downloads and builds the Fuchsia operating system using a toolchain installed +# in $RUST_INSTALL_DIR. + +set -euf -o pipefail + +INTEGRATION_SHA=66793c4894bf6204579bbee3b79956335f31c768 +PICK_REFS=() + +checkout=fuchsia +jiri=.jiri_root/bin/jiri + +set -x + +# This script will: +# - create a directory named "fuchsia" if it does not exist +# - download "jiri" to "fuchsia/.jiri_root/bin" +curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \ + | base64 --decode \ + | bash -s $checkout + +cd $checkout + +$jiri init \ + -partial=true \ + -analytics-opt=false \ + . + +$jiri import \ + -name=integration \ + -revision=$INTEGRATION_SHA \ + -overwrite=true \ + flower \ + "https://fuchsia.googlesource.com/integration" + +if [ -d ".git" ]; then + # Wipe out any local changes if we're reusing a checkout. + git checkout --force JIRI_HEAD +fi + +$jiri update -autoupdate=false + +echo integration commit = $(git -C integration rev-parse HEAD) + +for git_ref in "${PICK_REFS[@]}"; do + git fetch https://fuchsia.googlesource.com/fuchsia $git_ref + git cherry-pick --no-commit FETCH_HEAD +done + +bash scripts/rust/build_fuchsia_from_rust_ci.sh diff --git a/src/ci/docker/scripts/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh new file mode 100755 index 00000000000..beea2f522fd --- /dev/null +++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -ex +source shared.sh + +FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64 +FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC +FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7 +FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk +CLANG_DOWNLOAD_URL=\ +https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64 +CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C +CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd + +install_clang() { + mkdir -p clang_download + pushd clang_download > /dev/null + + # Download clang+llvm + curl -LO "${CLANG_DOWNLOAD_URL}/+/${CLANG_DOWNLOAD_ID}" + echo "$(echo ${CLANG_DOWNLOAD_SHA256}) ${CLANG_DOWNLOAD_ID}" | sha256sum --check --status + unzip -qq ${CLANG_DOWNLOAD_ID} -d clang-linux-amd64 + + # Other dists currently depend on our Clang... moving into /usr/local for other + # dist usage instead of a Fuchsia /usr/local directory + chmod -R 777 clang-linux-amd64/. + cp -a clang-linux-amd64/. /usr/local + + # CFLAGS and CXXFLAGS env variables in main Dockerfile handle sysroot linking + for arch in x86_64 aarch64; do + for tool in clang clang++; do + ln -s /usr/local/bin/${tool} /usr/local/bin/${arch}-unknown-fuchsia-${tool} + done + ln -s /usr/local/bin/llvm-ar /usr/local/bin/${arch}-unknown-fuchsia-ar + done + + popd > /dev/null + rm -rf clang_download +} + +install_zircon_libs() { + mkdir -p zircon + pushd zircon > /dev/null + + # Download Fuchsia SDK (with Zircon libs) + curl -LO "${FUCHSIA_SDK_URL}/+/${FUCHSIA_SDK_ID}" + echo "$(echo ${FUCHSIA_SDK_SHA256}) ${FUCHSIA_SDK_ID}" | sha256sum --check --status + unzip -qq ${FUCHSIA_SDK_ID} -d core-linux-amd64 + + # Moving SDK into Docker's user-space + mkdir -p ${FUCHSIA_SDK_USR_DIR} + chmod -R 777 core-linux-amd64/. + cp -r core-linux-amd64/* ${FUCHSIA_SDK_USR_DIR} + + popd > /dev/null + rm -rf zircon +} + +hide_output install_clang +hide_output install_zircon_libs diff --git a/src/ci/docker/scripts/shared.sh b/src/ci/docker/scripts/shared.sh new file mode 100644 index 00000000000..9969659088d --- /dev/null +++ b/src/ci/docker/scripts/shared.sh @@ -0,0 +1,42 @@ +#!/bin/false +# shellcheck shell=bash + +# This file is intended to be sourced with `. shared.sh` or +# `source shared.sh`, hence the invalid shebang and not being +# marked as an executable file in git. + +function hide_output { + { set +x; } 2>/dev/null + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + set -x +} + +# See https://unix.stackexchange.com/questions/82598 +# Duplicated in src/ci/shared.sh +function retry { + echo "Attempting with retry:" "$@" + local n=1 + local max=5 + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + sleep $n # don't retry immediately + ((n++)) + echo "Command failed. Attempt $n/$max:" + else + echo "The command has failed after $n attempts." + return 1 + fi + } + done +} diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 5b9c90273a3..800d8920951 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -470,6 +470,9 @@ jobs: - name: x86_64-gnu-aux <<: *job-linux-4c + - name: x86_64-gnu-integration + <<: *job-linux-16c + - name: x86_64-gnu-debug <<: *job-linux-8c @@ -730,6 +733,7 @@ jobs: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c + master: name: master runs-on: ubuntu-latest diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 8a88c56194c..720394af249 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -8,7 +8,7 @@ export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc" # See https://unix.stackexchange.com/questions/82598 -# Duplicated in docker/dist-various-2/shared.sh +# Duplicated in docker/scripts/shared.sh function retry { echo "Attempting with retry:" "$@" local n=1 -- cgit 1.4.1-3-g733a5