From 9ba34f83474ac011db5af6a4d9ef9aa632a6b6fc Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Wed, 8 Jan 2025 14:33:27 +0000 Subject: ci: Organise shared helper scripts Move shared helper scripts used by Docker builds under docker/scripts. --- src/ci/docker/scripts/build-clang.sh | 55 ++++++++++++++++++++++++++++++++ src/ci/docker/scripts/build-gcc.sh | 59 +++++++++++++++++++++++++++++++++++ src/ci/docker/scripts/build-gccjit.sh | 37 ++++++++++++++++++++++ src/ci/docker/scripts/build-zstd.sh | 29 +++++++++++++++++ 4 files changed, 180 insertions(+) create mode 100755 src/ci/docker/scripts/build-clang.sh create mode 100755 src/ci/docker/scripts/build-gcc.sh create mode 100755 src/ci/docker/scripts/build-gccjit.sh create mode 100755 src/ci/docker/scripts/build-zstd.sh (limited to 'src/ci/docker/scripts') diff --git a/src/ci/docker/scripts/build-clang.sh b/src/ci/docker/scripts/build-clang.sh new file mode 100755 index 00000000000..841a0adb2ab --- /dev/null +++ b/src/ci/docker/scripts/build-clang.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -ex + +source shared.sh + +# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh +LLVM=llvmorg-19.1.0-rc3 + +mkdir llvm-project +cd llvm-project + +curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \ + tar xzf - --strip-components=1 + +mkdir clang-build +cd clang-build + +# For whatever reason the default set of include paths for clang is different +# than that of gcc. As a result we need to manually include our sysroot's +# include path, /rustroot/include, to clang's default include path. +INC="/rustroot/include:/usr/include" + +GCC_PLUGIN_TARGET=$GCC_BUILD_TARGET +# We build gcc for the i686 job on x86_64 so the plugin will end up under an x86_64 path +if [[ $GCC_PLUGIN_TARGET == "i686-pc-linux-gnu" ]]; then + GCC_PLUGIN_TARGET=x86_64-pc-linux-gnu +fi + +# We need compiler-rt for the profile runtime (used later to PGO the LLVM build) +# but sanitizers aren't currently building. Since we don't need those, just +# disable them. BOLT is used for optimizing LLVM. +hide_output \ + cmake ../llvm \ + -DCMAKE_C_COMPILER=/rustroot/bin/gcc \ + -DCMAKE_CXX_COMPILER=/rustroot/bin/g++ \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/rustroot \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_BUILD_MEMPROF=OFF \ + -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ + -DLLVM_TARGETS_TO_BUILD=$LLVM_BUILD_TARGETS \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ + -DLLVM_BINUTILS_INCDIR="/rustroot/lib/gcc/$GCC_PLUGIN_TARGET/$GCC_VERSION/plugin/include/" \ + -DC_INCLUDE_DIRS="$INC" + +hide_output make -j$(nproc) +hide_output make install + +cd ../.. +rm -rf llvm-project diff --git a/src/ci/docker/scripts/build-gcc.sh b/src/ci/docker/scripts/build-gcc.sh new file mode 100755 index 00000000000..11db5aa811c --- /dev/null +++ b/src/ci/docker/scripts/build-gcc.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -eux + +source shared.sh + +# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. +# This version is specified in the Dockerfile +GCC=$GCC_VERSION + +curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - +cd gcc-$GCC + +# FIXME(#49246): Remove the `sed` below. +# +# On 2018 March 21st, two Travis builders' cache for Docker are suddenly invalidated. Normally this +# is fine, because we just need to rebuild the Docker image. However, it reveals a network issue: +# downloading from `ftp://gcc.gnu.org/` from Travis (using passive mode) often leads to "Connection +# timed out" error, and even when the download completed, the file is usually corrupted. This causes +# nothing to be landed that day. +# +# We observed that the `gcc-4.8.5.tar.bz2` above can be downloaded successfully, so as a stability +# improvement we try to download from the HTTPS mirror instead. Turns out this uncovered the third +# bug: the host `gcc.gnu.org` and `cygwin.com` share the same IP, and the TLS certificate of the +# latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server +# instead here. +# +# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to +# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or +# the expression will need to be updated. That new URL is available at: +# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 +# +sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites + +./contrib/download_prerequisites +mkdir ../gcc-build +cd ../gcc-build + +# '-fno-reorder-blocks-and-partition' is required to +# enable BOLT optimization of the C++ standard library, +# which is included in librustc_driver.so +hide_output ../gcc-$GCC/configure \ + --prefix=/rustroot \ + --enable-languages=c,c++ \ + --disable-gnu-unique-object \ + --enable-cxx-flags='-fno-reorder-blocks-and-partition' +hide_output make -j$(nproc) +hide_output make install +ln -s gcc /rustroot/bin/cc + +cd .. +rm -rf gcc-build +rm -rf gcc-$GCC + +if [[ $GCC_BUILD_TARGET == "i686-pc-linux-gnu" ]]; then + # FIXME: clang doesn't find 32-bit libraries in /rustroot/lib, + # but it does look all the way under /rustroot/lib/[...]/32, + # so we can link stuff there to help it out. + ln /rustroot/lib/*.{a,so} -rst /rustroot/lib/gcc/x86_64-pc-linux-gnu/$GCC/32/ +fi diff --git a/src/ci/docker/scripts/build-gccjit.sh b/src/ci/docker/scripts/build-gccjit.sh new file mode 100755 index 00000000000..c565922dcd1 --- /dev/null +++ b/src/ci/docker/scripts/build-gccjit.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +GIT_REPO="https://github.com/rust-lang/gcc" + +# This commit hash needs to be updated to use a more recent gcc fork version. +GIT_COMMIT="e744a9459d33864067214741daf5c5bc2a7b88c6" + +set -ex + +cd $1 + +source shared.sh + +# Setting up folders for GCC +curl -L "$GIT_REPO/archive/$GIT_COMMIT.tar.gz" | + tar -xz --transform "s/gcc-$GIT_COMMIT/gcc-src/" + +mkdir gcc-build gcc-install +pushd gcc-build + +# Building GCC. +hide_output \ + ../gcc-src/configure \ + --enable-host-shared \ + --enable-languages=jit \ + --enable-checking=release \ + --disable-bootstrap \ + --disable-multilib \ + --prefix=$(pwd)/../gcc-install \ + +hide_output make -j$(nproc) +hide_output make install + +popd +rm -rf gcc-src gcc-build +ln -s /scripts/gcc-install/lib/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so +ln -s /scripts/gcc-install/lib/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so.0 diff --git a/src/ci/docker/scripts/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh new file mode 100755 index 00000000000..a3d37ccc311 --- /dev/null +++ b/src/ci/docker/scripts/build-zstd.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -ex + +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/zstd_build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/zstd_build.log + trap - ERR + kill $PING_LOOP_PID + rm /tmp/zstd_build.log + set -x +} + +ZSTD=1.5.6 +curl -L https://github.com/facebook/zstd/releases/download/v$ZSTD/zstd-$ZSTD.tar.gz | tar xzf - + +cd zstd-$ZSTD +CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1 +hide_output make install + +cd .. +rm -rf zstd-$ZSTD -- cgit 1.4.1-3-g733a5