Auto merge of #83637 - bjorn3:sync_cg_clif-2021-03-29, r=bjorn3

Sync rustc_codegen_cranelift

The main highlight of this sync is support for cross-compiling to Windows using MinGW. Native compilation with MinGW would also work I think, but using the MSVC toolchain is not yet supported as PE TLS is not yet implemented. Another nice improvement is that crate metadata is now loaded using mmap instead of by reading files. This improves compilation time a bit.

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler

7 files changed, 193 insertions, 87 deletions

diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo.sh b/compiler/rustc_codegen_cranelift/scripts/cargo.sh
index 669d2d45b71..1daa5a78f7b 100755
--- a/compiler/rustc_codegen_cranelift/scripts/cargo.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/cargo.sh
@@ -4,7 +4,7 @@ dir=$(dirname "$0")
 source "$dir/config.sh"
 
 # read nightly compiler from rust-toolchain file
-TOOLCHAIN=$(cat "$dir/rust-toolchain")
+TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/")
 
 cmd=$1
 shift || true
diff --git a/compiler/rustc_codegen_cranelift/scripts/config.sh b/compiler/rustc_codegen_cranelift/scripts/config.sh
index c2ed2bf256d..99b302ee1d9 100644
--- a/compiler/rustc_codegen_cranelift/scripts/config.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/config.sh
@@ -2,15 +2,7 @@
 
 set -e
 
-unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then
-   dylib_ext='so'
-elif [[ "$unamestr" == 'Darwin' ]]; then
-   dylib_ext='dylib'
-else
-   echo "Unsupported os"
-   exit 1
-fi
+dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -)
 
 if echo "$RUSTC_WRAPPER" | grep sccache; then
 echo
@@ -24,10 +16,10 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
 export RUSTC=$dir"/bin/cg_clif"
 
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
-'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
+'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir
 
 # FIXME fix `#[linkage = "extern_weak"]` without this
-if [[ "$unamestr" == 'Darwin' ]]; then
+if [[ "$(uname)" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
index 694945a87c2..fa7557653d8 100755
--- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
@@ -8,7 +8,7 @@ case $1 in
 
         echo "=> Installing new nightly"
         rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
-        echo "nightly-${TOOLCHAIN}" > rust-toolchain
+        sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
         rustup component add rustfmt || true
 
         echo "=> Uninstalling all old nighlies"
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
new file mode 100644
index 00000000000..e8bedf625f7
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -e
+
+./build.sh
+source build/config.sh
+
+echo "[SETUP] Rust fork"
+git clone https://github.com/rust-lang/rust.git || true
+pushd rust
+git fetch
+git checkout -- .
+git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
+
+git apply - <<EOF
+diff --git a/Cargo.toml b/Cargo.toml
+index 5bd1147cad5..10d68a2ff14 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+ rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
+ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
+
++compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
++
+ [patch."https://github.com/rust-lang/rust-clippy"]
+ clippy_lints = { path = "src/tools/clippy/clippy_lints" }
+diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
+index 23e689fcae7..5f077b765b6 100644
+--- a/compiler/rustc_data_structures/Cargo.toml
++++ b/compiler/rustc_data_structures/Cargo.toml
+@@ -32,7 +32,6 @@ tempfile = "3.0.5"
+
+ [dependencies.parking_lot]
+ version = "0.11"
+-features = ["nightly"]
+
+ [target.'cfg(windows)'.dependencies]
+ winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
+
+ [dev-dependencies]
+ rand = "0.7"
+EOF
+
+cat > config.toml <<EOF
+[llvm]
+ninja = false
+
+[build]
+rustc = "$(pwd)/../build/bin/cg_clif"
+cargo = "$(rustup which cargo)"
+full-bootstrap = true
+local-rebuild = true
+
+[rust]
+codegen-backends = ["cranelift"]
+deny-warnings = false
+EOF
+popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
index 6473c6ad67d..791d457993d 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
@@ -3,70 +3,10 @@ set -e
 
 cd "$(dirname "$0")/../"
 
-./build.sh
-source build/config.sh
+source ./scripts/setup_rust_fork.sh
 
 echo "[TEST] Bootstrap of rustc"
-git clone https://github.com/rust-lang/rust.git || true
 pushd rust
-git fetch
-git checkout -- .
-git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
-
-git apply - <<EOF
-diff --git a/Cargo.toml b/Cargo.toml
-index 5bd1147cad5..10d68a2ff14 100644
---- a/Cargo.toml
-+++ b/Cargo.toml
-@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
- rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- 
-+compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
-+
- [patch."https://github.com/rust-lang/rust-clippy"]
- clippy_lints = { path = "src/tools/clippy/clippy_lints" }
-diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
-index 23e689fcae7..5f077b765b6 100644
---- a/compiler/rustc_data_structures/Cargo.toml
-+++ b/compiler/rustc_data_structures/Cargo.toml
-@@ -32,7 +32,6 @@ tempfile = "3.0.5"
-
- [dependencies.parking_lot]
- version = "0.11"
--features = ["nightly"]
-
- [target.'cfg(windows)'.dependencies]
- winapi = { version = "0.3", features = ["fileapi", "psapi"] }
-diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
-index d95b5b7f17f..00b6f0e3635 100644
---- a/library/alloc/Cargo.toml
-+++ b/library/alloc/Cargo.toml
-@@ -8,7 +8,7 @@ edition = "2018"
- 
- [dependencies]
- core = { path = "../core" }
--compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
- 
- [dev-dependencies]
- rand = "0.7"
-EOF
-
-cat > config.toml <<EOF
-[llvm]
-ninja = false
-
-[build]
-rustc = "$(pwd)/../build/bin/cg_clif"
-cargo = "$(rustup which cargo)"
-full-bootstrap = true
-local-rebuild = true
-
-[rust]
-codegen-backends = ["cranelift"]
-EOF
-
 rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
new file mode 100755
index 00000000000..fbc3feceec7
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+cd $(dirname "$0")/../
+
+source ./scripts/setup_rust_fork.sh
+
+echo "[TEST] Test suite of rustc"
+pushd rust
+
+cargo install ripgrep
+
+rm -r src/test/ui/{extern/,panics/,unsized-locals/,thinlto/,simd*,*lto*.rs,linkage*,unwind-*.rs} || true
+for test in $(rg --files-with-matches "asm!|catch_unwind|should_panic|lto" src/test/ui); do
+  rm $test
+done
+
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" src/test/ui); do
+  rm $test
+done
+
+git checkout -- src/test/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
+
+# these all depend on unwinding support
+rm src/test/ui/backtrace.rs
+rm src/test/ui/array-slice-vec/box-of-array-of-drop-*.rs
+rm src/test/ui/array-slice-vec/slice-panic-*.rs
+rm src/test/ui/array-slice-vec/nested-vec-3.rs
+rm src/test/ui/cleanup-rvalue-temp-during-incomplete-alloc.rs
+rm src/test/ui/issues/issue-26655.rs
+rm src/test/ui/issues/issue-29485.rs
+rm src/test/ui/issues/issue-30018-panic.rs
+rm src/test/ui/multi-panic.rs
+rm src/test/ui/sepcomp/sepcomp-unwind.rs
+rm src/test/ui/structs-enums/unit-like-struct-drop-run.rs
+rm src/test/ui/terminate-in-initializer.rs
+rm src/test/ui/threads-sendsync/task-stderr.rs
+rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs
+rm src/test/ui/drop/drop-trait-enum.rs
+rm src/test/ui/numbers-arithmetic/issue-8460.rs
+
+rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations
+rm src/test/ui/init-large-type.rs # same
+rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected
+rm src/test/ui/issues/issue-33992.rs # unsupported linkages
+rm src/test/ui/issues/issue-51947.rs # same
+rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result
+rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
+rm src/test/ui/mir/mir_raw_fat_ptr.rs # same
+rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte
+rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same
+rm src/test/ui/generator/size-moved-locals.rs # same
+rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
+rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-type.rs # "Cannot run dynamic test fn out-of-process"
+rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
+
+rm src/test/incremental/hashes/inline_asm.rs # inline asm
+rm src/test/incremental/issue-72386.rs # same
+rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind
+rm src/test/incremental/issue-49482.rs # same
+rm src/test/incremental/issue-54059.rs # same
+rm src/test/incremental/lto.rs # requires lto
+
+rm src/test/pretty/asm.rs # inline asm
+rm src/test/pretty/raw-str-nonexpr.rs # same
+
+rm -r src/test/run-pass-valgrind/unsized-locals
+
+rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning
+rm src/test/ui/json-bom-plus-crlf.rs # same
+rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep
+
+rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition
+rm src/test/ui/cfg/cfg-panic.rs
+rm src/test/ui/default-alloc-error-hook.rs
+rm -r src/test/ui/hygiene/
+
+rm -r src/test/ui/polymorphization/ # polymorphization not yet supported
+rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same
+
+rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization
+rm src/test/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
+rm src/test/ui/abi/variadic-ffi.rs # requires callee side vararg support
+
+echo "[TEST] rustc test suite"
+RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui}
+popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/tests.sh b/compiler/rustc_codegen_cranelift/scripts/tests.sh
index f9a9fb091fb..3afcea8f06b 100755
--- a/compiler/rustc_codegen_cranelift/scripts/tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/tests.sh
@@ -71,14 +71,20 @@ function base_sysroot_tests() {
     echo "[AOT] mod_bench"
     $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/mod_bench
+}
 
+function extended_sysroot_tests() {
     pushd rand
-    rm -r ./target || true
-    ../build/cargo.sh test --workspace
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-random/rand"
+        ../build/cargo.sh test --workspace
+    else
+        echo "[AOT] rust-random/rand"
+        ../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests
+    fi
     popd
-}
 
-function extended_sysroot_tests() {
     pushd simple-raytracer
     if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
         echo "[BENCH COMPILE] ebobby/simple-raytracer"
@@ -92,27 +98,40 @@ function extended_sysroot_tests() {
     else
         echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
         echo "[COMPILE] ebobby/simple-raytracer"
-        ../cargo.sh build
+        ../build/cargo.sh build --target $TARGET_TRIPLE
         echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
     fi
     popd
 
     pushd build_sysroot/sysroot_src/library/core/tests
     echo "[TEST] libcore"
-    rm -r ./target || true
-    ../../../../../build/cargo.sh test
+    cargo clean
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        ../../../../../build/cargo.sh test
+    else
+        ../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests
+    fi
     popd
 
     pushd regex
     echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo.sh clean
+    cargo clean
     # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo.sh build --example shootout-regex-dna
-    cat examples/regexdna-input.txt | ../build/cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt
-    diff -u res.txt examples/regexdna-output.txt
+    ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        cat examples/regexdna-input.txt \
+            | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \
+            | grep -v "Spawned thread" > res.txt
+        diff -u res.txt examples/regexdna-output.txt
+    fi
 
-    echo "[TEST] rust-lang/regex tests"
-    ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
+        echo "[TEST] rust-lang/regex tests"
+        ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
+    else
+        echo "[AOT] rust-lang/regex tests"
+        ../build/cargo.sh build --tests --target $TARGET_TRIPLE
+    fi
     popd
 }