Auto merge of #119278 - bjorn3:sync_cg_clif-2023-12-24, r=bjorn3

Subtree sync for rustc_codegen_cranelift

This implements simd_masked_store and fixes two borked subtree syncs that forgot to sync back some changes to the rust repo.

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler

6 files changed, 99 insertions, 290 deletions

diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index 901d1dbea66..74e7afee7bc 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76eb38f2af690b5a4411d9a8782b6d77dabff3ca939e0518453ab9f9a4392d41"
+checksum = "7c22542c0b95bd3302f7ed6839869c561f2324bac2fd5e7e99f5cfa65fdc8b92"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39526c036b92912417e8931f52c1e235796688068d3efdbbd8b164f299d19156"
+checksum = "6b3db903ef2e9c8a4de2ea6db5db052c7857282952f9df604aa55d169e6000d8"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -75,39 +75,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdb0deedc9fccf2db53a5a3c9c9d0163e44143b0d004dca9bf6ab6a0024cd79a"
+checksum = "6590feb5a1d6438f974bf6a5ac4dddf69fca14e1f07f3265d880f69e61a94463"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cea2d1b274e45aa8e61e9103efa1ba82d4b5a19d12bd1fd10744c3b7380ba3ff"
+checksum = "7239038c56fafe77fddc8788fc8533dd6c474dc5bdc5637216404f41ba807330"
 
 [[package]]
 name = "cranelift-control"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ea5977559a71e63db79a263f0e81a89b996e8a38212c4281e37dd1dbaa8b65c"
+checksum = "f7dc9c595341404d381d27a3d950160856b35b402275f0c3990cd1ad683c8053"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f871ada808b58158d84dfc43a6a2e2d2756baaf4ed1c51fd969ca8330e6ca5c"
+checksum = "44e3ee532fc4776c69bcedf7e62f9632cbb3f35776fa9a525cdade3195baa3f7"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8e6890f587ef59824b3debe577e68fdf9b307b3808c54b8d93a18fd0b70941b"
+checksum = "a612c94d09e653662ec37681dc2d6fd2b9856e6df7147be0afc9aabb0abf19df"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -117,15 +117,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8d5fc6d5d3b52d1917002b17a8ecce448c2621b5bf394bb4e77e2f676893537"
+checksum = "85db9830abeb1170b7d29b536ffd55af1d4d26ac8a77570b5d1aca003bf225cc"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a2d7744f743f59d9646d7589ad22ea17ed0d71e04906eb77c31e99bc13bd8b"
+checksum = "4946271f1055e26544ef8c90fa24776f201566419dfac4b3962c39d5a804ff67"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96cb196334698e612c197d7d0ae59af5e07667306ec20d7be414717db400873"
+checksum = "cb7e3bdae2597556e59edeb8ecb62eb32c7e054c4f042d393732902979db69c3"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e10c2e7faa65d4ae7de9a83b44f2c31aca7dc638e17d0a79572fdf8103d720b"
+checksum = "301ef0edafeaeda5771a5d2db64ac53e1818ae3111220a185677025fe91db4a1"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -165,9 +165,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.102.0"
+version = "0.103.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83ce94e18756058af8a66e3c0ba1123ae15517c72162d8060d0cb0974642adf2"
+checksum = "59e0ee3d013728903e0c513c31afa389b559bfd4fe8a44f80335c799e3132a41"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "15.0.0"
+version = "16.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b73ad1395eda136baec5ece7e079e0536a82ef73488e345456cc9b89858ad0ec"
+checksum = "6b6d197fcc34ad32ed440e1f9552fd57d1f377d9699d31dee1b5b457322c1f8a"
 dependencies = [
  "cfg-if",
  "libc",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 20fcd222732..fdac789423c 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.102", default-features = false, features = ["std", "unwind", "all-arch"] }
-cranelift-frontend = { version = "0.102" }
-cranelift-module = { version = "0.102" }
-cranelift-native = { version = "0.102" }
-cranelift-jit = { version = "0.102", optional = true }
-cranelift-object = { version = "0.102" }
+cranelift-codegen = { version = "0.103", default-features = false, features = ["std", "unwind", "all-arch"] }
+cranelift-frontend = { version = "0.103" }
+cranelift-module = { version = "0.103" }
+cranelift-native = { version = "0.103" }
+cranelift-jit = { version = "0.103", optional = true }
+cranelift-object = { version = "0.103" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
index 1d51b499c8b..a1cdf31c68a 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
@@ -538,261 +538,38 @@ pub enum E1 {
 pub enum E2<X> {
     V1 { f: bool },
 
-    /*_00*/ _01(X),
-    _02(X),
-    _03(X),
-    _04(X),
-    _05(X),
-    _06(X),
-    _07(X),
-    _08(X),
-    _09(X),
-    _0A(X),
-    _0B(X),
-    _0C(X),
-    _0D(X),
-    _0E(X),
-    _0F(X),
-    _10(X),
-    _11(X),
-    _12(X),
-    _13(X),
-    _14(X),
-    _15(X),
-    _16(X),
-    _17(X),
-    _18(X),
-    _19(X),
-    _1A(X),
-    _1B(X),
-    _1C(X),
-    _1D(X),
-    _1E(X),
-    _1F(X),
-    _20(X),
-    _21(X),
-    _22(X),
-    _23(X),
-    _24(X),
-    _25(X),
-    _26(X),
-    _27(X),
-    _28(X),
-    _29(X),
-    _2A(X),
-    _2B(X),
-    _2C(X),
-    _2D(X),
-    _2E(X),
-    _2F(X),
-    _30(X),
-    _31(X),
-    _32(X),
-    _33(X),
-    _34(X),
-    _35(X),
-    _36(X),
-    _37(X),
-    _38(X),
-    _39(X),
-    _3A(X),
-    _3B(X),
-    _3C(X),
-    _3D(X),
-    _3E(X),
-    _3F(X),
-    _40(X),
-    _41(X),
-    _42(X),
-    _43(X),
-    _44(X),
-    _45(X),
-    _46(X),
-    _47(X),
-    _48(X),
-    _49(X),
-    _4A(X),
-    _4B(X),
-    _4C(X),
-    _4D(X),
-    _4E(X),
-    _4F(X),
-    _50(X),
-    _51(X),
-    _52(X),
-    _53(X),
-    _54(X),
-    _55(X),
-    _56(X),
-    _57(X),
-    _58(X),
-    _59(X),
-    _5A(X),
-    _5B(X),
-    _5C(X),
-    _5D(X),
-    _5E(X),
-    _5F(X),
-    _60(X),
-    _61(X),
-    _62(X),
-    _63(X),
-    _64(X),
-    _65(X),
-    _66(X),
-    _67(X),
-    _68(X),
-    _69(X),
-    _6A(X),
-    _6B(X),
-    _6C(X),
-    _6D(X),
-    _6E(X),
-    _6F(X),
-    _70(X),
-    _71(X),
-    _72(X),
-    _73(X),
-    _74(X),
-    _75(X),
-    _76(X),
-    _77(X),
-    _78(X),
-    _79(X),
-    _7A(X),
-    _7B(X),
-    _7C(X),
-    _7D(X),
-    _7E(X),
-    _7F(X),
-    _80(X),
-    _81(X),
-    _82(X),
-    _83(X),
-    _84(X),
-    _85(X),
-    _86(X),
-    _87(X),
-    _88(X),
-    _89(X),
-    _8A(X),
-    _8B(X),
-    _8C(X),
-    _8D(X),
-    _8E(X),
-    _8F(X),
-    _90(X),
-    _91(X),
-    _92(X),
-    _93(X),
-    _94(X),
-    _95(X),
-    _96(X),
-    _97(X),
-    _98(X),
-    _99(X),
-    _9A(X),
-    _9B(X),
-    _9C(X),
-    _9D(X),
-    _9E(X),
-    _9F(X),
-    _A0(X),
-    _A1(X),
-    _A2(X),
-    _A3(X),
-    _A4(X),
-    _A5(X),
-    _A6(X),
-    _A7(X),
-    _A8(X),
-    _A9(X),
-    _AA(X),
-    _AB(X),
-    _AC(X),
-    _AD(X),
-    _AE(X),
-    _AF(X),
-    _B0(X),
-    _B1(X),
-    _B2(X),
-    _B3(X),
-    _B4(X),
-    _B5(X),
-    _B6(X),
-    _B7(X),
-    _B8(X),
-    _B9(X),
-    _BA(X),
-    _BB(X),
-    _BC(X),
-    _BD(X),
-    _BE(X),
-    _BF(X),
-    _C0(X),
-    _C1(X),
-    _C2(X),
-    _C3(X),
-    _C4(X),
-    _C5(X),
-    _C6(X),
-    _C7(X),
-    _C8(X),
-    _C9(X),
-    _CA(X),
-    _CB(X),
-    _CC(X),
-    _CD(X),
-    _CE(X),
-    _CF(X),
-    _D0(X),
-    _D1(X),
-    _D2(X),
-    _D3(X),
-    _D4(X),
-    _D5(X),
-    _D6(X),
-    _D7(X),
-    _D8(X),
-    _D9(X),
-    _DA(X),
-    _DB(X),
-    _DC(X),
-    _DD(X),
-    _DE(X),
-    _DF(X),
-    _E0(X),
-    _E1(X),
-    _E2(X),
-    _E3(X),
-    _E4(X),
-    _E5(X),
-    _E6(X),
-    _E7(X),
-    _E8(X),
-    _E9(X),
-    _EA(X),
-    _EB(X),
-    _EC(X),
-    _ED(X),
-    _EE(X),
-    _EF(X),
-    _F0(X),
-    _F1(X),
-    _F2(X),
-    _F3(X),
-    _F4(X),
-    _F5(X),
-    _F6(X),
-    _F7(X),
-    _F8(X),
-    _F9(X),
-    _FA(X),
-    _FB(X),
-    _FC(X),
-    _FD(X),
-    _FE(X),
-    _FF(X),
+    /*_00*/ _01(X), _02(X), _03(X), _04(X), _05(X), _06(X), _07(X),
+    _08(X), _09(X), _0A(X), _0B(X), _0C(X), _0D(X), _0E(X), _0F(X),
+    _10(X), _11(X), _12(X), _13(X), _14(X), _15(X), _16(X), _17(X),
+    _18(X), _19(X), _1A(X), _1B(X), _1C(X), _1D(X), _1E(X), _1F(X),
+    _20(X), _21(X), _22(X), _23(X), _24(X), _25(X), _26(X), _27(X),
+    _28(X), _29(X), _2A(X), _2B(X), _2C(X), _2D(X), _2E(X), _2F(X),
+    _30(X), _31(X), _32(X), _33(X), _34(X), _35(X), _36(X), _37(X),
+    _38(X), _39(X), _3A(X), _3B(X), _3C(X), _3D(X), _3E(X), _3F(X),
+    _40(X), _41(X), _42(X), _43(X), _44(X), _45(X), _46(X), _47(X),
+    _48(X), _49(X), _4A(X), _4B(X), _4C(X), _4D(X), _4E(X), _4F(X),
+    _50(X), _51(X), _52(X), _53(X), _54(X), _55(X), _56(X), _57(X),
+    _58(X), _59(X), _5A(X), _5B(X), _5C(X), _5D(X), _5E(X), _5F(X),
+    _60(X), _61(X), _62(X), _63(X), _64(X), _65(X), _66(X), _67(X),
+    _68(X), _69(X), _6A(X), _6B(X), _6C(X), _6D(X), _6E(X), _6F(X),
+    _70(X), _71(X), _72(X), _73(X), _74(X), _75(X), _76(X), _77(X),
+    _78(X), _79(X), _7A(X), _7B(X), _7C(X), _7D(X), _7E(X), _7F(X),
+    _80(X), _81(X), _82(X), _83(X), _84(X), _85(X), _86(X), _87(X),
+    _88(X), _89(X), _8A(X), _8B(X), _8C(X), _8D(X), _8E(X), _8F(X),
+    _90(X), _91(X), _92(X), _93(X), _94(X), _95(X), _96(X), _97(X),
+    _98(X), _99(X), _9A(X), _9B(X), _9C(X), _9D(X), _9E(X), _9F(X),
+    _A0(X), _A1(X), _A2(X), _A3(X), _A4(X), _A5(X), _A6(X), _A7(X),
+    _A8(X), _A9(X), _AA(X), _AB(X), _AC(X), _AD(X), _AE(X), _AF(X),
+    _B0(X), _B1(X), _B2(X), _B3(X), _B4(X), _B5(X), _B6(X), _B7(X),
+    _B8(X), _B9(X), _BA(X), _BB(X), _BC(X), _BD(X), _BE(X), _BF(X),
+    _C0(X), _C1(X), _C2(X), _C3(X), _C4(X), _C5(X), _C6(X), _C7(X),
+    _C8(X), _C9(X), _CA(X), _CB(X), _CC(X), _CD(X), _CE(X), _CF(X),
+    _D0(X), _D1(X), _D2(X), _D3(X), _D4(X), _D5(X), _D6(X), _D7(X),
+    _D8(X), _D9(X), _DA(X), _DB(X), _DC(X), _DD(X), _DE(X), _DF(X),
+    _E0(X), _E1(X), _E2(X), _E3(X), _E4(X), _E5(X), _E6(X), _E7(X),
+    _E8(X), _E9(X), _EA(X), _EB(X), _EC(X), _ED(X), _EE(X), _EF(X),
+    _F0(X), _F1(X), _F2(X), _F3(X), _F4(X), _F5(X), _F6(X), _F7(X),
+    _F8(X), _F9(X), _FA(X), _FB(X), _FC(X), _FD(X), _FE(X), _FF(X),
 
     V3,
     V4,
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index 4ba08f1af44..e1e1760c597 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-12-19"
+channel = "nightly-2023-12-24"
 components = ["rust-src", "rustc-dev", "llvm-tools"]
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index 7d7ffdadc7f..636f2875a68 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -31,6 +31,7 @@ rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR
 # FIXME add needs-unwind to these tests
 rm -r tests/run-make/libtest-junit
 rm tests/ui/asm/may_unwind.rs
+rm tests/ui/stable-mir-print/basic_function.rs
 
 # extra warning about -Cpanic=abort for proc macros
 rm tests/ui/proc-macro/crt-static.rs
@@ -44,7 +45,6 @@ rm tests/ui/proc-macro/no-mangle-in-proc-macro-issue-111888.rs
 # vendor intrinsics
 rm tests/ui/sse2.rs # CodegenBackend::target_features not yet implemented
 rm tests/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
-rm tests/ui/simd/masked-load-store.rs
 
 # exotic linkages
 rm tests/ui/issues/issue-33992.rs # unsupported linkages
@@ -80,6 +80,7 @@ rm -r tests/run-make/codegen-options-parsing
 rm -r tests/run-make/lto-*
 rm -r tests/run-make/reproducible-build-2
 rm -r tests/run-make/issue-109934-lto-debuginfo
+rm -r tests/run-make/no-builtins-lto
 
 # optimization tests
 # ==================
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index fe4f073f799..d06237f8d91 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -962,6 +962,37 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
         }
 
+        sym::simd_masked_store => {
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
+
+            let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(val_lane_count, mask_lane_count);
+            let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
+            let ptr_val = ptr.load_scalar(fx);
+
+            for lane_idx in 0..val_lane_count {
+                let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let if_enabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]);
+                fx.bcx.seal_block(if_enabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32;
+                fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_val, Offset32::new(offset));
+                fx.bcx.ins().jump(next, &[]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+
+                fx.bcx.ins().nop();
+            }
+        }
+
         sym::simd_gather => {
             intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
 
@@ -1057,7 +1088,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         }
 
         sym::simd_scatter => {
-            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
+            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
 
             let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
             let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);