Merge from rustc

3 files changed, 36 insertions, 5 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index df40a5eb475..8d3be19839e 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -179,7 +179,7 @@ pub(crate) fn compile_fn(
                 let early_dcx = rustc_session::EarlyDiagCtxt::new(
                     rustc_session::config::ErrorOutputType::default(),
                 );
-                early_dcx.early_error(format!(
+                early_dcx.early_fatal(format!(
                     "backend implementation limit exceeded while compiling {name}",
                     name = codegened_func.symbol_name
                 ));
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index fe4f073f799..d06237f8d91 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -962,6 +962,37 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
         }
 
+        sym::simd_masked_store => {
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
+
+            let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(val_lane_count, mask_lane_count);
+            let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
+            let ptr_val = ptr.load_scalar(fx);
+
+            for lane_idx in 0..val_lane_count {
+                let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let if_enabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]);
+                fx.bcx.seal_block(if_enabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32;
+                fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_val, Offset32::new(offset));
+                fx.bcx.ins().jump(next, &[]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+
+                fx.bcx.ins().nop();
+            }
+        }
+
         sym::simd_gather => {
             intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
 
@@ -1057,7 +1088,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         }
 
         sym::simd_scatter => {
-            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
+            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
 
             let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
             let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 148193b5a97..d0ce209be44 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -1,6 +1,6 @@
-#![cfg_attr(all(doc, not(bootstrap)), allow(internal_features))]
-#![cfg_attr(all(doc, not(bootstrap)), feature(rustdoc_internals))]
-#![cfg_attr(all(doc, not(bootstrap)), doc(rust_logo))]
+#![cfg_attr(doc, allow(internal_features))]
+#![cfg_attr(doc, feature(rustdoc_internals))]
+#![cfg_attr(doc, doc(rust_logo))]
 #![feature(rustc_private)]
 // Note: please avoid adding other feature gates where possible
 #![warn(rust_2018_idioms)]