Merge commit 'f2cdd4a78d89c009342197cf5844a21f8aa813df' into sync_cg_clif-2022-04-22

8 files changed, 70 insertions, 56 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 8c45993a8b7..65346cb3962 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -821,7 +821,8 @@ pub(crate) fn codegen_place<'tcx>(
                 if cplace.layout().ty.is_box() {
                     cplace = cplace
                         .place_field(fx, Field::new(0)) // Box<T> -> Unique<T>
-                        .place_field(fx, Field::new(0)) // Unique<T> -> *const T
+                        .place_field(fx, Field::new(0)) // Unique<T> -> NonNull<T>
+                        .place_field(fx, Field::new(0)) // NonNull<T> -> *mut T
                         .place_deref(fx);
                 } else {
                     cplace = cplace.place_deref(fx);
diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs
index e7e6afeb865..e19070774c6 100644
--- a/compiler/rustc_codegen_cranelift/src/cast.rs
+++ b/compiler/rustc_codegen_cranelift/src/cast.rs
@@ -84,7 +84,7 @@ pub(crate) fn clif_int_or_float_cast(
             fx.bcx.ins().fcvt_from_uint(to_ty, from)
         }
     } else if from_ty.is_float() && to_ty.is_int() {
-        if to_ty == types::I128 {
+        let val = if to_ty == types::I128 {
             // _____sssf___
             // __fix   sfti: f32 -> i128
             // __fix   dfti: f64 -> i128
@@ -109,13 +109,9 @@ pub(crate) fn clif_int_or_float_cast(
 
             let to_rust_ty = if to_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
 
-            return fx
-                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                .load_scalar(fx);
-        }
-
-        // float -> int-like
-        if to_ty == types::I8 || to_ty == types::I16 {
+            fx.easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
+                .load_scalar(fx)
+        } else if to_ty == types::I8 || to_ty == types::I16 {
             // FIXME implement fcvt_to_*int_sat.i8/i16
             let val = if to_signed {
                 fx.bcx.ins().fcvt_to_sint_sat(types::I32, from)
@@ -146,6 +142,23 @@ pub(crate) fn clif_int_or_float_cast(
             fx.bcx.ins().fcvt_to_sint_sat(to_ty, from)
         } else {
             fx.bcx.ins().fcvt_to_uint_sat(to_ty, from)
+        };
+
+        if let Some(false) = fx.tcx.sess.opts.debugging_opts.saturating_float_casts {
+            return val;
+        }
+
+        let is_not_nan = fx.bcx.ins().fcmp(FloatCC::Equal, from, from);
+        if to_ty == types::I128 {
+            // FIXME(bytecodealliance/wasmtime#3963): select.i128 on fcmp eq miscompiles
+            let (lsb, msb) = fx.bcx.ins().isplit(val);
+            let zero = fx.bcx.ins().iconst(types::I64, 0);
+            let lsb = fx.bcx.ins().select(is_not_nan, lsb, zero);
+            let msb = fx.bcx.ins().select(is_not_nan, msb, zero);
+            fx.bcx.ins().iconcat(lsb, msb)
+        } else {
+            let zero = fx.bcx.ins().iconst(to_ty, 0);
+            fx.bcx.ins().select(is_not_nan, val, zero)
         }
     } else if from_ty.is_float() && to_ty.is_float() {
         // float -> float
diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs
index eef3c8c8d6e..e59a0cb0a23 100644
--- a/compiler/rustc_codegen_cranelift/src/config.rs
+++ b/compiler/rustc_codegen_cranelift/src/config.rs
@@ -48,12 +48,6 @@ pub struct BackendConfig {
     /// Can be set using `-Cllvm-args=display_cg_time=...`.
     pub display_cg_time: bool,
 
-    /// The register allocator to use.
-    ///
-    /// Defaults to the value of `CG_CLIF_REGALLOC` or `backtracking` otherwise. Can be set using
-    /// `-Cllvm-args=regalloc=...`.
-    pub regalloc: String,
-
     /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run
     /// once before passing the clif ir to Cranelift for compilation.
     ///
@@ -80,8 +74,6 @@ impl Default for BackendConfig {
                 args.split(' ').map(|arg| arg.to_string()).collect()
             },
             display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"),
-            regalloc: std::env::var("CG_CLIF_REGALLOC")
-                .unwrap_or_else(|_| "backtracking".to_string()),
             enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
             disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),
         }
@@ -101,7 +93,6 @@ impl BackendConfig {
                 match name {
                     "mode" => config.codegen_mode = value.parse()?,
                     "display_cg_time" => config.display_cg_time = parse_bool(name, value)?,
-                    "regalloc" => config.regalloc = value.to_string(),
                     "enable_verifier" => config.enable_verifier = parse_bool(name, value)?,
                     "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?,
                     _ => return Err(format!("Unknown option `{}`", name)),
diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs
index 357cb4a6d24..f619bb5ed5e 100644
--- a/compiler/rustc_codegen_cranelift/src/discriminant.rs
+++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs
@@ -128,8 +128,16 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             let relative_discr = if niche_start == 0 {
                 tag
             } else {
-                // FIXME handle niche_start > i64::MAX
-                fx.bcx.ins().iadd_imm(tag, -i64::try_from(niche_start).unwrap())
+                let niche_start = match fx.bcx.func.dfg.value_type(tag) {
+                    types::I128 => {
+                        let lsb = fx.bcx.ins().iconst(types::I64, niche_start as u64 as i64);
+                        let msb =
+                            fx.bcx.ins().iconst(types::I64, (niche_start >> 64) as u64 as i64);
+                        fx.bcx.ins().iconcat(lsb, msb)
+                    }
+                    ty => fx.bcx.ins().iconst(ty, niche_start as i64),
+                };
+                fx.bcx.ins().isub(tag, niche_start)
             };
             let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
             let is_niche = {
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 310d27c6dec..d76dfca7960 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -1019,39 +1019,23 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, old);
         };
 
-        // In Rust floating point min and max don't propagate NaN. In Cranelift they do however.
-        // For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for `minnumf*`
-        // and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing
-        // a float against itself. Only in case of NaN is it not equal to itself.
         minnumf32, (v a, v b) {
-            let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
-            let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
-            let temp = fx.bcx.ins().select(a_ge_b, b, a);
-            let val = fx.bcx.ins().select(a_is_nan, b, temp);
+            let val = crate::num::codegen_float_min(fx, a, b);
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
             ret.write_cvalue(fx, val);
         };
         minnumf64, (v a, v b) {
-            let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
-            let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
-            let temp = fx.bcx.ins().select(a_ge_b, b, a);
-            let val = fx.bcx.ins().select(a_is_nan, b, temp);
+            let val = crate::num::codegen_float_min(fx, a, b);
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
             ret.write_cvalue(fx, val);
         };
         maxnumf32, (v a, v b) {
-            let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
-            let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
-            let temp = fx.bcx.ins().select(a_le_b, b, a);
-            let val = fx.bcx.ins().select(a_is_nan, b, temp);
+            let val = crate::num::codegen_float_max(fx, a, b);
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
             ret.write_cvalue(fx, val);
         };
         maxnumf64, (v a, v b) {
-            let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
-            let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
-            let temp = fx.bcx.ins().select(a_le_b, b, a);
-            let val = fx.bcx.ins().select(a_is_nan, b, temp);
+            let val = crate::num::codegen_float_max(fx, a, b);
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
             ret.write_cvalue(fx, val);
         };
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index bc21d736166..d1ca9edf2e0 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -322,20 +322,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
             assert_eq!(a.layout(), b.layout());
             assert_eq!(a.layout(), c.layout());
-            let layout = a.layout();
+            assert_eq!(a.layout(), ret.layout());
 
-            let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
-            assert_eq!(lane_count, ret_lane_count);
-            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+            let layout = a.layout();
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 
             for lane in 0..lane_count {
-                let a_lane = a.value_lane(fx, lane).load_scalar(fx);
-                let b_lane = b.value_lane(fx, lane).load_scalar(fx);
-                let c_lane = c.value_lane(fx, lane).load_scalar(fx);
+                let a_lane = a.value_lane(fx, lane);
+                let b_lane = b.value_lane(fx, lane);
+                let c_lane = c.value_lane(fx, lane);
 
-                let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
-                let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
+                let res_lane = match lane_ty.kind() {
+                    ty::Float(FloatTy::F32) => fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty),
+                    ty::Float(FloatTy::F64) => fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty),
+                    _ => unreachable!(),
+                };
 
                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
             }
@@ -354,8 +355,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                     _ => unreachable!("{:?}", lane_ty),
                 }
                 match intrinsic {
-                    sym::simd_fmin => fx.bcx.ins().fmin(x_lane, y_lane),
-                    sym::simd_fmax => fx.bcx.ins().fmax(x_lane, y_lane),
+                    sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
+                    sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
                     _ => unreachable!(),
                 }
             });
@@ -495,7 +496,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let lt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
-                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
+                    ty::Float(_) => return crate::num::codegen_float_min(fx, a, b),
                     _ => unreachable!(),
                 };
                 fx.bcx.ins().select(lt, a, b)
@@ -512,7 +513,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let gt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
-                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
+                    ty::Float(_) => return crate::num::codegen_float_max(fx, a, b),
                     _ => unreachable!(),
                 };
                 fx.bcx.ins().select(gt, a, b)
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 878b9390e13..9d2e12f9898 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -256,8 +256,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
 
     flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
 
-    flags_builder.set("regalloc", &backend_config.regalloc).unwrap();
-
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index 545d390e269..4ce8adb182e 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -420,3 +420,21 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
         CValue::by_val(fx.bcx.ins().bint(types::I8, res), fx.layout_of(fx.tcx.types.bool))
     }
 }
+
+// In Rust floating point min and max don't propagate NaN. In Cranelift they do however.
+// For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for `minnumf*`
+// and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing
+// a float against itself. Only in case of NaN is it not equal to itself.
+pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
+    let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
+    let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
+    let temp = fx.bcx.ins().select(a_ge_b, b, a);
+    fx.bcx.ins().select(a_is_nan, b, temp)
+}
+
+pub(crate) fn codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
+    let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
+    let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
+    let temp = fx.bcx.ins().select(a_le_b, b, a);
+    fx.bcx.ins().select(a_is_nan, b, temp)
+}