diff options
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 27 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/intrinsics.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_hir_analysis/src/check/intrinsic.rs | 14 | ||||
| -rw-r--r-- | library/core/src/intrinsics.rs | 47 | ||||
| -rw-r--r-- | library/core/src/num/nonzero.rs | 14 | ||||
| -rw-r--r-- | library/core/src/num/uint_macros.rs | 25 |
6 files changed, 117 insertions, 31 deletions
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 2bed7c1bd1c..ab9f20fdf63 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -315,25 +315,32 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { Some((width, signed)) => match name { sym::ctlz | sym::cttz => { let y = self.const_bool(false); - self.call_intrinsic( + let ret = self.call_intrinsic( &format!("llvm.{name}.i{width}"), &[args[0].immediate(), y], - ) + ); + + self.intcast(ret, llret_ty, false) } sym::ctlz_nonzero => { let y = self.const_bool(true); let llvm_name = &format!("llvm.ctlz.i{width}"); - self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) } sym::cttz_nonzero => { let y = self.const_bool(true); let llvm_name = &format!("llvm.cttz.i{width}"); - self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) + } + sym::ctpop => { + let ret = self.call_intrinsic( + &format!("llvm.ctpop.i{width}"), + &[args[0].immediate()], + ); + self.intcast(ret, llret_ty, false) } - sym::ctpop => self.call_intrinsic( - &format!("llvm.ctpop.i{width}"), - &[args[0].immediate()], - ), sym::bswap => { if width == 8 { args[0].immediate() // byte swap a u8/i8 is just a no-op @@ -355,6 +362,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { // rotate = funnel shift with first two args the same let llvm_name = &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + + // llvm expects shift to be the same type as the values, but rust always uses `u32` + let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) } sym::saturating_add | sym::saturating_sub => { diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 63c709d8aed..8a32d7bff1d 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -173,7 +173,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let ty = instance_args.type_at(0); let layout = self.layout_of(ty)?; let val = self.read_scalar(&args[0])?; - let out_val = self.numeric_intrinsic(intrinsic_name, val, layout)?; + + let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?; self.write_scalar(out_val, dest)?; } sym::saturating_add | sym::saturating_sub => { @@ -200,12 +201,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::rotate_left | sym::rotate_right => { // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW)) // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW)) - let layout = self.layout_of(instance_args.type_at(0))?; + let layout_val = self.layout_of(instance_args.type_at(0))?; let val = self.read_scalar(&args[0])?; - let val_bits = val.to_bits(layout.size)?; + let val_bits = val.to_bits(layout_val.size)?; + + let layout_raw_shift = self.layout_of(self.tcx.types.u32)?; let raw_shift = self.read_scalar(&args[1])?; - let raw_shift_bits = raw_shift.to_bits(layout.size)?; - let width_bits = u128::from(layout.size.bits()); + let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?; + + let width_bits = u128::from(layout_val.size.bits()); let shift_bits = raw_shift_bits % width_bits; let inv_shift_bits = (width_bits - shift_bits) % width_bits; let result_bits = if intrinsic_name == sym::rotate_left { @@ -213,8 +217,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } else { (val_bits >> shift_bits) | (val_bits << inv_shift_bits) }; - let truncated_bits = self.truncate(result_bits, layout); - let result = Scalar::from_uint(truncated_bits, layout.size); + let truncated_bits = self.truncate(result_bits, layout_val); + let result = Scalar::from_uint(truncated_bits, layout_val.size); self.write_scalar(result, dest)?; } sym::copy => { @@ -472,6 +476,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { name: Symbol, val: Scalar<M::Provenance>, layout: TyAndLayout<'tcx>, + ret_layout: TyAndLayout<'tcx>, ) -> InterpResult<'tcx, Scalar<M::Provenance>> { assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty); let bits = val.to_bits(layout.size)?; @@ -487,7 +492,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::bitreverse => (bits << extra).reverse_bits(), _ => bug!("not a numeric intrinsic: {}", name), }; - Ok(Scalar::from_uint(bits_out, layout.size)) + Ok(Scalar::from_uint(bits_out, ret_layout.size)) } pub fn exact_div( diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index bd64621f077..d73b157ec23 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -411,13 +411,11 @@ pub fn check_intrinsic_type( (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx)) } - sym::ctpop - | sym::ctlz - | sym::ctlz_nonzero - | sym::cttz - | sym::cttz_nonzero - | sym::bswap - | sym::bitreverse => (1, 0, vec![param(0)], param(0)), + sym::ctpop | sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => { + (1, 0, vec![param(0)], tcx.types.u32) + } + + sym::bswap | sym::bitreverse => (1, 0, vec![param(0)], param(0)), sym::three_way_compare => { (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span))) @@ -460,7 +458,7 @@ pub fn check_intrinsic_type( (1, 0, vec![param(0), param(0)], param(0)) } sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)), - sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), param(0)], param(0)), + sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)), sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => { (1, 0, vec![param(0), param(0)], param(0)) } diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 9406efd7ab2..23e444219c5 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1987,6 +1987,13 @@ extern "rust-intrinsic" { /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `count_ones` method. For example, /// [`u32::count_ones`] + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn ctpop<T: Copy>(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2028,6 +2035,13 @@ extern "rust-intrinsic" { /// let num_leading = ctlz(x); /// assert_eq!(num_leading, 16); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn ctlz<T: Copy>(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2050,6 +2064,12 @@ extern "rust-intrinsic" { /// let num_leading = unsafe { ctlz_nonzero(x) }; /// assert_eq!(num_leading, 3); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "constctlz", since = "1.50.0")] + #[rustc_nounwind] + pub fn ctlz_nonzero<T: Copy>(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "constctlz", since = "1.50.0")] #[rustc_nounwind] pub fn ctlz_nonzero<T: Copy>(x: T) -> T; @@ -2090,6 +2110,13 @@ extern "rust-intrinsic" { /// let num_trailing = cttz(x); /// assert_eq!(num_trailing, 16); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn cttz<T: Copy>(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2112,6 +2139,12 @@ extern "rust-intrinsic" { /// let num_trailing = unsafe { cttz_nonzero(x) }; /// assert_eq!(num_trailing, 3); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")] + #[rustc_nounwind] + pub fn cttz_nonzero<T: Copy>(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")] #[rustc_nounwind] pub fn cttz_nonzero<T: Copy>(x: T) -> T; @@ -2288,6 +2321,13 @@ extern "rust-intrinsic" { /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `rotate_left` method. For example, /// [`u32::rotate_left`] + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn rotate_left<T: Copy>(x: T, shift: u32) -> T; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2303,6 +2343,13 @@ extern "rust-intrinsic" { /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `rotate_right` method. For example, /// [`u32::rotate_right`] + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn rotate_right<T: Copy>(x: T, shift: u32) -> T; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] diff --git a/library/core/src/num/nonzero.rs b/library/core/src/num/nonzero.rs index 62ea7abf652..a6724baa774 100644 --- a/library/core/src/num/nonzero.rs +++ b/library/core/src/num/nonzero.rs @@ -528,7 +528,12 @@ macro_rules! nonzero_integer { #[inline] pub const fn leading_zeros(self) -> u32 { // SAFETY: since `self` cannot be zero, it is safe to call `ctlz_nonzero`. - unsafe { intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive) as u32 } + unsafe { + #[cfg(not(bootstrap))] + return intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive); + #[cfg(bootstrap)] + return intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive) as u32; + } } /// Returns the number of trailing zeros in the binary representation @@ -552,7 +557,12 @@ macro_rules! nonzero_integer { #[inline] pub const fn trailing_zeros(self) -> u32 { // SAFETY: since `self` cannot be zero, it is safe to call `cttz_nonzero`. - unsafe { intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive) as u32 } + unsafe { + #[cfg(not(bootstrap))] + return intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive); + #[cfg(bootstrap)] + return intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive) as u32; + } } /// Returns the number of ones in the binary representation of `self`. diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs index ba6a243041c..e82f94ea78b 100644 --- a/library/core/src/num/uint_macros.rs +++ b/library/core/src/num/uint_macros.rs @@ -77,7 +77,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn count_ones(self) -> u32 { - intrinsics::ctpop(self as $ActualT) as u32 + #[cfg(not(bootstrap))] + return intrinsics::ctpop(self as $ActualT); + #[cfg(bootstrap)] + return intrinsics::ctpop(self as $ActualT) as u32; } /// Returns the number of zeros in the binary representation of `self`. @@ -119,7 +122,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn leading_zeros(self) -> u32 { - intrinsics::ctlz(self as $ActualT) as u32 + #[cfg(not(bootstrap))] + return intrinsics::ctlz(self as $ActualT); + #[cfg(bootstrap)] + return intrinsics::ctlz(self as $ActualT) as u32; } /// Returns the number of trailing zeros in the binary representation @@ -140,7 +146,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn trailing_zeros(self) -> u32 { - intrinsics::cttz(self) as u32 + #[cfg(not(bootstrap))] + return intrinsics::cttz(self); + #[cfg(bootstrap)] + return intrinsics::cttz(self) as u32; } /// Returns the number of leading ones in the binary representation of `self`. @@ -205,7 +214,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn rotate_left(self, n: u32) -> Self { - intrinsics::rotate_left(self, n as $SelfT) + #[cfg(not(bootstrap))] + return intrinsics::rotate_left(self, n); + #[cfg(bootstrap)] + return intrinsics::rotate_left(self, n as $SelfT); } /// Shifts the bits to the right by a specified amount, `n`, @@ -230,7 +242,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn rotate_right(self, n: u32) -> Self { - intrinsics::rotate_right(self, n as $SelfT) + #[cfg(not(bootstrap))] + return intrinsics::rotate_right(self, n); + #[cfg(bootstrap)] + return intrinsics::rotate_right(self, n as $SelfT); } /// Reverses the byte order of the integer. |