From 28e0907111075dead53a931004b0ba6f1fe0d793 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 24 Jul 2024 20:55:43 +0200 Subject: nontemporal_store: make sure that the intrinsic is truly just a hint --- tests/codegen/intrinsics/nontemporal.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'tests/codegen') diff --git a/tests/codegen/intrinsics/nontemporal.rs b/tests/codegen/intrinsics/nontemporal.rs index 076d6d6d9da..828cb7e8287 100644 --- a/tests/codegen/intrinsics/nontemporal.rs +++ b/tests/codegen/intrinsics/nontemporal.rs @@ -1,13 +1,28 @@ //@ compile-flags: -O +//@ compile-flags: --target aarch64-unknown-linux-gnu +//@ needs-llvm-components: aarch64 -#![feature(core_intrinsics)] +#![feature(no_core, lang_items, intrinsics)] +#![no_core] #![crate_type = "lib"] +#[lang = "sized"] +pub trait Sized {} +#[lang = "copy"] +pub trait Copy {} + +impl Copy for u32 {} +impl Copy for *mut T {} + +extern "rust-intrinsic" { + pub fn nontemporal_store(ptr: *mut T, val: T); +} + #[no_mangle] pub fn a(a: &mut u32, b: u32) { // CHECK-LABEL: define{{.*}}void @a // CHECK: store i32 %b, ptr %a, align 4, !nontemporal unsafe { - std::intrinsics::nontemporal_store(a, b); + nontemporal_store(a, b); } } -- cgit 1.4.1-3-g733a5 From 1b587a6e76200fdd8364ef910246efa11c973e7b Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Tue, 6 Aug 2024 14:33:14 -0700 Subject: alloc: add ToString specialization for `&&str` Fixes #128690 --- library/alloc/src/string.rs | 43 ++++++++++++++++++++++------ tests/codegen/issues/str-to-string-128690.rs | 36 +++++++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 tests/codegen/issues/str-to-string-128690.rs (limited to 'tests/codegen') diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index 124230812df..d943901e9ec 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -2643,14 +2643,41 @@ impl ToString for i8 { } } -#[doc(hidden)] -#[cfg(not(no_global_oom_handling))] -#[stable(feature = "str_to_string_specialization", since = "1.9.0")] -impl ToString for str { - #[inline] - fn to_string(&self) -> String { - String::from(self) - } +// Generic/generated code can sometimes have multiple, nested references +// for strings, including `&&&str`s that would never be written +// by hand. This macro generates twelve layers of nested `&`-impl +// for primitive strings. +macro_rules! to_string_str { + {type ; x $($x:ident)*} => { + &to_string_str! { type ; $($x)* } + }; + {type ;} => { str }; + {impl ; x $($x:ident)*} => { + to_string_str! { $($x)* } + }; + {impl ;} => { }; + {$self:expr ; x $($x:ident)*} => { + *(to_string_str! { $self ; $($x)* }) + }; + {$self:expr ;} => { $self }; + {$($x:ident)*} => { + #[doc(hidden)] + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "str_to_string_specialization", since = "1.9.0")] + impl ToString for to_string_str!(type ; $($x)*) { + #[inline] + fn to_string(&self) -> String { + String::from(to_string_str!(self ; $($x)*)) + } + } + to_string_str! { impl ; $($x)* } + }; +} + +to_string_str! { + x x x x + x x x x + x x x x } #[doc(hidden)] diff --git a/tests/codegen/issues/str-to-string-128690.rs b/tests/codegen/issues/str-to-string-128690.rs new file mode 100644 index 00000000000..8b416306ba6 --- /dev/null +++ b/tests/codegen/issues/str-to-string-128690.rs @@ -0,0 +1,36 @@ +//@ compile-flags: -C opt-level=3 -Z merge-functions=disabled +#![crate_type = "lib"] + +//! Make sure str::to_string is specialized not to use fmt machinery. + +// CHECK-LABEL: define {{(dso_local )?}}void @one_ref +#[no_mangle] +pub fn one_ref(input: &str) -> String { + // CHECK-NOT: {{(call|invoke).*}}fmt + input.to_string() +} + +// CHECK-LABEL: define {{(dso_local )?}}void @two_ref +#[no_mangle] +pub fn two_ref(input: &&str) -> String { + // CHECK-NOT: {{(call|invoke).*}}fmt + input.to_string() +} + +// CHECK-LABEL: define {{(dso_local )?}}void @thirteen_ref +#[no_mangle] +pub fn thirteen_ref(input: &&&&&&&&&&&&&str) -> String { + // CHECK-NOT: {{(call|invoke).*}}fmt + input.to_string() +} + +// This is a known performance cliff because of the macro-generated +// specialized impl. If this test suddenly starts failing, +// consider removing the `to_string_str!` macro in `alloc/str/string.rs`. +// +// CHECK-LABEL: define {{(dso_local )?}}void @fourteen_ref +#[no_mangle] +pub fn fourteen_ref(input: &&&&&&&&&&&&&&str) -> String { + // CHECK: {{(call|invoke).*}}fmt + input.to_string() +} -- cgit 1.4.1-3-g733a5 From 27ca35aa1b949186d6cdec22c4d5cc4b18aae691 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Thu, 8 Aug 2024 11:15:03 +0100 Subject: const vector passed to codegen --- compiler/rustc_codegen_gcc/src/common.rs | 5 ++ compiler/rustc_codegen_llvm/src/common.rs | 10 +-- compiler/rustc_codegen_ssa/src/mir/block.rs | 6 +- compiler/rustc_codegen_ssa/src/mir/constant.rs | 39 ++++++--- compiler/rustc_codegen_ssa/src/mir/operand.rs | 19 ++++- compiler/rustc_codegen_ssa/src/traits/consts.rs | 1 + tests/codegen/const-vector.rs | 107 ++++++++++++++++++++++++ 7 files changed, 170 insertions(+), 17 deletions(-) create mode 100644 tests/codegen/const-vector.rs (limited to 'tests/codegen') diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs index 7a456e1c5d6..dca6b6494f9 100644 --- a/compiler/rustc_codegen_gcc/src/common.rs +++ b/compiler/rustc_codegen_gcc/src/common.rs @@ -160,6 +160,11 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> { self.context.new_struct_constructor(None, struct_type.as_type(), None, values) } + fn const_vector(&self, values: &[RValue<'gcc>]) -> RValue<'gcc> { + let typ = self.type_vector(values[0].get_type(), values.len() as u64); + self.context.new_rvalue_from_vector(None, typ, values) + } + fn const_to_opt_uint(&self, _v: RValue<'gcc>) -> Option { // TODO(antoyo) None diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 197bbb9ddbf..283518ed2bc 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -100,11 +100,6 @@ impl<'ll> CodegenCx<'ll, '_> { unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } - pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); - unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } - } - pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { bytes_in_context(self.llcx, bytes) } @@ -224,6 +219,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { struct_in_context(self.llcx, elts, packed) } + fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } + } + fn const_to_opt_uint(&self, v: &'ll Value) -> Option { try_as_const_integral(v).and_then(|v| unsafe { let mut i = 0u64; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index bc3076528da..772adf13ff1 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -923,8 +923,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // third argument must be constant. This is // checked by the type-checker. if i == 2 && intrinsic.name == sym::simd_shuffle { + // FIXME: the simd_shuffle argument is actually an array, + // not a vector, so we need this special hack to make sure + // it is passed as an immediate. We should pass the + // shuffle indices as a vector instead to avoid this hack. if let mir::Operand::Constant(constant) = &arg.node { - let (llval, ty) = self.simd_shuffle_indices(bx, constant); + let (llval, ty) = self.immediate_const_vector(bx, constant); return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty), diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index f4d974f7036..0aa85b82038 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -1,6 +1,6 @@ use rustc_middle::mir::interpret::ErrorHandled; use rustc_middle::ty::layout::HasTyCtxt; -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty::{self, Ty, ValTree}; use rustc_middle::{bug, mir, span_bug}; use rustc_target::abi::Abi; @@ -28,7 +28,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { .expect("erroneous constant missed by mono item collection") } - /// This is a convenience helper for `simd_shuffle_indices`. It has the precondition + /// This is a convenience helper for `immediate_const_vector`. It has the precondition /// that the given `constant` is an `Const::Unevaluated` and must be convertible to /// a `ValTree`. If you want a more general version of this, talk to `wg-const-eval` on zulip. /// @@ -59,23 +59,42 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.cx.tcx().const_eval_resolve_for_typeck(ty::ParamEnv::reveal_all(), uv, constant.span) } - /// process constant containing SIMD shuffle indices - pub fn simd_shuffle_indices( + /// process constant containing SIMD shuffle indices & constant vectors + pub fn immediate_const_vector( &mut self, bx: &Bx, constant: &mir::ConstOperand<'tcx>, ) -> (Bx::Value, Ty<'tcx>) { let ty = self.monomorphize(constant.ty()); + let ty_is_simd = ty.is_simd(); + // FIXME: ideally we'd assert that this is a SIMD type, but simd_shuffle + // in its current form relies on a regular array being passed as an + // immediate argument. This hack can be removed once that is fixed. + let field_ty = if ty_is_simd { + ty.simd_size_and_type(bx.tcx()).1 + } else { + ty.builtin_index().unwrap() + }; + let val = self .eval_unevaluated_mir_constant_to_valtree(constant) .ok() .map(|x| x.ok()) .flatten() .map(|val| { - let field_ty = ty.builtin_index().unwrap(); - let values: Vec<_> = val - .unwrap_branch() - .iter() + // Depending on whether this is a SIMD type with an array field + // or a type with many fields (one for each elements), the valtree + // is either a single branch with N children, or a root node + // with exactly one child which then in turn has many children. + // So we look at the first child to determine whether it is a + // leaf or whether we have to go one more layer down. + let branch_or_leaf = val.unwrap_branch(); + let first = branch_or_leaf.get(0).unwrap(); + let field_iter = match first { + ValTree::Branch(_) => first.unwrap_branch().iter(), + ValTree::Leaf(_) => branch_or_leaf.iter(), + }; + let values: Vec<_> = field_iter .map(|field| { if let Some(prim) = field.try_to_scalar() { let layout = bx.layout_of(field_ty); @@ -84,11 +103,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout)) } else { - bug!("simd shuffle field {:?}", field) + bug!("field is not a scalar {:?}", field) } }) .collect(); - bx.const_struct(&values, false) + if ty_is_simd { bx.const_vector(&values) } else { bx.const_struct(&values, false) } }) .unwrap_or_else(|| { bx.tcx().dcx().emit_err(errors::ShuffleIndicesEvaluation { span: constant.span }); diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 2bc2d0f70bf..6979478fda1 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -635,7 +635,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_consume(bx, place.as_ref()) } - mir::Operand::Constant(ref constant) => self.eval_mir_constant_to_operand(bx, constant), + mir::Operand::Constant(ref constant) => { + let constant_ty = self.monomorphize(constant.ty()); + // Most SIMD vector constants should be passed as immediates. + // (In particular, some intrinsics really rely on this.) + if constant_ty.is_simd() { + // However, some SIMD types do not actually use the vector ABI + // (in particular, packed SIMD types do not). Ensure we exclude those. + let layout = bx.layout_of(constant_ty); + if let Abi::Vector { .. } = layout.abi { + let (llval, ty) = self.immediate_const_vector(bx, constant); + return OperandRef { + val: OperandValue::Immediate(llval), + layout: bx.layout_of(ty), + }; + } + } + self.eval_mir_constant_to_operand(bx, constant) + } } } } diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index d93b3e06ca6..c15f1fa8e56 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -30,6 +30,7 @@ pub trait ConstMethods<'tcx>: BackendTypes { fn const_str(&self, s: &str) -> (Self::Value, Self::Value); fn const_struct(&self, elts: &[Self::Value], packed: bool) -> Self::Value; + fn const_vector(&self, elts: &[Self::Value]) -> Self::Value; fn const_to_opt_uint(&self, v: Self::Value) -> Option; fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; diff --git a/tests/codegen/const-vector.rs b/tests/codegen/const-vector.rs new file mode 100644 index 00000000000..d368838201e --- /dev/null +++ b/tests/codegen/const-vector.rs @@ -0,0 +1,107 @@ +//@ compile-flags: -C no-prepopulate-passes -Copt-level=0 + +// This test checks that constants of SIMD type are passed as immediate vectors. +// We ensure that both vector representations (struct with fields and struct wrapping array) work. +#![crate_type = "lib"] +#![feature(abi_unadjusted)] +#![feature(const_trait_impl)] +#![feature(repr_simd)] +#![feature(rustc_attrs)] +#![feature(simd_ffi)] +#![allow(non_camel_case_types)] + +// Setting up structs that can be used as const vectors +#[repr(simd)] +#[derive(Clone)] +pub struct i8x2(i8, i8); + +#[repr(simd)] +#[derive(Clone)] +pub struct i8x2_arr([i8; 2]); + +#[repr(simd)] +#[derive(Clone)] +pub struct f32x2(f32, f32); + +#[repr(simd)] +#[derive(Clone)] +pub struct f32x2_arr([f32; 2]); + +#[repr(simd, packed)] +#[derive(Copy, Clone)] +pub struct Simd([T; N]); + +// The following functions are required for the tests to ensure +// that they are called with a const vector + +extern "unadjusted" { + #[no_mangle] + fn test_i8x2(a: i8x2); +} + +extern "unadjusted" { + #[no_mangle] + fn test_i8x2_two_args(a: i8x2, b: i8x2); +} + +extern "unadjusted" { + #[no_mangle] + fn test_i8x2_mixed_args(a: i8x2, c: i32, b: i8x2); +} + +extern "unadjusted" { + #[no_mangle] + fn test_i8x2_arr(a: i8x2_arr); +} + +extern "unadjusted" { + #[no_mangle] + fn test_f32x2(a: f32x2); +} + +extern "unadjusted" { + #[no_mangle] + fn test_f32x2_arr(a: f32x2_arr); +} + +extern "unadjusted" { + #[no_mangle] + fn test_simd(a: Simd); +} + +extern "unadjusted" { + #[no_mangle] + fn test_simd_unaligned(a: Simd); +} + +// Ensure the packed variant of the simd struct does not become a const vector +// if the size is not a power of 2 +// CHECK: %"Simd" = type { [3 x i32] } + +pub fn do_call() { + unsafe { + // CHECK: call void @test_i8x2(<2 x i8> + test_i8x2(const { i8x2(32, 64) }); + + // CHECK: call void @test_i8x2_two_args(<2 x i8> , <2 x i8> + test_i8x2_two_args(const { i8x2(32, 64) }, const { i8x2(8, 16) }); + + // CHECK: call void @test_i8x2_mixed_args(<2 x i8> , i32 43, <2 x i8> + test_i8x2_mixed_args(const { i8x2(32, 64) }, 43, const { i8x2(8, 16) }); + + // CHECK: call void @test_i8x2_arr(<2 x i8> + test_i8x2_arr(const { i8x2_arr([32, 64]) }); + + // CHECK: call void @test_f32x2(<2 x float> + test_f32x2(const { f32x2(0.32, 0.64) }); + + // CHECK: void @test_f32x2_arr(<2 x float> + test_f32x2_arr(const { f32x2_arr([0.32, 0.64]) }); + + // CHECK: call void @test_simd(<4 x i32> + test_simd(const { Simd::([2, 4, 6, 8]) }); + + // CHECK: call void @test_simd_unaligned(%"Simd" %1 + test_simd_unaligned(const { Simd::([2, 4, 6]) }); + } +} -- cgit 1.4.1-3-g733a5 From cfadfabfcd9644628422396fcc8a67244df435c1 Mon Sep 17 00:00:00 2001 From: Andreas Jonson Date: Tue, 30 Jul 2024 11:06:26 +0200 Subject: Add range attribute to scalar function results and arguments --- compiler/rustc_codegen_llvm/src/abi.rs | 61 ++++++++++++++----- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 6 ++ compiler/rustc_codegen_llvm/src/llvm/mod.rs | 17 +++++- compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 12 ++++ tests/codegen/call-metadata.rs | 1 + tests/codegen/cast-optimized.rs | 2 - tests/codegen/common_prim_int_ptr.rs | 4 +- tests/codegen/enum/enum-match.rs | 2 +- tests/codegen/function-arguments.rs | 6 +- .../issues/issue-68667-unwrap-combinators.rs | 2 +- tests/codegen/range-attribute.rs | 68 ++++++++++++++++++++++ tests/codegen/repr/transparent.rs | 2 +- 12 files changed, 159 insertions(+), 24 deletions(-) create mode 100644 tests/codegen/range-attribute.rs (limited to 'tests/codegen') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b8f42031263..6d57d851d63 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -415,9 +415,32 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; + + let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| { + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (19, 0, 0) + && matches!(scalar.primitive(), Int(..)) + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + && !scalar.is_bool() + // LLVM also rejects full range. + && !scalar.is_always_valid(cx) + { + attributes::apply_to_llfn( + llfn, + idx, + &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))], + ); + } + }; + match &self.ret.mode { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); + } } PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); @@ -456,8 +479,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(attrs) - | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) => { + let i = apply(attrs); + if let abi::Abi::Scalar(scalar) = arg.layout.abi { + apply_range_attr(llvm::AttributePlace::Argument(i), scalar); + } + } + PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { apply(attrs); } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { @@ -466,8 +494,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { apply(meta_attrs); } PassMode::Pair(a, b) => { - apply(a); - apply(b); + let i = apply(a); + let ii = apply(b); + if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi { + apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a); + apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b); + } } PassMode::Cast { cast, pad_i32 } => { if *pad_i32 { @@ -517,15 +549,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } _ => {} } - if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { - // If the value is a boolean, the range is 0..2 and that ultimately - // become 0..0 when the type becomes i1, which would be rejected - // by the LLVM verifier. - if let Int(..) = scalar.primitive() { - if !scalar.is_bool() && !scalar.is_always_valid(bx) { - bx.range_metadata(callsite, scalar.valid_range(bx)); - } - } + if bx.cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() < (19, 0, 0) + && let abi::Abi::Scalar(scalar) = self.ret.layout.abi + && matches!(scalar.primitive(), Int(..)) + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + && !scalar.is_bool() + // LLVM also rejects full range. + && !scalar.is_always_valid(bx) + { + bx.range_metadata(callsite, scalar.valid_range(bx)); } for arg in self.args.iter() { match &arg.mode { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 80b13c0e1d4..faabbcb020d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1575,6 +1575,12 @@ extern "C" { pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute; + pub fn LLVMRustCreateRangeAttribute( + C: &Context, + num_bits: c_uint, + lower_words: *const u64, + upper_words: *const u64, + ) -> &Attribute; // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 72691907c0d..d0db350a149 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -8,7 +8,7 @@ use std::string::FromUtf8Error; use libc::c_uint; use rustc_data_structures::small_c_str::SmallCStr; use rustc_llvm::RustString; -use rustc_target::abi::Align; +use rustc_target::abi::{Align, Size, WrappingRange}; pub use self::AtomicRmwBinOp::*; pub use self::CallConv::*; @@ -105,6 +105,21 @@ pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribu unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } } +pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { + let lower = range.start; + let upper = range.end.wrapping_add(1); + let lower_words = [lower as u64, (lower >> 64) as u64]; + let upper_words = [upper as u64, (upper >> 64) as u64]; + unsafe { + LLVMRustCreateRangeAttribute( + llcx, + size.bits().try_into().unwrap(), + lower_words.as_ptr(), + upper_words.as_ptr(), + ) + } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 2ff7335a0fc..79a68b2ff0e 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -397,6 +397,18 @@ LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { std::nullopt)); } +extern "C" LLVMAttributeRef +LLVMRustCreateRangeAttribute(LLVMContextRef C, unsigned NumBits, + const uint64_t LowerWords[], + const uint64_t UpperWords[]) { +#if LLVM_VERSION_GE(19, 0) + return LLVMCreateConstantRangeAttribute(C, Attribute::Range, NumBits, + LowerWords, UpperWords); +#else + report_fatal_error("LLVM 19.0 is required for Range Attribute"); +#endif +} + // These values **must** match ffi::AllocKindFlags. // It _happens_ to match the LLVM values of llvm::AllocFnKind, // but that's happenstance and we do explicit conversions before diff --git a/tests/codegen/call-metadata.rs b/tests/codegen/call-metadata.rs index b2168990ff8..73c4b33e2cf 100644 --- a/tests/codegen/call-metadata.rs +++ b/tests/codegen/call-metadata.rs @@ -2,6 +2,7 @@ // scalar value. //@ compile-flags: -O -C no-prepopulate-passes +//@ ignore-llvm-version: 19 - 99 #![crate_type = "lib"] diff --git a/tests/codegen/cast-optimized.rs b/tests/codegen/cast-optimized.rs index 313b2b4f0d6..59cf40935cd 100644 --- a/tests/codegen/cast-optimized.rs +++ b/tests/codegen/cast-optimized.rs @@ -20,8 +20,6 @@ pub fn u32_index(c: u32) -> [bool; 22] { // CHECK-LABEL: @char_as_u32_index #[no_mangle] pub fn char_as_u32_index(c: char) -> [bool; 22] { - // CHECK: %[[B:.+]] = icmp ult i32 %c, 1114112 - // CHECK: call void @llvm.assume(i1 %[[B]]) let c = c as u32; let mut array = [false; 22]; diff --git a/tests/codegen/common_prim_int_ptr.rs b/tests/codegen/common_prim_int_ptr.rs index 87fa89abb86..aa7ebb4c911 100644 --- a/tests/codegen/common_prim_int_ptr.rs +++ b/tests/codegen/common_prim_int_ptr.rs @@ -28,7 +28,7 @@ pub fn insert_box(x: Box<()>) -> Result> { // CHECK-LABEL: @extract_int // CHECK-NOT: nonnull -// CHECK-SAME: (i{{[0-9]+}} {{[^,]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]]) +// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]]) #[no_mangle] pub unsafe fn extract_int(x: Result>) -> usize { // CHECK: [[TEMP:%.+]] = ptrtoint ptr [[PAYLOAD]] to [[USIZE:i[0-9]+]] @@ -40,7 +40,7 @@ pub unsafe fn extract_int(x: Result>) -> usize { } // CHECK-LABEL: @extract_box -// CHECK-SAME: (i{{[0-9]+}} {{[^,]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]]) +// CHECK-SAME: (i{{[0-9]+}} {{[^%]+}} [[DISCRIMINANT:%[0-9]+]], ptr {{[^,]+}} [[PAYLOAD:%[0-9]+]]) #[no_mangle] pub unsafe fn extract_box(x: Result>) -> Box { // CHECK: ret ptr [[PAYLOAD]] diff --git a/tests/codegen/enum/enum-match.rs b/tests/codegen/enum/enum-match.rs index 8da5de63e67..a24b98050d2 100644 --- a/tests/codegen/enum/enum-match.rs +++ b/tests/codegen/enum/enum-match.rs @@ -34,7 +34,7 @@ pub enum Enum1 { // CHECK: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match1{{.*}} // CHECK-NEXT: start: -// CHECK-NEXT: %1 = add i8 %0, -2 +// CHECK-NEXT: %1 = add{{( nsw)?}} i8 %0, -2 // CHECK-NEXT: %2 = zext i8 %1 to i64 // CHECK-NEXT: %3 = icmp ult i8 %1, 2 // CHECK-NEXT: %4 = add nuw nsw i64 %2, 1 diff --git a/tests/codegen/function-arguments.rs b/tests/codegen/function-arguments.rs index 56504df4034..bf9f405192b 100644 --- a/tests/codegen/function-arguments.rs +++ b/tests/codegen/function-arguments.rs @@ -50,7 +50,7 @@ pub fn maybeuninit_enum_bool(x: MaybeUninit) -> MaybeUninit { x } -// CHECK: noundef i32 @char(i32 noundef %x) +// CHECK: noundef{{( range\(i32 0, 1114112\))?}} i32 @char(i32 noundef{{( range\(i32 0, 1114112\))?}} %x) #[no_mangle] pub fn char(x: char) -> char { x @@ -68,7 +68,7 @@ pub fn int(x: u64) -> u64 { x } -// CHECK: noundef i64 @nonzero_int(i64 noundef %x) +// CHECK: noundef{{( range\(i64 1, 0\))?}} i64 @nonzero_int(i64 noundef{{( range\(i64 1, 0\))?}} %x) #[no_mangle] pub fn nonzero_int(x: NonZero) -> NonZero { x @@ -250,7 +250,7 @@ pub fn return_slice(x: &[u16]) -> &[u16] { x } -// CHECK: { i16, i16 } @enum_id_1(i16 noundef %x.0, i16 %x.1) +// CHECK: { i16, i16 } @enum_id_1(i16 noundef{{( range\(i16 0, 3\))?}} %x.0, i16 %x.1) #[no_mangle] pub fn enum_id_1(x: Option>) -> Option> { x diff --git a/tests/codegen/issues/issue-68667-unwrap-combinators.rs b/tests/codegen/issues/issue-68667-unwrap-combinators.rs index 6bd4c566a0c..21a5a5bf4ee 100644 --- a/tests/codegen/issues/issue-68667-unwrap-combinators.rs +++ b/tests/codegen/issues/issue-68667-unwrap-combinators.rs @@ -5,7 +5,7 @@ // MIR inlining now optimizes this code. // CHECK-LABEL: @unwrap_combinators -// CHECK: icmp +// CHECK: {{icmp|trunc}} // CHECK-NEXT: icmp // CHECK-NEXT: select i1 // CHECK-NEXT: ret i1 diff --git a/tests/codegen/range-attribute.rs b/tests/codegen/range-attribute.rs new file mode 100644 index 00000000000..bb19bec0fb9 --- /dev/null +++ b/tests/codegen/range-attribute.rs @@ -0,0 +1,68 @@ +// Checks that range metadata gets emitted on functions result and arguments +// with scalar value. + +//@ compile-flags: -O -C no-prepopulate-passes +//@ min-llvm-version: 19 + +#![crate_type = "lib"] + +use std::num::NonZero; + +// Hack to get the correct size for usize +// CHECK: @helper([[USIZE:i[0-9]+]] noundef %_1) +#[no_mangle] +pub fn helper(_: usize) {} + +// CHECK: noundef range(i128 1, 0) i128 @nonzero_int(i128 noundef range(i128 1, 0) %x) +#[no_mangle] +pub fn nonzero_int(x: NonZero) -> NonZero { + x +} + +// CHECK: noundef range(i8 0, 3) i8 @optional_bool(i8 noundef range(i8 0, 3) %x) +#[no_mangle] +pub fn optional_bool(x: Option) -> Option { + x +} + +pub enum Enum0 { + A(bool), + B, + C, +} + +// CHECK: noundef range(i8 0, 4) i8 @enum0_value(i8 noundef range(i8 0, 4) %x) +#[no_mangle] +pub fn enum0_value(x: Enum0) -> Enum0 { + x +} + +pub enum Enum1 { + A(u64), + B(u64), + C(u64), +} + +// CHECK: { [[ENUM1_TYP:i[0-9]+]], i64 } @enum1_value([[ENUM1_TYP]] noundef range([[ENUM1_TYP]] 0, 3) %x.0, i64 noundef %x.1) +#[no_mangle] +pub fn enum1_value(x: Enum1) -> Enum1 { + x +} + +pub enum Enum2 { + A(Enum0), + B(Enum0), + C(Enum0), +} + +// CHECK: { i8, i8 } @enum2_value(i8 noundef range(i8 0, 3) %x.0, i8 noundef %x.1) +#[no_mangle] +pub fn enum2_value(x: Enum2) -> Enum2 { + x +} + +// CHECK: noundef [[USIZE]] @takes_slice(ptr noalias noundef nonnull readonly align 4 %x.0, [[USIZE]] noundef %x.1) +#[no_mangle] +pub fn takes_slice(x: &[i32]) -> usize { + x.len() +} diff --git a/tests/codegen/repr/transparent.rs b/tests/codegen/repr/transparent.rs index 4b41332db45..9140b8542ec 100644 --- a/tests/codegen/repr/transparent.rs +++ b/tests/codegen/repr/transparent.rs @@ -74,7 +74,7 @@ pub enum Bool { FileNotFound, } -// CHECK: define{{( dso_local)?}} noundef{{( zeroext)?}} i8 @test_Gpz(i8 noundef{{( zeroext)?}} %_1) +// CHECK: define{{( dso_local)?}} noundef{{( zeroext)?( range\(i8 0, 3\))?}} i8 @test_Gpz(i8 noundef{{( zeroext)?( range\(i8 0, 3\))?}} %_1) #[no_mangle] pub extern "C" fn test_Gpz(_: GenericPlusZst) -> GenericPlusZst { loop {} -- cgit 1.4.1-3-g733a5 From 75743dc5a057cdc0678d88523edbbf3fdd1bf901 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Mon, 12 Aug 2024 11:10:26 +0200 Subject: make the codegen test also cover an ill-behaved arch, and add links --- compiler/rustc_codegen_llvm/src/builder.rs | 2 ++ tests/codegen/intrinsics/nontemporal.rs | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'tests/codegen') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 20bf580a716..8e7a99e46a5 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -737,6 +737,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // a hint, and use regular stores everywhere else. // (In the future, we could alternatively ensure that an sfence gets emitted after a sequence of movnt // before any kind of synchronizing operation. But it's not clear how to do that with LLVM.) + // For more context, see and + // . const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] = &["aarch64", "arm", "riscv32", "riscv64"]; diff --git a/tests/codegen/intrinsics/nontemporal.rs b/tests/codegen/intrinsics/nontemporal.rs index 828cb7e8287..ff2d6296066 100644 --- a/tests/codegen/intrinsics/nontemporal.rs +++ b/tests/codegen/intrinsics/nontemporal.rs @@ -1,6 +1,14 @@ //@ compile-flags: -O -//@ compile-flags: --target aarch64-unknown-linux-gnu -//@ needs-llvm-components: aarch64 +//@revisions: with_nontemporal without_nontemporal +//@[with_nontemporal] compile-flags: --target aarch64-unknown-linux-gnu +//@[with_nontemporal] needs-llvm-components: aarch64 +//@[without_nontemporal] compile-flags: --target x86_64-unknown-linux-gnu +//@[without_nontemporal] needs-llvm-components: x86 + +// Ensure that we *do* emit the `!nontemporal` flag on architectures where it +// is well-behaved, but do *not* emit it on architectures where it is ill-behaved. +// For more context, see and +// . #![feature(no_core, lang_items, intrinsics)] #![no_core] @@ -21,7 +29,8 @@ extern "rust-intrinsic" { #[no_mangle] pub fn a(a: &mut u32, b: u32) { // CHECK-LABEL: define{{.*}}void @a - // CHECK: store i32 %b, ptr %a, align 4, !nontemporal + // with_nontemporal: store i32 %b, ptr %a, align 4, !nontemporal + // without_nontemporal-NOT: nontemporal unsafe { nontemporal_store(a, b); } -- cgit 1.4.1-3-g733a5