From d25910eaeb3eab25d3bbf9b9815e6d215c202d1f Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 19 Aug 2025 22:16:58 +0200 Subject: make `prefetch` intrinsics safe --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4935f8d7dff..06c3d8ed6bc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -330,10 +330,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); + let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32; self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], - &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)], + &[ + ptr, + self.const_i32(rw), + self.const_i32(locality), + self.const_i32(cache_type), + ], ) } sym::carrying_mul_add => { -- cgit 1.4.1-3-g733a5 From 8ea3b093819aabd92a605b42989341da0c97c0d6 Mon Sep 17 00:00:00 2001 From: clubby789 Date: Thu, 17 Jul 2025 21:47:00 +0100 Subject: Pass `alloc-variant-zeroed` to LLVM --- compiler/rustc_codegen_llvm/src/attributes.rs | 10 ++++++++++ compiler/rustc_feature/src/builtin_attrs.rs | 4 ++++ compiler/rustc_span/src/symbol.rs | 1 + library/alloc/src/alloc.rs | 1 + tests/codegen-llvm/vec-calloc.rs | 20 ++++++++++++++++++++ 5 files changed, 36 insertions(+) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c548f467583..984db43c67f 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -436,6 +436,16 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); + if let Some(zv) = + cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + && let Some(name) = zv.value_str() + { + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "alloc-variant-zeroed", + &mangle_internal_symbol(cx.tcx, name.as_str()), + )); + } // apply to argument place instead of function let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index ab6b8f92802..6cd0c43a3e0 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -993,6 +993,10 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ rustc_allocator_zeroed, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::No, ), + rustc_attr!( + rustc_allocator_zeroed_variant, Normal, template!(NameValueStr: "function"), ErrorPreceding, + EncodeCrossCrate::Yes, + ), gated!( default_lib_allocator, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::No, allocator_internals, experimental!(default_lib_allocator), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index acbed7a9eed..21ba6c2779d 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1816,6 +1816,7 @@ symbols! { rustc_align, rustc_allocator, rustc_allocator_zeroed, + rustc_allocator_zeroed_variant, rustc_allow_const_fn_unstable, rustc_allow_incoherent_impl, rustc_allowed_through_unstable_modules, diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index c9b98fa4e5a..76630a746dd 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -17,6 +17,7 @@ unsafe extern "Rust" { #[rustc_allocator] #[rustc_nounwind] #[rustc_std_internal_symbol] + #[rustc_allocator_zeroed_variant = "__rust_alloc_zeroed"] fn __rust_alloc(size: usize, align: usize) -> *mut u8; #[rustc_deallocator] #[rustc_nounwind] diff --git a/tests/codegen-llvm/vec-calloc.rs b/tests/codegen-llvm/vec-calloc.rs index d1c320ead01..15971bbfa00 100644 --- a/tests/codegen-llvm/vec-calloc.rs +++ b/tests/codegen-llvm/vec-calloc.rs @@ -1,4 +1,6 @@ +//@ revisions: normal llvm21 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled +//@ [llvm21] min-llvm-version: 21 //@ only-x86_64 #![crate_type = "lib"] @@ -176,6 +178,24 @@ pub fn vec_option_i32(n: usize) -> Vec> { vec![None; n] } +// LLVM21-LABEL: @vec_array +#[cfg(llvm21)] +#[no_mangle] +pub fn vec_array(n: usize) -> Vec<[u32; 1_000_000]> { + // LLVM21-NOT: call {{.*}}alloc::vec::from_elem + // LLVM21-NOT: call {{.*}}reserve + // LLVM21-NOT: call {{.*}}__rust_alloc( + + // LLVM21: call {{.*}}__rust_alloc_zeroed( + + // LLVM21-NOT: call {{.*}}alloc::vec::from_elem + // LLVM21-NOT: call {{.*}}reserve + // LLVM21-NOT: call {{.*}}__rust_alloc( + + // LLVM21: ret void + vec![[0; 1_000_000]; 3] +} + // Ensure that __rust_alloc_zeroed gets the right attributes for LLVM to optimize it away. // CHECK: declare noalias noundef ptr @{{.*}}__rust_alloc_zeroed(i64 noundef, i64 allocalign noundef) unnamed_addr [[RUST_ALLOC_ZEROED_ATTRS:#[0-9]+]] -- cgit 1.4.1-3-g733a5 From d71ed8d19bcfdbd6eb3bf2620071f2ab95470a57 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 11 Aug 2025 17:18:23 +0200 Subject: Tell LLVM about read-only captures `&Freeze` parameters are not only `readonly` within the function, but any captures of the pointer can also only be used for reads. This can now be encoded using the `captures(address, read_provenance)` attribute. --- compiler/rustc_codegen_llvm/src/abi.rs | 7 ++++++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 8 ++++++++ compiler/rustc_target/src/callconv/mod.rs | 1 + compiler/rustc_ty_utils/src/abi.rs | 1 + tests/codegen-llvm/function-arguments.rs | 20 ++++++++++---------- tests/codegen-llvm/range-attribute.rs | 2 +- tests/codegen-llvm/read-only-capture-opt.rs | 18 ++++++++++++++++++ tests/ui/abi/debug.generic.stderr | 2 +- tests/ui/abi/debug.riscv64.stderr | 2 +- 10 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 tests/codegen-llvm/read-only-capture-opt.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 043123fcab2..399f8b6e762 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -42,12 +42,13 @@ trait ArgAttributesExt { const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; -const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 6] = [ (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), + (ArgAttribute::CapturesReadOnly, llvm::AttributeKind::CapturesReadOnly), ]; fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { @@ -83,6 +84,10 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' } for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { if regular.contains(attr) { + // captures(address, read_provenance) is only available since LLVM 21. + if attr == ArgAttribute::CapturesReadOnly && llvm_util::get_version() < (21, 0, 0) { + continue; + } attrs.push(llattr.create_attr(cx.llcx)); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 55d26a97e2d..2461f70a86e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -251,6 +251,7 @@ pub(crate) enum AttributeKind { Writable = 42, DeadOnUnwind = 43, DeadOnReturn = 44, + CapturesReadOnly = 45, } /// LLVMIntPredicate diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index e4fe1fc2e42..e699e4b9c13 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -278,6 +278,7 @@ enum class LLVMRustAttributeKind { Writable = 42, DeadOnUnwind = 43, DeadOnReturn = 44, + CapturesReadOnly = 45, }; static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { @@ -376,6 +377,8 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { #else report_fatal_error("DeadOnReturn attribute requires LLVM 21 or later"); #endif + case LLVMRustAttributeKind::CapturesReadOnly: + report_fatal_error("Should be handled separately"); } report_fatal_error("bad LLVMRustAttributeKind"); } @@ -430,6 +433,11 @@ LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttributeKind RustAttr) { if (RustAttr == LLVMRustAttributeKind::NoCapture) { return wrap(Attribute::getWithCaptureInfo(*unwrap(C), CaptureInfo::none())); } + if (RustAttr == LLVMRustAttributeKind::CapturesReadOnly) { + return wrap(Attribute::getWithCaptureInfo( + *unwrap(C), CaptureInfo(CaptureComponents::Address | + CaptureComponents::ReadProvenance))); + } #endif return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); } diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 63e56744aec..5f2a6f7ba38 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -119,6 +119,7 @@ mod attr_impl { const ReadOnly = 1 << 4; const InReg = 1 << 5; const NoUndef = 1 << 6; + const CapturesReadOnly = 1 << 7; } } rustc_data_structures::external_bitflags_debug! { ArgAttribute } diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 89d1dd8cf23..e4ed084b073 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -356,6 +356,7 @@ fn arg_attrs_for_rust_scalar<'tcx>( if matches!(kind, PointerKind::SharedRef { frozen: true }) && !is_return { attrs.set(ArgAttribute::ReadOnly); + attrs.set(ArgAttribute::CapturesReadOnly); } } } diff --git a/tests/codegen-llvm/function-arguments.rs b/tests/codegen-llvm/function-arguments.rs index a3fafbe6f82..db508682862 100644 --- a/tests/codegen-llvm/function-arguments.rs +++ b/tests/codegen-llvm/function-arguments.rs @@ -80,7 +80,7 @@ pub fn option_nonzero_int(x: Option>) -> Option> { x } -// CHECK: @readonly_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @readonly_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn readonly_borrow(_: &i32) {} @@ -91,12 +91,12 @@ pub fn readonly_borrow_ret() -> &'static i32 { loop {} } -// CHECK: @static_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @static_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // static borrow may be captured #[no_mangle] pub fn static_borrow(_: &'static i32) {} -// CHECK: @named_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @named_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // borrow with named lifetime may be captured #[no_mangle] pub fn named_borrow<'r>(_: &'r i32) {} @@ -129,7 +129,7 @@ pub fn mutable_borrow_ret() -> &'static mut i32 { // . pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {} -// CHECK: @notunpin_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @notunpin_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // But `&NotUnpin` behaves perfectly normal. #[no_mangle] pub fn notunpin_borrow(_: &NotUnpin) {} @@ -138,12 +138,12 @@ pub fn notunpin_borrow(_: &NotUnpin) {} #[no_mangle] pub fn indirect_struct(_: S) {} -// CHECK: @borrowed_struct(ptr noalias noundef readonly align 4 dereferenceable(32) %_1) +// CHECK: @borrowed_struct(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(32) %_1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn borrowed_struct(_: &S) {} -// CHECK: @option_borrow(ptr noalias noundef readonly align 4 dereferenceable_or_null(4) %_x) +// CHECK: @option_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable_or_null(4) %_x) #[no_mangle] pub fn option_borrow(_x: Option<&i32>) {} @@ -185,7 +185,7 @@ pub fn _box(x: Box) -> Box { // With a custom allocator, it should *not* have `noalias`. (See // for why.) The second argument is the allocator, // which is a reference here that still carries `noalias` as usual. -// CHECK: @_box_custom(ptr noundef nonnull align 4 %x.0, ptr noalias noundef nonnull readonly align 1 %x.1) +// CHECK: @_box_custom(ptr noundef nonnull align 4 %x.0, ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %x.1) #[no_mangle] pub fn _box_custom(x: Box) { drop(x) @@ -208,7 +208,7 @@ pub fn struct_return() -> S { #[no_mangle] pub fn helper(_: usize) {} -// CHECK: @slice(ptr noalias noundef nonnull readonly align 1 %_1.0, [[USIZE]] noundef %_1.1) +// CHECK: @slice(ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %_1.0, [[USIZE]] noundef %_1.1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn slice(_: &[u8]) {} @@ -227,7 +227,7 @@ pub fn unsafe_slice(_: &[UnsafeInner]) {} #[no_mangle] pub fn raw_slice(_: *const [u8]) {} -// CHECK: @str(ptr noalias noundef nonnull readonly align 1 %_1.0, [[USIZE]] noundef %_1.1) +// CHECK: @str(ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %_1.0, [[USIZE]] noundef %_1.1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn str(_: &[u8]) {} @@ -259,7 +259,7 @@ pub fn trait_option(x: Option>) -> Option &[u16] { x diff --git a/tests/codegen-llvm/range-attribute.rs b/tests/codegen-llvm/range-attribute.rs index b81ff9ab3e2..865d36d4747 100644 --- a/tests/codegen-llvm/range-attribute.rs +++ b/tests/codegen-llvm/range-attribute.rs @@ -67,7 +67,7 @@ pub fn enum2_value(x: Enum2) -> Enum2 { x } -// CHECK: noundef [[USIZE]] @takes_slice(ptr noalias noundef nonnull readonly align 4 %x.0, [[USIZE]] noundef %x.1) +// CHECK: noundef [[USIZE]] @takes_slice(ptr {{.*}} %x.0, [[USIZE]] noundef %x.1) #[no_mangle] pub fn takes_slice(x: &[i32]) -> usize { x.len() diff --git a/tests/codegen-llvm/read-only-capture-opt.rs b/tests/codegen-llvm/read-only-capture-opt.rs new file mode 100644 index 00000000000..78d56f8efc2 --- /dev/null +++ b/tests/codegen-llvm/read-only-capture-opt.rs @@ -0,0 +1,18 @@ +//@ compile-flags: -C opt-level=3 -Z mir-opt-level=0 +//@ min-llvm-version: 21 + +#![crate_type = "lib"] + +unsafe extern "C" { + safe fn do_something(p: &i32); +} + +#[unsafe(no_mangle)] +pub fn test() -> i32 { + // CHECK-LABEL: @test( + // CHECK: ret i32 0 + let i = 0; + do_something(&i); + do_something(&i); + i +} diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index 3b29efc8102..8375de3e817 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef, + regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index 2417396de2f..bddf0aea3d7 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef, + regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some( -- cgit 1.4.1-3-g733a5