From 8f648d7185a8455151951f30357b93dee726c6cb Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 8 Aug 2025 11:24:28 +0000 Subject: Remove bitcode_llvm_cmdline It used to be necessary on Apple platforms to ship with the App Store, but XCode 15 has stopped embedding LLVM bitcode and the App Store no longer accepts apps with bitcode embedded. --- compiler/rustc_codegen_llvm/src/back/write.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 85a06f457eb..62998003ca1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -862,7 +862,7 @@ pub(crate) fn codegen( .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); let thin_bc = module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode"); - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); + embed_bitcode(cgcx, llcx, llmod, &thin_bc); } } @@ -1058,7 +1058,6 @@ fn embed_bitcode( cgcx: &CodegenContext, llcx: &llvm::Context, llmod: &llvm::Module, - cmdline: &str, bitcode: &[u8], ) { // We're adding custom sections to the output object file, but we definitely @@ -1074,7 +1073,9 @@ fn embed_bitcode( // * Mach-O - this is for macOS. Inspecting the source code for the native // linker here shows that the `.llvmbc` and `.llvmcmd` sections are // automatically skipped by the linker. In that case there's nothing extra - // that we need to do here. + // that we need to do here. We do need to make sure that the + // `__LLVM,__cmdline` section exists even though it is empty as otherwise + // ld64 rejects the object file. // // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and // `.llvmcmd` sections, so there's nothing extra we need to do. @@ -1111,7 +1112,7 @@ fn embed_bitcode( llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); - let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llconst = common::bytes_in_context(llcx, &[]); let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); llvm::set_initializer(llglobal, llconst); let section = if cgcx.target_is_like_darwin { @@ -1128,7 +1129,7 @@ fn embed_bitcode( let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); llvm::append_module_inline_asm(llmod, &asm); - let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]); llvm::append_module_inline_asm(llmod, &asm); } } -- cgit 1.4.1-3-g733a5 From 8831c5b9946561a37dc7ee6fd826ef1470e8f380 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 8 Aug 2025 21:59:28 -0700 Subject: Stop using uadd.with.overflow --- compiler/rustc_codegen_llvm/src/builder.rs | 26 +++++++++++++++++++------- tests/assembly-llvm/x86_64-bigint-helpers.rs | 23 ++++++++++------------- tests/codegen-llvm/bigint-helpers.rs | 15 ++++++++++++--- 3 files changed, 41 insertions(+), 23 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 32cdef075e7..917d07e3c61 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -557,13 +557,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let (size, signed) = ty.int_size_and_signed(self.tcx); let width = size.bits(); - if oop == OverflowOp::Sub && !signed { - // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these - // to be the canonical form. It will attempt to reform llvm.usub.with.overflow - // in the backend if profitable. - let sub = self.sub(lhs, rhs); - let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); - return (sub, cmp); + if !signed { + match oop { + OverflowOp::Sub => { + // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these + // to be the canonical form. It will attempt to reform llvm.usub.with.overflow + // in the backend if profitable. + let sub = self.sub(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); + return (sub, cmp); + } + OverflowOp::Add => { + // Like with sub above, using icmp is the preferred form. See + // + let add = self.add(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, add, lhs); + return (add, cmp); + } + OverflowOp::Mul => {} + } } let oop_str = match oop { diff --git a/tests/assembly-llvm/x86_64-bigint-helpers.rs b/tests/assembly-llvm/x86_64-bigint-helpers.rs index 58785932bc2..c5efda58fd6 100644 --- a/tests/assembly-llvm/x86_64-bigint-helpers.rs +++ b/tests/assembly-llvm/x86_64-bigint-helpers.rs @@ -2,9 +2,7 @@ //@ assembly-output: emit-asm //@ compile-flags: --crate-type=lib -Copt-level=3 -C target-cpu=x86-64-v4 //@ compile-flags: -C llvm-args=-x86-asm-syntax=intel -//@ revisions: llvm-pre-20 llvm-20 -//@ [llvm-20] min-llvm-version: 20 -//@ [llvm-pre-20] max-llvm-major-version: 19 +//@ min-llvm-version: 20 #![no_std] #![feature(bigint_helper_methods)] @@ -23,16 +21,15 @@ pub unsafe extern "sysv64" fn bigint_chain_carrying_add( n: usize, mut carry: bool, ) -> bool { - // llvm-pre-20: mov [[TEMP:r..]], qword ptr [rsi + 8*[[IND:r..]] + 8] - // llvm-pre-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8] - // llvm-pre-20: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP]] - // llvm-pre-20: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 16] - // llvm-pre-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 16] - // llvm-pre-20: mov qword ptr [rdi + 8*[[IND]] + 16], [[TEMP]] - // llvm-20: adc [[TEMP:r..]], qword ptr [rdx + 8*[[IND:r..]]] - // llvm-20: mov qword ptr [rdi + 8*[[IND]]], [[TEMP]] - // llvm-20: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 8] - // llvm-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8] + // Even if we emit A+B, LLVM will sometimes reorder that to B+A, so this + // test doesn't actually check which register is mov vs which is adc. + + // CHECK: mov [[TEMP1:.+]], qword ptr [{{rdx|rsi}} + 8*[[IND:.+]] + 8] + // CHECK: adc [[TEMP1]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 8] + // CHECK: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP1]] + // CHECK: mov [[TEMP2:.+]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 16] + // CHECK: adc [[TEMP2]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 16] + // CHECK: mov qword ptr [rdi + 8*[[IND]] + 16], [[TEMP2]] for i in 0..n { (*dest.add(i), carry) = u64::carrying_add(*src1.add(i), *src2.add(i), carry); } diff --git a/tests/codegen-llvm/bigint-helpers.rs b/tests/codegen-llvm/bigint-helpers.rs index 355cccb8150..ec70a3eabed 100644 --- a/tests/codegen-llvm/bigint-helpers.rs +++ b/tests/codegen-llvm/bigint-helpers.rs @@ -3,11 +3,20 @@ #![crate_type = "lib"] #![feature(bigint_helper_methods)] +// Note that there's also an assembly test for this, which is what checks for +// the `ADC` (Add with Carry) instruction on x86 now that the IR we emit uses +// the preferred instruction phrasing instead of the intrinsic. + // CHECK-LABEL: @u32_carrying_add #[no_mangle] pub fn u32_carrying_add(a: u32, b: u32, c: bool) -> (u32, bool) { - // CHECK: @llvm.uadd.with.overflow.i32 - // CHECK: @llvm.uadd.with.overflow.i32 - // CHECK: or disjoint i1 + // CHECK: %[[AB:.+]] = add i32 {{%a, %b|%b, %a}} + // CHECK: %[[O1:.+]] = icmp ult i32 %[[AB]], %a + // CHECK: %[[CEXT:.+]] = zext i1 %c to i32 + // CHECK: %[[ABC:.+]] = add i32 %[[AB]], %[[CEXT]] + // CHECK: %[[O2:.+]] = icmp ult i32 %[[ABC]], %[[AB]] + // CHECK: %[[O:.+]] = or disjoint i1 %[[O1]], %[[O2]] + // CHECK: insertvalue {{.+}}, i32 %[[ABC]], 0 + // CHECK: insertvalue {{.+}}, i1 %[[O]], 1 u32::carrying_add(a, b, c) } -- cgit 1.4.1-3-g733a5 From 0246245420bbbb378db020c0492561cf701165cd Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 8 Aug 2025 21:59:51 +0800 Subject: rustc_target: Add the `32s` target feature for LoongArch --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 1 + compiler/rustc_target/src/target_features.rs | 1 + library/std_detect/src/detect/arch/loongarch.rs | 3 +++ library/std_detect/src/detect/os/linux/loongarch.rs | 13 ++++++------- library/std_detect/tests/macro_trailing_commas.rs | 2 ++ tests/ui/check-cfg/target_feature.stderr | 1 + 6 files changed, 14 insertions(+), 7 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 3b290e5a129..5c463555406 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -277,6 +277,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option None, // Filter out features that are not supported by the current LLVM version ("riscv32" | "riscv64", "zacas") if get_version().0 < 20 => None, ( diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs index 297d9ed84c5..bff8e4d37e9 100644 --- a/compiler/rustc_target/src/target_features.rs +++ b/compiler/rustc_target/src/target_features.rs @@ -763,6 +763,7 @@ static CSKY_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ static LOONGARCH_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start + ("32s", Unstable(sym::loongarch_target_feature), &[]), ("d", Stable, &["f"]), ("div32", Unstable(sym::loongarch_target_feature), &[]), ("f", Stable, &[]), diff --git a/library/std_detect/src/detect/arch/loongarch.rs b/library/std_detect/src/detect/arch/loongarch.rs index 68fc600fa8e..d5a442fbbb8 100644 --- a/library/std_detect/src/detect/arch/loongarch.rs +++ b/library/std_detect/src/detect/arch/loongarch.rs @@ -8,6 +8,7 @@ features! { /// Checks if `loongarch` feature is enabled. /// Supported arguments are: /// + /// * `"32s"` /// * `"f"` /// * `"d"` /// * `"frecipe"` @@ -22,6 +23,8 @@ features! { /// * `"lvz"` /// * `"ual"` #[stable(feature = "stdarch_loongarch_feature", since = "1.89.0")] + @FEATURE: #[unstable(feature = "stdarch_loongarch_feature_detection", issue = "117425")] _32s: "32s"; + /// 32S @FEATURE: #[stable(feature = "stdarch_loongarch_feature", since = "1.89.0")] f: "f"; /// F @FEATURE: #[stable(feature = "stdarch_loongarch_feature", since = "1.89.0")] d: "d"; diff --git a/library/std_detect/src/detect/os/linux/loongarch.rs b/library/std_detect/src/detect/os/linux/loongarch.rs index e97fda11d08..74415266f8b 100644 --- a/library/std_detect/src/detect/os/linux/loongarch.rs +++ b/library/std_detect/src/detect/os/linux/loongarch.rs @@ -17,22 +17,21 @@ pub(crate) fn detect_features() -> cache::Initializer { // The values are part of the platform-specific [cpucfg] // // [cpucfg]: LoongArch Reference Manual Volume 1: Basic Architecture v1.1 + let cpucfg1: usize; let cpucfg2: usize; - unsafe { - asm!( - "cpucfg {}, {}", - out(reg) cpucfg2, in(reg) 2, - options(pure, nomem, preserves_flags, nostack) - ); - } let cpucfg3: usize; unsafe { asm!( "cpucfg {}, {}", + "cpucfg {}, {}", + "cpucfg {}, {}", + out(reg) cpucfg1, in(reg) 1, + out(reg) cpucfg2, in(reg) 2, out(reg) cpucfg3, in(reg) 3, options(pure, nomem, preserves_flags, nostack) ); } + enable_feature(&mut value, Feature::_32s, bit::test(cpucfg1, 0) || bit::test(cpucfg1, 1)); enable_feature(&mut value, Feature::frecipe, bit::test(cpucfg2, 25)); enable_feature(&mut value, Feature::div32, bit::test(cpucfg2, 26)); enable_feature(&mut value, Feature::lam_bh, bit::test(cpucfg2, 27)); diff --git a/library/std_detect/tests/macro_trailing_commas.rs b/library/std_detect/tests/macro_trailing_commas.rs index 2fee0abdd57..6072ddf5ac4 100644 --- a/library/std_detect/tests/macro_trailing_commas.rs +++ b/library/std_detect/tests/macro_trailing_commas.rs @@ -69,6 +69,8 @@ fn aarch64() { #[test] #[cfg(any(target_arch = "loongarch32", target_arch = "loongarch64"))] fn loongarch() { + let _ = is_loongarch_feature_detected!("32s"); + let _ = is_loongarch_feature_detected!("32s",); let _ = is_loongarch_feature_detected!("lsx"); let _ = is_loongarch_feature_detected!("lsx",); } diff --git a/tests/ui/check-cfg/target_feature.stderr b/tests/ui/check-cfg/target_feature.stderr index 44fc23b6390..f845f9a3c25 100644 --- a/tests/ui/check-cfg/target_feature.stderr +++ b/tests/ui/check-cfg/target_feature.stderr @@ -6,6 +6,7 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); | = note: expected values for `target_feature` are: `10e60` `2e3` +`32s` `3e3r1` `3e3r2` `3e3r3` -- cgit 1.4.1-3-g733a5 From ebef9d7f6387fb0541a0dab48d1cd1f86556df04 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 8 Aug 2025 12:39:30 +0200 Subject: Set dead_on_return attribute for indirect arguments Set the dead_on_return attribute (added in LLVM 21) for arguments that are passed indirectly, but not byval. This indicates that the value of the argument on return does not matter, enabling additional dead store elimination. --- compiler/rustc_codegen_llvm/src/abi.rs | 12 ++++++++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 7 +++++ tests/codegen-llvm/addr-of-mutate.rs | 6 ++--- tests/codegen-llvm/dead_on_return.rs | 31 ++++++++++++++++++++++ tests/codegen-llvm/function-arguments.rs | 2 +- .../loongarch-abi/loongarch64-lp64d-abi.rs | 2 +- 7 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 tests/codegen-llvm/dead_on_return.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 009e7e2487b..043123fcab2 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -24,6 +24,7 @@ use crate::attributes::{self, llfn_attrs_from_instance}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -500,7 +501,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { - apply(attrs); + let i = apply(attrs); + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (21, 0, 0) + { + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[llvm::AttributeKind::DeadOnReturn.create_attr(cx.llcx)], + ); + } } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 75d3d27f74e..ad3c3d5932e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -249,6 +249,7 @@ pub(crate) enum AttributeKind { FnRetThunkExtern = 41, Writable = 42, DeadOnUnwind = 43, + DeadOnReturn = 44, } /// LLVMIntPredicate diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 588d867bbbf..cd4f80f808c 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -277,6 +277,7 @@ enum class LLVMRustAttributeKind { FnRetThunkExtern = 41, Writable = 42, DeadOnUnwind = 43, + DeadOnReturn = 44, }; static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { @@ -369,6 +370,12 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { return Attribute::Writable; case LLVMRustAttributeKind::DeadOnUnwind: return Attribute::DeadOnUnwind; + case LLVMRustAttributeKind::DeadOnReturn: +#if LLVM_VERSION_GE(21, 0) + return Attribute::DeadOnReturn; +#else + report_fatal_error("DeadOnReturn attribute requires LLVM 21 or later"); +#endif } report_fatal_error("bad LLVMRustAttributeKind"); } diff --git a/tests/codegen-llvm/addr-of-mutate.rs b/tests/codegen-llvm/addr-of-mutate.rs index 14bc4b8ab28..71669065289 100644 --- a/tests/codegen-llvm/addr-of-mutate.rs +++ b/tests/codegen-llvm/addr-of-mutate.rs @@ -5,7 +5,7 @@ // Test for the absence of `readonly` on the argument when it is mutated via `&raw const`. // See . -// CHECK: i8 @foo(ptr noalias{{( nocapture)?}} noundef align 1{{( captures\(none\))?}} dereferenceable(128) %x) +// CHECK: i8 @foo(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align 1{{( captures\(none\))?}} dereferenceable(128) %x) #[no_mangle] pub fn foo(x: [u8; 128]) -> u8 { let ptr = core::ptr::addr_of!(x).cast_mut(); @@ -15,7 +15,7 @@ pub fn foo(x: [u8; 128]) -> u8 { x[0] } -// CHECK: i1 @second(ptr noalias{{( nocapture)?}} noundef align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) +// CHECK: i1 @second(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) #[no_mangle] pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { let b_bool_ptr = core::ptr::addr_of!(a_ptr_and_b.1.1).cast_mut(); @@ -24,7 +24,7 @@ pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { } // If going through a deref (and there are no other mutating accesses), then `readonly` is fine. -// CHECK: i1 @third(ptr noalias{{( nocapture)?}} noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) +// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) #[no_mangle] pub unsafe fn third(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { let b_bool_ptr = core::ptr::addr_of!((*a_ptr_and_b.0).1).cast_mut(); diff --git a/tests/codegen-llvm/dead_on_return.rs b/tests/codegen-llvm/dead_on_return.rs new file mode 100644 index 00000000000..3c1940d6ba7 --- /dev/null +++ b/tests/codegen-llvm/dead_on_return.rs @@ -0,0 +1,31 @@ +//@ compile-flags: -C opt-level=3 +//@ min-llvm-version: 21 + +#![crate_type = "lib"] +#![allow(unused_assignments, unused_variables)] + +// Check that the old string is deallocated, but a new one is not initialized. +#[unsafe(no_mangle)] +pub fn test_str_new(mut s: String) { + // CHECK-LABEL: @test_str_new + // CHECK: __rust_dealloc + // CHECK-NOT: store + s = String::new(); +} + +#[unsafe(no_mangle)] +pub fn test_str_take(mut x: String) -> String { + // CHECK-LABEL: @test_str_take + // CHECK-NEXT: {{.*}}: + // CHECK-NEXT: call void @llvm.memcpy + // CHECK-NEXT: ret + core::mem::take(&mut x) +} + +#[unsafe(no_mangle)] +pub fn test_array_store(mut x: [u32; 100]) { + // CHECK-LABEL: @test_array_store + // CHECK-NEXT: {{.*}}: + // CHECK-NEXT: ret + x[0] = 1; +} diff --git a/tests/codegen-llvm/function-arguments.rs b/tests/codegen-llvm/function-arguments.rs index c8cd8526ae5..a3fafbe6f82 100644 --- a/tests/codegen-llvm/function-arguments.rs +++ b/tests/codegen-llvm/function-arguments.rs @@ -134,7 +134,7 @@ pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {} #[no_mangle] pub fn notunpin_borrow(_: &NotUnpin) {} -// CHECK: @indirect_struct(ptr noalias{{( nocapture)?}} noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1) +// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1) #[no_mangle] pub fn indirect_struct(_: S) {} diff --git a/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs b/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs index 93c8d60930b..4666342a16a 100644 --- a/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs +++ b/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs @@ -256,7 +256,7 @@ pub struct IntDoubleInt { c: i32, } -// CHECK: define void @f_int_double_int_s_arg(ptr noalias{{( nocapture)?}} noundef align 8{{( captures\(none\))?}} dereferenceable(24) %a) +// CHECK: define void @f_int_double_int_s_arg(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align 8{{( captures\(none\))?}} dereferenceable(24) %a) #[no_mangle] pub extern "C" fn f_int_double_int_s_arg(a: IntDoubleInt) {} -- cgit 1.4.1-3-g733a5 From a4cf5e9eca00527082fd58356e4adc8775aeee03 Mon Sep 17 00:00:00 2001 From: Chen Zhongyao Date: Mon, 14 Jul 2025 13:35:53 +0000 Subject: Add new Tier-3 target: riscv64a23-unknown-linux-gnu --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 6 +- compiler/rustc_target/src/spec/mod.rs | 1 + .../spec/targets/riscv64a23_unknown_linux_gnu.rs | 27 +++++ compiler/rustc_target/src/target_features.rs | 113 +++++++++++++++++++++ src/bootstrap/src/core/sanity.rs | 1 + tests/assembly-llvm/targets/targets-elf.rs | 3 + tests/ui/check-cfg/target_feature.stderr | 16 +++ 7 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 compiler/rustc_target/src/spec/targets/riscv64a23_unknown_linux_gnu.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 28d2100f478..a85a894e42c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -278,7 +278,11 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option None, + ("riscv32" | "riscv64", "zacas" | "rva23s64" | "rva23u64" | "sha" | "ssnpm" | "supm") + if get_version().0 < 20 => + { + None + } ( "s390x", "message-security-assist-extension12" diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index b9fbff8db05..d72d8bd2d08 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -2137,6 +2137,7 @@ supported_targets! { ("riscv64gc-unknown-none-elf", riscv64gc_unknown_none_elf), ("riscv64gc-unknown-linux-gnu", riscv64gc_unknown_linux_gnu), ("riscv64gc-unknown-linux-musl", riscv64gc_unknown_linux_musl), + ("riscv64a23-unknown-linux-gnu", riscv64a23_unknown_linux_gnu), ("sparc-unknown-none-elf", sparc_unknown_none_elf), diff --git a/compiler/rustc_target/src/spec/targets/riscv64a23_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/riscv64a23_unknown_linux_gnu.rs new file mode 100644 index 00000000000..27a0e27c969 --- /dev/null +++ b/compiler/rustc_target/src/spec/targets/riscv64a23_unknown_linux_gnu.rs @@ -0,0 +1,27 @@ +use std::borrow::Cow; + +use crate::spec::{CodeModel, SplitDebuginfo, Target, TargetMetadata, TargetOptions, base}; + +pub(crate) fn target() -> Target { + Target { + llvm_target: "riscv64-unknown-linux-gnu".into(), + metadata: TargetMetadata { + description: Some("RISC-V Linux (kernel 4.20, glibc 2.29)".into()), + tier: Some(3), + host_tools: Some(true), + std: Some(true), + }, + pointer_width: 64, + data_layout: "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128".into(), + arch: "riscv64".into(), + options: TargetOptions { + code_model: Some(CodeModel::Medium), + cpu: "generic-rv64".into(), + features: "+rva23u64,+rva23s64".into(), + llvm_abiname: "lp64d".into(), + max_atomic_width: Some(64), + supported_split_debuginfo: Cow::Borrowed(&[SplitDebuginfo::Off]), + ..base::linux_gnu::opts() + }, + } +} diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs index 297d9ed84c5..434f6a840b6 100644 --- a/compiler/rustc_target/src/target_features.rs +++ b/compiler/rustc_target/src/target_features.rs @@ -597,6 +597,119 @@ static RISCV_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ ), ("m", Stable, &[]), ("relax", Unstable(sym::riscv_target_feature), &[]), + ( + "rva23s64", + Unstable(sym::riscv_target_feature), + &[ + "m", + "a", + "f", + "d", + "c", + "b", + "v", + "zicsr", + "zicntr", + "zihpm", + "ziccif", + "ziccrse", + "ziccamoa", + "zicclsm", + "zic64b", + "za64rs", + "zihintpause", + "zba", + "zbb", + "zbs", + "zicbom", + "zicbop", + "zicboz", + "zfhmin", + "zkt", + "zvfhmin", + "zvbb", + "zvkt", + "zihintntl", + "zicond", + "zimop", + "zcmop", + "zcb", + "zfa", + "zawrs", + "svbare", + "svade", + "ssccptr", + "sstvecd", + "sstvala", + "sscounterenw", + "svpbmt", + "svinval", + "svnapot", + "sstc", + "sscofpmf", + "ssnpm", + "ssu64xl", + "sha", + "supm", + ], + ), + ( + "rva23u64", + Unstable(sym::riscv_target_feature), + &[ + "m", + "a", + "f", + "d", + "c", + "b", + "v", + "zicsr", + "zicntr", + "zihpm", + "ziccif", + "ziccrse", + "ziccamoa", + "zicclsm", + "zic64b", + "za64rs", + "zihintpause", + "zba", + "zbb", + "zbs", + "zicbom", + "zicbop", + "zicboz", + "zfhmin", + "zkt", + "zvfhmin", + "zvbb", + "zvkt", + "zihintntl", + "zicond", + "zimop", + "zcmop", + "zcb", + "zfa", + "zawrs", + "supm", + ], + ), + ("sha", Unstable(sym::riscv_target_feature), &[]), + ("ssccptr", Unstable(sym::riscv_target_feature), &[]), + ("sscofpmf", Unstable(sym::riscv_target_feature), &[]), + ("sscounterenw", Unstable(sym::riscv_target_feature), &[]), + ("ssnpm", Unstable(sym::riscv_target_feature), &[]), + ("sstc", Unstable(sym::riscv_target_feature), &[]), + ("sstvala", Unstable(sym::riscv_target_feature), &[]), + ("sstvecd", Unstable(sym::riscv_target_feature), &[]), + ("ssu64xl", Unstable(sym::riscv_target_feature), &[]), + ("supm", Unstable(sym::riscv_target_feature), &[]), + ("svade", Unstable(sym::riscv_target_feature), &[]), + ("svbare", Unstable(sym::riscv_target_feature), &[]), + ("svinval", Unstable(sym::riscv_target_feature), &[]), + ("svnapot", Unstable(sym::riscv_target_feature), &[]), + ("svpbmt", Unstable(sym::riscv_target_feature), &[]), ("unaligned-scalar-mem", Unstable(sym::riscv_target_feature), &[]), ("unaligned-vector-mem", Unstable(sym::riscv_target_feature), &[]), ("v", Unstable(sym::riscv_target_feature), &["zvl128b", "zve64d"]), diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index 3080e641b5b..a2d5d3f0e37 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -34,6 +34,7 @@ pub struct Finder { // Targets can be removed from this list once they are present in the stage0 compiler (usually by updating the beta compiler of the bootstrap). const STAGE0_MISSING_TARGETS: &[&str] = &[ "armv7a-vex-v5", + "riscv64a23-unknown-linux-gnu", // just a dummy comment so the list doesn't get onelined ]; diff --git a/tests/assembly-llvm/targets/targets-elf.rs b/tests/assembly-llvm/targets/targets-elf.rs index ee63dffe9ea..44f536b6856 100644 --- a/tests/assembly-llvm/targets/targets-elf.rs +++ b/tests/assembly-llvm/targets/targets-elf.rs @@ -475,6 +475,9 @@ //@ revisions: riscv64gc_unknown_linux_gnu //@ [riscv64gc_unknown_linux_gnu] compile-flags: --target riscv64gc-unknown-linux-gnu //@ [riscv64gc_unknown_linux_gnu] needs-llvm-components: riscv +//@ revisions: riscv64a23_unknown_linux_gnu +//@ [riscv64a23_unknown_linux_gnu] compile-flags: --target riscv64a23-unknown-linux-gnu +//@ [riscv64a23_unknown_linux_gnu] needs-llvm-components: riscv //@ revisions: riscv64gc_unknown_linux_musl //@ [riscv64gc_unknown_linux_musl] compile-flags: --target riscv64gc-unknown-linux-musl //@ [riscv64gc_unknown_linux_musl] needs-llvm-components: riscv diff --git a/tests/ui/check-cfg/target_feature.stderr b/tests/ui/check-cfg/target_feature.stderr index 44fc23b6390..a484fb0e5f0 100644 --- a/tests/ui/check-cfg/target_feature.stderr +++ b/tests/ui/check-cfg/target_feature.stderr @@ -241,6 +241,8 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); `relax` `relaxed-simd` `rtm` +`rva23s64` +`rva23u64` `sb` `scq` `sha` @@ -292,16 +294,27 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); `soft-float` `spe` `ssbs` +`ssccptr` +`sscofpmf` +`sscounterenw` `sse` `sse2` `sse3` `sse4.1` `sse4.2` `sse4a` +`ssnpm` `ssse3` +`sstc` +`sstvala` +`sstvecd` +`ssu64xl` `ssve-fp8dot2` `ssve-fp8dot4` `ssve-fp8fma` +`supm` +`svade` +`svbare` `sve` `sve-b16b16` `sve2` @@ -310,6 +323,9 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); `sve2-sha3` `sve2-sm4` `sve2p1` +`svinval` +`svnapot` +`svpbmt` `tail-call` `tbm` `thumb-mode` -- cgit 1.4.1-3-g733a5 From 258915a55539593423c3d4c30f7b741f65c56e51 Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Fri, 8 Aug 2025 17:38:03 +0000 Subject: llvm: Accept new LLVM lifetime format LLVM removed the size parameter from the lifetime format. Tolerate not having that size parameter. --- compiler/rustc_codegen_llvm/src/builder.rs | 6 +++++- .../codegen-llvm/align-byval-alignment-mismatch.rs | 4 ++-- tests/codegen-llvm/call-tmps-lifetime.rs | 16 +++++++-------- tests/codegen-llvm/intrinsics/transmute-simd.rs | 24 +++++++++++----------- tests/codegen-llvm/intrinsics/transmute.rs | 8 ++++---- .../issues/issue-105386-ub-in-debuginfo.rs | 2 +- tests/codegen-llvm/lifetime_start_end.rs | 16 +++++++-------- .../uninhabited-transparent-return-abi.rs | 4 ++-- 8 files changed, 42 insertions(+), 38 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index bd175e560c7..35a619da710 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1686,7 +1686,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + if crate::llvm_util::get_version() >= (22, 0, 0) { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]); + } else { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + } } } impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { diff --git a/tests/codegen-llvm/align-byval-alignment-mismatch.rs b/tests/codegen-llvm/align-byval-alignment-mismatch.rs index c69fc2de9d2..8eaf2751ed7 100644 --- a/tests/codegen-llvm/align-byval-alignment-mismatch.rs +++ b/tests/codegen-llvm/align-byval-alignment-mismatch.rs @@ -55,10 +55,10 @@ extern "C" { pub unsafe fn rust_to_c_increases_alignment(x: Align1) { // i686-linux: start: // i686-linux-NEXT: [[ALLOCA:%[0-9a-z]+]] = alloca [48 x i8], align 4 - // i686-linux-NEXT: call void @llvm.lifetime.start.p0(i64 48, ptr {{.*}}[[ALLOCA]]) + // i686-linux-NEXT: call void @llvm.lifetime.start.p0({{(i64 48, )?}}ptr {{.*}}[[ALLOCA]]) // i686-linux-NEXT: call void @llvm.memcpy.{{.+}}(ptr {{.*}}align 4 {{.*}}[[ALLOCA]], ptr {{.*}}align 1 {{.*}}%x // i686-linux-NEXT: call void @extern_c_align1({{.+}} [[ALLOCA]]) - // i686-linux-NEXT: call void @llvm.lifetime.end.p0(i64 48, ptr {{.*}}[[ALLOCA]]) + // i686-linux-NEXT: call void @llvm.lifetime.end.p0({{(i64 48, )?}}ptr {{.*}}[[ALLOCA]]) // x86_64-linux: start: // x86_64-linux-NEXT: call void @extern_c_align1 diff --git a/tests/codegen-llvm/call-tmps-lifetime.rs b/tests/codegen-llvm/call-tmps-lifetime.rs index 7b7b6e17bdd..0d7657ed758 100644 --- a/tests/codegen-llvm/call-tmps-lifetime.rs +++ b/tests/codegen-llvm/call-tmps-lifetime.rs @@ -16,14 +16,14 @@ use minicore::*; // CHECK-NEXT: start: // CHECK-NEXT: [[B:%.*]] = alloca // CHECK-NEXT: [[A:%.*]] = alloca -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[A]]) +// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 4096, )?}}ptr [[A]]) // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 4096, i1 false) // CHECK-NEXT: call void %h(ptr {{.*}} [[A]]) -// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[A]]) -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[B]]) +// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 4096, )?}}ptr [[A]]) +// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 4096, )?}}ptr [[B]]) // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 4096, i1 false) // CHECK-NEXT: call void %h(ptr {{.*}} [[B]]) -// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[B]]) +// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 4096, )?}}ptr [[B]]) #[no_mangle] pub fn const_indirect(h: extern "C" fn([u32; 1024])) { const C: [u32; 1024] = [0; 1024]; @@ -42,12 +42,12 @@ pub struct Str { // CHECK-LABEL: define void @immediate_indirect(ptr {{.*}}%s.0, i32 {{.*}}%s.1, ptr {{.*}}%g) // CHECK-NEXT: start: // CHECK-NEXT: [[A:%.*]] = alloca -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[A]]) +// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 8, )?}}ptr [[A]]) // CHECK-NEXT: store ptr %s.0, ptr [[A]] // CHECK-NEXT: [[B:%.]] = getelementptr inbounds i8, ptr [[A]], i32 4 // CHECK-NEXT: store i32 %s.1, ptr [[B]] // CHECK-NEXT: call void %g(ptr {{.*}} [[A]]) -// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[A]]) +// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 8, )?}}ptr [[A]]) #[no_mangle] pub fn immediate_indirect(s: Str, g: extern "C" fn(Str)) { g(s); @@ -58,10 +58,10 @@ pub fn immediate_indirect(s: Str, g: extern "C" fn(Str)) { // CHECK-LABEL: define void @align_indirect(ptr{{.*}} align 1{{.*}} %a, ptr{{.*}} %fun) // CHECK-NEXT: start: // CHECK-NEXT: [[A:%.*]] = alloca [1024 x i8], align 4 -// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 1024, ptr [[A]]) +// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 1024, )?}}ptr [[A]]) // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 1 %a, i32 1024, i1 false) // CHECK-NEXT: call void %fun(ptr {{.*}} [[A]]) -// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1024, ptr [[A]]) +// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 1024, )?}}ptr [[A]]) #[no_mangle] pub fn align_indirect(a: [u8; 1024], fun: extern "C" fn([u8; 1024])) { fun(a); diff --git a/tests/codegen-llvm/intrinsics/transmute-simd.rs b/tests/codegen-llvm/intrinsics/transmute-simd.rs index e34b27e1333..92af16945ab 100644 --- a/tests/codegen-llvm/intrinsics/transmute-simd.rs +++ b/tests/codegen-llvm/intrinsics/transmute-simd.rs @@ -97,10 +97,10 @@ pub extern "C" fn float_ptr_same_lanes(v: f64x2) -> PtrX2 { // CHECK-NOT: alloca // CHECK: %[[TEMP:.+]] = alloca [16 x i8] // CHECK-NOT: alloca - // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: store <2 x double> %v, ptr %[[TEMP]] // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]] - // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: ret <2 x ptr> %[[RET]] unsafe { transmute(v) } } @@ -111,10 +111,10 @@ pub extern "C" fn ptr_float_same_lanes(v: PtrX2) -> f64x2 { // CHECK-NOT: alloca // CHECK: %[[TEMP:.+]] = alloca [16 x i8] // CHECK-NOT: alloca - // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: store <2 x ptr> %v, ptr %[[TEMP]] // CHECK: %[[RET:.+]] = load <2 x double>, ptr %[[TEMP]] - // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: ret <2 x double> %[[RET]] unsafe { transmute(v) } } @@ -125,10 +125,10 @@ pub extern "C" fn int_ptr_same_lanes(v: i64x2) -> PtrX2 { // CHECK-NOT: alloca // CHECK: %[[TEMP:.+]] = alloca [16 x i8] // CHECK-NOT: alloca - // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: store <2 x i64> %v, ptr %[[TEMP]] // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]] - // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: ret <2 x ptr> %[[RET]] unsafe { transmute(v) } } @@ -139,10 +139,10 @@ pub extern "C" fn ptr_int_same_lanes(v: PtrX2) -> i64x2 { // CHECK-NOT: alloca // CHECK: %[[TEMP:.+]] = alloca [16 x i8] // CHECK-NOT: alloca - // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: store <2 x ptr> %v, ptr %[[TEMP]] // CHECK: %[[RET:.+]] = load <2 x i64>, ptr %[[TEMP]] - // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: ret <2 x i64> %[[RET]] unsafe { transmute(v) } } @@ -153,10 +153,10 @@ pub extern "C" fn float_ptr_widen(v: f32x4) -> PtrX2 { // CHECK-NOT: alloca // CHECK: %[[TEMP:.+]] = alloca [16 x i8] // CHECK-NOT: alloca - // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: store <4 x float> %v, ptr %[[TEMP]] // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]] - // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: ret <2 x ptr> %[[RET]] unsafe { transmute(v) } } @@ -167,10 +167,10 @@ pub extern "C" fn int_ptr_widen(v: i32x4) -> PtrX2 { // CHECK-NOT: alloca // CHECK: %[[TEMP:.+]] = alloca [16 x i8] // CHECK-NOT: alloca - // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: store <4 x i32> %v, ptr %[[TEMP]] // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]] - // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]]) // CHECK: ret <2 x ptr> %[[RET]] unsafe { transmute(v) } } diff --git a/tests/codegen-llvm/intrinsics/transmute.rs b/tests/codegen-llvm/intrinsics/transmute.rs index 91cff38773d..e327c100d7d 100644 --- a/tests/codegen-llvm/intrinsics/transmute.rs +++ b/tests/codegen-llvm/intrinsics/transmute.rs @@ -192,12 +192,12 @@ pub unsafe fn check_byte_from_bool(x: bool) -> u8 { #[no_mangle] pub unsafe fn check_to_pair(x: u64) -> Option { // CHECK: %[[TEMP:.+]] = alloca [8 x i8], align 8 - // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 8, )?}}ptr %[[TEMP]]) // CHECK: store i64 %x, ptr %[[TEMP]], align 8 // CHECK: %[[PAIR0:.+]] = load i32, ptr %[[TEMP]], align 8 // CHECK: %[[PAIR1P:.+]] = getelementptr inbounds i8, ptr %[[TEMP]], i64 4 // CHECK: %[[PAIR1:.+]] = load i32, ptr %[[PAIR1P]], align 4 - // CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 8, )?}}ptr %[[TEMP]]) // CHECK: insertvalue {{.+}}, i32 %[[PAIR0]], 0 // CHECK: insertvalue {{.+}}, i32 %[[PAIR1]], 1 transmute(x) @@ -207,12 +207,12 @@ pub unsafe fn check_to_pair(x: u64) -> Option { #[no_mangle] pub unsafe fn check_from_pair(x: Option) -> u64 { // CHECK: %[[TEMP:.+]] = alloca [8 x i8], align 8 - // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.start.p0({{(i64 8, )?}}ptr %[[TEMP]]) // CHECK: store i32 %x.0, ptr %[[TEMP]], align 8 // CHECK: %[[PAIR1P:.+]] = getelementptr inbounds i8, ptr %[[TEMP]], i64 4 // CHECK: store i32 %x.1, ptr %[[PAIR1P]], align 4 // CHECK: %[[R:.+]] = load i64, ptr %[[TEMP]], align 8 - // CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TEMP]]) + // CHECK: call void @llvm.lifetime.end.p0({{(i64 8, )?}}ptr %[[TEMP]]) // CHECK: ret i64 %[[R]] transmute(x) } diff --git a/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs b/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs index 848aa910b58..95e70bf5678 100644 --- a/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs +++ b/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs @@ -19,6 +19,6 @@ pub fn outer_function(x: S, y: S) -> usize { // CHECK: [[spill:%.*]] = alloca // CHECK-NOT: [[ptr_tmp:%.*]] = getelementptr inbounds i8, ptr [[spill]] // CHECK-NOT: [[load:%.*]] = load ptr, ptr -// CHECK: call void @llvm.lifetime.start{{.*}}({{.*}}, ptr [[spill]]) +// CHECK: call void @llvm.lifetime.start{{.*}}({{(.*, )?}}ptr [[spill]]) // CHECK: [[inner:%.*]] = getelementptr inbounds i8, ptr [[spill]] // CHECK: call void @llvm.memcpy{{.*}}(ptr {{align .*}} [[inner]], ptr {{align .*}} %x diff --git a/tests/codegen-llvm/lifetime_start_end.rs b/tests/codegen-llvm/lifetime_start_end.rs index 0639e7640aa..2df5acf54df 100644 --- a/tests/codegen-llvm/lifetime_start_end.rs +++ b/tests/codegen-llvm/lifetime_start_end.rs @@ -8,27 +8,27 @@ pub fn test() { let a = 0u8; &a; // keep variable in an alloca - // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, ptr %a) + // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}ptr %a) { let b = &Some(a); &b; // keep variable in an alloca - // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, {{.*}}) + // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}{{.*}}) - // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, {{.*}}) + // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}{{.*}}) - // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, {{.*}}) + // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}{{.*}}) - // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, {{.*}}) + // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}{{.*}}) } let c = 1u8; &c; // keep variable in an alloca - // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, ptr %c) + // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}ptr %c) - // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, ptr %c) + // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}ptr %c) - // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, ptr %a) + // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}ptr %a) } diff --git a/tests/codegen-llvm/uninhabited-transparent-return-abi.rs b/tests/codegen-llvm/uninhabited-transparent-return-abi.rs index face1577c3f..83d9a7a32ab 100644 --- a/tests/codegen-llvm/uninhabited-transparent-return-abi.rs +++ b/tests/codegen-llvm/uninhabited-transparent-return-abi.rs @@ -23,7 +23,7 @@ extern "Rust" { #[no_mangle] pub fn test_uninhabited_ret_by_ref() { // CHECK: %_1 = alloca [24 x i8], align {{8|4}} - // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %_1) + // CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 24, )?}}ptr nonnull %_1) // CHECK-NEXT: call void @opaque({{.*}} sret([24 x i8]) {{.*}} %_1) #2 // CHECK-NEXT: unreachable unsafe { @@ -35,7 +35,7 @@ pub fn test_uninhabited_ret_by_ref() { #[no_mangle] pub fn test_uninhabited_ret_by_ref_with_arg(rsi: u32) { // CHECK: %_2 = alloca [24 x i8], align {{8|4}} - // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %_2) + // CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 24, )?}}ptr nonnull %_2) // CHECK-NEXT: call void @opaque_with_arg({{.*}} sret([24 x i8]) {{.*}} %_2, i32 noundef %rsi) #2 // CHECK-NEXT: unreachable unsafe { -- cgit 1.4.1-3-g733a5 From 2563e4a7ffe47e93c69283c14e6d660b08834cf1 Mon Sep 17 00:00:00 2001 From: Tom Vijlbrief Date: Mon, 11 Aug 2025 11:19:47 +0200 Subject: [AVR] Changed data_layout --- compiler/rustc_codegen_llvm/src/context.rs | 6 ++++++ compiler/rustc_target/src/spec/targets/avr_none.rs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ee77774c688..27ae729a531 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -213,6 +213,12 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("p8:128:128:128:48", "p8:128:128") } } + if llvm_version < (22, 0, 0) { + if sess.target.arch == "avr" { + // LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010 + target_data_layout = target_data_layout.replace("n8:16", "n8") + } + } // Ensure the data-layout values hardcoded remain the defaults. { diff --git a/compiler/rustc_target/src/spec/targets/avr_none.rs b/compiler/rustc_target/src/spec/targets/avr_none.rs index 07ed2a37803..ad056d02326 100644 --- a/compiler/rustc_target/src/spec/targets/avr_none.rs +++ b/compiler/rustc_target/src/spec/targets/avr_none.rs @@ -9,7 +9,7 @@ pub(crate) fn target() -> Target { host_tools: None, std: None, }, - data_layout: "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8".into(), + data_layout: "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8".into(), llvm_target: "avr-unknown-unknown".into(), pointer_width: 16, options: TargetOptions { -- cgit 1.4.1-3-g733a5 From e13e1e4c0c3253d25e2571ed7e58b3a02e28e922 Mon Sep 17 00:00:00 2001 From: Adwin White Date: Tue, 12 Aug 2025 17:13:13 +0800 Subject: fix(debuginfo): handle false positives in overflow check --- .../src/debuginfo/metadata/type_map.rs | 54 +++++++++++++++------- tests/debuginfo/recursive-struct.rs | 9 ++++ 2 files changed, 47 insertions(+), 16 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index d1502d2b1e6..18a783a348a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -276,7 +276,7 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( && let ty::Adt(adt_def, args) = ty.kind() { let def_id = adt_def.did(); - // If any sub type reference the original type definition and the sub type has a type + // If any child type references the original type definition and the child type has a type // parameter that strictly contains the original parameter, the original type is a recursive // type that can expanding indefinitely. Example, // ``` @@ -285,21 +285,43 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( // Item(T), // } // ``` - let is_expanding_recursive = adt_def.is_enum() - && debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| { - if def_id == *parent_def_id { - args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| { - if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type()) - { - arg != parent_arg && arg.contains(parent_arg) - } else { - false - } - }) - } else { - false - } - }); + let is_expanding_recursive = { + let stack = debug_context(cx).adt_stack.borrow(); + stack + .iter() + .enumerate() + .rev() + .skip(1) + .filter(|(_, (ancestor_def_id, _))| def_id == *ancestor_def_id) + .any(|(ancestor_index, (_, ancestor_args))| { + args.iter() + .zip(ancestor_args.iter()) + .filter_map(|(arg, ancestor_arg)| arg.as_type().zip(ancestor_arg.as_type())) + .any(|(arg, ancestor_arg)| + // Strictly contains. + (arg != ancestor_arg && arg.contains(ancestor_arg)) + // Check all types between current and ancestor use the + // ancestor_arg. + // Otherwise, duplicate wrappers in normal recursive type may be + // regarded as expanding. + // ``` + // struct Recursive { + // a: Box>, + // } + // ``` + // It can produce an ADT stack like this, + // - Box + // - Recursive + // - Box> + && stack[ancestor_index + 1..stack.len()].iter().all( + |(_, intermediate_args)| + intermediate_args + .iter() + .filter_map(|arg| arg.as_type()) + .any(|mid_arg| mid_arg.contains(ancestor_arg)) + )) + }) + }; if is_expanding_recursive { // FIXME: indicate that this is an expanding recursive type in stub metadata? return DINodeCreationResult::new(stub_info.metadata, false); diff --git a/tests/debuginfo/recursive-struct.rs b/tests/debuginfo/recursive-struct.rs index 5be90992848..427a7100a4f 100644 --- a/tests/debuginfo/recursive-struct.rs +++ b/tests/debuginfo/recursive-struct.rs @@ -62,6 +62,7 @@ use self::Opt::{Empty, Val}; use std::boxed::Box as B; +use std::marker::PhantomData; enum Opt { Empty, @@ -98,6 +99,11 @@ struct LongCycleWithAnonymousTypes { value: usize, } +struct Expanding { + a: PhantomData, + b: *const Expanding<(T, T)>, +} + // This test case makes sure that recursive structs are properly described. The Node structs are // generic so that we can have a new type (that newly needs to be described) for the different // cases. The potential problem with recursive types is that the DI generation algorithm gets @@ -205,6 +211,9 @@ fn main() { value: 30 }))))); + // This type can generate new instances infinitely if not handled properly. + std::hint::black_box(Expanding::<()> { a: PhantomData, b: std::ptr::null() }); + zzz(); // #break } -- cgit 1.4.1-3-g733a5 From d4eb0947f133d8f44594d7551d99a48daff21c02 Mon Sep 17 00:00:00 2001 From: Cameron Steffen Date: Tue, 12 Aug 2025 17:57:02 -0500 Subject: Cleanup assoc parent utils --- .../src/diagnostics/mutability_errors.rs | 3 +- compiler/rustc_borrowck/src/type_check/mod.rs | 5 +-- compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 43 ++++++++++------------ compiler/rustc_const_eval/src/interpret/call.rs | 2 +- .../src/check/compare_impl_item.rs | 6 +-- compiler/rustc_hir_analysis/src/check/wfcheck.rs | 3 +- compiler/rustc_lint/src/pass_by_value.rs | 6 +-- compiler/rustc_lint/src/types.rs | 5 +-- compiler/rustc_middle/src/middle/privacy.rs | 6 +-- compiler/rustc_middle/src/ty/mod.rs | 39 +++++++++++++++++--- .../src/check_call_recursion.rs | 4 +- .../rustc_mir_transform/src/check_packed_ref.rs | 2 +- compiler/rustc_mir_transform/src/shim.rs | 2 +- compiler/rustc_monomorphize/src/partitioning.rs | 13 ++++--- compiler/rustc_passes/src/dead.rs | 2 +- .../rustc_trait_selection/src/traits/project.rs | 2 +- 16 files changed, 79 insertions(+), 64 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs index 5d9416b59fc..c0ca35f9ff8 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs @@ -677,12 +677,13 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { /// - is the trait from the local crate? If not, we can't suggest changing signatures /// - `Span` of the argument in the trait definition fn is_error_in_trait(&self, local: Local) -> (bool, bool, Option) { + let tcx = self.infcx.tcx; if self.body.local_kind(local) != LocalKind::Arg { return (false, false, None); } let my_def = self.body.source.def_id(); let Some(td) = - self.infcx.tcx.impl_of_assoc(my_def).and_then(|x| self.infcx.tcx.trait_id_of_impl(x)) + tcx.trait_impl_of_assoc(my_def).and_then(|id| self.infcx.tcx.trait_id_of_impl(id)) else { return (false, false, None); }; diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index c3aa205d5aa..a960b96b91c 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1773,10 +1773,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { locations, ); - assert!(!matches!( - tcx.impl_of_assoc(def_id).map(|imp| tcx.def_kind(imp)), - Some(DefKind::Impl { of_trait: true }) - )); + assert_eq!(tcx.trait_impl_of_assoc(def_id), None); self.prove_predicates( args.types().map(|ty| ty::ClauseKind::WellFormed(ty.into())), locations, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 6cbf2dbf7d3..2c3a84499ac 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -533,31 +533,26 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - if let Some(impl_def_id) = cx.tcx.impl_of_assoc(instance.def_id()) { - // If the method does *not* belong to a trait, proceed - if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( - instance.args, - cx.typing_env(), - cx.tcx.type_of(impl_def_id), - ); - - // Only "class" methods are generally understood by LLVM, - // so avoid methods on other types (e.g., `<*mut T>::null`). - if let ty::Adt(def, ..) = impl_self_ty.kind() - && !def.is_box() - { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() - { - return (type_di_node(cx, impl_self_ty), true); - } else { - return (namespace::item_namespace(cx, def.did()), false); - } + // For trait method impls we still use the "parallel namespace" + // strategy + if let Some(imp_def_id) = cx.tcx.inherent_impl_of_assoc(instance.def_id()) { + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + cx.tcx.type_of(imp_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - } else { - // For trait method impls we still use the "parallel namespace" - // strategy } } diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index b1cc0cc2878..21237cfe492 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -731,7 +731,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) { let tcx = *self.tcx; - let trait_def_id = tcx.trait_of_assoc(def_id).unwrap(); + let trait_def_id = tcx.parent(def_id); let virtual_trait_ref = ty::TraitRef::from_assoc(tcx, trait_def_id, virtual_instance.args); let existential_trait_ref = ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref); let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty); diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs index 6767e5ed88d..e4827256193 100644 --- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs +++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs @@ -445,10 +445,10 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>( tcx: TyCtxt<'tcx>, impl_m_def_id: LocalDefId, ) -> Result<&'tcx DefIdMap>>, ErrorGuaranteed> { - let impl_m = tcx.opt_associated_item(impl_m_def_id.to_def_id()).unwrap(); - let trait_m = tcx.opt_associated_item(impl_m.trait_item_def_id.unwrap()).unwrap(); + let impl_m = tcx.associated_item(impl_m_def_id.to_def_id()); + let trait_m = tcx.associated_item(impl_m.trait_item_def_id.unwrap()); let impl_trait_ref = - tcx.impl_trait_ref(impl_m.impl_container(tcx).unwrap()).unwrap().instantiate_identity(); + tcx.impl_trait_ref(tcx.parent(impl_m_def_id.to_def_id())).unwrap().instantiate_identity(); // First, check a few of the same things as `compare_impl_method`, // just so we don't ICE during instantiation later. check_method_is_structurally_compatible(tcx, impl_m, trait_m, impl_trait_ref, true)?; diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs index c642435b989..e6a1f6d8d8b 100644 --- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs +++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs @@ -2287,8 +2287,7 @@ fn lint_redundant_lifetimes<'tcx>( // Proceed } DefKind::AssocFn | DefKind::AssocTy | DefKind::AssocConst => { - let parent_def_id = tcx.local_parent(owner_id); - if matches!(tcx.def_kind(parent_def_id), DefKind::Impl { of_trait: true }) { + if tcx.trait_impl_of_assoc(owner_id.to_def_id()).is_some() { // Don't check for redundant lifetimes for associated items of trait // implementations, since the signature is required to be compatible // with the trait, even if the implementation implies some lifetimes diff --git a/compiler/rustc_lint/src/pass_by_value.rs b/compiler/rustc_lint/src/pass_by_value.rs index 4f65acd8001..29006732aad 100644 --- a/compiler/rustc_lint/src/pass_by_value.rs +++ b/compiler/rustc_lint/src/pass_by_value.rs @@ -24,10 +24,8 @@ impl<'tcx> LateLintPass<'tcx> for PassByValue { fn check_ty(&mut self, cx: &LateContext<'_>, ty: &'tcx hir::Ty<'tcx, AmbigArg>) { match &ty.kind { TyKind::Ref(_, hir::MutTy { ty: inner_ty, mutbl: hir::Mutability::Not }) => { - if let Some(impl_did) = cx.tcx.impl_of_assoc(ty.hir_id.owner.to_def_id()) { - if cx.tcx.impl_trait_ref(impl_did).is_some() { - return; - } + if cx.tcx.trait_impl_of_assoc(ty.hir_id.owner.to_def_id()).is_some() { + return; } if let Some(t) = path_for_pass_by_value(cx, inner_ty) { cx.emit_span_lint( diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs index b0afc333ebe..f8a692313f0 100644 --- a/compiler/rustc_lint/src/types.rs +++ b/compiler/rustc_lint/src/types.rs @@ -1904,10 +1904,9 @@ impl InvalidAtomicOrdering { if let ExprKind::MethodCall(method_path, _, args, _) = &expr.kind && recognized_names.contains(&method_path.ident.name) && let Some(m_def_id) = cx.typeck_results().type_dependent_def_id(expr.hir_id) - && let Some(impl_did) = cx.tcx.impl_of_assoc(m_def_id) - && let Some(adt) = cx.tcx.type_of(impl_did).instantiate_identity().ty_adt_def() // skip extension traits, only lint functions from the standard library - && cx.tcx.trait_id_of_impl(impl_did).is_none() + && let Some(impl_did) = cx.tcx.inherent_impl_of_assoc(m_def_id) + && let Some(adt) = cx.tcx.type_of(impl_did).instantiate_identity().ty_adt_def() && let parent = cx.tcx.parent(adt.did()) && cx.tcx.is_diagnostic_item(sym::atomic_mod, parent) && ATOMIC_TYPES.contains(&cx.tcx.item_name(adt.did())) diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs index 785ddd1ee29..e3e04c9d180 100644 --- a/compiler/rustc_middle/src/middle/privacy.rs +++ b/compiler/rustc_middle/src/middle/privacy.rs @@ -181,11 +181,7 @@ impl EffectiveVisibilities { // nominal visibility. For some items nominal visibility doesn't make sense so we // don't check this condition for them. let is_impl = matches!(tcx.def_kind(def_id), DefKind::Impl { .. }); - let is_associated_item_in_trait_impl = tcx - .impl_of_assoc(def_id.to_def_id()) - .and_then(|impl_id| tcx.trait_id_of_impl(impl_id)) - .is_some(); - if !is_impl && !is_associated_item_in_trait_impl { + if !is_impl && tcx.trait_impl_of_assoc(def_id.to_def_id()).is_none() { let nominal_vis = tcx.visibility(def_id); if !nominal_vis.is_at_least(ev.reachable, tcx) { span_bug!( diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 73e1661106e..e70c98ab704 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1925,21 +1925,50 @@ impl<'tcx> TyCtxt<'tcx> { self.impl_trait_ref(def_id).map(|tr| tr.skip_binder().def_id) } - /// If the given `DefId` is an associated item, returns the `DefId` of the parent trait or impl. - pub fn assoc_parent(self, def_id: DefId) -> Option { - self.def_kind(def_id).is_assoc().then(|| self.parent(def_id)) + /// If the given `DefId` is an associated item, returns the `DefId` and `DefKind` of the parent trait or impl. + pub fn assoc_parent(self, def_id: DefId) -> Option<(DefId, DefKind)> { + if !self.def_kind(def_id).is_assoc() { + return None; + } + let parent = self.parent(def_id); + let def_kind = self.def_kind(parent); + Some((parent, def_kind)) } /// If the given `DefId` is an associated item of a trait, /// returns the `DefId` of the trait; otherwise, returns `None`. pub fn trait_of_assoc(self, def_id: DefId) -> Option { - self.assoc_parent(def_id).filter(|id| self.def_kind(id) == DefKind::Trait) + match self.assoc_parent(def_id) { + Some((id, DefKind::Trait)) => Some(id), + _ => None, + } } /// If the given `DefId` is an associated item of an impl, /// returns the `DefId` of the impl; otherwise returns `None`. pub fn impl_of_assoc(self, def_id: DefId) -> Option { - self.assoc_parent(def_id).filter(|id| matches!(self.def_kind(id), DefKind::Impl { .. })) + match self.assoc_parent(def_id) { + Some((id, DefKind::Impl { .. })) => Some(id), + _ => None, + } + } + + /// If the given `DefId` is an associated item of an inherent impl, + /// returns the `DefId` of the impl; otherwise, returns `None`. + pub fn inherent_impl_of_assoc(self, def_id: DefId) -> Option { + match self.assoc_parent(def_id) { + Some((id, DefKind::Impl { of_trait: false })) => Some(id), + _ => None, + } + } + + /// If the given `DefId` is an associated item of a trait impl, + /// returns the `DefId` of the impl; otherwise, returns `None`. + pub fn trait_impl_of_assoc(self, def_id: DefId) -> Option { + match self.assoc_parent(def_id) { + Some((id, DefKind::Impl { of_trait: true })) => Some(id), + _ => None, + } } pub fn is_exportable(self, def_id: DefId) -> bool { diff --git a/compiler/rustc_mir_transform/src/check_call_recursion.rs b/compiler/rustc_mir_transform/src/check_call_recursion.rs index 6d61ac2dd80..a9acb1da5a3 100644 --- a/compiler/rustc_mir_transform/src/check_call_recursion.rs +++ b/compiler/rustc_mir_transform/src/check_call_recursion.rs @@ -43,8 +43,8 @@ impl<'tcx> MirLint<'tcx> for CheckDropRecursion { // First check if `body` is an `fn drop()` of `Drop` if let DefKind::AssocFn = tcx.def_kind(def_id) - && let Some(trait_ref) = - tcx.impl_of_assoc(def_id.to_def_id()).and_then(|def_id| tcx.impl_trait_ref(def_id)) + && let Some(impl_id) = tcx.trait_impl_of_assoc(def_id.to_def_id()) + && let trait_ref = tcx.impl_trait_ref(impl_id).unwrap() && tcx.is_lang_item(trait_ref.instantiate_identity().def_id, LangItem::Drop) // avoid erroneous `Drop` impls from causing ICEs below && let sig = tcx.fn_sig(def_id).instantiate_identity() diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index dcb812c7899..100104e9de0 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -40,7 +40,7 @@ impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> { if context.is_borrow() && util::is_disaligned(self.tcx, self.body, self.typing_env, *place) { let def_id = self.body.source.instance.def_id(); - if let Some(impl_def_id) = self.tcx.impl_of_assoc(def_id) + if let Some(impl_def_id) = self.tcx.trait_impl_of_assoc(def_id) && self.tcx.is_builtin_derived(impl_def_id) { // If we ever reach here it means that the generated derive diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index c687036f544..c6760b3583f 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -75,7 +75,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id)) } ty::InstanceKind::FnPtrShim(def_id, ty) => { - let trait_ = tcx.trait_of_assoc(def_id).unwrap(); + let trait_ = tcx.parent(def_id); // Supports `Fn` or `async Fn` traits. let adjustment = match tcx .fn_trait_kind_from_def_id(trait_) diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index d76b27d9970..54d77b182f8 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -650,17 +650,18 @@ fn characteristic_def_id_of_mono_item<'tcx>( // its self-type. If the self-type does not provide a characteristic // DefId, we use the location of the impl after all. - if tcx.trait_of_assoc(def_id).is_some() { + let assoc_parent = tcx.assoc_parent(def_id); + + if let Some((_, DefKind::Trait)) = assoc_parent { let self_ty = instance.args.type_at(0); // This is a default implementation of a trait method. return characteristic_def_id_of_type(self_ty).or(Some(def_id)); } - if let Some(impl_def_id) = tcx.impl_of_assoc(def_id) { - if tcx.sess.opts.incremental.is_some() - && tcx - .trait_id_of_impl(impl_def_id) - .is_some_and(|def_id| tcx.is_lang_item(def_id, LangItem::Drop)) + if let Some((impl_def_id, DefKind::Impl { of_trait })) = assoc_parent { + if of_trait + && tcx.sess.opts.incremental.is_some() + && tcx.is_lang_item(tcx.trait_id_of_impl(impl_def_id).unwrap(), LangItem::Drop) { // Put `Drop::drop` into the same cgu as `drop_in_place` // since `drop_in_place` is the only thing that can diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs index de52973acbb..08d06402000 100644 --- a/compiler/rustc_passes/src/dead.rs +++ b/compiler/rustc_passes/src/dead.rs @@ -371,7 +371,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> { /// will be ignored for the purposes of dead code analysis (see PR #85200 /// for discussion). fn should_ignore_item(&mut self, def_id: DefId) -> bool { - if let Some(impl_of) = self.tcx.impl_of_assoc(def_id) { + if let Some(impl_of) = self.tcx.trait_impl_of_assoc(def_id) { if !self.tcx.is_automatically_derived(impl_of) { return false; } diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index 581191b2036..884d53732fe 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -1507,7 +1507,7 @@ fn confirm_builtin_candidate<'cx, 'tcx>( let tcx = selcx.tcx(); let self_ty = obligation.predicate.self_ty(); let item_def_id = obligation.predicate.def_id; - let trait_def_id = tcx.trait_of_assoc(item_def_id).unwrap(); + let trait_def_id = tcx.parent(item_def_id); let args = tcx.mk_args(&[self_ty.into()]); let (term, obligations) = if tcx.is_lang_item(trait_def_id, LangItem::DiscriminantKind) { let discriminant_def_id = -- cgit 1.4.1-3-g733a5 From 04ff1444bb89379d7489bda9edc65ebaae4db037 Mon Sep 17 00:00:00 2001 From: winstonallo Date: Wed, 13 Aug 2025 08:51:58 +0200 Subject: Set NumRegisterParameters LLVM module flag to `N` when `-Zregparm=N` is set * Enforce the `-Zregparm=N` flag by setting the NumRegisterParameters LLVM module flag * Add assembly tests verifying that the parameters are passed in registers for reparm values 1, 2, and 3, for both LLVM intrinsics and non-builtin functions * Add c_void type to minicore --- compiler/rustc_codegen_llvm/src/context.rs | 9 ++++ tests/assembly-llvm/regparm-module-flag.rs | 70 ++++++++++++++++++++++++++++++ tests/auxiliary/minicore.rs | 7 +++ 3 files changed, 86 insertions(+) create mode 100644 tests/assembly-llvm/regparm-module-flag.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ee77774c688..215508f8095 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -372,6 +372,15 @@ pub(crate) unsafe fn create_module<'ll>( } } + if let Some(regparm_count) = sess.opts.unstable_opts.regparm { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "NumRegisterParameters", + regparm_count, + ); + } + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch == "aarch64" { llvm::add_module_flag_u32( diff --git a/tests/assembly-llvm/regparm-module-flag.rs b/tests/assembly-llvm/regparm-module-flag.rs new file mode 100644 index 00000000000..67ef44285ea --- /dev/null +++ b/tests/assembly-llvm/regparm-module-flag.rs @@ -0,0 +1,70 @@ +// Test the regparm ABI with builtin and non-builtin calls +// Issue: https://github.com/rust-lang/rust/issues/145271 +//@ add-core-stubs +//@ assembly-output: emit-asm +//@ compile-flags: -O --target=i686-unknown-linux-gnu -Crelocation-model=static +//@ revisions: REGPARM1 REGPARM2 REGPARM3 +//@[REGPARM1] compile-flags: -Zregparm=1 +//@[REGPARM2] compile-flags: -Zregparm=2 +//@[REGPARM3] compile-flags: -Zregparm=3 +//@ needs-llvm-components: x86 +#![feature(no_core)] +#![no_std] +#![no_core] +#![crate_type = "lib"] + +extern crate minicore; +use minicore::*; + +unsafe extern "C" { + fn memset(p: *mut c_void, val: i32, len: usize) -> *mut c_void; + fn non_builtin_memset(p: *mut c_void, val: i32, len: usize) -> *mut c_void; +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn entrypoint(len: usize, ptr: *mut c_void, val: i32) -> *mut c_void { + // REGPARM1-LABEL: entrypoint + // REGPARM1: movl %e{{.*}}, %ecx + // REGPARM1: pushl + // REGPARM1: pushl + // REGPARM1: calll memset + + // REGPARM2-LABEL: entrypoint + // REGPARM2: movl 16(%esp), %edx + // REGPARM2: movl %e{{.*}}, (%esp) + // REGPARM2: movl %e{{.*}}, %eax + // REGPARM2: calll memset + + // REGPARM3-LABEL: entrypoint + // REGPARM3: movl %e{{.*}}, %esi + // REGPARM3: movl %e{{.*}}, %eax + // REGPARM3: movl %e{{.*}}, %ecx + // REGPARM3: jmp memset + unsafe { memset(ptr, val, len) } +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn non_builtin_entrypoint( + len: usize, + ptr: *mut c_void, + val: i32, +) -> *mut c_void { + // REGPARM1-LABEL: non_builtin_entrypoint + // REGPARM1: movl %e{{.*}}, %ecx + // REGPARM1: pushl + // REGPARM1: pushl + // REGPARM1: calll non_builtin_memset + + // REGPARM2-LABEL: non_builtin_entrypoint + // REGPARM2: movl 16(%esp), %edx + // REGPARM2: movl %e{{.*}}, (%esp) + // REGPARM2: movl %e{{.*}}, %eax + // REGPARM2: calll non_builtin_memset + + // REGPARM3-LABEL: non_builtin_entrypoint + // REGPARM3: movl %e{{.*}}, %esi + // REGPARM3: movl %e{{.*}}, %eax + // REGPARM3: movl %e{{.*}}, %ecx + // REGPARM3: jmp non_builtin_memset + unsafe { non_builtin_memset(ptr, val, len) } +} diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs index 47dadd51ce0..da880100a10 100644 --- a/tests/auxiliary/minicore.rs +++ b/tests/auxiliary/minicore.rs @@ -225,3 +225,10 @@ pub mod mem { #[rustc_intrinsic] pub unsafe fn transmute(src: Src) -> Dst; } + +#[lang = "c_void"] +#[repr(u8)] +pub enum c_void { + __variant1, + __variant2, +} -- cgit 1.4.1-3-g733a5 From f978932903cac8cf508ef78b8133fafd9fe5a5b8 Mon Sep 17 00:00:00 2001 From: StackOverflowExcept1on <109800286+StackOverflowExcept1on@users.noreply.github.com> Date: Tue, 12 Aug 2025 00:39:24 +0300 Subject: fix(compiler/rustc_codegen_llvm): apply `target-cpu` attribute --- Cargo.lock | 2 +- compiler/rustc_codegen_llvm/src/allocator.rs | 17 +++++++++- src/tools/run-make-support/Cargo.toml | 2 +- tests/run-make/wasm-unexpected-features/rmake.rs | 38 ++++++++++++++++++++++ .../wasm32_test/Cargo.toml | 13 ++++++++ .../wasm32_test/src/lib.rs | 34 +++++++++++++++++++ 6 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/run-make/wasm-unexpected-features/rmake.rs create mode 100644 tests/run-make/wasm-unexpected-features/wasm32_test/Cargo.toml create mode 100644 tests/run-make/wasm-unexpected-features/wasm32_test/src/lib.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/Cargo.lock b/Cargo.lock index 4eb246995b1..1a196172cdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3283,7 +3283,7 @@ dependencies = [ "regex", "serde_json", "similar", - "wasmparser 0.219.2", + "wasmparser 0.236.0", ] [[package]] diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 2b5090ed6db..23610aa856c 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -8,11 +8,12 @@ use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; +use smallvec::SmallVec; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; use crate::llvm::{self, False, True, Type, Value}; -use crate::{SimpleCx, attributes, debuginfo}; +use crate::{SimpleCx, attributes, debuginfo, llvm_util}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, @@ -147,6 +148,20 @@ fn create_wrapper_function( llvm::Visibility::from_generic(tcx.sess.default_visibility()), ty, ); + + let mut attrs = SmallVec::<[_; 2]>::new(); + + let target_cpu = llvm_util::target_cpu(tcx.sess); + let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu); + + let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess) + .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)); + + attrs.push(target_cpu_attr); + attrs.extend(tune_cpu_attr); + + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + let no_return = if no_return { // -> ! DIFlagNoReturn let no_return = llvm::AttributeKind::NoReturn.create_attr(cx.llcx); diff --git a/src/tools/run-make-support/Cargo.toml b/src/tools/run-make-support/Cargo.toml index a4e7534137d..250e0f65a9f 100644 --- a/src/tools/run-make-support/Cargo.toml +++ b/src/tools/run-make-support/Cargo.toml @@ -17,7 +17,7 @@ object = "0.37" regex = "1.11" serde_json = "1.0" similar = "2.7" -wasmparser = { version = "0.219", default-features = false, features = ["std"] } +wasmparser = { version = "0.236", default-features = false, features = ["std", "features", "validate"] } # tidy-alphabetical-end # Shared with bootstrap and compiletest diff --git a/tests/run-make/wasm-unexpected-features/rmake.rs b/tests/run-make/wasm-unexpected-features/rmake.rs new file mode 100644 index 00000000000..142860c47ec --- /dev/null +++ b/tests/run-make/wasm-unexpected-features/rmake.rs @@ -0,0 +1,38 @@ +//@ only-wasm32-wasip1 + +use std::path::Path; + +use run_make_support::{cargo, path, rfs, target, wasmparser}; + +fn main() { + let target_dir = path("target"); + + cargo() + .args([ + "rustc", + "--manifest-path", + "wasm32_test/Cargo.toml", + "--profile", + "release", + "--target", + "wasm32-wasip1", + "-Zbuild-std=core,alloc,panic_abort", + "--", + "-Clink-arg=--import-memory", + "-Clinker-plugin-lto=on", + ]) + .env("RUSTFLAGS", "-Ctarget-cpu=mvp") + .env("CARGO_TARGET_DIR", &target_dir) + .run(); + + let wasm32_program_path = target_dir.join(target()).join("release").join("wasm32_program.wasm"); + verify_features(&wasm32_program_path); +} + +fn verify_features(path: &Path) { + eprintln!("verify {path:?}"); + let file = rfs::read(&path); + + let mut validator = wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::WASM1); + validator.validate_all(&file).unwrap(); +} diff --git a/tests/run-make/wasm-unexpected-features/wasm32_test/Cargo.toml b/tests/run-make/wasm-unexpected-features/wasm32_test/Cargo.toml new file mode 100644 index 00000000000..9f1070e1c54 --- /dev/null +++ b/tests/run-make/wasm-unexpected-features/wasm32_test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "wasm32_test" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] +name = "wasm32_program" + +[profile.release] +codegen-units = 1 +lto = "fat" +opt-level = "z" diff --git a/tests/run-make/wasm-unexpected-features/wasm32_test/src/lib.rs b/tests/run-make/wasm-unexpected-features/wasm32_test/src/lib.rs new file mode 100644 index 00000000000..123c2c1b6ca --- /dev/null +++ b/tests/run-make/wasm-unexpected-features/wasm32_test/src/lib.rs @@ -0,0 +1,34 @@ +#![no_std] + +extern crate alloc; + +use core::alloc::{GlobalAlloc, Layout}; +use core::mem::MaybeUninit; + +#[global_allocator] +static ALLOC: GlobalDlmalloc = GlobalDlmalloc; + +struct GlobalDlmalloc; + +unsafe impl GlobalAlloc for GlobalDlmalloc { + #[inline] + unsafe fn alloc(&self, _layout: Layout) -> *mut u8 { + core::ptr::null_mut() + } + + #[inline] + unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {} +} + +#[used] +static mut BUF: MaybeUninit<[u8; 1024]> = MaybeUninit::uninit(); + +#[unsafe(no_mangle)] +extern "C" fn init() { + alloc::alloc::handle_alloc_error(Layout::new::<[u8; 64 * 1024]>()); +} + +#[panic_handler] +fn my_panic(_: &core::panic::PanicInfo) -> ! { + loop {} +} -- cgit 1.4.1-3-g733a5 From d435197afcb019a692aa1faf8b7169e167ac1de8 Mon Sep 17 00:00:00 2001 From: Sasha Pourcelot Date: Tue, 12 Aug 2025 20:22:45 +0200 Subject: Port the `#[linkage]` attribute to the new attribute system --- .../src/attributes/link_attrs.rs | 76 ++++++++++++++++++++- compiler/rustc_attr_parsing/src/context.rs | 3 +- compiler/rustc_codegen_cranelift/src/constant.rs | 8 +-- compiler/rustc_codegen_cranelift/src/driver/aot.rs | 5 +- compiler/rustc_codegen_cranelift/src/linkage.rs | 3 +- compiler/rustc_codegen_gcc/src/base.rs | 2 +- compiler/rustc_codegen_gcc/src/consts.rs | 2 +- compiler/rustc_codegen_gcc/src/mono_item.rs | 3 +- compiler/rustc_codegen_llvm/src/base.rs | 3 +- compiler/rustc_codegen_llvm/src/consts.rs | 3 +- compiler/rustc_codegen_llvm/src/mono_item.rs | 3 +- .../rustc_codegen_ssa/src/back/symbol_export.rs | 3 +- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 77 +++++++--------------- compiler/rustc_codegen_ssa/src/mir/naked_asm.rs | 4 +- compiler/rustc_codegen_ssa/src/mono_item.rs | 3 +- compiler/rustc_codegen_ssa/src/traits/declare.rs | 3 +- compiler/rustc_hir/src/attrs/data_structures.rs | 21 ++++++ compiler/rustc_hir/src/attrs/encode_cross_crate.rs | 1 + compiler/rustc_hir/src/hir.rs | 1 + .../rustc_middle/src/middle/codegen_fn_attrs.rs | 3 +- compiler/rustc_middle/src/mir/mono.rs | 18 +---- compiler/rustc_monomorphize/src/partitioning.rs | 4 +- compiler/rustc_passes/src/check_attr.rs | 8 ++- compiler/rustc_span/src/symbol.rs | 8 +++ tests/ui/attributes/malformed-attrs.stderr | 37 +++++------ tests/ui/linkage-attr/linkage3.rs | 2 +- tests/ui/linkage-attr/linkage3.stderr | 27 ++++++-- 27 files changed, 207 insertions(+), 124 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs index d406c30b83e..e4ced2e37c5 100644 --- a/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs @@ -1,6 +1,6 @@ use rustc_feature::{AttributeTemplate, template}; -use rustc_hir::attrs::AttributeKind; use rustc_hir::attrs::AttributeKind::{LinkName, LinkOrdinal, LinkSection}; +use rustc_hir::attrs::{AttributeKind, Linkage}; use rustc_span::{Span, Symbol, sym}; use crate::attributes::{ @@ -129,3 +129,77 @@ impl SingleAttributeParser for LinkOrdinalParser { Some(LinkOrdinal { ordinal, span: cx.attr_span }) } } + +pub(crate) struct LinkageParser; + +impl SingleAttributeParser for LinkageParser { + const PATH: &[Symbol] = &[sym::linkage]; + + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + + const TEMPLATE: AttributeTemplate = template!(NameValueStr: [ + "available_externally", + "common", + "extern_weak", + "external", + "internal", + "linkonce", + "linkonce_odr", + "weak", + "weak_odr", + ]); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let Some(name_value) = args.name_value() else { + cx.expected_name_value(cx.attr_span, Some(sym::linkage)); + return None; + }; + + let Some(value) = name_value.value_as_str() else { + cx.expected_string_literal(name_value.value_span, Some(name_value.value_as_lit())); + return None; + }; + + // Use the names from src/llvm/docs/LangRef.rst here. Most types are only + // applicable to variable declarations and may not really make sense for + // Rust code in the first place but allow them anyway and trust that the + // user knows what they're doing. Who knows, unanticipated use cases may pop + // up in the future. + // + // ghost, dllimport, dllexport and linkonce_odr_autohide are not supported + // and don't have to be, LLVM treats them as no-ops. + let linkage = match value { + sym::available_externally => Linkage::AvailableExternally, + sym::common => Linkage::Common, + sym::extern_weak => Linkage::ExternalWeak, + sym::external => Linkage::External, + sym::internal => Linkage::Internal, + sym::linkonce => Linkage::LinkOnceAny, + sym::linkonce_odr => Linkage::LinkOnceODR, + sym::weak => Linkage::WeakAny, + sym::weak_odr => Linkage::WeakODR, + + _ => { + cx.expected_specific_argument( + name_value.value_span, + vec![ + "available_externally", + "common", + "extern_weak", + "external", + "internal", + "linkonce", + "linkonce_odr", + "weak", + "weak_odr", + ], + ); + return None; + } + }; + + Some(AttributeKind::Linkage(linkage, cx.attr_span)) + } +} diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs index 1420753a44e..6045a8b28a2 100644 --- a/compiler/rustc_attr_parsing/src/context.rs +++ b/compiler/rustc_attr_parsing/src/context.rs @@ -27,7 +27,7 @@ use crate::attributes::dummy::DummyParser; use crate::attributes::inline::{InlineParser, RustcForceInlineParser}; use crate::attributes::link_attrs::{ ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser, - LinkSectionParser, StdInternalSymbolParser, + LinkSectionParser, LinkageParser, StdInternalSymbolParser, }; use crate::attributes::lint_helpers::{ AsPtrParser, AutomaticallyDerivedParser, PassByValueParser, PubTransparentParser, @@ -167,6 +167,7 @@ attribute_parsers!( Single, Single, Single, + Single, Single, Single, Single, diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index bec546badc9..a56466750e7 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -281,8 +281,8 @@ fn data_id_for_static( .abi .bytes(); - let linkage = if import_linkage == rustc_middle::mir::mono::Linkage::ExternalWeak - || import_linkage == rustc_middle::mir::mono::Linkage::WeakAny + let linkage = if import_linkage == rustc_hir::attrs::Linkage::ExternalWeak + || import_linkage == rustc_hir::attrs::Linkage::WeakAny { Linkage::Preemptible } else { @@ -332,8 +332,8 @@ fn data_id_for_static( let linkage = if definition { crate::linkage::get_static_linkage(tcx, def_id) - } else if attrs.linkage == Some(rustc_middle::mir::mono::Linkage::ExternalWeak) - || attrs.linkage == Some(rustc_middle::mir::mono::Linkage::WeakAny) + } else if attrs.linkage == Some(rustc_hir::attrs::Linkage::ExternalWeak) + || attrs.linkage == Some(rustc_hir::attrs::Linkage::WeakAny) { Linkage::Preemptible } else { diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 8ec3599b63d..7e77781dc2f 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -18,12 +18,11 @@ use rustc_codegen_ssa::{ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; +use rustc_hir::attrs::Linkage as RLinkage; use rustc_metadata::fs::copy_to_stdout; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::mono::{ - CodegenUnit, Linkage as RLinkage, MonoItem, MonoItemData, Visibility, -}; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem, MonoItemData, Visibility}; use rustc_session::Session; use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType}; diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs index ca853aac158..d76ab9d0109 100644 --- a/compiler/rustc_codegen_cranelift/src/linkage.rs +++ b/compiler/rustc_codegen_cranelift/src/linkage.rs @@ -1,4 +1,5 @@ -use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; +use rustc_hir::attrs::Linkage as RLinkage; +use rustc_middle::mir::mono::{MonoItem, Visibility}; use crate::prelude::*; diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs index c105916bbb2..e9d72e457a0 100644 --- a/compiler/rustc_codegen_gcc/src/base.rs +++ b/compiler/rustc_codegen_gcc/src/base.rs @@ -8,8 +8,8 @@ use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::DebugInfoCodegenMethods; +use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; -use rustc_middle::mir::mono::Linkage; #[cfg(feature = "master")] use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs index 873f1f1951c..619277eba8b 100644 --- a/compiler/rustc_codegen_gcc/src/consts.rs +++ b/compiler/rustc_codegen_gcc/src/consts.rs @@ -5,13 +5,13 @@ use rustc_abi::{self as abi, Align, HasDataLayout, Primitive, Size, WrappingRang use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, }; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint, }; -use rustc_middle::mir::mono::Linkage; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; diff --git a/compiler/rustc_codegen_gcc/src/mono_item.rs b/compiler/rustc_codegen_gcc/src/mono_item.rs index ff188c437da..35d44d21bcb 100644 --- a/compiler/rustc_codegen_gcc/src/mono_item.rs +++ b/compiler/rustc_codegen_gcc/src/mono_item.rs @@ -1,11 +1,12 @@ #[cfg(feature = "master")] use gccjit::{FnAttribute, VarAttribute}; use rustc_codegen_ssa::traits::PreDefineCodegenMethods; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5dda836988c..9cc5d8dbc21 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -18,9 +18,10 @@ use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; use rustc_session::config::DebugInfo; use rustc_span::Symbol; diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 6b06daf3477..9ec7b0f80ae 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -4,6 +4,7 @@ use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -11,7 +12,7 @@ use rustc_middle::mir::interpret::{ Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar, read_target_uint, }; -use rustc_middle::mir::mono::{Linkage, MonoItem}; +use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f9edaded60d..5075befae8a 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,8 +1,9 @@ use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 0494666bda9..77096822fdc 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -306,7 +306,8 @@ fn exported_generic_symbols_provider_local<'tcx>( let mut symbols: Vec<_> = vec![]; if tcx.local_crate_exports_generics() { - use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; + use rustc_hir::attrs::Linkage; + use rustc_middle::mir::mono::{MonoItem, Visibility}; use rustc_middle::ty::InstanceKind; // Normally, we require that shared monomorphizations are not hidden, diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 287787eb3d1..a36a772bc97 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -11,7 +11,6 @@ use rustc_hir::{self as hir, Attribute, LangItem, find_attr, lang_items}; use rustc_middle::middle::codegen_fn_attrs::{ CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, }; -use rustc_middle::mir::mono::Linkage; use rustc_middle::query::Providers; use rustc_middle::span_bug; use rustc_middle::ty::{self as ty, TyCtxt}; @@ -26,31 +25,6 @@ use crate::target_features::{ check_target_feature_trait_unsafe, check_tied_features, from_target_feature_attr, }; -fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage { - use rustc_middle::mir::mono::Linkage::*; - - // Use the names from src/llvm/docs/LangRef.rst here. Most types are only - // applicable to variable declarations and may not really make sense for - // Rust code in the first place but allow them anyway and trust that the - // user knows what they're doing. Who knows, unanticipated use cases may pop - // up in the future. - // - // ghost, dllimport, dllexport and linkonce_odr_autohide are not supported - // and don't have to be, LLVM treats them as no-ops. - match name { - "available_externally" => AvailableExternally, - "common" => Common, - "extern_weak" => ExternalWeak, - "external" => External, - "internal" => Internal, - "linkonce" => LinkOnceAny, - "linkonce_odr" => LinkOnceODR, - "weak" => WeakAny, - "weak_odr" => WeakODR, - _ => tcx.dcx().span_fatal(tcx.def_span(def_id), "invalid linkage specified"), - } -} - /// In some cases, attributes are only valid on functions, but it's the `check_attr` /// pass that checks that they aren't used anywhere else, rather than this module. /// In these cases, we bail from performing further checks that are only meaningful for @@ -103,13 +77,6 @@ fn parse_instruction_set_attr(tcx: TyCtxt<'_>, attr: &Attribute) -> Option, did: LocalDefId, attr: &Attribute) -> Option { - let val = attr.value_str()?; - let linkage = linkage_by_name(tcx, did, val.as_str()); - Some(linkage) -} - // FIXME(jdonszelmann): remove when no_sanitize becomes a parsed attr fn parse_no_sanitize_attr(tcx: TyCtxt<'_>, attr: &Attribute) -> Option { let list = attr.meta_item_list()?; @@ -332,6 +299,28 @@ fn process_builtin_attrs( AttributeKind::StdInternalSymbol(_) => { codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL } + AttributeKind::Linkage(linkage, _) => { + let linkage = Some(*linkage); + + if tcx.is_foreign_item(did) { + codegen_fn_attrs.import_linkage = linkage; + + if tcx.is_mutable_static(did.into()) { + let mut diag = tcx.dcx().struct_span_err( + attr.span(), + "extern mutable statics are not allowed with `#[linkage]`", + ); + diag.note( + "marking the extern static mutable would allow changing which \ + symbol the static references rather than make the target of the \ + symbol mutable", + ); + diag.emit(); + } + } else { + codegen_fn_attrs.linkage = linkage; + } + } _ => {} } } @@ -349,28 +338,6 @@ fn process_builtin_attrs( codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED } sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL, - sym::linkage => { - let linkage = parse_linkage_attr(tcx, did, attr); - - if tcx.is_foreign_item(did) { - codegen_fn_attrs.import_linkage = linkage; - - if tcx.is_mutable_static(did.into()) { - let mut diag = tcx.dcx().struct_span_err( - attr.span(), - "extern mutable statics are not allowed with `#[linkage]`", - ); - diag.note( - "marking the extern static mutable would allow changing which \ - symbol the static references rather than make the target of the \ - symbol mutable", - ); - diag.emit(); - } - } else { - codegen_fn_attrs.linkage = linkage; - } - } sym::no_sanitize => { interesting_spans.no_sanitize = Some(attr.span()); codegen_fn_attrs.no_sanitize |= diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index 2a9b5c9019b..31784cabf4a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -1,6 +1,6 @@ use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind}; -use rustc_hir::attrs::InstructionSetAttr; -use rustc_middle::mir::mono::{Linkage, MonoItemData, Visibility}; +use rustc_hir::attrs::{InstructionSetAttr, Linkage}; +use rustc_middle::mir::mono::{MonoItemData, Visibility}; use rustc_middle::mir::{InlineAsmOperand, START_BLOCK}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::{Instance, Ty, TyCtxt, TypeVisitableExt}; diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index b9040c330fb..8f03dc1e6b5 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -1,5 +1,6 @@ +use rustc_hir::attrs::Linkage; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility}; +use rustc_middle::mir::mono::{MonoItem, MonoItemData, Visibility}; use rustc_middle::ty::layout::HasTyCtxt; use tracing::debug; diff --git a/compiler/rustc_codegen_ssa/src/traits/declare.rs b/compiler/rustc_codegen_ssa/src/traits/declare.rs index 9f735546558..8d5f0a5b939 100644 --- a/compiler/rustc_codegen_ssa/src/traits/declare.rs +++ b/compiler/rustc_codegen_ssa/src/traits/declare.rs @@ -1,5 +1,6 @@ +use rustc_hir::attrs::Linkage; use rustc_hir::def_id::DefId; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::Instance; pub trait PreDefineCodegenMethods<'tcx> { diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs index e02edf5fe24..510fc832978 100644 --- a/compiler/rustc_hir/src/attrs/data_structures.rs +++ b/compiler/rustc_hir/src/attrs/data_structures.rs @@ -187,6 +187,24 @@ pub enum CfgEntry { Version(Option, Span), } +/// Possible values for the `#[linkage]` attribute, allowing to specify the +/// linkage type for a `MonoItem`. +/// +/// See for more details about these variants. +#[derive(Encodable, Decodable, Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive(HashStable_Generic, PrintAttribute)] +pub enum Linkage { + AvailableExternally, + Common, + ExternalWeak, + External, + Internal, + LinkOnceAny, + LinkOnceODR, + WeakAny, + WeakODR, +} + /// Represents parsed *built-in* inert attributes. /// /// ## Overview @@ -360,6 +378,9 @@ pub enum AttributeKind { /// Represents [`#[link_section]`](https://doc.rust-lang.org/reference/abi.html#the-link_section-attribute) LinkSection { name: Symbol, span: Span }, + /// Represents `#[linkage]`. + Linkage(Linkage, Span), + /// Represents `#[loop_match]`. LoopMatch(Span), diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs index 7ce624dcc55..84a975523f2 100644 --- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs +++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs @@ -46,6 +46,7 @@ impl AttributeKind { LinkName { .. } => Yes, // Needed for rustdoc LinkOrdinal { .. } => No, LinkSection { .. } => Yes, // Needed for rustdoc + Linkage(..) => No, LoopMatch(..) => No, MacroEscape(..) => No, MacroTransparency(..) => Yes, diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index b27c223527e..260cfc1052b 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -1311,6 +1311,7 @@ impl AttributeExt for Attribute { Attribute::Parsed(AttributeKind::ShouldPanic { span, .. }) => *span, Attribute::Parsed(AttributeKind::AutomaticallyDerived(span)) => *span, Attribute::Parsed(AttributeKind::AllowInternalUnsafe(span)) => *span, + Attribute::Parsed(AttributeKind::Linkage(_, span)) => *span, a => panic!("can't get the span of an arbitrary parsed attribute: {a:?}"), } } diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs index 52341df0740..2852c4cbd34 100644 --- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs +++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs @@ -2,13 +2,12 @@ use std::borrow::Cow; use rustc_abi::Align; use rustc_ast::expand::autodiff_attrs::AutoDiffAttrs; -use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; +use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, Linkage, OptimizeAttr}; use rustc_hir::def_id::DefId; use rustc_macros::{HashStable, TyDecodable, TyEncodable}; use rustc_span::Symbol; use rustc_target::spec::SanitizerSet; -use crate::mir::mono::Linkage; use crate::ty::{InstanceKind, TyCtxt}; impl<'tcx> TyCtxt<'tcx> { diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 0d98e055d95..440771b3d68 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -10,7 +10,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHas use rustc_data_structures::unord::UnordMap; use rustc_hashes::Hash128; use rustc_hir::ItemId; -use rustc_hir::attrs::InlineAttr; +use rustc_hir::attrs::{InlineAttr, Linkage}; use rustc_hir::def_id::{CrateNum, DefId, DefIdSet, LOCAL_CRATE}; use rustc_macros::{HashStable, TyDecodable, TyEncodable}; use rustc_query_system::ich::StableHashingContext; @@ -368,22 +368,6 @@ pub struct MonoItemData { pub size_estimate: usize, } -/// Specifies the linkage type for a `MonoItem`. -/// -/// See for more details about these variants. -#[derive(Copy, Clone, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)] -pub enum Linkage { - External, - AvailableExternally, - LinkOnceAny, - LinkOnceODR, - WeakAny, - WeakODR, - Internal, - ExternalWeak, - Common, -} - /// Specifies the symbol visibility with regards to dynamic linking. /// /// Visibility doesn't have any effect when linkage is internal. diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index d76b27d9970..c80d058c503 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -104,7 +104,7 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::sync; use rustc_data_structures::unord::{UnordMap, UnordSet}; use rustc_hir::LangItem; -use rustc_hir::attrs::InlineAttr; +use rustc_hir::attrs::{InlineAttr, Linkage}; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE}; use rustc_hir::definitions::DefPathDataName; @@ -112,7 +112,7 @@ use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_middle::mir::mono::{ - CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, MonoItem, MonoItemData, + CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, MonoItem, MonoItemData, MonoItemPartitions, Visibility, }; use rustc_middle::ty::print::{characteristic_def_id_of_type, with_no_trimmed_paths}; diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 165f8fe1995..df1db65c06e 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -332,6 +332,9 @@ impl<'tcx> CheckAttrVisitor<'tcx> { &Attribute::Parsed(AttributeKind::Coroutine(attr_span)) => { self.check_coroutine(attr_span, target) } + &Attribute::Parsed(AttributeKind::Linkage(_, attr_span)) => { + self.check_linkage(attr_span, span, target); + } Attribute::Unparsed(attr_item) => { style = Some(attr_item.style); match attr.path().as_slice() { @@ -395,7 +398,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> { [sym::autodiff_forward, ..] | [sym::autodiff_reverse, ..] => { self.check_autodiff(hir_id, attr, span, target) } - [sym::linkage, ..] => self.check_linkage(attr, span, target), [ // ok sym::allow @@ -2707,7 +2709,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { } } - fn check_linkage(&self, attr: &Attribute, span: Span, target: Target) { + fn check_linkage(&self, attr_span: Span, span: Span, target: Target) { match target { Target::Fn | Target::Method(..) @@ -2715,7 +2717,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { | Target::ForeignStatic | Target::ForeignFn => {} _ => { - self.dcx().emit_err(errors::Linkage { attr_span: attr.span(), span }); + self.dcx().emit_err(errors::Linkage { attr_span, span }); } } } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index acbed7a9eed..416ce27367e 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -545,6 +545,7 @@ symbols! { autodiff_forward, autodiff_reverse, automatically_derived, + available_externally, avx, avx10_target_feature, avx512_target_feature, @@ -676,6 +677,7 @@ symbols! { cold_path, collapse_debuginfo, column, + common, compare_bytes, compare_exchange, compare_exchange_weak, @@ -956,6 +958,7 @@ symbols! { extern_prelude, extern_system_varargs, extern_types, + extern_weak, external, external_doc, f, @@ -1213,6 +1216,7 @@ symbols! { instruction_set, integer_: "integer", // underscore to avoid clashing with the function `sym::integer` below integral, + internal, internal_features, into_async_iter_into_iter, into_future, @@ -1287,6 +1291,8 @@ symbols! { linkage, linker, linker_messages, + linkonce, + linkonce_odr, lint_reasons, literal, load, @@ -2360,6 +2366,8 @@ symbols! { wasm_abi, wasm_import_module, wasm_target_feature, + weak, + weak_odr, where_clause_attrs, while_let, width, diff --git a/tests/ui/attributes/malformed-attrs.stderr b/tests/ui/attributes/malformed-attrs.stderr index 705050e9a7d..aa4891459aa 100644 --- a/tests/ui/attributes/malformed-attrs.stderr +++ b/tests/ui/attributes/malformed-attrs.stderr @@ -90,25 +90,6 @@ error: malformed `cfi_encoding` attribute input LL | #[cfi_encoding] | ^^^^^^^^^^^^^^^ help: must be of the form: `#[cfi_encoding = "encoding"]` -error: malformed `linkage` attribute input - --> $DIR/malformed-attrs.rs:170:5 - | -LL | #[linkage] - | ^^^^^^^^^^ - | - = note: for more information, visit -help: the following are the possible correct uses - | -LL | #[linkage = "available_externally"] - | ++++++++++++++++++++++++ -LL | #[linkage = "common"] - | ++++++++++ -LL | #[linkage = "extern_weak"] - | +++++++++++++++ -LL | #[linkage = "external"] - | ++++++++++++ - = and 5 other candidates - error: malformed `allow` attribute input --> $DIR/malformed-attrs.rs:175:1 | @@ -659,6 +640,24 @@ LL | #[unsafe(ffi_const = 1)] | | didn't expect any arguments here | help: must be of the form: `#[ffi_const]` +error[E0539]: malformed `linkage` attribute input + --> $DIR/malformed-attrs.rs:170:5 + | +LL | #[linkage] + | ^^^^^^^^^^ expected this to be of the form `linkage = "..."` + | +help: try changing it to one of the following valid forms of the attribute + | +LL | #[linkage = "available_externally"] + | ++++++++++++++++++++++++ +LL | #[linkage = "common"] + | ++++++++++ +LL | #[linkage = "extern_weak"] + | +++++++++++++++ +LL | #[linkage = "external"] + | ++++++++++++ + = and 5 other candidates + error[E0565]: malformed `automatically_derived` attribute input --> $DIR/malformed-attrs.rs:188:1 | diff --git a/tests/ui/linkage-attr/linkage3.rs b/tests/ui/linkage-attr/linkage3.rs index f95e5eecc48..74a6fd6600b 100644 --- a/tests/ui/linkage-attr/linkage3.rs +++ b/tests/ui/linkage-attr/linkage3.rs @@ -5,7 +5,7 @@ extern "C" { #[linkage = "foo"] static foo: *const i32; -//~^ ERROR: invalid linkage specified +//~^^ ERROR: malformed `linkage` attribute input [E0539] } fn main() { diff --git a/tests/ui/linkage-attr/linkage3.stderr b/tests/ui/linkage-attr/linkage3.stderr index 5f7b7ef227c..f1215f09aea 100644 --- a/tests/ui/linkage-attr/linkage3.stderr +++ b/tests/ui/linkage-attr/linkage3.stderr @@ -1,8 +1,27 @@ -error: invalid linkage specified - --> $DIR/linkage3.rs:7:5 +error[E0539]: malformed `linkage` attribute input + --> $DIR/linkage3.rs:6:5 | -LL | static foo: *const i32; - | ^^^^^^^^^^^^^^^^^^^^^^ +LL | #[linkage = "foo"] + | ^^^^^^^^^^^^-----^ + | | + | valid arguments are `available_externally`, `common`, `extern_weak`, `external`, `internal`, `linkonce`, `linkonce_odr`, `weak` or `weak_odr` + | +help: try changing it to one of the following valid forms of the attribute + | +LL - #[linkage = "foo"] +LL + #[linkage = "available_externally"] + | +LL - #[linkage = "foo"] +LL + #[linkage = "common"] + | +LL - #[linkage = "foo"] +LL + #[linkage = "extern_weak"] + | +LL - #[linkage = "foo"] +LL + #[linkage = "external"] + | + = and 5 other candidates error: aborting due to 1 previous error +For more information about this error, try `rustc --explain E0539`. -- cgit 1.4.1-3-g733a5 From 5c631041aa0b0ad9e161b966b78e6dfdb8011023 Mon Sep 17 00:00:00 2001 From: Marcelo Domínguez Date: Thu, 14 Aug 2025 15:22:45 +0000 Subject: Basic implementation of `autodiff` intrinsic --- compiler/rustc_builtin_macros/src/autodiff.rs | 139 +++++++++- .../rustc_codegen_llvm/src/builder/autodiff.rs | 302 +++++---------------- compiler/rustc_codegen_llvm/src/context.rs | 2 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 76 +++++- compiler/rustc_hir_analysis/src/check/intrinsic.rs | 5 +- compiler/rustc_span/src/symbol.rs | 1 + library/core/src/intrinsics/mod.rs | 4 + 7 files changed, 284 insertions(+), 245 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs index a662840eda5..3f8585d35bc 100644 --- a/compiler/rustc_builtin_macros/src/autodiff.rs +++ b/compiler/rustc_builtin_macros/src/autodiff.rs @@ -329,17 +329,22 @@ mod llvm_enzyme { .filter(|a| **a == DiffActivity::Active || **a == DiffActivity::ActiveOnly) .count() as u32; let (d_sig, new_args, idents, errored) = gen_enzyme_decl(ecx, &sig, &x, span); - let d_body = gen_enzyme_body( + + // TODO(Sa4dUs): Remove this and all the related logic + let _d_body = gen_enzyme_body( ecx, &x, n_active, &sig, &d_sig, primal, &new_args, span, sig_span, idents, errored, &generics, ); + let d_body = + call_autodiff(ecx, primal, first_ident(&meta_item_vec[0]), span, &d_sig); + // The first element of it is the name of the function to be generated let asdf = Box::new(ast::Fn { defaultness: ast::Defaultness::Final, sig: d_sig, ident: first_ident(&meta_item_vec[0]), - generics, + generics: generics.clone(), contract: None, body: Some(d_body), define_opaque: None, @@ -428,12 +433,15 @@ mod llvm_enzyme { tokens: ts, }); + let vis_clone = vis.clone(); + + let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let d_attr = outer_normal_attr(&rustc_ad_attr, new_id, span); let d_annotatable = match &item { Annotatable::AssocItem(_, _) => { let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(asdf); let d_fn = Box::new(ast::AssocItem { - attrs: thin_vec![d_attr, inline_never], + attrs: thin_vec![d_attr], id: ast::DUMMY_NODE_ID, span, vis, @@ -443,13 +451,13 @@ mod llvm_enzyme { Annotatable::AssocItem(d_fn, Impl { of_trait: false }) } Annotatable::Item(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr, inline_never], ItemKind::Fn(asdf)); + let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(asdf)); d_fn.vis = vis; Annotatable::Item(d_fn) } Annotatable::Stmt(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr, inline_never], ItemKind::Fn(asdf)); + let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(asdf)); d_fn.vis = vis; Annotatable::Stmt(Box::new(ast::Stmt { @@ -463,7 +471,9 @@ mod llvm_enzyme { } }; - return vec![orig_annotatable, d_annotatable]; + let dummy_const_annotatable = gen_dummy_const(ecx, span, primal, sig, generics, vis_clone); + + return vec![orig_annotatable, dummy_const_annotatable, d_annotatable]; } // shadow arguments (the extra ones which were not in the original (primal) function), in reverse mode must be @@ -484,6 +494,123 @@ mod llvm_enzyme { ty } + // Generate `autodiff` intrinsic call + // ``` + // std::intrinsics::autodiff(source, diff, (args)) + // ``` + fn call_autodiff( + ecx: &ExtCtxt<'_>, + primal: Ident, + diff: Ident, + span: Span, + d_sig: &FnSig, + ) -> P { + let primal_path_expr = ecx.expr_path(ecx.path_ident(span, primal)); + let diff_path_expr = ecx.expr_path(ecx.path_ident(span, diff)); + + let tuple_expr = ecx.expr_tuple( + span, + d_sig + .decl + .inputs + .iter() + .map(|arg| match arg.pat.kind { + PatKind::Ident(_, ident, _) => ecx.expr_path(ecx.path_ident(span, ident)), + _ => todo!(), + }) + .collect::>() + .into(), + ); + + let enzyme_path = ecx.path( + span, + vec![ + Ident::from_str("std"), + Ident::from_str("intrinsics"), + Ident::from_str("autodiff"), + ], + ); + let call_expr = ecx.expr_call( + span, + ecx.expr_path(enzyme_path), + vec![primal_path_expr, diff_path_expr, tuple_expr].into(), + ); + + let block = ecx.block_expr(call_expr); + + block + } + + // Generate dummy const to prevent primal function + // from being optimized away before applying enzyme + // ``` + // const _: () = + // { + // #[used] + // pub static DUMMY_PTR: fn_type = primal_fn; + // }; + // ``` + fn gen_dummy_const( + ecx: &ExtCtxt<'_>, + span: Span, + primal: Ident, + sig: FnSig, + generics: Generics, + vis: Visibility, + ) -> Annotatable { + // #[used] + let used_attr = P(ast::NormalAttr::from_ident(Ident::with_dummy_span(sym::used))); + let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); + let used_attr = outer_normal_attr(&used_attr, new_id, span); + + // static DUMMY_PTR: = + let static_ident = Ident::from_str_and_span("DUMMY_PTR", span); + let fn_ptr_ty = ast::TyKind::BareFn(Box::new(ast::BareFnTy { + safety: sig.header.safety, + ext: sig.header.ext, + generic_params: generics.params, + decl: sig.decl, + decl_span: sig.span, + })); + let static_ty = ecx.ty(span, fn_ptr_ty); + + let static_expr = ecx.expr_path(ecx.path(span, vec![primal])); + let static_item_kind = ast::ItemKind::Static(Box::new(ast::StaticItem { + ident: static_ident, + ty: static_ty, + safety: ast::Safety::Default, + mutability: ast::Mutability::Not, + expr: Some(static_expr), + define_opaque: None, + })); + + let static_item = ast::Item { + attrs: thin_vec![used_attr], + id: ast::DUMMY_NODE_ID, + span, + vis, + kind: static_item_kind, + tokens: None, + }; + + let block_expr = ecx.expr_block(Box::new(ast::Block { + stmts: thin_vec![ecx.stmt_item(span, P(static_item))], + id: ast::DUMMY_NODE_ID, + rules: ast::BlockCheckMode::Default, + span, + tokens: None, + })); + + let const_item = ecx.item_const( + span, + Ident::from_str_and_span("_", span), + ecx.ty(span, ast::TyKind::Tup(thin_vec![])), + block_expr, + ); + + Annotatable::Item(const_item) + } + // Will generate a body of the type: // ``` // { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 829b3c513c2..66c34fbcfb1 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -3,22 +3,22 @@ use std::ptr; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode}; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::common::TypeKind; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; use rustc_errors::FatalError; use rustc_middle::bug; use tracing::{debug, trace}; use crate::back::write::llvm_err; -use crate::builder::{SBuilder, UNNAMED}; +use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; use crate::errors::{AutoDiffWithoutEnable, LlvmError}; use crate::llvm::AttributePlace::Function; -use crate::llvm::{Metadata, True}; +use crate::llvm::{Metadata, True, Type}; use crate::value::Value; use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm}; -fn get_params(fnc: &Value) -> Vec<&Value> { +fn _get_params(fnc: &Value) -> Vec<&Value> { let param_num = llvm::LLVMCountParams(fnc) as usize; let mut fnc_args: Vec<&Value> = vec![]; fnc_args.reserve(param_num); @@ -29,7 +29,7 @@ fn get_params(fnc: &Value) -> Vec<&Value> { fnc_args } -fn has_sret(fnc: &Value) -> bool { +fn _has_sret(fnc: &Value) -> bool { let num_args = llvm::LLVMCountParams(fnc) as usize; if num_args == 0 { false @@ -48,14 +48,13 @@ fn has_sret(fnc: &Value) -> bool { // need to match those. // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it // using iterators and peek()? -fn match_args_from_caller_to_enzyme<'ll>( +fn match_args_from_caller_to_enzyme<'ll, 'tcx>( cx: &SimpleCx<'ll>, - builder: &SBuilder<'ll, 'll>, + builder: &mut Builder<'_, 'll, 'tcx>, width: u32, args: &mut Vec<&'ll llvm::Value>, inputs: &[DiffActivity], outer_args: &[&'ll llvm::Value], - has_sret: bool, ) { debug!("matching autodiff arguments"); // We now handle the issue that Rust level arguments not always match the llvm-ir level @@ -67,20 +66,12 @@ fn match_args_from_caller_to_enzyme<'ll>( let mut outer_pos: usize = 0; let mut activity_pos = 0; - if has_sret { - // Then the first outer arg is the sret pointer. Enzyme doesn't know about sret, so the - // inner function will still return something. We increase our outer_pos by one, - // and once we're done with all other args we will take the return of the inner call and - // update the sret pointer with it - outer_pos = 1; - } - - let enzyme_const = cx.create_metadata(b"enzyme_const"); - let enzyme_out = cx.create_metadata(b"enzyme_out"); - let enzyme_dup = cx.create_metadata(b"enzyme_dup"); - let enzyme_dupv = cx.create_metadata(b"enzyme_dupv"); - let enzyme_dupnoneed = cx.create_metadata(b"enzyme_dupnoneed"); - let enzyme_dupnoneedv = cx.create_metadata(b"enzyme_dupnoneedv"); + let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap(); + let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap(); + let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap(); + let enzyme_dupv = cx.create_metadata("enzyme_dupv".to_string()).unwrap(); + let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap(); + let enzyme_dupnoneedv = cx.create_metadata("enzyme_dupnoneedv".to_string()).unwrap(); while activity_pos < inputs.len() { let diff_activity = inputs[activity_pos as usize]; @@ -193,92 +184,6 @@ fn match_args_from_caller_to_enzyme<'ll>( } } -// On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input -// arguments. We do however need to declare them with their correct return type. -// We already figured the correct return type out in our frontend, when generating the outer_fn, -// so we can now just go ahead and use that. This is not always trivial, e.g. because sret. -// Beyond sret, this article describes our challenges nicely: -// -// I.e. (i32, f32) will get merged into i64, but we don't handle that yet. -fn compute_enzyme_fn_ty<'ll>( - cx: &SimpleCx<'ll>, - attrs: &AutoDiffAttrs, - fn_to_diff: &'ll Value, - outer_fn: &'ll Value, -) -> &'ll llvm::Type { - let fn_ty = cx.get_type_of_global(outer_fn); - let mut ret_ty = cx.get_return_type(fn_ty); - - let has_sret = has_sret(outer_fn); - - if has_sret { - // Now we don't just forward the return type, so we have to figure it out based on the - // primal return type, in combination with the autodiff settings. - let fn_ty = cx.get_type_of_global(fn_to_diff); - let inner_ret_ty = cx.get_return_type(fn_ty); - - let void_ty = unsafe { llvm::LLVMVoidTypeInContext(cx.llcx) }; - if inner_ret_ty == void_ty { - // This indicates that even the inner function has an sret. - // Right now I only look for an sret in the outer function. - // This *probably* needs some extra handling, but I never ran - // into such a case. So I'll wait for user reports to have a test case. - bug!("sret in inner function"); - } - - if attrs.width == 1 { - // Enzyme returns a struct of style: - // `{ original_ret(if requested), float, float, ... }` - let mut struct_elements = vec![]; - if attrs.has_primal_ret() { - struct_elements.push(inner_ret_ty); - } - // Next, we push the list of active floats, since they will be lowered to `enzyme_out`, - // and therefore part of the return struct. - let param_tys = cx.func_params_types(fn_ty); - for (act, param_ty) in attrs.input_activity.iter().zip(param_tys) { - if matches!(act, DiffActivity::Active) { - // Now find the float type at position i based on the fn_ty, - // to know what (f16/f32/f64/...) to add to the struct. - struct_elements.push(param_ty); - } - } - ret_ty = cx.type_struct(&struct_elements, false); - } else { - // First we check if we also have to deal with the primal return. - match attrs.mode { - DiffMode::Forward => match attrs.ret_activity { - DiffActivity::Dual => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64 + 1) }; - ret_ty = arr_ty; - } - DiffActivity::DualOnly => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64) }; - ret_ty = arr_ty; - } - DiffActivity::Const => { - todo!("Not sure, do we need to do something here?"); - } - _ => { - bug!("unreachable"); - } - }, - DiffMode::Reverse => { - todo!("Handle sret for reverse mode"); - } - _ => { - bug!("unreachable"); - } - } - } - } - - // LLVM can figure out the input types on it's own, so we take a shortcut here. - unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) } -} - /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another /// function with expected naming and calling conventions[^1] which will be /// discovered by the enzyme LLVM pass and its body populated with the differentiated @@ -288,11 +193,15 @@ fn compute_enzyme_fn_ty<'ll>( /// [^1]: // FIXME(ZuseZ4): `outer_fn` should include upstream safety checks to // cover some assumptions of enzyme/autodiff, which could lead to UB otherwise. -fn generate_enzyme_call<'ll>( +pub(crate) fn generate_enzyme_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, cx: &SimpleCx<'ll>, fn_to_diff: &'ll Value, - outer_fn: &'ll Value, + outer_name: &str, + ret_ty: &'ll Type, + fn_args: &[&'ll Value], attrs: AutoDiffAttrs, + dest: PlaceRef<'tcx, &'ll Value>, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -302,11 +211,9 @@ fn generate_enzyme_call<'ll>( } .to_string(); - // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple + // add outer_name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. - let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::str::from_utf8(&name).unwrap(); - ad_name.push_str(outer_fn_name); + ad_name.push_str(outer_name); // Let us assume the user wrote the following function square: // @@ -317,13 +224,7 @@ fn generate_enzyme_call<'ll>( // ret double %0 // } // ``` - // - // The user now applies autodiff to the function square, in which case fn_to_diff will be `square`. - // Our macro generates the following placeholder code (slightly simplified): - // - // ```llvm // define double @dsquare(double %x) { - // ; placeholder code // return 0.0; // } // ``` @@ -340,120 +241,52 @@ fn generate_enzyme_call<'ll>( // ret double %0 // } // ``` - unsafe { - let enzyme_ty = compute_enzyme_fn_ty(cx, &attrs, fn_to_diff, outer_fn); - - // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and - // think a bit more about what should go here. - let cc = llvm::LLVMGetFunctionCallConv(outer_fn); - let ad_fn = declare_simple_fn( - cx, - &ad_name, - llvm::CallConv::try_from(cc).expect("invalid callconv"), - llvm::UnnamedAddr::No, - llvm::Visibility::Default, - enzyme_ty, - ); - - // Otherwise LLVM might inline our temporary code before the enzyme pass has a chance to - // do it's work. - let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); - attributes::apply_to_llfn(ad_fn, Function, &[attr]); - - // We add a made-up attribute just such that we can recognize it after AD to update - // (no)-inline attributes. We'll then also remove this attribute. - let enzyme_marker_attr = llvm::CreateAttrString(cx.llcx, "enzyme_marker"); - attributes::apply_to_llfn(outer_fn, Function, &[enzyme_marker_attr]); - - // first, remove all calls from fnc - let entry = llvm::LLVMGetFirstBasicBlock(outer_fn); - let br = llvm::LLVMRustGetTerminator(entry); - llvm::LLVMRustEraseInstFromParent(br); - - let last_inst = llvm::LLVMRustGetLastInstruction(entry).unwrap(); - let mut builder = SBuilder::build(cx, entry); - - let num_args = llvm::LLVMCountParams(&fn_to_diff); - let mut args = Vec::with_capacity(num_args as usize + 1); - args.push(fn_to_diff); - - let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); - if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { - args.push(cx.get_metadata_value(enzyme_primal_ret)); - } - if attrs.width > 1 { - let enzyme_width = cx.create_metadata(b"enzyme_width"); - args.push(cx.get_metadata_value(enzyme_width)); - args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); - } - - let has_sret = has_sret(outer_fn); - let outer_args: Vec<&llvm::Value> = get_params(outer_fn); - match_args_from_caller_to_enzyme( - &cx, - &builder, - attrs.width, - &mut args, - &attrs.input_activity, - &outer_args, - has_sret, - ); - - let call = builder.call(enzyme_ty, ad_fn, &args, None); - - // This part is a bit iffy. LLVM requires that a call to an inlineable function has some - // metadata attached to it, but we just created this code oota. Given that the - // differentiated function already has partly confusing metadata, and given that this - // affects nothing but the auttodiff IR, we take a shortcut and just steal metadata from the - // dummy code which we inserted at a higher level. - // FIXME(ZuseZ4): Work with Enzyme core devs to clarify what debug metadata issues we have, - // and how to best improve it for enzyme core and rust-enzyme. - let md_ty = cx.get_md_kind_id("dbg"); - if llvm::LLVMRustHasMetadata(last_inst, md_ty) { - let md = llvm::LLVMRustDIGetInstMetadata(last_inst) - .expect("failed to get instruction metadata"); - let md_todiff = cx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, md_ty, md_todiff); - } else { - // We don't panic, since depending on whether we are in debug or release mode, we might - // have no debug info to copy, which would then be ok. - trace!("no dbg info"); - } - - // Now that we copied the metadata, get rid of dummy code. - llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst); + let enzyme_ty = unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) }; + + // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and + // think a bit more about what should go here. + // FIXME(Sa4dUs): have to find a way to get the cc, using `FastCallConv` for now + let cc = 8; + let ad_fn = declare_simple_fn( + cx, + &ad_name, + llvm::CallConv::try_from(cc).expect("invalid callconv"), + llvm::UnnamedAddr::No, + llvm::Visibility::Default, + enzyme_ty, + ); + + // Otherwise LLVM might inline our temporary code before the enzyme pass has a chance to + // do it's work. + let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); + attributes::apply_to_llfn(ad_fn, Function, &[attr]); + + let num_args = llvm::LLVMCountParams(&fn_to_diff); + let mut args = Vec::with_capacity(num_args as usize + 1); + args.push(fn_to_diff); + + let enzyme_primal_ret = cx.create_metadata("enzyme_primal_return".to_string()).unwrap(); + if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { + args.push(cx.get_metadata_value(enzyme_primal_ret)); + } + if attrs.width > 1 { + let enzyme_width = cx.create_metadata("enzyme_width".to_string()).unwrap(); + args.push(cx.get_metadata_value(enzyme_width)); + args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); + } - if cx.val_ty(call) == cx.type_void() || has_sret { - if has_sret { - // This is what we already have in our outer_fn (shortened): - // define void @_foo(ptr <..> sret([32 x i8]) initializes((0, 32)) %0, <...>) { - // %7 = call [4 x double] (...) @__enzyme_fwddiff_foo(ptr @square, metadata !"enzyme_width", i64 4, <...>) - // - // store [4 x double] %7, ptr %0, align 8 - // ret void - // } + match_args_from_caller_to_enzyme( + &cx, + builder, + attrs.width, + &mut args, + &attrs.input_activity, + fn_args, + ); - // now store the result of the enzyme call into the sret pointer. - let sret_ptr = outer_args[0]; - let call_ty = cx.val_ty(call); - if attrs.width == 1 { - assert_eq!(cx.type_kind(call_ty), TypeKind::Struct); - } else { - assert_eq!(cx.type_kind(call_ty), TypeKind::Array); - } - llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr); - } - builder.ret_void(); - } else { - builder.ret(call); - } + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); - // Let's crash in case that we messed something up above and generated invalid IR. - llvm::LLVMRustVerifyFunction( - outer_fn, - llvm::LLVMRustVerifierFailureAction::LLVMAbortProcessAction, - ); - } + builder.store_to_place(call, dest.val); } pub(crate) fn differentiate<'ll>( @@ -461,6 +294,7 @@ pub(crate) fn differentiate<'ll>( cgcx: &CodegenContext, diff_items: Vec, ) -> Result<(), FatalError> { + // TODO(Sa4dUs): delete all this logic for item in &diff_items { trace!("{}", item); } @@ -480,7 +314,7 @@ pub(crate) fn differentiate<'ll>( for item in diff_items.iter() { let name = item.source.clone(); let fn_def: Option<&llvm::Value> = cx.get_function(&name); - let Some(fn_def) = fn_def else { + let Some(_fn_def) = fn_def else { return Err(llvm_err( diag_handler.handle(), LlvmError::PrepareAutoDiff { @@ -492,7 +326,7 @@ pub(crate) fn differentiate<'ll>( }; debug!(?item.target); let fn_target: Option<&llvm::Value> = cx.get_function(&item.target); - let Some(fn_target) = fn_target else { + let Some(_fn_target) = fn_target else { return Err(llvm_err( diag_handler.handle(), LlvmError::PrepareAutoDiff { @@ -503,7 +337,7 @@ pub(crate) fn differentiate<'ll>( )); }; - generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); + // generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); } // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 27ae729a531..8eb15571e82 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -660,7 +660,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } } impl<'ll> SimpleCx<'ll> { - pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type { + pub(crate) fn _get_return_type(&self, ty: &'ll Type) -> &'ll Type { assert_eq!(self.type_kind(ty), TypeKind::Function); unsafe { llvm::LLVMGetReturnType(ty) } } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7b27e496986..1102fc1d0c8 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -3,23 +3,26 @@ use std::cmp::Ordering; use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; +use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::mir::BinOp; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; -use rustc_middle::ty::{self, GenericArgsRef, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; -use rustc_symbol_mangling::mangle_internal_symbol; +use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; use rustc_target::spec::PanicStrategy; use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; +use crate::builder::autodiff::generate_enzyme_call; use crate::context::CodegenCx; use crate::llvm::{self, Metadata}; use crate::type_::Type; @@ -174,10 +177,17 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { span: Span, ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; + let callee_ty = instance.ty(tcx, self.typing_env()); - let name = tcx.item_name(instance.def_id()); let fn_args = instance.args; + let sig = callee_ty.fn_sig(tcx); + let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig); + let ret_ty = sig.output(); + let name = tcx.item_name(instance.def_id()); + + let llret_ty = self.layout_of(ret_ty).llvm_type(self); + let simple = call_simple_intrinsic(self, name, args); let llval = match name { _ if simple.is_some() => simple.unwrap(), @@ -189,6 +199,66 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &[ptr, args[1].immediate()], ) } + sym::autodiff => { + let val_arr: Vec<&'ll Value> = match args[2].val { + crate::intrinsic::OperandValue::Ref(ref place_value) => { + let mut ret_arr = vec![]; + let tuple_place = PlaceRef { val: *place_value, layout: args[2].layout }; + + for i in 0..tuple_place.layout.layout.0.fields.count() { + let field_place = tuple_place.project_field(self, i); + let field_layout = tuple_place.layout.field(self, i); + let llvm_ty = field_layout.llvm_type(self.cx); + + let field_val = + self.load(llvm_ty, field_place.val.llval, field_place.val.align); + + ret_arr.push(field_val) + } + + ret_arr + } + crate::intrinsic::OperandValue::Pair(v1, v2) => vec![v1, v2], + OperandValue::Immediate(v) => vec![v], + OperandValue::ZeroSized => bug!("unexpected `ZeroSized` arg"), + }; + + // Get source, diff, and attrs + let source_id = match fn_args.into_type_list(tcx)[0].kind() { + ty::FnDef(def_id, _) => def_id, + _ => bug!("invalid args"), + }; + let fn_source = Instance::mono(tcx, *source_id); + let source_symbol = + symbol_name_for_instance_in_crate(tcx, fn_source.clone(), LOCAL_CRATE); + let fn_to_diff: Option<&'ll llvm::Value> = self.cx.get_function(&source_symbol); + let Some(fn_to_diff) = fn_to_diff else { bug!("could not find source function") }; + + let diff_id = match fn_args.into_type_list(tcx)[1].kind() { + ty::FnDef(def_id, _) => def_id, + _ => bug!("invalid args"), + }; + let fn_diff = Instance::mono(tcx, *diff_id); + let diff_symbol = + symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE); + + let diff_attrs = autodiff_attrs(tcx, *diff_id); + let Some(diff_attrs) = diff_attrs else { bug!("could not find autodiff attrs") }; + + // Build body + generate_enzyme_call( + self, + self.cx, + fn_to_diff, + &diff_symbol, + llret_ty, + &val_arr, + diff_attrs.clone(), + result, + ); + + return Ok(()); + } sym::is_val_statically_known => { if let OperandValue::Immediate(imm) = args[0].val { self.call_intrinsic( diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 4441dd6ebd6..46371cfe591 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -135,6 +135,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::round_ties_even_f32 | sym::round_ties_even_f64 | sym::round_ties_even_f128 + | sym::autodiff | sym::const_eval_select => hir::Safety::Safe, _ => hir::Safety::Unsafe, }; @@ -171,6 +172,8 @@ pub(crate) fn check_intrinsic_type( } }; + let has_autodiff = tcx.has_attr(intrinsic_id, sym::rustc_autodiff); + let bound_vars = tcx.mk_bound_variable_kinds(&[ ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon), ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon), @@ -195,9 +198,9 @@ pub(crate) fn check_intrinsic_type( (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty) }; - let safety = intrinsic_operation_unsafety(tcx, intrinsic_id); let n_lts = 0; let (n_tps, n_cts, inputs, output) = match intrinsic_name { + sym::autodiff => (4, 0, vec![param(0), param(1), param(2)], param(3)), sym::abort => (0, 0, vec![], tcx.types.never), sym::unreachable => (0, 0, vec![], tcx.types.never), sym::breakpoint => (0, 0, vec![], tcx.types.unit), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 416ce27367e..f7a8258a9d8 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -542,6 +542,7 @@ symbols! { audit_that, augmented_assignments, auto_traits, + autodiff, autodiff_forward, autodiff_reverse, automatically_derived, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 7228ad0ed6d..6c389c55a5f 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3157,6 +3157,10 @@ pub const unsafe fn copysignf64(x: f64, y: f64) -> f64; #[rustc_intrinsic] pub const unsafe fn copysignf128(x: f128, y: f128) -> f128; +#[rustc_nounwind] +#[rustc_intrinsic] +pub const fn autodiff(f: F, df: G, args: T) -> R; + /// Inform Miri that a given pointer definitely has a certain alignment. #[cfg(miri)] #[rustc_allow_const_fn_unstable(const_eval_select)] -- cgit 1.4.1-3-g733a5 From 250d77e5d72fde69a6406050a3b037635f685378 Mon Sep 17 00:00:00 2001 From: Marcelo Domínguez Date: Thu, 14 Aug 2025 15:27:57 +0000 Subject: Complete functionality and general cleanup --- Cargo.lock | 2 - compiler/rustc_builtin_macros/src/autodiff.rs | 492 ++++----------------- compiler/rustc_codegen_gcc/src/lib.rs | 6 - .../rustc_codegen_llvm/src/builder/autodiff.rs | 178 ++++---- compiler/rustc_codegen_llvm/src/context.rs | 5 - compiler/rustc_codegen_llvm/src/intrinsic.rs | 212 ++++++--- compiler/rustc_codegen_llvm/src/lib.rs | 6 - compiler/rustc_codegen_ssa/src/back/write.rs | 18 +- compiler/rustc_codegen_ssa/src/base.rs | 7 +- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 10 +- compiler/rustc_codegen_ssa/src/traits/write.rs | 2 - compiler/rustc_hir_analysis/src/check/intrinsic.rs | 3 +- .../rustc_middle/src/middle/codegen_fn_attrs.rs | 4 - compiler/rustc_middle/src/mir/mono.rs | 2 - compiler/rustc_monomorphize/Cargo.toml | 2 - compiler/rustc_monomorphize/src/collector.rs | 5 + .../rustc_monomorphize/src/collector/autodiff.rs | 48 ++ compiler/rustc_monomorphize/src/partitioning.rs | 34 +- .../src/partitioning/autodiff.rs | 143 ------ library/core/src/intrinsics/mod.rs | 34 ++ library/core/src/macros/mod.rs | 2 + src/doc/rustc-dev-guide/src/SUMMARY.md | 1 - .../rustc-dev-guide/src/autodiff/limitations.md | 27 -- triagebot.toml | 3 - 24 files changed, 419 insertions(+), 827 deletions(-) create mode 100644 compiler/rustc_monomorphize/src/collector/autodiff.rs delete mode 100644 compiler/rustc_monomorphize/src/partitioning/autodiff.rs delete mode 100644 src/doc/rustc-dev-guide/src/autodiff/limitations.md (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/Cargo.lock b/Cargo.lock index 8a878faecbc..e8590fa484b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4308,7 +4308,6 @@ name = "rustc_monomorphize" version = "0.0.0" dependencies = [ "rustc_abi", - "rustc_ast", "rustc_data_structures", "rustc_errors", "rustc_fluent_macro", @@ -4317,7 +4316,6 @@ dependencies = [ "rustc_middle", "rustc_session", "rustc_span", - "rustc_symbol_mangling", "rustc_target", "serde", "serde_json", diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs index 3f8585d35bc..c260dca87c0 100644 --- a/compiler/rustc_builtin_macros/src/autodiff.rs +++ b/compiler/rustc_builtin_macros/src/autodiff.rs @@ -15,11 +15,12 @@ mod llvm_enzyme { use rustc_ast::tokenstream::*; use rustc_ast::visit::AssocCtxt::*; use rustc_ast::{ - self as ast, AssocItemKind, BindingMode, ExprKind, FnRetTy, FnSig, Generics, ItemKind, - MetaItemInner, PatKind, QSelf, TyKind, Visibility, + self as ast, AngleBracketedArg, AngleBracketedArgs, AnonConst, AssocItemKind, BindingMode, + FnRetTy, FnSig, GenericArg, GenericArgs, GenericParamKind, Generics, ItemKind, + MetaItemInner, PatKind, Path, PathSegment, TyKind, Visibility, }; use rustc_expand::base::{Annotatable, ExtCtxt}; - use rustc_span::{Ident, Span, Symbol, kw, sym}; + use rustc_span::{Ident, Span, Symbol, sym}; use thin_vec::{ThinVec, thin_vec}; use tracing::{debug, trace}; @@ -179,11 +180,8 @@ mod llvm_enzyme { } /// We expand the autodiff macro to generate a new placeholder function which passes - /// type-checking and can be called by users. The function body of the placeholder function will - /// later be replaced on LLVM-IR level, so the design of the body is less important and for now - /// should just prevent early inlining and optimizations which alter the function signature. - /// The exact signature of the generated function depends on the configuration provided by the - /// user, but here is an example: + /// type-checking and can be called by users. The exact signature of the generated function + /// depends on the configuration provided by the user, but here is an example: /// /// ``` /// #[autodiff(cos_box, Reverse, Duplicated, Active)] @@ -199,14 +197,8 @@ mod llvm_enzyme { /// f32::sin(**x) /// } /// #[rustc_autodiff(Reverse, Duplicated, Active)] - /// #[inline(never)] /// fn cos_box(x: &Box, dx: &mut Box, dret: f32) -> f32 { - /// unsafe { - /// asm!("NOP"); - /// }; - /// ::core::hint::black_box(sin(x)); - /// ::core::hint::black_box((dx, dret)); - /// ::core::hint::black_box(sin(x)) + /// std::intrinsics::autodiff(sin::<>, cos_box::<>, (x, dx, dret)) /// } /// ``` /// FIXME(ZuseZ4): Once autodiff is enabled by default, make this a doc comment which is checked @@ -227,16 +219,24 @@ mod llvm_enzyme { // first get information about the annotable item: visibility, signature, name and generic // parameters. // these will be used to generate the differentiated version of the function - let Some((vis, sig, primal, generics)) = (match &item { - Annotatable::Item(iitem) => extract_item_info(iitem), + let Some((vis, sig, primal, generics, impl_of_trait)) = (match &item { + Annotatable::Item(iitem) => { + extract_item_info(iitem).map(|(v, s, p, g)| (v, s, p, g, false)) + } Annotatable::Stmt(stmt) => match &stmt.kind { - ast::StmtKind::Item(iitem) => extract_item_info(iitem), + ast::StmtKind::Item(iitem) => { + extract_item_info(iitem).map(|(v, s, p, g)| (v, s, p, g, false)) + } _ => None, }, - Annotatable::AssocItem(assoc_item, Impl { .. }) => match &assoc_item.kind { - ast::AssocItemKind::Fn(box ast::Fn { sig, ident, generics, .. }) => { - Some((assoc_item.vis.clone(), sig.clone(), ident.clone(), generics.clone())) - } + Annotatable::AssocItem(assoc_item, Impl { of_trait }) => match &assoc_item.kind { + ast::AssocItemKind::Fn(box ast::Fn { sig, ident, generics, .. }) => Some(( + assoc_item.vis.clone(), + sig.clone(), + ident.clone(), + generics.clone(), + *of_trait, + )), _ => None, }, _ => None, @@ -254,7 +254,6 @@ mod llvm_enzyme { }; let has_ret = has_ret(&sig.decl.output); - let sig_span = ecx.with_call_site_ctxt(sig.span); // create TokenStream from vec elemtents: // meta_item doesn't have a .tokens field @@ -323,28 +322,27 @@ mod llvm_enzyme { } let span = ecx.with_def_site_ctxt(expand_span); - let n_active: u32 = x - .input_activity - .iter() - .filter(|a| **a == DiffActivity::Active || **a == DiffActivity::ActiveOnly) - .count() as u32; - let (d_sig, new_args, idents, errored) = gen_enzyme_decl(ecx, &sig, &x, span); - - // TODO(Sa4dUs): Remove this and all the related logic - let _d_body = gen_enzyme_body( - ecx, &x, n_active, &sig, &d_sig, primal, &new_args, span, sig_span, idents, errored, - &generics, - ); + let d_sig = gen_enzyme_decl(ecx, &sig, &x, span); - let d_body = - call_autodiff(ecx, primal, first_ident(&meta_item_vec[0]), span, &d_sig); + let d_body = ecx.block( + span, + thin_vec![call_autodiff( + ecx, + primal, + first_ident(&meta_item_vec[0]), + span, + &d_sig, + &generics, + impl_of_trait, + )], + ); // The first element of it is the name of the function to be generated - let asdf = Box::new(ast::Fn { + let d_fn = Box::new(ast::Fn { defaultness: ast::Defaultness::Final, sig: d_sig, ident: first_ident(&meta_item_vec[0]), - generics: generics.clone(), + generics, contract: None, body: Some(d_body), define_opaque: None, @@ -433,13 +431,11 @@ mod llvm_enzyme { tokens: ts, }); - let vis_clone = vis.clone(); - let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let d_attr = outer_normal_attr(&rustc_ad_attr, new_id, span); let d_annotatable = match &item { Annotatable::AssocItem(_, _) => { - let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(asdf); + let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(d_fn); let d_fn = Box::new(ast::AssocItem { attrs: thin_vec![d_attr], id: ast::DUMMY_NODE_ID, @@ -451,13 +447,13 @@ mod llvm_enzyme { Annotatable::AssocItem(d_fn, Impl { of_trait: false }) } Annotatable::Item(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(asdf)); + let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(d_fn)); d_fn.vis = vis; Annotatable::Item(d_fn) } Annotatable::Stmt(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(asdf)); + let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(d_fn)); d_fn.vis = vis; Annotatable::Stmt(Box::new(ast::Stmt { @@ -471,9 +467,7 @@ mod llvm_enzyme { } }; - let dummy_const_annotatable = gen_dummy_const(ecx, span, primal, sig, generics, vis_clone); - - return vec![orig_annotatable, dummy_const_annotatable, d_annotatable]; + return vec![orig_annotatable, d_annotatable]; } // shadow arguments (the extra ones which were not in the original (primal) function), in reverse mode must be @@ -504,9 +498,11 @@ mod llvm_enzyme { diff: Ident, span: Span, d_sig: &FnSig, - ) -> P { - let primal_path_expr = ecx.expr_path(ecx.path_ident(span, primal)); - let diff_path_expr = ecx.expr_path(ecx.path_ident(span, diff)); + generics: &Generics, + is_impl: bool, + ) -> rustc_ast::Stmt { + let primal_path_expr = gen_turbofish_expr(ecx, primal, generics, span, is_impl); + let diff_path_expr = gen_turbofish_expr(ecx, diff, generics, span, is_impl); let tuple_expr = ecx.expr_tuple( span, @@ -522,371 +518,65 @@ mod llvm_enzyme { .into(), ); - let enzyme_path = ecx.path( - span, - vec![ - Ident::from_str("std"), - Ident::from_str("intrinsics"), - Ident::from_str("autodiff"), - ], - ); + let enzyme_path_idents = ecx.std_path(&[sym::intrinsics, sym::autodiff]); + let enzyme_path = ecx.path(span, enzyme_path_idents); let call_expr = ecx.expr_call( span, ecx.expr_path(enzyme_path), vec![primal_path_expr, diff_path_expr, tuple_expr].into(), ); - let block = ecx.block_expr(call_expr); - - block - } - - // Generate dummy const to prevent primal function - // from being optimized away before applying enzyme - // ``` - // const _: () = - // { - // #[used] - // pub static DUMMY_PTR: fn_type = primal_fn; - // }; - // ``` - fn gen_dummy_const( - ecx: &ExtCtxt<'_>, - span: Span, - primal: Ident, - sig: FnSig, - generics: Generics, - vis: Visibility, - ) -> Annotatable { - // #[used] - let used_attr = P(ast::NormalAttr::from_ident(Ident::with_dummy_span(sym::used))); - let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); - let used_attr = outer_normal_attr(&used_attr, new_id, span); - - // static DUMMY_PTR: = - let static_ident = Ident::from_str_and_span("DUMMY_PTR", span); - let fn_ptr_ty = ast::TyKind::BareFn(Box::new(ast::BareFnTy { - safety: sig.header.safety, - ext: sig.header.ext, - generic_params: generics.params, - decl: sig.decl, - decl_span: sig.span, - })); - let static_ty = ecx.ty(span, fn_ptr_ty); - - let static_expr = ecx.expr_path(ecx.path(span, vec![primal])); - let static_item_kind = ast::ItemKind::Static(Box::new(ast::StaticItem { - ident: static_ident, - ty: static_ty, - safety: ast::Safety::Default, - mutability: ast::Mutability::Not, - expr: Some(static_expr), - define_opaque: None, - })); - - let static_item = ast::Item { - attrs: thin_vec![used_attr], - id: ast::DUMMY_NODE_ID, - span, - vis, - kind: static_item_kind, - tokens: None, - }; - - let block_expr = ecx.expr_block(Box::new(ast::Block { - stmts: thin_vec![ecx.stmt_item(span, P(static_item))], - id: ast::DUMMY_NODE_ID, - rules: ast::BlockCheckMode::Default, - span, - tokens: None, - })); - - let const_item = ecx.item_const( - span, - Ident::from_str_and_span("_", span), - ecx.ty(span, ast::TyKind::Tup(thin_vec![])), - block_expr, - ); - - Annotatable::Item(const_item) + ecx.stmt_expr(call_expr) } - // Will generate a body of the type: - // ``` - // { - // unsafe { - // asm!("NOP"); - // } - // ::core::hint::black_box(primal(args)); - // ::core::hint::black_box((args, ret)); - // - // } - // ``` - fn init_body_helper( + // Generate turbofish expression from fn name and generics + // Given `foo` and `` params, gen `foo::` + // We use this expression when passing primal and diff function to the autodiff intrinsic + fn gen_turbofish_expr( ecx: &ExtCtxt<'_>, - span: Span, - primal: Ident, - new_names: &[String], - sig_span: Span, - new_decl_span: Span, - idents: &[Ident], - errored: bool, + ident: Ident, generics: &Generics, - ) -> (Box, Box, Box, Box) { - let blackbox_path = ecx.std_path(&[sym::hint, sym::black_box]); - let noop = ast::InlineAsm { - asm_macro: ast::AsmMacro::Asm, - template: vec![ast::InlineAsmTemplatePiece::String("NOP".into())], - template_strs: Box::new([]), - operands: vec![], - clobber_abis: vec![], - options: ast::InlineAsmOptions::PURE | ast::InlineAsmOptions::NOMEM, - line_spans: vec![], - }; - let noop_expr = ecx.expr_asm(span, Box::new(noop)); - let unsf = ast::BlockCheckMode::Unsafe(ast::UnsafeSource::CompilerGenerated); - let unsf_block = ast::Block { - stmts: thin_vec![ecx.stmt_semi(noop_expr)], - id: ast::DUMMY_NODE_ID, - tokens: None, - rules: unsf, - span, - }; - let unsf_expr = ecx.expr_block(Box::new(unsf_block)); - let blackbox_call_expr = ecx.expr_path(ecx.path(span, blackbox_path)); - let primal_call = gen_primal_call(ecx, span, primal, idents, generics); - let black_box_primal_call = ecx.expr_call( - new_decl_span, - blackbox_call_expr.clone(), - thin_vec![primal_call.clone()], - ); - let tup_args = new_names - .iter() - .map(|arg| ecx.expr_path(ecx.path_ident(span, Ident::from_str(arg)))) - .collect(); - - let black_box_remaining_args = ecx.expr_call( - sig_span, - blackbox_call_expr.clone(), - thin_vec![ecx.expr_tuple(sig_span, tup_args)], - ); - - let mut body = ecx.block(span, ThinVec::new()); - body.stmts.push(ecx.stmt_semi(unsf_expr)); - - // This uses primal args which won't be available if we errored before - if !errored { - body.stmts.push(ecx.stmt_semi(black_box_primal_call.clone())); - } - body.stmts.push(ecx.stmt_semi(black_box_remaining_args)); - - (body, primal_call, black_box_primal_call, blackbox_call_expr) - } - - /// We only want this function to type-check, since we will replace the body - /// later on llvm level. Using `loop {}` does not cover all return types anymore, - /// so instead we manually build something that should pass the type checker. - /// We also add a inline_asm line, as one more barrier for rustc to prevent inlining - /// or const propagation. inline_asm will also triggers an Enzyme crash if due to another - /// bug would ever try to accidentally differentiate this placeholder function body. - /// Finally, we also add back_box usages of all input arguments, to prevent rustc - /// from optimizing any arguments away. - fn gen_enzyme_body( - ecx: &ExtCtxt<'_>, - x: &AutoDiffAttrs, - n_active: u32, - sig: &ast::FnSig, - d_sig: &ast::FnSig, - primal: Ident, - new_names: &[String], span: Span, - sig_span: Span, - idents: Vec, - errored: bool, - generics: &Generics, - ) -> Box { - let new_decl_span = d_sig.span; - - // Just adding some default inline-asm and black_box usages to prevent early inlining - // and optimizations which alter the function signature. - // - // The bb_primal_call is the black_box call of the primal function. We keep it around, - // since it has the convenient property of returning the type of the primal function, - // Remember, we only care to match types here. - // No matter which return we pick, we always wrap it into a std::hint::black_box call, - // to prevent rustc from propagating it into the caller. - let (mut body, primal_call, bb_primal_call, bb_call_expr) = init_body_helper( - ecx, - span, - primal, - new_names, - sig_span, - new_decl_span, - &idents, - errored, - generics, - ); - - if !has_ret(&d_sig.decl.output) { - // there is no return type that we have to match, () works fine. - return body; - } - - // Everything from here onwards just tries to fulfil the return type. Fun! - - // having an active-only return means we'll drop the original return type. - // So that can be treated identical to not having one in the first place. - let primal_ret = has_ret(&sig.decl.output) && !x.has_active_only_ret(); - - if primal_ret && n_active == 0 && x.mode.is_rev() { - // We only have the primal ret. - body.stmts.push(ecx.stmt_expr(bb_primal_call)); - return body; - } - - if !primal_ret && n_active == 1 { - // Again no tuple return, so return default float val. - let ty = match d_sig.decl.output { - FnRetTy::Ty(ref ty) => ty.clone(), - FnRetTy::Default(span) => { - panic!("Did not expect Default ret ty: {:?}", span); + is_impl: bool, + ) -> Box { + let generic_args = generics + .params + .iter() + .filter_map(|p| match &p.kind { + GenericParamKind::Type { .. } => { + let path = ast::Path::from_ident(p.ident); + let ty = ecx.ty_path(path); + Some(AngleBracketedArg::Arg(GenericArg::Type(ty))) } - }; - let arg = ty.kind.is_simple_path().unwrap(); - let tmp = ecx.def_site_path(&[arg, kw::Default]); - let default_call_expr = ecx.expr_path(ecx.path(span, tmp)); - let default_call_expr = ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]); - body.stmts.push(ecx.stmt_expr(default_call_expr)); - return body; - } - - let mut exprs: Box = primal_call; - let d_ret_ty = match d_sig.decl.output { - FnRetTy::Ty(ref ty) => ty.clone(), - FnRetTy::Default(span) => { - panic!("Did not expect Default ret ty: {:?}", span); - } - }; - if x.mode.is_fwd() { - // Fwd mode is easy. If the return activity is Const, we support arbitrary types. - // Otherwise, we only support a scalar, a pair of scalars, or an array of scalars. - // We checked that (on a best-effort base) in the preceding gen_enzyme_decl function. - // In all three cases, we can return `std::hint::black_box(::default())`. - if x.ret_activity == DiffActivity::Const { - // Here we call the primal function, since our dummy function has the same return - // type due to the Const return activity. - exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]); - } else { - let q = QSelf { ty: d_ret_ty, path_span: span, position: 0 }; - let y = ExprKind::Path( - Some(Box::new(q)), - ecx.path_ident(span, Ident::with_dummy_span(kw::Default)), - ); - let default_call_expr = ecx.expr(span, y); - let default_call_expr = - ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]); - exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![default_call_expr]); - } - } else if x.mode.is_rev() { - if x.width == 1 { - // We either have `-> ArbitraryType` or `-> (ArbitraryType, repeated_float_scalars)`. - match d_ret_ty.kind { - TyKind::Tup(ref args) => { - // We have a tuple return type. We need to create a tuple of the same size - // and fill it with default values. - let mut exprs2 = thin_vec![exprs]; - for arg in args.iter().skip(1) { - let arg = arg.kind.is_simple_path().unwrap(); - let tmp = ecx.def_site_path(&[arg, kw::Default]); - let default_call_expr = ecx.expr_path(ecx.path(span, tmp)); - let default_call_expr = - ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]); - exprs2.push(default_call_expr); - } - exprs = ecx.expr_tuple(new_decl_span, exprs2); - } - _ => { - // Interestingly, even the `-> ArbitraryType` case - // ends up getting matched and handled correctly above, - // so we don't have to handle any other case for now. - panic!("Unsupported return type: {:?}", d_ret_ty); - } + GenericParamKind::Const { .. } => { + let expr = ecx.expr_path(ast::Path::from_ident(p.ident)); + let anon_const = AnonConst { id: ast::DUMMY_NODE_ID, value: expr }; + Some(AngleBracketedArg::Arg(GenericArg::Const(anon_const))) } - } - exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]); - } else { - unreachable!("Unsupported mode: {:?}", x.mode); - } - - body.stmts.push(ecx.stmt_expr(exprs)); + GenericParamKind::Lifetime { .. } => None, + }) + .collect::>(); - body - } + let args: AngleBracketedArgs = AngleBracketedArgs { span, args: generic_args }; - fn gen_primal_call( - ecx: &ExtCtxt<'_>, - span: Span, - primal: Ident, - idents: &[Ident], - generics: &Generics, - ) -> Box { - let has_self = idents.len() > 0 && idents[0].name == kw::SelfLower; + let segment = PathSegment { + ident, + id: ast::DUMMY_NODE_ID, + args: Some(Box::new(GenericArgs::AngleBracketed(args))), + }; - if has_self { - let args: ThinVec<_> = - idents[1..].iter().map(|arg| ecx.expr_path(ecx.path_ident(span, *arg))).collect(); - let self_expr = ecx.expr_self(span); - ecx.expr_method_call(span, self_expr, primal, args) + let segments = if is_impl { + thin_vec![ + PathSegment { ident: Ident::from_str("Self"), id: ast::DUMMY_NODE_ID, args: None }, + segment, + ] } else { - let args: ThinVec<_> = - idents.iter().map(|arg| ecx.expr_path(ecx.path_ident(span, *arg))).collect(); - let mut primal_path = ecx.path_ident(span, primal); - - let is_generic = !generics.params.is_empty(); - - match (is_generic, primal_path.segments.last_mut()) { - (true, Some(function_path)) => { - let primal_generic_types = generics - .params - .iter() - .filter(|param| matches!(param.kind, ast::GenericParamKind::Type { .. })); - - let generated_generic_types = primal_generic_types - .map(|type_param| { - let generic_param = TyKind::Path( - None, - ast::Path { - span, - segments: thin_vec![ast::PathSegment { - ident: type_param.ident, - args: None, - id: ast::DUMMY_NODE_ID, - }], - tokens: None, - }, - ); - - ast::AngleBracketedArg::Arg(ast::GenericArg::Type(Box::new(ast::Ty { - id: type_param.id, - span, - kind: generic_param, - tokens: None, - }))) - }) - .collect(); - - function_path.args = - Some(Box::new(ast::GenericArgs::AngleBracketed(ast::AngleBracketedArgs { - span, - args: generated_generic_types, - }))); - } - _ => {} - } + thin_vec![segment] + }; - let primal_call_expr = ecx.expr_path(primal_path); - ecx.expr_call(span, primal_call_expr, args) - } + let path = Path { span, segments, tokens: None }; + + ecx.expr_path(path) } // Generate the new function declaration. Const arguments are kept as is. Duplicated arguments must @@ -905,7 +595,7 @@ mod llvm_enzyme { sig: &ast::FnSig, x: &AutoDiffAttrs, span: Span, - ) -> (ast::FnSig, Vec, Vec, bool) { + ) -> ast::FnSig { let dcx = ecx.sess.dcx(); let has_ret = has_ret(&sig.decl.output); let sig_args = sig.decl.inputs.len() + if has_ret { 1 } else { 0 }; @@ -917,7 +607,7 @@ mod llvm_enzyme { found: num_activities, }); // This is not the right signature, but we can continue parsing. - return (sig.clone(), vec![], vec![], true); + return sig.clone(); } assert!(sig.decl.inputs.len() == x.input_activity.len()); assert!(has_ret == x.has_ret_activity()); @@ -960,7 +650,7 @@ mod llvm_enzyme { if errors { // This is not the right signature, but we can continue parsing. - return (sig.clone(), new_inputs, idents, true); + return sig.clone(); } let unsafe_activities = x @@ -1174,7 +864,7 @@ mod llvm_enzyme { } let d_sig = FnSig { header: d_header, decl: d_decl, span }; trace!("Generated signature: {:?}", d_sig); - (d_sig, new_inputs, idents, false) + d_sig } } diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index b11f11d38e3..4025aba82da 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -93,7 +93,6 @@ use gccjit::{CType, Context, OptimizationLevel}; #[cfg(feature = "master")] use gccjit::{TargetInfo, Version}; use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn, @@ -363,12 +362,7 @@ impl WriteBackendMethods for GccCodegenBackend { _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - diff_functions: Vec, ) -> Result, FatalError> { - if !diff_functions.is_empty() { - unimplemented!(); - } - back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules) } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 66c34fbcfb1..56116959a62 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,40 +1,93 @@ use std::ptr; -use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode}; -use rustc_codegen_ssa::ModuleCodegen; +use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; -use rustc_errors::FatalError; -use rustc_middle::bug; -use tracing::{debug, trace}; +use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; +use rustc_middle::{bug, ty}; +use tracing::debug; -use crate::back::write::llvm_err; use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; -use crate::errors::{AutoDiffWithoutEnable, LlvmError}; use crate::llvm::AttributePlace::Function; use crate::llvm::{Metadata, True, Type}; use crate::value::Value; -use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm}; +use crate::{attributes, llvm}; -fn _get_params(fnc: &Value) -> Vec<&Value> { - let param_num = llvm::LLVMCountParams(fnc) as usize; - let mut fnc_args: Vec<&Value> = vec![]; - fnc_args.reserve(param_num); - unsafe { - llvm::LLVMGetParams(fnc, fnc_args.as_mut_ptr()); - fnc_args.set_len(param_num); +pub(crate) fn adjust_activity_to_abi<'tcx>( + tcx: TyCtxt<'tcx>, + fn_ty: Ty<'tcx>, + da: &mut Vec, +) { + if !matches!(fn_ty.kind(), ty::FnDef(..)) { + bug!("expected fn def for autodiff, got {:?}", fn_ty); } - fnc_args -} -fn _has_sret(fnc: &Value) -> bool { - let num_args = llvm::LLVMCountParams(fnc) as usize; - if num_args == 0 { - false - } else { - unsafe { llvm::LLVMRustHasAttributeAtIndex(fnc, 0, llvm::AttributeKind::StructRet) } + // We don't actually pass the types back into the type system. + // All we do is decide how to handle the arguments. + let sig = fn_ty.fn_sig(tcx).skip_binder(); + + let mut new_activities = vec![]; + let mut new_positions = vec![]; + for (i, ty) in sig.inputs().iter().enumerate() { + if let Some(inner_ty) = ty.builtin_deref(true) { + if inner_ty.is_slice() { + // Now we need to figure out the size of each slice element in memory to allow + // safety checks and usability improvements in the backend. + let sty = match inner_ty.builtin_index() { + Some(sty) => sty, + None => { + panic!("slice element type unknown"); + } + }; + let pci = PseudoCanonicalInput { + typing_env: TypingEnv::fully_monomorphized(), + value: sty, + }; + + let layout = tcx.layout_of(pci); + let elem_size = match layout { + Ok(layout) => layout.size, + Err(_) => { + bug!("autodiff failed to compute slice element size"); + } + }; + let elem_size: u32 = elem_size.bytes() as u32; + + // We know that the length will be passed as extra arg. + if !da.is_empty() { + // We are looking at a slice. The length of that slice will become an + // extra integer on llvm level. Integers are always const. + // However, if the slice get's duplicated, we want to know to later check the + // size. So we mark the new size argument as FakeActivitySize. + // There is one FakeActivitySize per slice, so for convenience we store the + // slice element size in bytes in it. We will use the size in the backend. + let activity = match da[i] { + DiffActivity::DualOnly + | DiffActivity::Dual + | DiffActivity::Dualv + | DiffActivity::DuplicatedOnly + | DiffActivity::Duplicated => { + DiffActivity::FakeActivitySize(Some(elem_size)) + } + DiffActivity::Const => DiffActivity::Const, + _ => bug!("unexpected activity for ptr/ref"), + }; + new_activities.push(activity); + new_positions.push(i + 1); + } + + continue; + } + } + } + // now add the extra activities coming from slices + // Reverse order to not invalidate the indices + for _ in 0..new_activities.len() { + let pos = new_positions.pop().unwrap(); + let activity = new_activities.pop().unwrap(); + da.insert(pos, activity); } } @@ -66,12 +119,12 @@ fn match_args_from_caller_to_enzyme<'ll, 'tcx>( let mut outer_pos: usize = 0; let mut activity_pos = 0; - let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap(); - let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap(); - let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap(); - let enzyme_dupv = cx.create_metadata("enzyme_dupv".to_string()).unwrap(); - let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap(); - let enzyme_dupnoneedv = cx.create_metadata("enzyme_dupnoneedv".to_string()).unwrap(); + let enzyme_const = cx.create_metadata(b"enzyme_const"); + let enzyme_out = cx.create_metadata(b"enzyme_out"); + let enzyme_dup = cx.create_metadata(b"enzyme_dup"); + let enzyme_dupv = cx.create_metadata(b"enzyme_dupv"); + let enzyme_dupnoneed = cx.create_metadata(b"enzyme_dupnoneed"); + let enzyme_dupnoneedv = cx.create_metadata(b"enzyme_dupnoneedv"); while activity_pos < inputs.len() { let diff_activity = inputs[activity_pos as usize]; @@ -223,7 +276,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( // %0 = fmul double %x, %x // ret double %0 // } - // ``` + // // define double @dsquare(double %x) { // return 0.0; // } @@ -245,8 +298,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and // think a bit more about what should go here. - // FIXME(Sa4dUs): have to find a way to get the cc, using `FastCallConv` for now - let cc = 8; + let cc = unsafe { llvm::LLVMGetFunctionCallConv(fn_to_diff) }; let ad_fn = declare_simple_fn( cx, &ad_name, @@ -265,12 +317,12 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( let mut args = Vec::with_capacity(num_args as usize + 1); args.push(fn_to_diff); - let enzyme_primal_ret = cx.create_metadata("enzyme_primal_return".to_string()).unwrap(); + let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { args.push(cx.get_metadata_value(enzyme_primal_ret)); } if attrs.width > 1 { - let enzyme_width = cx.create_metadata("enzyme_width".to_string()).unwrap(); + let enzyme_width = cx.create_metadata(b"enzyme_width"); args.push(cx.get_metadata_value(enzyme_width)); args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); } @@ -288,61 +340,3 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( builder.store_to_place(call, dest.val); } - -pub(crate) fn differentiate<'ll>( - module: &'ll ModuleCodegen, - cgcx: &CodegenContext, - diff_items: Vec, -) -> Result<(), FatalError> { - // TODO(Sa4dUs): delete all this logic - for item in &diff_items { - trace!("{}", item); - } - - let diag_handler = cgcx.create_dcx(); - - let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - - // First of all, did the user try to use autodiff without using the -Zautodiff=Enable flag? - if !diff_items.is_empty() - && !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) - { - return Err(diag_handler.handle().emit_almost_fatal(AutoDiffWithoutEnable)); - } - - // Here we replace the placeholder code with the actual autodiff code, which calls Enzyme. - for item in diff_items.iter() { - let name = item.source.clone(); - let fn_def: Option<&llvm::Value> = cx.get_function(&name); - let Some(_fn_def) = fn_def else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find source function".to_owned(), - }, - )); - }; - debug!(?item.target); - let fn_target: Option<&llvm::Value> = cx.get_function(&item.target); - let Some(_fn_target) = fn_target else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find target function".to_owned(), - }, - )); - }; - - // generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); - } - - // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts - - trace!("done with differentiate()"); - - Ok(()) -} diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 8eb15571e82..da8c1e5f47b 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use std::str; use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx}; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; -use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN}; @@ -660,10 +659,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } } impl<'ll> SimpleCx<'ll> { - pub(crate) fn _get_return_type(&self, ty: &'ll Type) -> &'ll Type { - assert_eq!(self.type_kind(ty), TypeKind::Function); - unsafe { llvm::LLVMGetReturnType(ty) } - } pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type { unsafe { llvm::LLVMGlobalGetValueType(val) } } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 1102fc1d0c8..4935f8d7dff 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -9,21 +9,23 @@ use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphizati use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; -use rustc_hir as hir; use rustc_hir::def_id::LOCAL_CRATE; +use rustc_hir::{self as hir}; use rustc_middle::mir::BinOp; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; -use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty, TyCtxt, TypingEnv}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; +use rustc_target::callconv::PassMode; use rustc_target::spec::PanicStrategy; use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; -use crate::builder::autodiff::generate_enzyme_call; +use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::context::CodegenCx; +use crate::errors::AutoDiffWithoutEnable; use crate::llvm::{self, Metadata}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -177,16 +179,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { span: Span, ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; - let callee_ty = instance.ty(tcx, self.typing_env()); - let fn_args = instance.args; - - let sig = callee_ty.fn_sig(tcx); - let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig); - let ret_ty = sig.output(); let name = tcx.item_name(instance.def_id()); - - let llret_ty = self.layout_of(ret_ty).llvm_type(self); + let fn_args = instance.args; let simple = call_simple_intrinsic(self, name, args); let llval = match name { @@ -200,63 +195,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ) } sym::autodiff => { - let val_arr: Vec<&'ll Value> = match args[2].val { - crate::intrinsic::OperandValue::Ref(ref place_value) => { - let mut ret_arr = vec![]; - let tuple_place = PlaceRef { val: *place_value, layout: args[2].layout }; - - for i in 0..tuple_place.layout.layout.0.fields.count() { - let field_place = tuple_place.project_field(self, i); - let field_layout = tuple_place.layout.field(self, i); - let llvm_ty = field_layout.llvm_type(self.cx); - - let field_val = - self.load(llvm_ty, field_place.val.llval, field_place.val.align); - - ret_arr.push(field_val) - } - - ret_arr - } - crate::intrinsic::OperandValue::Pair(v1, v2) => vec![v1, v2], - OperandValue::Immediate(v) => vec![v], - OperandValue::ZeroSized => bug!("unexpected `ZeroSized` arg"), - }; - - // Get source, diff, and attrs - let source_id = match fn_args.into_type_list(tcx)[0].kind() { - ty::FnDef(def_id, _) => def_id, - _ => bug!("invalid args"), - }; - let fn_source = Instance::mono(tcx, *source_id); - let source_symbol = - symbol_name_for_instance_in_crate(tcx, fn_source.clone(), LOCAL_CRATE); - let fn_to_diff: Option<&'ll llvm::Value> = self.cx.get_function(&source_symbol); - let Some(fn_to_diff) = fn_to_diff else { bug!("could not find source function") }; - - let diff_id = match fn_args.into_type_list(tcx)[1].kind() { - ty::FnDef(def_id, _) => def_id, - _ => bug!("invalid args"), - }; - let fn_diff = Instance::mono(tcx, *diff_id); - let diff_symbol = - symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE); - - let diff_attrs = autodiff_attrs(tcx, *diff_id); - let Some(diff_attrs) = diff_attrs else { bug!("could not find autodiff attrs") }; - - // Build body - generate_enzyme_call( - self, - self.cx, - fn_to_diff, - &diff_symbol, - llret_ty, - &val_arr, - diff_attrs.clone(), - result, - ); - + codegen_autodiff(self, tcx, instance, args, result); return Ok(()); } sym::is_val_statically_known => { @@ -1183,6 +1122,143 @@ fn get_rust_try_fn<'a, 'll, 'tcx>( rust_try } +fn codegen_autodiff<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + result: PlaceRef<'tcx, &'ll Value>, +) { + if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) { + let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable); + } + + let fn_args = instance.args; + let callee_ty = instance.ty(tcx, bx.typing_env()); + + let sig = callee_ty.fn_sig(tcx).skip_binder(); + + let ret_ty = sig.output(); + let llret_ty = bx.layout_of(ret_ty).llvm_type(bx); + + // Get source, diff, and attrs + let (source_id, source_args) = match fn_args.into_type_list(tcx)[0].kind() { + ty::FnDef(def_id, source_params) => (def_id, source_params), + _ => bug!("invalid autodiff intrinsic args"), + }; + + let fn_source = match Instance::try_resolve(tcx, bx.cx.typing_env(), *source_id, source_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + source_id, + source_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let source_symbol = symbol_name_for_instance_in_crate(tcx, fn_source.clone(), LOCAL_CRATE); + let Some(fn_to_diff) = bx.cx.get_function(&source_symbol) else { + bug!("could not find source function") + }; + + let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() { + ty::FnDef(def_id, diff_args) => (def_id, diff_args), + _ => bug!("invalid args"), + }; + + let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + diff_id, + diff_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let val_arr = get_args_from_tuple(bx, args[2], fn_diff); + let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE); + + let Some(mut diff_attrs) = autodiff_attrs(tcx, fn_diff.def_id()) else { + bug!("could not find autodiff attrs") + }; + + adjust_activity_to_abi( + tcx, + fn_source.ty(tcx, TypingEnv::fully_monomorphized()), + &mut diff_attrs.input_activity, + ); + + // Build body + generate_enzyme_call( + bx, + bx.cx, + fn_to_diff, + &diff_symbol, + llret_ty, + &val_arr, + diff_attrs.clone(), + result, + ); +} + +fn get_args_from_tuple<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tuple_op: OperandRef<'tcx, &'ll Value>, + fn_instance: Instance<'tcx>, +) -> Vec<&'ll Value> { + let cx = bx.cx; + let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty()); + + match tuple_op.val { + OperandValue::Immediate(val) => vec![val], + OperandValue::Pair(v1, v2) => vec![v1, v2], + OperandValue::Ref(ptr) => { + let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout }; + + let mut result = Vec::with_capacity(fn_abi.args.len()); + let mut tuple_index = 0; + + for arg in &fn_abi.args { + match arg.mode { + PassMode::Ignore => {} + PassMode::Direct(_) | PassMode::Cast { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(val); + tuple_index += 1; + } + PassMode::Pair(_, _) => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(bx.extract_value(pair_val, 0)); + result.push(bx.extract_value(pair_val, 1)); + tuple_index += 1; + } + PassMode::Indirect { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + result.push(field.val.llval); + tuple_index += 1; + } + } + } + + result + } + + OperandValue::ZeroSized => vec![], + } +} + fn generic_simd_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ca84b6de8b1..79e80db6f55 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -30,7 +30,6 @@ use context::SimpleCx; use errors::ParseTargetMachineConfig; use llvm_util::target_config; use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -173,15 +172,10 @@ impl WriteBackendMethods for LlvmCodegenBackend { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - diff_fncs: Vec, ) -> Result, FatalError> { let mut module = back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?; - if !diff_fncs.is_empty() { - builder::autodiff::differentiate(&module, cgcx, diff_fncs)?; - } - let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); back::lto::run_pass_manager(cgcx, dcx, &mut module, false)?; diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index aa29afb7f5b..2e8122798d1 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -7,7 +7,6 @@ use std::{fs, io, mem, str, thread}; use rustc_abi::Size; use rustc_ast::attr; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::jobserver::{self, Acquired}; use rustc_data_structures::memmap::Mmap; @@ -38,7 +37,7 @@ use tracing::debug; use super::link::{self, ensure_removed}; use super::lto::{self, SerializedModule}; use crate::back::lto::check_lto_allowed; -use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir}; +use crate::errors::ErrorCreatingRemarkDir; use crate::traits::*; use crate::{ CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, @@ -454,7 +453,6 @@ pub(crate) fn start_async_codegen( backend: B, tcx: TyCtxt<'_>, target_cpu: String, - autodiff_items: &[AutoDiffItem], ) -> OngoingCodegen { let (coordinator_send, coordinator_receive) = channel(); @@ -473,7 +471,6 @@ pub(crate) fn start_async_codegen( backend.clone(), tcx, &crate_info, - autodiff_items, shared_emitter, codegen_worker_send, coordinator_receive, @@ -728,7 +725,6 @@ pub(crate) enum WorkItem { each_linked_rlib_for_lto: Vec, needs_fat_lto: Vec>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, - autodiff: Vec, }, /// Performs thin-LTO on the given module. ThinLto(lto::ThinModule), @@ -1001,7 +997,6 @@ fn execute_fat_lto_work_item( each_linked_rlib_for_lto: &[PathBuf], mut needs_fat_lto: Vec>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, - autodiff: Vec, module_config: &ModuleConfig, ) -> Result, FatalError> { for (module, wp) in import_only_modules { @@ -1013,7 +1008,6 @@ fn execute_fat_lto_work_item( exported_symbols_for_lto, each_linked_rlib_for_lto, needs_fat_lto, - autodiff, )?; let module = B::codegen(cgcx, module, module_config)?; Ok(WorkItemResult::Finished(module)) @@ -1105,7 +1099,6 @@ fn start_executing_work( backend: B, tcx: TyCtxt<'_>, crate_info: &CrateInfo, - autodiff_items: &[AutoDiffItem], shared_emitter: SharedEmitter, codegen_worker_send: Sender, coordinator_receive: Receiver>, @@ -1115,7 +1108,6 @@ fn start_executing_work( ) -> thread::JoinHandle> { let coordinator_send = tx_to_llvm_workers; let sess = tcx.sess; - let autodiff_items = autodiff_items.to_vec(); let mut each_linked_rlib_for_lto = Vec::new(); let mut each_linked_rlib_file_for_lto = Vec::new(); @@ -1448,7 +1440,6 @@ fn start_executing_work( each_linked_rlib_for_lto: each_linked_rlib_file_for_lto, needs_fat_lto, import_only_modules, - autodiff: autodiff_items.clone(), }, 0, )); @@ -1456,11 +1447,6 @@ fn start_executing_work( helper.request_token(); } } else { - if !autodiff_items.is_empty() { - let dcx = cgcx.create_dcx(); - dcx.handle().emit_fatal(AutodiffWithoutLto {}); - } - for (work, cost) in generate_thin_lto_work( &cgcx, &exported_symbols_for_lto, @@ -1795,7 +1781,6 @@ fn spawn_work<'a, B: ExtraBackendMethods>( each_linked_rlib_for_lto, needs_fat_lto, import_only_modules, - autodiff, } => { let _timer = cgcx .prof @@ -1806,7 +1791,6 @@ fn spawn_work<'a, B: ExtraBackendMethods>( &each_linked_rlib_for_lto, needs_fat_lto, import_only_modules, - autodiff, module_config, ) } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b4556ced0b3..b483c01da59 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -647,7 +647,7 @@ pub fn codegen_crate( ) -> OngoingCodegen { // Skip crate items and just output metadata in -Z no-codegen mode. if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, &[]); + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu); ongoing_codegen.codegen_finished(tcx); @@ -665,8 +665,7 @@ pub fn codegen_crate( // Run the monomorphization collector and partition the collected items into // codegen units. - let MonoItemPartitions { codegen_units, autodiff_items, .. } = - tcx.collect_and_partition_mono_items(()); + let MonoItemPartitions { codegen_units, .. } = tcx.collect_and_partition_mono_items(()); // Force all codegen_unit queries so they are already either red or green // when compile_codegen_unit accesses them. We are not able to re-execute @@ -679,7 +678,7 @@ pub fn codegen_crate( } } - let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu, autodiff_items); + let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu); // Codegen an allocator shim, if necessary. if let Some(kind) = allocator_kind_for_codegen(tcx) { diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index a36a772bc97..af70f0deb07 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -177,14 +177,6 @@ fn process_builtin_attrs( let mut interesting_spans = InterestingAttributeDiagnosticSpans::default(); let rust_target_features = tcx.rust_target_features(LOCAL_CRATE); - // If our rustc version supports autodiff/enzyme, then we call our handler - // to check for any `#[rustc_autodiff(...)]` attributes. - // FIXME(jdonszelmann): merge with loop below - if cfg!(llvm_enzyme) { - let ad = autodiff_attrs(tcx, did.into()); - codegen_fn_attrs.autodiff_item = ad; - } - for attr in attrs.iter() { if let hir::Attribute::Parsed(p) = attr { match p { @@ -612,7 +604,7 @@ fn inherited_align<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option { /// placeholder functions. We wrote the rustc_autodiff attributes ourself, so this should never /// panic, unless we introduced a bug when parsing the autodiff macro. //FIXME(jdonszelmann): put in the main loop. No need to have two..... :/ Let's do that when we make autodiff parsed. -fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option { +pub fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option { let attrs = tcx.get_attrs(id, sym::rustc_autodiff); let attrs = attrs.filter(|attr| attr.has_name(sym::rustc_autodiff)).collect::>(); diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index f391c198e1a..c29ad90735b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -1,6 +1,5 @@ use std::path::PathBuf; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_errors::{DiagCtxtHandle, FatalError}; use rustc_middle::dep_graph::WorkProduct; @@ -23,7 +22,6 @@ pub trait WriteBackendMethods: Clone + 'static { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - diff_fncs: Vec, ) -> Result, FatalError>; /// Performs thin LTO by performing necessary global analysis and returning two /// lists, one of the modules that need optimization and another for modules that diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 46371cfe591..f50aed0b3c2 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -172,8 +172,6 @@ pub(crate) fn check_intrinsic_type( } }; - let has_autodiff = tcx.has_attr(intrinsic_id, sym::rustc_autodiff); - let bound_vars = tcx.mk_bound_variable_kinds(&[ ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon), ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon), @@ -198,6 +196,7 @@ pub(crate) fn check_intrinsic_type( (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty) }; + let safety = intrinsic_operation_unsafety(tcx, intrinsic_id); let n_lts = 0; let (n_tps, n_cts, inputs, output) = match intrinsic_name { sym::autodiff => (4, 0, vec![param(0), param(1), param(2)], param(3)), diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs index 2852c4cbd34..7d2fc0995aa 100644 --- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs +++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs @@ -1,7 +1,6 @@ use std::borrow::Cow; use rustc_abi::Align; -use rustc_ast::expand::autodiff_attrs::AutoDiffAttrs; use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, Linkage, OptimizeAttr}; use rustc_hir::def_id::DefId; use rustc_macros::{HashStable, TyDecodable, TyEncodable}; @@ -75,8 +74,6 @@ pub struct CodegenFnAttrs { /// The `#[patchable_function_entry(...)]` attribute. Indicates how many nops should be around /// the function entry. pub patchable_function_entry: Option, - /// For the `#[autodiff]` macros. - pub autodiff_item: Option, } #[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)] @@ -182,7 +179,6 @@ impl CodegenFnAttrs { instruction_set: None, alignment: None, patchable_function_entry: None, - autodiff_item: None, } } diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 440771b3d68..0e6f797b1e4 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -2,7 +2,6 @@ use std::borrow::Cow; use std::fmt; use std::hash::Hash; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_data_structures::base_n::{BaseNString, CASE_INSENSITIVE, ToBaseN}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxIndexMap; @@ -336,7 +335,6 @@ impl ToStableHashKey> for MonoItem<'_> { pub struct MonoItemPartitions<'tcx> { pub codegen_units: &'tcx [CodegenUnit<'tcx>], pub all_mono_items: &'tcx DefIdSet, - pub autodiff_items: &'tcx [AutoDiffItem], } #[derive(Debug, HashStable)] diff --git a/compiler/rustc_monomorphize/Cargo.toml b/compiler/rustc_monomorphize/Cargo.toml index 0ed5b4fc0d0..09a55f0b5f8 100644 --- a/compiler/rustc_monomorphize/Cargo.toml +++ b/compiler/rustc_monomorphize/Cargo.toml @@ -6,7 +6,6 @@ edition = "2024" [dependencies] # tidy-alphabetical-start rustc_abi = { path = "../rustc_abi" } -rustc_ast = { path = "../rustc_ast" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_fluent_macro = { path = "../rustc_fluent_macro" } @@ -15,7 +14,6 @@ rustc_macros = { path = "../rustc_macros" } rustc_middle = { path = "../rustc_middle" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } -rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_target = { path = "../rustc_target" } serde = "1" serde_json = "1" diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 26ca8518434..af2c3177067 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -205,6 +205,8 @@ //! this is not implemented however: a mono item will be produced //! regardless of whether it is actually needed or not. +mod autodiff; + use std::cell::OnceCell; use rustc_data_structures::fx::FxIndexMap; @@ -235,6 +237,7 @@ use rustc_span::source_map::{Spanned, dummy_spanned, respan}; use rustc_span::{DUMMY_SP, Span}; use tracing::{debug, instrument, trace}; +use crate::collector::autodiff::collect_autodiff_fn; use crate::errors::{ self, EncounteredErrorWhileInstantiating, EncounteredErrorWhileInstantiatingGlobalAsm, NoOptimizedMir, RecursionLimit, @@ -911,6 +914,8 @@ fn visit_instance_use<'tcx>( return; } if let Some(intrinsic) = tcx.intrinsic(instance.def_id()) { + collect_autodiff_fn(tcx, instance, intrinsic, output); + if let Some(_requirement) = ValidityRequirement::from_intrinsic(intrinsic.name) { // The intrinsics assert_inhabited, assert_zero_valid, and assert_mem_uninitialized_valid will // be lowered in codegen to nothing or a call to panic_nounwind. So if we encounter any diff --git a/compiler/rustc_monomorphize/src/collector/autodiff.rs b/compiler/rustc_monomorphize/src/collector/autodiff.rs new file mode 100644 index 00000000000..13868cca944 --- /dev/null +++ b/compiler/rustc_monomorphize/src/collector/autodiff.rs @@ -0,0 +1,48 @@ +use rustc_middle::bug; +use rustc_middle::ty::{self, GenericArg, IntrinsicDef, TyCtxt}; + +use crate::collector::{MonoItems, create_fn_mono_item}; + +// Here, we force both primal and diff function to be collected in +// mono so this does not interfere in `autodiff` intrinsics +// codegen process. If they are unused, LLVM will remove them when +// compiling with O3. +pub(crate) fn collect_autodiff_fn<'tcx>( + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + intrinsic: IntrinsicDef, + output: &mut MonoItems<'tcx>, +) { + if intrinsic.name != rustc_span::sym::autodiff { + return; + }; + + collect_autodiff_fn_from_arg(instance.args[0], tcx, output); +} + +fn collect_autodiff_fn_from_arg<'tcx>( + arg: GenericArg<'tcx>, + tcx: TyCtxt<'tcx>, + output: &mut MonoItems<'tcx>, +) { + let (instance, span) = match arg.kind() { + ty::GenericArgKind::Type(ty) => match ty.kind() { + ty::FnDef(def_id, substs) => { + let span = tcx.def_span(def_id); + let instance = ty::Instance::expect_resolve( + tcx, + ty::TypingEnv::non_body_analysis(tcx, def_id), + *def_id, + substs, + span, + ); + + (instance, span) + } + _ => bug!("expected autodiff function"), + }, + _ => bug!("expected type when matching autodiff arg"), + }; + + output.push(create_fn_mono_item(tcx, instance, span)); +} diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index 628ea2b63de..d784d3540c4 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -92,8 +92,6 @@ //! source-level module, functions from the same module will be available for //! inlining, even when they are not marked `#[inline]`. -mod autodiff; - use std::cmp; use std::collections::hash_map::Entry; use std::fs::{self, File}; @@ -251,17 +249,7 @@ where always_export_generics, ); - // We can't differentiate a function that got inlined. - let autodiff_active = cfg!(llvm_enzyme) - && matches!(mono_item, MonoItem::Fn(_)) - && cx - .tcx - .codegen_fn_attrs(mono_item.def_id()) - .autodiff_item - .as_ref() - .is_some_and(|ad| ad.is_active()); - - if !autodiff_active && visibility == Visibility::Hidden && can_be_internalized { + if visibility == Visibility::Hidden && can_be_internalized { internalization_candidates.insert(mono_item); } let size_estimate = mono_item.size_estimate(cx.tcx); @@ -1157,27 +1145,15 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> MonoItemPartitio } } - #[cfg(not(llvm_enzyme))] - let autodiff_mono_items: Vec<_> = vec![]; - #[cfg(llvm_enzyme)] - let mut autodiff_mono_items: Vec<_> = vec![]; let mono_items: DefIdSet = items .iter() .filter_map(|mono_item| match *mono_item { - MonoItem::Fn(ref instance) => { - #[cfg(llvm_enzyme)] - autodiff_mono_items.push((mono_item, instance)); - Some(instance.def_id()) - } + MonoItem::Fn(ref instance) => Some(instance.def_id()), MonoItem::Static(def_id) => Some(def_id), _ => None, }) .collect(); - let autodiff_items = - autodiff::find_autodiff_source_functions(tcx, &usage_map, autodiff_mono_items); - let autodiff_items = tcx.arena.alloc_from_iter(autodiff_items); - // Output monomorphization stats per def_id if let SwitchWithOptPath::Enabled(ref path) = tcx.sess.opts.unstable_opts.dump_mono_stats && let Err(err) = @@ -1235,11 +1211,7 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> MonoItemPartitio } } - MonoItemPartitions { - all_mono_items: tcx.arena.alloc(mono_items), - codegen_units, - autodiff_items, - } + MonoItemPartitions { all_mono_items: tcx.arena.alloc(mono_items), codegen_units } } /// Outputs stats about instantiation counts and estimated size, per `MonoItem`'s diff --git a/compiler/rustc_monomorphize/src/partitioning/autodiff.rs b/compiler/rustc_monomorphize/src/partitioning/autodiff.rs deleted file mode 100644 index 22d593b80b8..00000000000 --- a/compiler/rustc_monomorphize/src/partitioning/autodiff.rs +++ /dev/null @@ -1,143 +0,0 @@ -use rustc_ast::expand::autodiff_attrs::{AutoDiffItem, DiffActivity}; -use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; -use rustc_middle::mir::mono::MonoItem; -use rustc_middle::ty::{self, Instance, PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; -use rustc_symbol_mangling::symbol_name_for_instance_in_crate; -use tracing::{debug, trace}; - -use crate::partitioning::UsageMap; - -fn adjust_activity_to_abi<'tcx>(tcx: TyCtxt<'tcx>, fn_ty: Ty<'tcx>, da: &mut Vec) { - if !matches!(fn_ty.kind(), ty::FnDef(..)) { - bug!("expected fn def for autodiff, got {:?}", fn_ty); - } - - // We don't actually pass the types back into the type system. - // All we do is decide how to handle the arguments. - let sig = fn_ty.fn_sig(tcx).skip_binder(); - - let mut new_activities = vec![]; - let mut new_positions = vec![]; - for (i, ty) in sig.inputs().iter().enumerate() { - if let Some(inner_ty) = ty.builtin_deref(true) { - if inner_ty.is_slice() { - // Now we need to figure out the size of each slice element in memory to allow - // safety checks and usability improvements in the backend. - let sty = match inner_ty.builtin_index() { - Some(sty) => sty, - None => { - panic!("slice element type unknown"); - } - }; - let pci = PseudoCanonicalInput { - typing_env: TypingEnv::fully_monomorphized(), - value: sty, - }; - - let layout = tcx.layout_of(pci); - let elem_size = match layout { - Ok(layout) => layout.size, - Err(_) => { - bug!("autodiff failed to compute slice element size"); - } - }; - let elem_size: u32 = elem_size.bytes() as u32; - - // We know that the length will be passed as extra arg. - if !da.is_empty() { - // We are looking at a slice. The length of that slice will become an - // extra integer on llvm level. Integers are always const. - // However, if the slice get's duplicated, we want to know to later check the - // size. So we mark the new size argument as FakeActivitySize. - // There is one FakeActivitySize per slice, so for convenience we store the - // slice element size in bytes in it. We will use the size in the backend. - let activity = match da[i] { - DiffActivity::DualOnly - | DiffActivity::Dual - | DiffActivity::Dualv - | DiffActivity::DuplicatedOnly - | DiffActivity::Duplicated => { - DiffActivity::FakeActivitySize(Some(elem_size)) - } - DiffActivity::Const => DiffActivity::Const, - _ => bug!("unexpected activity for ptr/ref"), - }; - new_activities.push(activity); - new_positions.push(i + 1); - } - - continue; - } - } - } - // now add the extra activities coming from slices - // Reverse order to not invalidate the indices - for _ in 0..new_activities.len() { - let pos = new_positions.pop().unwrap(); - let activity = new_activities.pop().unwrap(); - da.insert(pos, activity); - } -} - -pub(crate) fn find_autodiff_source_functions<'tcx>( - tcx: TyCtxt<'tcx>, - usage_map: &UsageMap<'tcx>, - autodiff_mono_items: Vec<(&MonoItem<'tcx>, &Instance<'tcx>)>, -) -> Vec { - let mut autodiff_items: Vec = vec![]; - for (item, instance) in autodiff_mono_items { - let target_id = instance.def_id(); - let cg_fn_attr = &tcx.codegen_fn_attrs(target_id).autodiff_item; - let Some(target_attrs) = cg_fn_attr else { - continue; - }; - let mut input_activities: Vec = target_attrs.input_activity.clone(); - if target_attrs.is_source() { - trace!("source found: {:?}", target_id); - } - if !target_attrs.apply_autodiff() { - continue; - } - - let target_symbol = symbol_name_for_instance_in_crate(tcx, instance.clone(), LOCAL_CRATE); - - let source = - usage_map.used_map.get(&item).unwrap().into_iter().find_map(|item| match *item { - MonoItem::Fn(ref instance_s) => { - let source_id = instance_s.def_id(); - if let Some(ad) = &tcx.codegen_fn_attrs(source_id).autodiff_item - && ad.is_active() - { - return Some(instance_s); - } - None - } - _ => None, - }); - let inst = match source { - Some(source) => source, - None => continue, - }; - - debug!("source_id: {:?}", inst.def_id()); - let fn_ty = inst.ty(tcx, ty::TypingEnv::fully_monomorphized()); - assert!(fn_ty.is_fn()); - adjust_activity_to_abi(tcx, fn_ty, &mut input_activities); - let symb = symbol_name_for_instance_in_crate(tcx, inst.clone(), LOCAL_CRATE); - - let mut new_target_attrs = target_attrs.clone(); - new_target_attrs.input_activity = input_activities; - let itm = new_target_attrs.into_item(symb, target_symbol); - autodiff_items.push(itm); - } - - if !autodiff_items.is_empty() { - trace!("AUTODIFF ITEMS EXIST"); - for item in &mut *autodiff_items { - trace!("{}", &item); - } - } - - autodiff_items -} diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 6c389c55a5f..dd838d494bc 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -3157,6 +3157,40 @@ pub const unsafe fn copysignf64(x: f64, y: f64) -> f64; #[rustc_intrinsic] pub const unsafe fn copysignf128(x: f128, y: f128) -> f128; +/// Generates the LLVM body for the automatic differentiation of `f` using Enzyme, +/// with `df` as the derivative function and `args` as its arguments. +/// +/// Used internally as the body of `df` when expanding the `#[autodiff_forward]` +/// and `#[autodiff_reverse]` attribute macros. +/// +/// Type Parameters: +/// - `F`: The original function to differentiate. Must be a function item. +/// - `G`: The derivative function. Must be a function item. +/// - `T`: A tuple of arguments passed to `df`. +/// - `R`: The return type of the derivative function. +/// +/// This shows where the `autodiff` intrinsic is used during macro expansion: +/// +/// ```rust,ignore (macro example) +/// #[autodiff_forward(df1, Dual, Const, Dual)] +/// pub fn f1(x: &[f64], y: f64) -> f64 { +/// unimplemented!() +/// } +/// ``` +/// +/// expands to: +/// +/// ```rust,ignore (macro example) +/// #[rustc_autodiff] +/// #[inline(never)] +/// pub fn f1(x: &[f64], y: f64) -> f64 { +/// ::core::panicking::panic("not implemented") +/// } +/// #[rustc_autodiff(Forward, 1, Dual, Const, Dual)] +/// pub fn df1(x: &[f64], bx_0: &[f64], y: f64) -> (f64, f64) { +/// ::core::intrinsics::autodiff(f1::<>, df1::<>, (x, bx_0, y)) +/// } +/// ``` #[rustc_nounwind] #[rustc_intrinsic] pub const fn autodiff(f: F, df: G, args: T) -> R; diff --git a/library/core/src/macros/mod.rs b/library/core/src/macros/mod.rs index c59290a757b..888369d73f4 100644 --- a/library/core/src/macros/mod.rs +++ b/library/core/src/macros/mod.rs @@ -1494,6 +1494,7 @@ pub(crate) mod builtin { /// (or explicitly returns `-> ()`). Otherwise, it must be set to one of the allowed activities. #[unstable(feature = "autodiff", issue = "124509")] #[allow_internal_unstable(rustc_attrs)] + #[allow_internal_unstable(core_intrinsics)] #[rustc_builtin_macro] pub macro autodiff_forward($item:item) { /* compiler built-in */ @@ -1512,6 +1513,7 @@ pub(crate) mod builtin { /// (or explicitly returns `-> ()`). Otherwise, it must be set to one of the allowed activities. #[unstable(feature = "autodiff", issue = "124509")] #[allow_internal_unstable(rustc_attrs)] + #[allow_internal_unstable(core_intrinsics)] #[rustc_builtin_macro] pub macro autodiff_reverse($item:item) { /* compiler built-in */ diff --git a/src/doc/rustc-dev-guide/src/SUMMARY.md b/src/doc/rustc-dev-guide/src/SUMMARY.md index 9ded467d5cd..025a078ae5b 100644 --- a/src/doc/rustc-dev-guide/src/SUMMARY.md +++ b/src/doc/rustc-dev-guide/src/SUMMARY.md @@ -107,7 +107,6 @@ - [Installation](./autodiff/installation.md) - [How to debug](./autodiff/debugging.md) - [Autodiff flags](./autodiff/flags.md) - - [Current limitations](./autodiff/limitations.md) # Source Code Representation diff --git a/src/doc/rustc-dev-guide/src/autodiff/limitations.md b/src/doc/rustc-dev-guide/src/autodiff/limitations.md deleted file mode 100644 index 90afbd51f3f..00000000000 --- a/src/doc/rustc-dev-guide/src/autodiff/limitations.md +++ /dev/null @@ -1,27 +0,0 @@ -# Current limitations - -## Safety and Soundness - -Enzyme currently assumes that the user passes shadow arguments (`dx`, `dy`, ...) of appropriate size. Under Reverse Mode, we additionally assume that shadow arguments are mutable. In Reverse Mode we adjust the outermost pointer or reference to be mutable. Therefore `&f32` will receive the shadow type `&mut f32`. However, we do not check length for other types than slices (e.g. enums, Vec). We also do not enforce mutability of inner references, but will warn if we recognize them. We do intend to add additional checks over time. - -## ABI adjustments - -In some cases, a function parameter might get lowered in a way that we currently don't handle correctly, leading to a compile time type mismatch in the `rustc_codegen_llvm` backend. Here are some [examples](https://github.com/EnzymeAD/rust/issues/105). - -## Compile Times - -Enzyme will often achieve excellent runtime performance, but might increase your compile time by a large factor. For Rust, we already have made significant improvements and have a list of further improvements planed - please reach out if you have time to help here. - -### Type Analysis - -Most of the times, Type Analysis (TA) is the reason of large (>5x) compile time increases when using Enzyme. This poster explains why we need to run Type Analysis in the bottom left part: [Poster Link](https://c.wsmoses.com/posters/Enzyme-llvmdev.pdf). - -We intend to increase the number of locations where we pass down Type information based on Rust types, which in turn will reduce the number of locations where Enzyme has to run Type Analysis, which will help compile times. - -### Duplicated Optimizations - -The key reason for Enzyme offering often excellent performance is that Enzyme differentiates already optimized LLVM-IR. However, we also (have to) run LLVM's optimization pipeline after differentiating, to make sure that the code which Enzyme generates is optimized properly. As a result you should have excellent runtime performance (please fill an issue if not), but at a compile time cost for running optimizations twice. - -### Fat-LTO - -The usage of `#[autodiff(...)]` currently requires compiling your project with Fat-LTO. We technically only need LTO if the function being differentiated calls functions in other compilation units. Therefore, other solutions are possible, but this is the most simple one to get started. diff --git a/triagebot.toml b/triagebot.toml index 6f6e95c5b50..df81bb71160 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -283,7 +283,6 @@ trigger_files = [ "src/tools/enzyme", "src/doc/unstable-book/src/compiler-flags/autodiff.md", "compiler/rustc_ast/src/expand/autodiff_attrs.rs", - "compiler/rustc_monomorphize/src/partitioning/autodiff.rs", "compiler/rustc_codegen_llvm/src/builder/autodiff.rs", "compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs", ] @@ -1285,8 +1284,6 @@ cc = ["@ZuseZ4"] cc = ["@ZuseZ4"] [mentions."compiler/rustc_builtin_macros/src/autodiff.rs"] cc = ["@ZuseZ4"] -[mentions."compiler/rustc_monomorphize/src/partitioning/autodiff.rs"] -cc = ["@ZuseZ4"] [mentions."compiler/rustc_codegen_llvm/src/builder/autodiff.rs"] cc = ["@ZuseZ4"] [mentions."compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs"] -- cgit 1.4.1-3-g733a5 From c9c1c171289aa575040678f3d0c005342b1e29e1 Mon Sep 17 00:00:00 2001 From: Marcelo Domínguez Date: Thu, 14 Aug 2025 15:29:37 +0000 Subject: Remove inlining for autodiff handling --- compiler/rustc_builtin_macros/src/autodiff.rs | 32 ++++++++++++++-------- .../rustc_codegen_llvm/src/builder/autodiff.rs | 8 +----- 2 files changed, 21 insertions(+), 19 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs index c260dca87c0..48d0795af5e 100644 --- a/compiler/rustc_builtin_macros/src/autodiff.rs +++ b/compiler/rustc_builtin_macros/src/autodiff.rs @@ -192,7 +192,6 @@ mod llvm_enzyme { /// which becomes expanded to: /// ``` /// #[rustc_autodiff] - /// #[inline(never)] /// fn sin(x: &Box) -> f32 { /// f32::sin(**x) /// } @@ -371,7 +370,7 @@ mod llvm_enzyme { let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let inline_never = outer_normal_attr(&inline_never_attr, new_id, span); - // We're avoid duplicating the attributes `#[rustc_autodiff]` and `#[inline(never)]`. + // We're avoid duplicating the attribute `#[rustc_autodiff]`. fn same_attribute(attr: &ast::AttrKind, item: &ast::AttrKind) -> bool { match (attr, item) { (ast::AttrKind::Normal(a), ast::AttrKind::Normal(b)) => { @@ -384,14 +383,16 @@ mod llvm_enzyme { } } + let mut has_inline_never = false; + // Don't add it multiple times: let orig_annotatable: Annotatable = match item { Annotatable::Item(ref mut iitem) => { if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) { iitem.attrs.push(attr); } - if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { - iitem.attrs.push(inline_never.clone()); + if iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { + has_inline_never = true; } Annotatable::Item(iitem.clone()) } @@ -399,8 +400,8 @@ mod llvm_enzyme { if !assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) { assoc_item.attrs.push(attr); } - if !assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { - assoc_item.attrs.push(inline_never.clone()); + if assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { + has_inline_never = true; } Annotatable::AssocItem(assoc_item.clone(), i) } @@ -410,9 +411,8 @@ mod llvm_enzyme { if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) { iitem.attrs.push(attr); } - if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) - { - iitem.attrs.push(inline_never.clone()); + if iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { + has_inline_never = true; } } _ => unreachable!("stmt kind checked previously"), @@ -433,11 +433,19 @@ mod llvm_enzyme { let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let d_attr = outer_normal_attr(&rustc_ad_attr, new_id, span); + + // If the source function has the `#[inline(never)]` attribute, we'll also add it to the diff function + let mut d_attrs = thin_vec![d_attr]; + + if has_inline_never { + d_attrs.push(inline_never); + } + let d_annotatable = match &item { Annotatable::AssocItem(_, _) => { let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(d_fn); let d_fn = Box::new(ast::AssocItem { - attrs: thin_vec![d_attr], + attrs: d_attrs, id: ast::DUMMY_NODE_ID, span, vis, @@ -447,13 +455,13 @@ mod llvm_enzyme { Annotatable::AssocItem(d_fn, Impl { of_trait: false }) } Annotatable::Item(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(d_fn)); + let mut d_fn = ecx.item(span, d_attrs, ItemKind::Fn(d_fn)); d_fn.vis = vis; Annotatable::Item(d_fn) } Annotatable::Stmt(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr], ItemKind::Fn(d_fn)); + let mut d_fn = ecx.item(span, d_attrs, ItemKind::Fn(d_fn)); d_fn.vis = vis; Annotatable::Stmt(Box::new(ast::Stmt { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 56116959a62..e2df3265f6f 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -10,10 +10,9 @@ use tracing::debug; use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; -use crate::llvm::AttributePlace::Function; +use crate::llvm; use crate::llvm::{Metadata, True, Type}; use crate::value::Value; -use crate::{attributes, llvm}; pub(crate) fn adjust_activity_to_abi<'tcx>( tcx: TyCtxt<'tcx>, @@ -308,11 +307,6 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( enzyme_ty, ); - // Otherwise LLVM might inline our temporary code before the enzyme pass has a chance to - // do it's work. - let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); - attributes::apply_to_llfn(ad_fn, Function, &[attr]); - let num_args = llvm::LLVMCountParams(&fn_to_diff); let mut args = Vec::with_capacity(num_args as usize + 1); args.push(fn_to_diff); -- cgit 1.4.1-3-g733a5 From e1d79b9aad9cfcfcf23c68ec8625411819d4e3f7 Mon Sep 17 00:00:00 2001 From: Marcelo Domínguez Date: Thu, 14 Aug 2025 08:21:15 +0000 Subject: Remove lto inline logic --- compiler/rustc_codegen_llvm/src/attributes.rs | 16 --------------- compiler/rustc_codegen_llvm/src/back/lto.rs | 28 +-------------------------- compiler/rustc_codegen_llvm/src/context.rs | 10 ---------- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 26 ------------------------- 4 files changed, 1 insertion(+), 79 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c548f467583..a6daacd95ef 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -28,22 +28,6 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } } -pub(crate) fn has_attr(llfn: &Value, idx: AttributePlace, attr: AttributeKind) -> bool { - llvm::HasAttributeAtIndex(llfn, idx, attr) -} - -pub(crate) fn has_string_attr(llfn: &Value, name: &str) -> bool { - llvm::HasStringAttribute(llfn, name) -} - -pub(crate) fn remove_from_llfn(llfn: &Value, place: AttributePlace, kind: AttributeKind) { - llvm::RemoveRustEnumAttributeAtIndex(llfn, place, kind); -} - -pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: &str) { - llvm::RemoveStringAttrFromFn(llfn, name); -} - /// Get LLVM attribute for the provided inline heuristic. #[inline] fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index c269f11e931..853d0295238 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -24,9 +24,8 @@ use crate::back::write::{ self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode, }; use crate::errors::{LlvmError, LtoBitcodeFromRlib}; -use crate::llvm::AttributePlace::Function; use crate::llvm::{self, build_string}; -use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes}; +use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx}; /// We keep track of the computed LTO cache keys from the previous /// session to determine which CGUs we can reuse. @@ -593,31 +592,6 @@ pub(crate) fn run_pass_manager( } if cfg!(llvm_enzyme) && enable_ad && !thin { - let cx = - SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); - - for function in cx.get_functions() { - let enzyme_marker = "enzyme_marker"; - if attributes::has_string_attr(function, enzyme_marker) { - // Sanity check: Ensure 'noinline' is present before replacing it. - assert!( - attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), - "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'" - ); - - attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline); - attributes::remove_string_attr_from_llfn(function, enzyme_marker); - - assert!( - !attributes::has_string_attr(function, enzyme_marker), - "Expected function to not have 'enzyme_marker'" - ); - - let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx); - attributes::apply_to_llfn(function, Function, &[always_inline]); - } - } - let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index da8c1e5f47b..b0f3494ea68 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -722,16 +722,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) } } - - pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { - let mut functions = vec![]; - let mut func = unsafe { llvm::LLVMGetFirstFunction(self.llmod()) }; - while let Some(f) = func { - functions.push(f); - func = unsafe { llvm::LLVMGetNextFunction(f) } - } - functions - } } impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 154ba4fd690..0ea0af0c9af 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -42,32 +42,6 @@ pub(crate) fn AddFunctionAttributes<'ll>( } } -pub(crate) fn HasAttributeAtIndex<'ll>( - llfn: &'ll Value, - idx: AttributePlace, - kind: AttributeKind, -) -> bool { - unsafe { LLVMRustHasAttributeAtIndex(llfn, idx.as_uint(), kind) } -} - -pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: &str) -> bool { - unsafe { LLVMRustHasFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) { - unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveRustEnumAttributeAtIndex( - llfn: &Value, - place: AttributePlace, - kind: AttributeKind, -) { - unsafe { - LLVMRustRemoveEnumAttributeAtIndex(llfn, place.as_uint(), kind); - } -} - pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, -- cgit 1.4.1-3-g733a5 From c64c6d85e12d227869014c42abde50aaf935bc87 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Thu, 14 Aug 2025 22:34:35 +1000 Subject: Use `LLVMSetTailCallKind` --- compiler/rustc_codegen_llvm/src/builder.rs | 2 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 3 ++- compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 26 ------------------------ 3 files changed, 3 insertions(+), 28 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 917d07e3c61..2983927ca1c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1453,7 +1453,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { instance: Option>, ) { let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance); - llvm::LLVMRustSetTailCallKind(call, llvm::TailCallKind::MustTail); + llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); match &fn_abi.ret.mode { PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(), diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index ad3c3d5932e..439626263e9 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -97,6 +97,7 @@ pub(crate) enum ModuleFlagMergeBehavior { // Consts for the LLVM CallConv type, pre-cast to usize. +/// Must match the layout of `LLVMTailCallKind`. #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] #[allow(dead_code)] @@ -1197,7 +1198,7 @@ unsafe extern "C" { pub(crate) safe fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub(crate) safe fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); - pub(crate) safe fn LLVMRustSetTailCallKind(CallInst: &Value, Kind: TailCallKind); + pub(crate) safe fn LLVMSetTailCallKind(CallInst: &Value, kind: TailCallKind); // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index cd4f80f808c..7cc50d00a63 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1993,29 +1993,3 @@ extern "C" void LLVMRustSetNoSanitizeHWAddress(LLVMValueRef Global) { MD.NoHWAddress = true; GV.setSanitizerMetadata(MD); } - -enum class LLVMRustTailCallKind { - None = 0, - Tail = 1, - MustTail = 2, - NoTail = 3 -}; - -extern "C" void LLVMRustSetTailCallKind(LLVMValueRef Call, - LLVMRustTailCallKind Kind) { - CallInst *CI = unwrap(Call); - switch (Kind) { - case LLVMRustTailCallKind::None: - CI->setTailCallKind(CallInst::TCK_None); - break; - case LLVMRustTailCallKind::Tail: - CI->setTailCallKind(CallInst::TCK_Tail); - break; - case LLVMRustTailCallKind::MustTail: - CI->setTailCallKind(CallInst::TCK_MustTail); - break; - case LLVMRustTailCallKind::NoTail: - CI->setTailCallKind(CallInst::TCK_NoTail); - break; - } -} -- cgit 1.4.1-3-g733a5 From e193b5342b170f9e3cc6e7ee3bd863652f1244a2 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Thu, 14 Aug 2025 22:51:10 +1000 Subject: Use `LLVMGetTypeKind` --- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 16 ++++--- compiler/rustc_codegen_llvm/src/type_.rs | 2 +- compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 54 ------------------------ 3 files changed, 11 insertions(+), 61 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 439626263e9..8265b0114ce 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -333,10 +333,15 @@ impl RealPredicate { } } -/// LLVMTypeKind -#[derive(Copy, Clone, PartialEq, Debug)] +/// Must match the layout of `LLVMTypeKind`. +/// +/// Use [`RawEnum`] for values of `LLVMTypeKind` returned from LLVM, +/// to avoid risk of UB if LLVM adds new enum values. +/// +/// All of LLVM's variants should be declared here, even if no Rust-side code refers +/// to them, because unknown variants will cause [`RawEnum::to_rust`] to panic. +#[derive(Copy, Clone, PartialEq, Debug, TryFromU32)] #[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] pub(crate) enum TypeKind { Void = 0, Half = 1, @@ -1047,6 +1052,8 @@ unsafe extern "C" { CanThrow: llvm::Bool, ) -> &'ll Value; + pub(crate) safe fn LLVMGetTypeKind(Ty: &Type) -> RawEnum; + // Operations on integer types pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; @@ -1842,9 +1849,6 @@ unsafe extern "C" { // Create and destroy contexts. pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; - /// See llvm::LLVMTypeKind::getTypeID. - pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on all values pub(crate) fn LLVMRustGlobalAddMetadata<'a>( Val: &'a Value, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 89365503138..f02d16baf94 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -204,7 +204,7 @@ impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_kind(&self, ty: &'ll Type) -> TypeKind { - unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } + llvm::LLVMGetTypeKind(ty).to_rust().to_generic() } fn type_ptr(&self) -> &'ll Type { diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 7cc50d00a63..e4fe1fc2e42 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1460,60 +1460,6 @@ LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { return toRust((DiagnosticKind)unwrap(DI)->getKind()); } -// This is kept distinct from LLVMGetTypeKind, because when -// a new type kind is added, the Rust-side enum must be -// updated or UB will result. -extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { - switch (unwrap(Ty)->getTypeID()) { - case Type::VoidTyID: - return LLVMVoidTypeKind; - case Type::HalfTyID: - return LLVMHalfTypeKind; - case Type::FloatTyID: - return LLVMFloatTypeKind; - case Type::DoubleTyID: - return LLVMDoubleTypeKind; - case Type::X86_FP80TyID: - return LLVMX86_FP80TypeKind; - case Type::FP128TyID: - return LLVMFP128TypeKind; - case Type::PPC_FP128TyID: - return LLVMPPC_FP128TypeKind; - case Type::LabelTyID: - return LLVMLabelTypeKind; - case Type::MetadataTyID: - return LLVMMetadataTypeKind; - case Type::IntegerTyID: - return LLVMIntegerTypeKind; - case Type::FunctionTyID: - return LLVMFunctionTypeKind; - case Type::StructTyID: - return LLVMStructTypeKind; - case Type::ArrayTyID: - return LLVMArrayTypeKind; - case Type::PointerTyID: - return LLVMPointerTypeKind; - case Type::FixedVectorTyID: - return LLVMVectorTypeKind; - case Type::TokenTyID: - return LLVMTokenTypeKind; - case Type::ScalableVectorTyID: - return LLVMScalableVectorTypeKind; - case Type::BFloatTyID: - return LLVMBFloatTypeKind; - case Type::X86_AMXTyID: - return LLVMX86_AMXTypeKind; - default: { - std::string error; - auto stream = llvm::raw_string_ostream(error); - stream << "Rust does not support the TypeID: " << unwrap(Ty)->getTypeID() - << " for the type: " << *unwrap(Ty); - stream.flush(); - report_fatal_error(error.c_str()); - } - } -} - DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic(LLVMDiagnosticInfoRef DI, -- cgit 1.4.1-3-g733a5 From 460519a7f509756d07dcf187e671a11fde7b3fca Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 15 Aug 2025 09:37:24 +0000 Subject: Merge link_name and export_name --- compiler/rustc_codegen_llvm/src/attributes.rs | 2 +- compiler/rustc_codegen_ssa/src/base.rs | 2 +- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 17 +++++++---- compiler/rustc_lint/src/foreign_modules.rs | 2 +- compiler/rustc_metadata/src/native_libs.rs | 2 +- .../rustc_middle/src/middle/codegen_fn_attrs.rs | 17 ++++------- compiler/rustc_symbol_mangling/src/lib.rs | 33 ++++++++-------------- src/tools/miri/src/shims/foreign_items.rs | 2 +- 8 files changed, 33 insertions(+), 44 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c548f467583..35cc013b852 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -513,7 +513,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b4556ced0b3..883212cd07a 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -857,7 +857,7 @@ pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>( instance: Instance<'tcx>, ) -> bool { fn is_llvm_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - if let Some(name) = tcx.codegen_fn_attrs(def_id).link_name { + if let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name { name.as_str().starts_with("llvm.") } else { false diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 7fa37198387..136b8591f83 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -189,7 +189,7 @@ fn process_builtin_attrs( match p { AttributeKind::Cold(_) => codegen_fn_attrs.flags |= CodegenFnAttrFlags::COLD, AttributeKind::ExportName { name, .. } => { - codegen_fn_attrs.export_name = Some(*name) + codegen_fn_attrs.symbol_name = Some(*name) } AttributeKind::Inline(inline, span) => { codegen_fn_attrs.inline = *inline; @@ -197,7 +197,13 @@ fn process_builtin_attrs( } AttributeKind::Naked(_) => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NAKED, AttributeKind::Align { align, .. } => codegen_fn_attrs.alignment = Some(*align), - AttributeKind::LinkName { name, .. } => codegen_fn_attrs.link_name = Some(*name), + AttributeKind::LinkName { name, .. } => { + // FIXME Remove check for foreign functions once #[link_name] on non-foreign + // functions is a hard error + if tcx.is_foreign_item(did) { + codegen_fn_attrs.symbol_name = Some(*name); + } + } AttributeKind::LinkOrdinal { ordinal, span } => { codegen_fn_attrs.link_ordinal = Some(*ordinal); interesting_spans.link_ordinal = Some(*span); @@ -417,7 +423,7 @@ fn apply_overrides(tcx: TyCtxt<'_>, did: LocalDefId, codegen_fn_attrs: &mut Code // * `#[rustc_std_internal_symbol]` mangles the symbol name in a special way // both for exports and imports through foreign items. This is handled further, // during symbol mangling logic. - } else if codegen_fn_attrs.link_name.is_some() { + } else if codegen_fn_attrs.symbol_name.is_some() { // * This can be overridden with the `#[link_name]` attribute } else { // NOTE: there's one more exception that we cannot apply here. On wasm, @@ -472,7 +478,7 @@ fn check_result( } // error when specifying link_name together with link_ordinal - if let Some(_) = codegen_fn_attrs.link_name + if let Some(_) = codegen_fn_attrs.symbol_name && let Some(_) = codegen_fn_attrs.link_ordinal { let msg = "cannot use `#[link_name]` with `#[link_ordinal]`"; @@ -523,8 +529,7 @@ fn handle_lang_items( && let Some(link_name) = lang_item.link_name() { codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL; - codegen_fn_attrs.export_name = Some(link_name); - codegen_fn_attrs.link_name = Some(link_name); + codegen_fn_attrs.symbol_name = Some(link_name); } // error when using no_mangle on a lang item item diff --git a/compiler/rustc_lint/src/foreign_modules.rs b/compiler/rustc_lint/src/foreign_modules.rs index 759e6c927b8..3267e70f1de 100644 --- a/compiler/rustc_lint/src/foreign_modules.rs +++ b/compiler/rustc_lint/src/foreign_modules.rs @@ -179,7 +179,7 @@ impl ClashingExternDeclarations { /// symbol's name. fn name_of_extern_decl(tcx: TyCtxt<'_>, fi: hir::OwnerId) -> SymbolName { if let Some((overridden_link_name, overridden_link_name_span)) = - tcx.codegen_fn_attrs(fi).link_name.map(|overridden_link_name| { + tcx.codegen_fn_attrs(fi).symbol_name.map(|overridden_link_name| { // FIXME: Instead of searching through the attributes again to get span // information, we could have codegen_fn_attrs also give span information back for // where the attribute was defined. However, until this is found to be a diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs index 958e314efab..63f1b51df1c 100644 --- a/compiler/rustc_metadata/src/native_libs.rs +++ b/compiler/rustc_metadata/src/native_libs.rs @@ -701,7 +701,7 @@ impl<'tcx> Collector<'tcx> { .link_ordinal .map_or(import_name_type, |ord| Some(PeImportNameType::Ordinal(ord))); - let name = codegen_fn_attrs.link_name.unwrap_or_else(|| self.tcx.item_name(item)); + let name = codegen_fn_attrs.symbol_name.unwrap_or_else(|| self.tcx.item_name(item)); if self.tcx.sess.target.binary_format == BinaryFormat::Elf { let name = name.as_str(); diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs index 2852c4cbd34..b3a26f3b8ef 100644 --- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs +++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs @@ -36,14 +36,10 @@ pub struct CodegenFnAttrs { pub inline: InlineAttr, /// Parsed representation of the `#[optimize]` attribute pub optimize: OptimizeAttr, - /// The `#[export_name = "..."]` attribute, indicating a custom symbol a - /// function should be exported under - pub export_name: Option, - /// The `#[link_name = "..."]` attribute, indicating a custom symbol an - /// imported function should be imported as. Note that `export_name` - /// probably isn't set when this is set, this is for foreign items while - /// `#[export_name]` is for Rust-defined functions. - pub link_name: Option, + /// The name this function will be imported/exported under. This can be set + /// using the `#[export_name = "..."]` or `#[link_name = "..."]` attribute + /// depending on if this is a function definition or foreign function. + pub symbol_name: Option, /// The `#[link_ordinal = "..."]` attribute, indicating an ordinal an /// imported function has in the dynamic library. Note that this must not /// be set when `link_name` is set. This is for foreign items with the @@ -170,8 +166,7 @@ impl CodegenFnAttrs { flags: CodegenFnAttrFlags::empty(), inline: InlineAttr::None, optimize: OptimizeAttr::Default, - export_name: None, - link_name: None, + symbol_name: None, link_ordinal: None, target_features: vec![], safe_target_features: false, @@ -200,7 +195,7 @@ impl CodegenFnAttrs { self.flags.contains(CodegenFnAttrFlags::NO_MANGLE) || self.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) - || self.export_name.is_some() + || self.symbol_name.is_some() || match self.linkage { // These are private, so make sure we don't try to consider // them external. diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs index 96a501fb0ea..d97ee956525 100644 --- a/compiler/rustc_symbol_mangling/src/lib.rs +++ b/compiler/rustc_symbol_mangling/src/lib.rs @@ -204,13 +204,9 @@ fn compute_symbol_name<'tcx>( // mangling scheme can't be used. For example both the GCC backend and // Rust-for-Linux don't support some of the characters used by the // legacy symbol mangling scheme. - let name = if tcx.is_foreign_item(def_id) { - if let Some(name) = attrs.link_name { name } else { tcx.item_name(def_id) } - } else { - if let Some(name) = attrs.export_name { name } else { tcx.item_name(def_id) } - }; + let name = if let Some(name) = attrs.symbol_name { name } else { tcx.item_name(def_id) }; - return v0::mangle_internal_symbol(tcx, name.as_str()); + return v0::mangle_internal_symbol(tcx, name.as_str()); } let wasm_import_module_exception_force_mangling = { @@ -235,23 +231,16 @@ fn compute_symbol_name<'tcx>( && tcx.wasm_import_module_map(LOCAL_CRATE).contains_key(&def_id.into()) }; - if let Some(name) = attrs.link_name - && !wasm_import_module_exception_force_mangling - { - // Use provided name - return name.to_string(); - } - - if let Some(name) = attrs.export_name { - // Use provided name - return name.to_string(); - } + if !wasm_import_module_exception_force_mangling { + if let Some(name) = attrs.symbol_name { + // Use provided name + return name.to_string(); + } - if attrs.flags.contains(CodegenFnAttrFlags::NO_MANGLE) - && !wasm_import_module_exception_force_mangling - { - // Don't mangle - return tcx.item_name(def_id).to_string(); + if attrs.flags.contains(CodegenFnAttrFlags::NO_MANGLE) { + // Don't mangle + return tcx.item_name(def_id).to_string(); + } } // If we're dealing with an instance of a function that's inlined from diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 21545b68029..23b1d0c4f6e 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -146,7 +146,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { return interp_ok(()); } // Skip over items without an explicitly defined symbol name. - if !(attrs.export_name.is_some() + if !(attrs.symbol_name.is_some() || attrs.flags.contains(CodegenFnAttrFlags::NO_MANGLE) || attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL)) { -- cgit 1.4.1-3-g733a5 From 8d0a04966c97eb2015c950396e54f31f5212a888 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 15 Aug 2025 20:06:01 +1000 Subject: Declare module `rustc_codegen_llvm::back` in the normal way Declaring these submodules directly in `lib.rs` was needlessly confusing. --- compiler/rustc_codegen_llvm/src/back/mod.rs | 5 +++++ compiler/rustc_codegen_llvm/src/lib.rs | 9 +-------- 2 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 compiler/rustc_codegen_llvm/src/back/mod.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/mod.rs b/compiler/rustc_codegen_llvm/src/back/mod.rs new file mode 100644 index 00000000000..6cb89f80ab8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod archive; +pub(crate) mod lto; +pub(crate) mod owned_target_machine; +mod profiling; +pub(crate) mod write; diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ca84b6de8b1..3264a747e3a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -47,18 +47,11 @@ use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_span::Symbol; -mod back { - pub(crate) mod archive; - pub(crate) mod lto; - pub(crate) mod owned_target_machine; - mod profiling; - pub(crate) mod write; -} - mod abi; mod allocator; mod asm; mod attributes; +mod back; mod base; mod builder; mod callee; -- cgit 1.4.1-3-g733a5 From 44f5ec7d56acb21cad9bcc0dbc13e70a749f35aa Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 15 Aug 2025 19:51:33 +1000 Subject: Avoid an explicit cast from `*const c_uchar` to `*const c_char` As noted in the `ffi` module docs, passing pointer/length byte strings from Rust to C++ is easier if we declare them as `*const c_uchar` on the Rust side, but `const char *` (possibly signed) on the C++ side. This is allowed because both pointer types are ABI-compatible, regardless of char signedness. --- compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 4 ++-- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 8e82013e94a..30b1c243de0 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,4 +1,4 @@ -use std::ffi::{CStr, c_char}; +use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -71,7 +71,7 @@ impl OwnedTargetMachine { output_obj_file.as_ptr(), debug_info_compression.as_ptr(), use_emulated_tls, - args_cstr_buff.as_ptr() as *const c_char, + args_cstr_buff.as_ptr(), args_cstr_buff.len(), use_wasm_eh, ) diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index ad3c3d5932e..0ed6a9dc33c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2438,7 +2438,7 @@ unsafe extern "C" { OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, UseEmulatedTls: bool, - ArgsCstrBuff: *const c_char, + ArgsCstrBuff: *const c_uchar, // See "PTR_LEN_STR". ArgsCstrBuffLen: usize, UseWasmEH: bool, ) -> *mut TargetMachine; -- cgit 1.4.1-3-g733a5 From 61932e12221fb8a76a8dedc4a21e5f6e1e332d9d Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 15 Aug 2025 19:54:59 +1000 Subject: Avoid an unnecessary intermediate `&mut` reference The `NonNull::as_mut` method returns a mut *reference*, rather than the mut *pointer* that is intended here. --- compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 30b1c243de0..b073bafc0aa 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -99,7 +99,7 @@ impl Drop for OwnedTargetMachine { // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut()); + llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_ptr()); } } } -- cgit 1.4.1-3-g733a5 From 9e7d06692803af171633b90799495be880dc3508 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Fri, 15 Aug 2025 20:13:42 +1000 Subject: Simplify the `args_cstr_buff` assertion --- compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index b073bafc0aa..6d8178320fe 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,3 +1,4 @@ +use std::assert_matches::assert_matches; use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -41,11 +42,9 @@ impl OwnedTargetMachine { args_cstr_buff: &[u8], use_wasm_eh: bool, ) -> Result> { - assert!(args_cstr_buff.len() > 0); - assert!( - *args_cstr_buff.last().unwrap() == 0, - "The last character must be a null terminator." - ); + // The argument list is passed as the concatenation of one or more C strings. + // This implies that there must be a last byte, and it must be 0. + assert_matches!(args_cstr_buff, [.., b'\0'], "the last byte must be a NUL terminator"); // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { -- cgit 1.4.1-3-g733a5 From cf8ec6798f53a67b811a08f0dd45646f1e075126 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Sat, 16 Aug 2025 13:14:52 +1000 Subject: Remove `LlvmArchiveBuilder` and supporting code/bindings --- compiler/rustc_codegen_llvm/src/back/archive.rs | 183 +----------------- compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs | 94 ---------- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 46 ----- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 17 -- compiler/rustc_llvm/build.rs | 1 - .../rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp | 208 --------------------- 6 files changed, 6 insertions(+), 543 deletions(-) delete mode 100644 compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs delete mode 100644 compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 0a161442933..7a340ae83f3 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,104 +1,21 @@ //! A helper class for dealing with static archives -use std::ffi::{CStr, CString, c_char, c_void}; -use std::path::{Path, PathBuf}; -use std::{io, mem, ptr, str}; +use std::ffi::{CStr, c_char, c_void}; +use std::io; use rustc_codegen_ssa::back::archive::{ - ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, - DEFAULT_OBJECT_READER, ObjectReader, UnknownArchiveKind, try_extract_macho_fat_archive, + ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, DEFAULT_OBJECT_READER, ObjectReader, }; use rustc_session::Session; -use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind, last_error}; - -/// Helper for adding many files to an archive. -#[must_use = "must call build() to finish building the archive"] -pub(crate) struct LlvmArchiveBuilder<'a> { - sess: &'a Session, - additions: Vec, -} - -enum Addition { - File { path: PathBuf, name_in_archive: String }, - Archive { path: PathBuf, archive: ArchiveRO, skip: Box bool> }, -} - -impl Addition { - fn path(&self) -> &Path { - match self { - Addition::File { path, .. } | Addition::Archive { path, .. } => path, - } - } -} - -fn is_relevant_child(c: &Child<'_>) -> bool { - match c.name() { - Some(name) => !name.contains("SYMDEF"), - None => false, - } -} - -impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { - fn add_archive( - &mut self, - archive: &Path, - skip: Box bool + 'static>, - ) -> io::Result<()> { - let mut archive = archive.to_path_buf(); - if self.sess.target.llvm_target.contains("-apple-macosx") { - if let Some(new_archive) = try_extract_macho_fat_archive(self.sess, &archive)? { - archive = new_archive - } - } - let archive_ro = match ArchiveRO::open(&archive) { - Ok(ar) => ar, - Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), - }; - if self.additions.iter().any(|ar| ar.path() == archive) { - return Ok(()); - } - self.additions.push(Addition::Archive { - path: archive, - archive: archive_ro, - skip: Box::new(skip), - }); - Ok(()) - } - - /// Adds an arbitrary file to this archive - fn add_file(&mut self, file: &Path) { - let name = file.file_name().unwrap().to_str().unwrap(); - self.additions - .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); - } - - /// Combine the provided files, rlibs, and native libraries into a single - /// `Archive`. - fn build(mut self: Box, output: &Path) -> bool { - match self.build_with_llvm(output) { - Ok(any_members) => any_members, - Err(error) => { - self.sess.dcx().emit_fatal(ArchiveBuildFailure { path: output.to_owned(), error }) - } - } - } -} +use crate::llvm; pub(crate) struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box { - // Keeping LlvmArchiveBuilder around in case of a regression caused by using - // ArArchiveBuilder. - // FIXME(#128955) remove a couple of months after #128936 gets merged in case - // no regression is found. - if false { - Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) - } else { - Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) - } + // Use the `object` crate to build archives, with a little bit of help from LLVM. + Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) } } @@ -178,91 +95,3 @@ fn llvm_is_64_bit_object_file(buf: &[u8]) -> bool { fn llvm_is_ec_object_file(buf: &[u8]) -> bool { unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } - -impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self, output: &Path) -> io::Result { - let kind = &*self.sess.target.archive_format; - let kind = kind - .parse::() - .map_err(|_| kind) - .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); - - let mut additions = mem::take(&mut self.additions); - // Values in the `members` list below will contain pointers to the strings allocated here. - // So they need to get dropped after all elements of `members` get freed. - let mut strings = Vec::new(); - let mut members = Vec::new(); - - let dst = CString::new(output.to_str().unwrap())?; - - unsafe { - for addition in &mut additions { - match addition { - Addition::File { path, name_in_archive } => { - let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.as_bytes())?; - members.push(llvm::LLVMRustArchiveMemberNew( - path.as_ptr(), - name.as_ptr(), - None, - )); - strings.push(path); - strings.push(name); - } - Addition::Archive { archive, skip, .. } => { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - if !is_relevant_child(&child) { - continue; - } - let child_name = child.name().unwrap(); - if skip(child_name) { - continue; - } - - // It appears that LLVM's archive writer is a little - // buggy if the name we pass down isn't just the - // filename component, so chop that off here and - // pass it in. - // - // See LLVM bug 25877 for more info. - let child_name = - Path::new(child_name).file_name().unwrap().to_str().unwrap(); - let name = CString::new(child_name)?; - let m = llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - ); - members.push(m); - strings.push(name); - } - } - } - } - - let r = llvm::LLVMRustWriteArchive( - dst.as_ptr(), - members.len() as libc::size_t, - members.as_ptr() as *const &_, - true, - kind, - self.sess.target.arch == "arm64ec", - ); - let ret = if r.into_result().is_err() { - let msg = last_error().unwrap_or_else(|| "failed to write archive".into()); - Err(io::Error::new(io::ErrorKind::Other, msg)) - } else { - Ok(!members.is_empty()) - }; - for member in members { - llvm::LLVMRustArchiveMemberFree(member); - } - ret - } - } -} - -fn string_to_io_error(s: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("bad archive: {s}")) -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs deleted file mode 100644 index 51bcc4d123d..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! A wrapper around LLVM's archive (.a) code - -use std::path::Path; -use std::{slice, str}; - -use rustc_fs_util::path_to_c_string; - -pub(crate) struct ArchiveRO { - pub raw: &'static mut super::Archive, -} - -unsafe impl Send for ArchiveRO {} - -pub(crate) struct Iter<'a> { - raw: &'a mut super::ArchiveIterator<'a>, -} - -pub(crate) struct Child<'a> { - pub raw: &'a mut super::ArchiveChild<'a>, -} - -impl ArchiveRO { - /// Opens a static archive for read-only purposes. This is more optimized - /// than the `open` method because it uses LLVM's internal `Archive` class - /// rather than shelling out to `ar` for everything. - /// - /// If this archive is used with a mutable method, then an error will be - /// raised. - pub(crate) fn open(dst: &Path) -> Result { - unsafe { - let s = path_to_c_string(dst); - let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { - super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) - })?; - Ok(ArchiveRO { raw: ar }) - } - } - - pub(crate) fn iter(&self) -> Iter<'_> { - unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } - } -} - -impl Drop for ArchiveRO { - fn drop(&mut self) { - unsafe { - super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Iterator for Iter<'a> { - type Item = Result, String>; - - fn next(&mut self) -> Option, String>> { - unsafe { - match super::LLVMRustArchiveIteratorNext(self.raw) { - Some(raw) => Some(Ok(Child { raw })), - None => super::last_error().map(Err), - } - } - } -} - -impl<'a> Drop for Iter<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Child<'a> { - pub(crate) fn name(&self) -> Option<&'a str> { - unsafe { - let mut name_len = 0; - let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); - if name_ptr.is_null() { - None - } else { - let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); - str::from_utf8(name).ok().map(|s| s.trim()) - } - } - } -} - -impl<'a> Drop for Child<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index ad3c3d5932e..cdedfcc846f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -610,17 +610,6 @@ pub(crate) enum DiagnosticLevel { Remark, } -/// LLVMRustArchiveKind -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) enum ArchiveKind { - K_GNU, - K_BSD, - K_DARWIN, - K_COFF, - K_AIXBIG, -} - unsafe extern "C" { // LLVMRustThinLTOData pub(crate) type ThinLTOData; @@ -769,19 +758,12 @@ pub(crate) struct Builder<'a>(InvariantOpaque<'a>); pub(crate) struct PassManager<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub type TargetMachine; - pub(crate) type Archive; } -#[repr(C)] -pub(crate) struct ArchiveIterator<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub(crate) struct ArchiveChild<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub(crate) type Twine; pub(crate) type DiagnosticInfo; pub(crate) type SMDiagnostic; } -#[repr(C)] -pub(crate) struct RustArchiveMember<'a>(InvariantOpaque<'a>); /// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); @@ -2505,19 +2487,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub(crate) fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - pub(crate) fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub(crate) fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub(crate) fn LLVMRustArchiveIteratorNext<'a>( - AIR: &ArchiveIterator<'a>, - ) -> Option<&'a mut ArchiveChild<'a>>; - pub(crate) fn LLVMRustArchiveChildName( - ACR: &ArchiveChild<'_>, - size: &mut size_t, - ) -> *const c_char; - pub(crate) fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub(crate) fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub(crate) fn LLVMRustDestroyArchive(AR: &'static mut Archive); - pub(crate) fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); pub(crate) fn LLVMRustUnpackOptimizationDiagnostic<'a>( @@ -2555,21 +2524,6 @@ unsafe extern "C" { num_ranges: &mut usize, ) -> bool; - pub(crate) fn LLVMRustWriteArchive( - Dst: *const c_char, - NumMembers: size_t, - Members: *const &RustArchiveMember<'_>, - WriteSymbtab: bool, - Kind: ArchiveKind, - isEC: bool, - ) -> LLVMRustResult; - pub(crate) fn LLVMRustArchiveMemberNew<'a>( - Filename: *const c_char, - Name: *const c_char, - Child: Option<&ArchiveChild<'a>>, - ) -> &'a mut RustArchiveMember<'a>; - pub(crate) fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); pub(crate) fn LLVMRustPositionBuilderPastAllocas<'a>(B: &Builder<'a>, Fn: &'a Value); diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 0ea0af0c9af..7fea7b79a8c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -3,7 +3,6 @@ use std::ffi::{CStr, CString}; use std::num::NonZero; use std::ptr; -use std::str::FromStr; use std::string::FromUtf8Error; use libc::c_uint; @@ -16,7 +15,6 @@ pub(crate) use self::MetadataType::*; pub(crate) use self::ffi::*; use crate::common::AsCCharPtr; -pub(crate) mod archive_ro; pub(crate) mod diagnostic; pub(crate) mod enzyme_ffi; mod ffi; @@ -152,21 +150,6 @@ pub(crate) enum CodeGenOptSize { CodeGenOptSizeAggressive = 2, } -impl FromStr for ArchiveKind { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "gnu" => Ok(ArchiveKind::K_GNU), - "bsd" => Ok(ArchiveKind::K_BSD), - "darwin" => Ok(ArchiveKind::K_DARWIN), - "coff" => Ok(ArchiveKind::K_COFF), - "aix_big" => Ok(ArchiveKind::K_AIXBIG), - _ => Err(()), - } - } -} - pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index 1394edcee6b..d01f79dcade 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -226,7 +226,6 @@ fn main() { rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper")); cfg.file("llvm-wrapper/PassWrapper.cpp") .file("llvm-wrapper/RustWrapper.cpp") - .file("llvm-wrapper/ArchiveWrapper.cpp") .file("llvm-wrapper/CoverageMappingWrapper.cpp") .file("llvm-wrapper/SymbolWrapper.cpp") .file("llvm-wrapper/Linker.cpp") diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp deleted file mode 100644 index feac6a5649c..00000000000 --- a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp +++ /dev/null @@ -1,208 +0,0 @@ -#include "LLVMWrapper.h" - -#include "llvm/Object/Archive.h" -#include "llvm/Object/ArchiveWriter.h" -#include "llvm/Support/Path.h" - -using namespace llvm; -using namespace llvm::object; - -struct RustArchiveMember { - const char *Filename; - const char *Name; - Archive::Child Child; - - RustArchiveMember() - : Filename(nullptr), Name(nullptr), Child(nullptr, nullptr, nullptr) {} - ~RustArchiveMember() {} -}; - -struct RustArchiveIterator { - bool First; - Archive::child_iterator Cur; - Archive::child_iterator End; - std::unique_ptr Err; - - RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End, - std::unique_ptr Err) - : First(true), Cur(Cur), End(End), Err(std::move(Err)) {} -}; - -enum class LLVMRustArchiveKind { - GNU, - BSD, - DARWIN, - COFF, - AIX_BIG, -}; - -static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { - switch (Kind) { - case LLVMRustArchiveKind::GNU: - return Archive::K_GNU; - case LLVMRustArchiveKind::BSD: - return Archive::K_BSD; - case LLVMRustArchiveKind::DARWIN: - return Archive::K_DARWIN; - case LLVMRustArchiveKind::COFF: - return Archive::K_COFF; - case LLVMRustArchiveKind::AIX_BIG: - return Archive::K_AIXBIG; - default: - report_fatal_error("Bad ArchiveKind."); - } -} - -typedef OwningBinary *LLVMRustArchiveRef; -typedef RustArchiveMember *LLVMRustArchiveMemberRef; -typedef Archive::Child *LLVMRustArchiveChildRef; -typedef Archive::Child const *LLVMRustArchiveChildConstRef; -typedef RustArchiveIterator *LLVMRustArchiveIteratorRef; - -extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) { - ErrorOr> BufOr = MemoryBuffer::getFile( - Path, /*IsText*/ false, /*RequiresNullTerminator=*/false); - if (!BufOr) { - LLVMRustSetLastError(BufOr.getError().message().c_str()); - return nullptr; - } - - Expected> ArchiveOr = - Archive::create(BufOr.get()->getMemBufferRef()); - - if (!ArchiveOr) { - LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str()); - return nullptr; - } - - OwningBinary *Ret = new OwningBinary( - std::move(ArchiveOr.get()), std::move(BufOr.get())); - - return Ret; -} - -extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) { - delete RustArchive; -} - -extern "C" LLVMRustArchiveIteratorRef -LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) { - Archive *Archive = RustArchive->getBinary(); - std::unique_ptr Err = std::make_unique(Error::success()); - auto Cur = Archive->child_begin(*Err); - if (*Err) { - LLVMRustSetLastError(toString(std::move(*Err)).c_str()); - return nullptr; - } - auto End = Archive->child_end(); - return new RustArchiveIterator(Cur, End, std::move(Err)); -} - -extern "C" LLVMRustArchiveChildConstRef -LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) { - if (RAI->Cur == RAI->End) - return nullptr; - - // Advancing the iterator validates the next child, and this can - // uncover an error. LLVM requires that we check all Errors, - // so we only advance the iterator if we actually need to fetch - // the next child. - // This means we must not advance the iterator in the *first* call, - // but instead advance it *before* fetching the child in all later calls. - if (!RAI->First) { - ++RAI->Cur; - if (*RAI->Err) { - LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str()); - return nullptr; - } - } else { - RAI->First = false; - } - - if (RAI->Cur == RAI->End) - return nullptr; - - const Archive::Child &Child = *RAI->Cur.operator->(); - Archive::Child *Ret = new Archive::Child(Child); - - return Ret; -} - -extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) { - delete Child; -} - -extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) { - delete RAI; -} - -extern "C" const char * -LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { - Expected NameOrErr = Child->getName(); - if (!NameOrErr) { - // rustc_codegen_llvm currently doesn't use this error string, but it might - // be useful in the future, and in the meantime this tells LLVM that the - // error was not ignored and that it shouldn't abort the process. - LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str()); - return nullptr; - } - StringRef Name = NameOrErr.get(); - *Size = Name.size(); - return Name.data(); -} - -extern "C" LLVMRustArchiveMemberRef -LLVMRustArchiveMemberNew(char *Filename, char *Name, - LLVMRustArchiveChildRef Child) { - RustArchiveMember *Member = new RustArchiveMember; - Member->Filename = Filename; - Member->Name = Name; - if (Child) - Member->Child = *Child; - return Member; -} - -extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { - delete Member; -} - -extern "C" LLVMRustResult LLVMRustWriteArchive( - char *Dst, size_t NumMembers, const LLVMRustArchiveMemberRef *NewMembers, - bool WriteSymbtab, LLVMRustArchiveKind RustKind, bool isEC) { - - std::vector Members; - auto Kind = fromRust(RustKind); - - for (size_t I = 0; I < NumMembers; I++) { - auto Member = NewMembers[I]; - assert(Member->Name); - if (Member->Filename) { - Expected MOrErr = - NewArchiveMember::getFile(Member->Filename, true); - if (!MOrErr) { - LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); - return LLVMRustResult::Failure; - } - MOrErr->MemberName = sys::path::filename(MOrErr->MemberName); - Members.push_back(std::move(*MOrErr)); - } else { - Expected MOrErr = - NewArchiveMember::getOldMember(Member->Child, true); - if (!MOrErr) { - LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); - return LLVMRustResult::Failure; - } - Members.push_back(std::move(*MOrErr)); - } - } - - auto SymtabMode = WriteSymbtab ? SymtabWritingMode::NormalSymtab - : SymtabWritingMode::NoSymtab; - auto Result = - writeArchive(Dst, Members, SymtabMode, Kind, true, false, nullptr, isEC); - if (!Result) - return LLVMRustResult::Success; - LLVMRustSetLastError(toString(std::move(Result)).c_str()); - - return LLVMRustResult::Failure; -} -- cgit 1.4.1-3-g733a5 From 1cd7080c3a7f29297675a72a157575ae12717304 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 May 2025 07:57:01 +0000 Subject: Add -Zindirect-branch-cs-prefix (from draft PR) --- compiler/rustc_codegen_llvm/src/context.rs | 9 +++++++++ compiler/rustc_interface/src/tests.rs | 1 + compiler/rustc_session/src/options.rs | 2 ++ tests/codegen-llvm/indirect-branch-cs-prefix.rs | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+) create mode 100644 tests/codegen-llvm/indirect-branch-cs-prefix.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index b0f3494ea68..ea3a21dfdb5 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -462,6 +462,15 @@ pub(crate) unsafe fn create_module<'ll>( } } + if sess.opts.unstable_opts.indirect_branch_cs_prefix { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "indirect_branch_cs_prefix", + 1, + ); + } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) { // Set up the small-data optimization limit for architectures that use diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 0a764808f95..4425877308a 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -807,6 +807,7 @@ fn test_unstable_options_tracking_hash() { tracked!(hint_mostly_unused, true); tracked!(human_readable_cgu_names, true); tracked!(incremental_ignore_spans, true); + tracked!(indirect_branch_cs_prefix, true); tracked!(inline_mir, Some(true)); tracked!(inline_mir_hint_threshold, Some(123)); tracked!(inline_mir_threshold, Some(123)); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 0e112edc733..fce4d18a1d8 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2295,6 +2295,8 @@ options! { - hashes of green query instances - hash collisions of query keys - hash collisions when creating dep-nodes"), + indirect_branch_cs_prefix: bool = (false, parse_bool, [TRACKED TARGET_MODIFIER], + "add cs prefix to call and jmp to indirect thunk (default: no)"), inline_llvm: bool = (true, parse_bool, [TRACKED], "enable LLVM inlining (default: yes)"), inline_mir: Option = (None, parse_opt_bool, [TRACKED], diff --git a/tests/codegen-llvm/indirect-branch-cs-prefix.rs b/tests/codegen-llvm/indirect-branch-cs-prefix.rs new file mode 100644 index 00000000000..a00b63a4970 --- /dev/null +++ b/tests/codegen-llvm/indirect-branch-cs-prefix.rs @@ -0,0 +1,18 @@ +// Test that the `indirect_branch_cs_prefix` module attribute is (not) emitted when the +// `-Zindirect-branch-cs-prefix` flag is (not) set. + +//@ add-core-stubs +//@ revisions: unset set +//@ needs-llvm-components: x86 +//@ compile-flags: --target x86_64-unknown-linux-gnu +//@ [set] compile-flags: -Zindirect-branch-cs-prefix + +#![crate_type = "lib"] +#![feature(no_core, lang_items)] +#![no_core] + +#[lang = "sized"] +trait Sized {} + +// unset-NOT: !{{[0-9]+}} = !{i32 4, !"indirect_branch_cs_prefix", i32 1} +// set: !{{[0-9]+}} = !{i32 4, !"indirect_branch_cs_prefix", i32 1} -- cgit 1.4.1-3-g733a5 From 45ea228c42cfb6712be36a0b4b64b43b8651a54f Mon Sep 17 00:00:00 2001 From: Zhongyao Chen Date: Tue, 19 Aug 2025 10:33:54 +0800 Subject: completely remove rva23s64 --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 6 +-- compiler/rustc_target/src/target_features.rs | 70 ---------------------------- tests/ui/check-cfg/target_feature.stderr | 14 ------ 3 files changed, 1 insertion(+), 89 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index a85a894e42c..d81c4c329b3 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -278,11 +278,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option - { - None - } + ("riscv32" | "riscv64", "zacas" | "rva23u64" | "supm") if get_version().0 < 20 => None, ( "s390x", "message-security-assist-extension12" diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs index 434f6a840b6..0729c44bc91 100644 --- a/compiler/rustc_target/src/target_features.rs +++ b/compiler/rustc_target/src/target_features.rs @@ -597,62 +597,6 @@ static RISCV_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ ), ("m", Stable, &[]), ("relax", Unstable(sym::riscv_target_feature), &[]), - ( - "rva23s64", - Unstable(sym::riscv_target_feature), - &[ - "m", - "a", - "f", - "d", - "c", - "b", - "v", - "zicsr", - "zicntr", - "zihpm", - "ziccif", - "ziccrse", - "ziccamoa", - "zicclsm", - "zic64b", - "za64rs", - "zihintpause", - "zba", - "zbb", - "zbs", - "zicbom", - "zicbop", - "zicboz", - "zfhmin", - "zkt", - "zvfhmin", - "zvbb", - "zvkt", - "zihintntl", - "zicond", - "zimop", - "zcmop", - "zcb", - "zfa", - "zawrs", - "svbare", - "svade", - "ssccptr", - "sstvecd", - "sstvala", - "sscounterenw", - "svpbmt", - "svinval", - "svnapot", - "sstc", - "sscofpmf", - "ssnpm", - "ssu64xl", - "sha", - "supm", - ], - ), ( "rva23u64", Unstable(sym::riscv_target_feature), @@ -695,21 +639,7 @@ static RISCV_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ "supm", ], ), - ("sha", Unstable(sym::riscv_target_feature), &[]), - ("ssccptr", Unstable(sym::riscv_target_feature), &[]), - ("sscofpmf", Unstable(sym::riscv_target_feature), &[]), - ("sscounterenw", Unstable(sym::riscv_target_feature), &[]), - ("ssnpm", Unstable(sym::riscv_target_feature), &[]), - ("sstc", Unstable(sym::riscv_target_feature), &[]), - ("sstvala", Unstable(sym::riscv_target_feature), &[]), - ("sstvecd", Unstable(sym::riscv_target_feature), &[]), - ("ssu64xl", Unstable(sym::riscv_target_feature), &[]), ("supm", Unstable(sym::riscv_target_feature), &[]), - ("svade", Unstable(sym::riscv_target_feature), &[]), - ("svbare", Unstable(sym::riscv_target_feature), &[]), - ("svinval", Unstable(sym::riscv_target_feature), &[]), - ("svnapot", Unstable(sym::riscv_target_feature), &[]), - ("svpbmt", Unstable(sym::riscv_target_feature), &[]), ("unaligned-scalar-mem", Unstable(sym::riscv_target_feature), &[]), ("unaligned-vector-mem", Unstable(sym::riscv_target_feature), &[]), ("v", Unstable(sym::riscv_target_feature), &["zvl128b", "zve64d"]), diff --git a/tests/ui/check-cfg/target_feature.stderr b/tests/ui/check-cfg/target_feature.stderr index a484fb0e5f0..218755a01f7 100644 --- a/tests/ui/check-cfg/target_feature.stderr +++ b/tests/ui/check-cfg/target_feature.stderr @@ -241,7 +241,6 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); `relax` `relaxed-simd` `rtm` -`rva23s64` `rva23u64` `sb` `scq` @@ -294,27 +293,17 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); `soft-float` `spe` `ssbs` -`ssccptr` -`sscofpmf` -`sscounterenw` `sse` `sse2` `sse3` `sse4.1` `sse4.2` `sse4a` -`ssnpm` `ssse3` -`sstc` -`sstvala` -`sstvecd` -`ssu64xl` `ssve-fp8dot2` `ssve-fp8dot4` `ssve-fp8fma` `supm` -`svade` -`svbare` `sve` `sve-b16b16` `sve2` @@ -323,9 +312,6 @@ LL | cfg!(target_feature = "_UNEXPECTED_VALUE"); `sve2-sha3` `sve2-sm4` `sve2p1` -`svinval` -`svnapot` -`svpbmt` `tail-call` `tbm` `thumb-mode` -- cgit 1.4.1-3-g733a5 From d25910eaeb3eab25d3bbf9b9815e6d215c202d1f Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 19 Aug 2025 22:16:58 +0200 Subject: make `prefetch` intrinsics safe --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 8 ++- compiler/rustc_hir_analysis/src/check/intrinsic.rs | 6 +- library/core/src/intrinsics/mod.rs | 28 ++++----- src/tools/miri/tests/pass/prefetch.rs | 28 ++++----- tests/codegen-llvm/intrinsics/prefetch.rs | 72 ++++++++++------------ 5 files changed, 70 insertions(+), 72 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4935f8d7dff..06c3d8ed6bc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -330,10 +330,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); + let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32; self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], - &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)], + &[ + ptr, + self.const_i32(rw), + self.const_i32(locality), + self.const_i32(cache_type), + ], ) } sym::carrying_mul_add => { diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index f50aed0b3c2..cfc6bc2f3a0 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -136,6 +136,10 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::round_ties_even_f64 | sym::round_ties_even_f128 | sym::autodiff + | sym::prefetch_read_data + | sym::prefetch_write_data + | sym::prefetch_read_instruction + | sym::prefetch_write_instruction | sym::const_eval_select => hir::Safety::Safe, _ => hir::Safety::Unsafe, }; @@ -218,7 +222,7 @@ pub(crate) fn check_intrinsic_type( | sym::prefetch_write_data | sym::prefetch_read_instruction | sym::prefetch_write_instruction => { - (1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit) + (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.unit) } sym::needs_drop => (1, 0, vec![], tcx.types.bool), diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 346c7825b74..904aa52c784 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -265,17 +265,16 @@ pub unsafe fn atomic_singlethreadfence(); /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_read_data(data: *const T, locality: i32) { +pub const fn prefetch_read_data(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction @@ -283,17 +282,16 @@ pub unsafe fn prefetch_read_data(data: *const T, locality: i32) { /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_write_data(data: *const T, locality: i32) { +pub const fn prefetch_write_data(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction @@ -301,17 +299,16 @@ pub unsafe fn prefetch_write_data(data: *const T, locality: i32) { /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32) { +pub const fn prefetch_read_instruction(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction @@ -319,17 +316,16 @@ pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32) { /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_write_instruction(data: *const T, locality: i32) { +pub const fn prefetch_write_instruction(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// Executes a breakpoint trap, for inspection by a debugger. diff --git a/src/tools/miri/tests/pass/prefetch.rs b/src/tools/miri/tests/pass/prefetch.rs index 09352695989..99c75c38bde 100644 --- a/src/tools/miri/tests/pass/prefetch.rs +++ b/src/tools/miri/tests/pass/prefetch.rs @@ -1,23 +1,23 @@ #![feature(core_intrinsics)] +// Test that these intrinsics work. Their behavior should be a no-op. + fn main() { static X: [u8; 8] = [0; 8]; - unsafe { - ::std::intrinsics::prefetch_read_data(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_read_data(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_read_data(X.as_ptr(), 3); + ::std::intrinsics::prefetch_read_data::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_read_data::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_read_data::<_, 3>(X.as_ptr()); - ::std::intrinsics::prefetch_write_data(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_write_data(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_write_data(X.as_ptr(), 3); + ::std::intrinsics::prefetch_write_data::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_write_data::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_write_data::<_, 3>(X.as_ptr()); - ::std::intrinsics::prefetch_read_instruction(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_read_instruction(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_read_instruction(X.as_ptr(), 3); + ::std::intrinsics::prefetch_read_instruction::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_read_instruction::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_read_instruction::<_, 3>(X.as_ptr()); - ::std::intrinsics::prefetch_write_instruction(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_write_instruction(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_write_instruction(X.as_ptr(), 3); - } + ::std::intrinsics::prefetch_write_instruction::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_write_instruction::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_write_instruction::<_, 3>(X.as_ptr()); } diff --git a/tests/codegen-llvm/intrinsics/prefetch.rs b/tests/codegen-llvm/intrinsics/prefetch.rs index 3f9f21c85cb..41877872019 100644 --- a/tests/codegen-llvm/intrinsics/prefetch.rs +++ b/tests/codegen-llvm/intrinsics/prefetch.rs @@ -9,56 +9,48 @@ use std::intrinsics::{ #[no_mangle] pub fn check_prefetch_read_data(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1) - prefetch_read_data(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1) - prefetch_read_data(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1) - prefetch_read_data(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1) - prefetch_read_data(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1) + prefetch_read_data::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1) + prefetch_read_data::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1) + prefetch_read_data::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1) + prefetch_read_data::<_, 3>(data.as_ptr()); } #[no_mangle] pub fn check_prefetch_write_data(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1) - prefetch_write_data(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1) - prefetch_write_data(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1) - prefetch_write_data(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1) - prefetch_write_data(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1) + prefetch_write_data::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1) + prefetch_write_data::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1) + prefetch_write_data::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1) + prefetch_write_data::<_, 3>(data.as_ptr()); } #[no_mangle] pub fn check_prefetch_read_instruction(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0) - prefetch_read_instruction(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0) - prefetch_read_instruction(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0) - prefetch_read_instruction(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0) - prefetch_read_instruction(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0) + prefetch_read_instruction::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0) + prefetch_read_instruction::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0) + prefetch_read_instruction::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0) + prefetch_read_instruction::<_, 3>(data.as_ptr()); } #[no_mangle] pub fn check_prefetch_write_instruction(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0) - prefetch_write_instruction(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0) - prefetch_write_instruction(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0) - prefetch_write_instruction(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0) - prefetch_write_instruction(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0) + prefetch_write_instruction::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0) + prefetch_write_instruction::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0) + prefetch_write_instruction::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0) + prefetch_write_instruction::<_, 3>(data.as_ptr()); } -- cgit 1.4.1-3-g733a5 From 8ea3b093819aabd92a605b42989341da0c97c0d6 Mon Sep 17 00:00:00 2001 From: clubby789 Date: Thu, 17 Jul 2025 21:47:00 +0100 Subject: Pass `alloc-variant-zeroed` to LLVM --- compiler/rustc_codegen_llvm/src/attributes.rs | 10 ++++++++++ compiler/rustc_feature/src/builtin_attrs.rs | 4 ++++ compiler/rustc_span/src/symbol.rs | 1 + library/alloc/src/alloc.rs | 1 + tests/codegen-llvm/vec-calloc.rs | 20 ++++++++++++++++++++ 5 files changed, 36 insertions(+) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c548f467583..984db43c67f 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -436,6 +436,16 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); + if let Some(zv) = + cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + && let Some(name) = zv.value_str() + { + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "alloc-variant-zeroed", + &mangle_internal_symbol(cx.tcx, name.as_str()), + )); + } // apply to argument place instead of function let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index ab6b8f92802..6cd0c43a3e0 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -993,6 +993,10 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ rustc_allocator_zeroed, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::No, ), + rustc_attr!( + rustc_allocator_zeroed_variant, Normal, template!(NameValueStr: "function"), ErrorPreceding, + EncodeCrossCrate::Yes, + ), gated!( default_lib_allocator, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::No, allocator_internals, experimental!(default_lib_allocator), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index acbed7a9eed..21ba6c2779d 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1816,6 +1816,7 @@ symbols! { rustc_align, rustc_allocator, rustc_allocator_zeroed, + rustc_allocator_zeroed_variant, rustc_allow_const_fn_unstable, rustc_allow_incoherent_impl, rustc_allowed_through_unstable_modules, diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs index c9b98fa4e5a..76630a746dd 100644 --- a/library/alloc/src/alloc.rs +++ b/library/alloc/src/alloc.rs @@ -17,6 +17,7 @@ unsafe extern "Rust" { #[rustc_allocator] #[rustc_nounwind] #[rustc_std_internal_symbol] + #[rustc_allocator_zeroed_variant = "__rust_alloc_zeroed"] fn __rust_alloc(size: usize, align: usize) -> *mut u8; #[rustc_deallocator] #[rustc_nounwind] diff --git a/tests/codegen-llvm/vec-calloc.rs b/tests/codegen-llvm/vec-calloc.rs index d1c320ead01..15971bbfa00 100644 --- a/tests/codegen-llvm/vec-calloc.rs +++ b/tests/codegen-llvm/vec-calloc.rs @@ -1,4 +1,6 @@ +//@ revisions: normal llvm21 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled +//@ [llvm21] min-llvm-version: 21 //@ only-x86_64 #![crate_type = "lib"] @@ -176,6 +178,24 @@ pub fn vec_option_i32(n: usize) -> Vec> { vec![None; n] } +// LLVM21-LABEL: @vec_array +#[cfg(llvm21)] +#[no_mangle] +pub fn vec_array(n: usize) -> Vec<[u32; 1_000_000]> { + // LLVM21-NOT: call {{.*}}alloc::vec::from_elem + // LLVM21-NOT: call {{.*}}reserve + // LLVM21-NOT: call {{.*}}__rust_alloc( + + // LLVM21: call {{.*}}__rust_alloc_zeroed( + + // LLVM21-NOT: call {{.*}}alloc::vec::from_elem + // LLVM21-NOT: call {{.*}}reserve + // LLVM21-NOT: call {{.*}}__rust_alloc( + + // LLVM21: ret void + vec![[0; 1_000_000]; 3] +} + // Ensure that __rust_alloc_zeroed gets the right attributes for LLVM to optimize it away. // CHECK: declare noalias noundef ptr @{{.*}}__rust_alloc_zeroed(i64 noundef, i64 allocalign noundef) unnamed_addr [[RUST_ALLOC_ZEROED_ATTRS:#[0-9]+]] -- cgit 1.4.1-3-g733a5 From d71ed8d19bcfdbd6eb3bf2620071f2ab95470a57 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 11 Aug 2025 17:18:23 +0200 Subject: Tell LLVM about read-only captures `&Freeze` parameters are not only `readonly` within the function, but any captures of the pointer can also only be used for reads. This can now be encoded using the `captures(address, read_provenance)` attribute. --- compiler/rustc_codegen_llvm/src/abi.rs | 7 ++++++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 8 ++++++++ compiler/rustc_target/src/callconv/mod.rs | 1 + compiler/rustc_ty_utils/src/abi.rs | 1 + tests/codegen-llvm/function-arguments.rs | 20 ++++++++++---------- tests/codegen-llvm/range-attribute.rs | 2 +- tests/codegen-llvm/read-only-capture-opt.rs | 18 ++++++++++++++++++ tests/ui/abi/debug.generic.stderr | 2 +- tests/ui/abi/debug.riscv64.stderr | 2 +- 10 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 tests/codegen-llvm/read-only-capture-opt.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 043123fcab2..399f8b6e762 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -42,12 +42,13 @@ trait ArgAttributesExt { const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; -const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 6] = [ (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), + (ArgAttribute::CapturesReadOnly, llvm::AttributeKind::CapturesReadOnly), ]; fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { @@ -83,6 +84,10 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' } for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { if regular.contains(attr) { + // captures(address, read_provenance) is only available since LLVM 21. + if attr == ArgAttribute::CapturesReadOnly && llvm_util::get_version() < (21, 0, 0) { + continue; + } attrs.push(llattr.create_attr(cx.llcx)); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 55d26a97e2d..2461f70a86e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -251,6 +251,7 @@ pub(crate) enum AttributeKind { Writable = 42, DeadOnUnwind = 43, DeadOnReturn = 44, + CapturesReadOnly = 45, } /// LLVMIntPredicate diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index e4fe1fc2e42..e699e4b9c13 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -278,6 +278,7 @@ enum class LLVMRustAttributeKind { Writable = 42, DeadOnUnwind = 43, DeadOnReturn = 44, + CapturesReadOnly = 45, }; static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { @@ -376,6 +377,8 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { #else report_fatal_error("DeadOnReturn attribute requires LLVM 21 or later"); #endif + case LLVMRustAttributeKind::CapturesReadOnly: + report_fatal_error("Should be handled separately"); } report_fatal_error("bad LLVMRustAttributeKind"); } @@ -430,6 +433,11 @@ LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttributeKind RustAttr) { if (RustAttr == LLVMRustAttributeKind::NoCapture) { return wrap(Attribute::getWithCaptureInfo(*unwrap(C), CaptureInfo::none())); } + if (RustAttr == LLVMRustAttributeKind::CapturesReadOnly) { + return wrap(Attribute::getWithCaptureInfo( + *unwrap(C), CaptureInfo(CaptureComponents::Address | + CaptureComponents::ReadProvenance))); + } #endif return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); } diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 63e56744aec..5f2a6f7ba38 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -119,6 +119,7 @@ mod attr_impl { const ReadOnly = 1 << 4; const InReg = 1 << 5; const NoUndef = 1 << 6; + const CapturesReadOnly = 1 << 7; } } rustc_data_structures::external_bitflags_debug! { ArgAttribute } diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 89d1dd8cf23..e4ed084b073 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -356,6 +356,7 @@ fn arg_attrs_for_rust_scalar<'tcx>( if matches!(kind, PointerKind::SharedRef { frozen: true }) && !is_return { attrs.set(ArgAttribute::ReadOnly); + attrs.set(ArgAttribute::CapturesReadOnly); } } } diff --git a/tests/codegen-llvm/function-arguments.rs b/tests/codegen-llvm/function-arguments.rs index a3fafbe6f82..db508682862 100644 --- a/tests/codegen-llvm/function-arguments.rs +++ b/tests/codegen-llvm/function-arguments.rs @@ -80,7 +80,7 @@ pub fn option_nonzero_int(x: Option>) -> Option> { x } -// CHECK: @readonly_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @readonly_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn readonly_borrow(_: &i32) {} @@ -91,12 +91,12 @@ pub fn readonly_borrow_ret() -> &'static i32 { loop {} } -// CHECK: @static_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @static_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // static borrow may be captured #[no_mangle] pub fn static_borrow(_: &'static i32) {} -// CHECK: @named_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @named_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // borrow with named lifetime may be captured #[no_mangle] pub fn named_borrow<'r>(_: &'r i32) {} @@ -129,7 +129,7 @@ pub fn mutable_borrow_ret() -> &'static mut i32 { // . pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {} -// CHECK: @notunpin_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1) +// CHECK: @notunpin_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1) // But `&NotUnpin` behaves perfectly normal. #[no_mangle] pub fn notunpin_borrow(_: &NotUnpin) {} @@ -138,12 +138,12 @@ pub fn notunpin_borrow(_: &NotUnpin) {} #[no_mangle] pub fn indirect_struct(_: S) {} -// CHECK: @borrowed_struct(ptr noalias noundef readonly align 4 dereferenceable(32) %_1) +// CHECK: @borrowed_struct(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(32) %_1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn borrowed_struct(_: &S) {} -// CHECK: @option_borrow(ptr noalias noundef readonly align 4 dereferenceable_or_null(4) %_x) +// CHECK: @option_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable_or_null(4) %_x) #[no_mangle] pub fn option_borrow(_x: Option<&i32>) {} @@ -185,7 +185,7 @@ pub fn _box(x: Box) -> Box { // With a custom allocator, it should *not* have `noalias`. (See // for why.) The second argument is the allocator, // which is a reference here that still carries `noalias` as usual. -// CHECK: @_box_custom(ptr noundef nonnull align 4 %x.0, ptr noalias noundef nonnull readonly align 1 %x.1) +// CHECK: @_box_custom(ptr noundef nonnull align 4 %x.0, ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %x.1) #[no_mangle] pub fn _box_custom(x: Box) { drop(x) @@ -208,7 +208,7 @@ pub fn struct_return() -> S { #[no_mangle] pub fn helper(_: usize) {} -// CHECK: @slice(ptr noalias noundef nonnull readonly align 1 %_1.0, [[USIZE]] noundef %_1.1) +// CHECK: @slice(ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %_1.0, [[USIZE]] noundef %_1.1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn slice(_: &[u8]) {} @@ -227,7 +227,7 @@ pub fn unsafe_slice(_: &[UnsafeInner]) {} #[no_mangle] pub fn raw_slice(_: *const [u8]) {} -// CHECK: @str(ptr noalias noundef nonnull readonly align 1 %_1.0, [[USIZE]] noundef %_1.1) +// CHECK: @str(ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %_1.0, [[USIZE]] noundef %_1.1) // FIXME #25759 This should also have `nocapture` #[no_mangle] pub fn str(_: &[u8]) {} @@ -259,7 +259,7 @@ pub fn trait_option(x: Option>) -> Option &[u16] { x diff --git a/tests/codegen-llvm/range-attribute.rs b/tests/codegen-llvm/range-attribute.rs index b81ff9ab3e2..865d36d4747 100644 --- a/tests/codegen-llvm/range-attribute.rs +++ b/tests/codegen-llvm/range-attribute.rs @@ -67,7 +67,7 @@ pub fn enum2_value(x: Enum2) -> Enum2 { x } -// CHECK: noundef [[USIZE]] @takes_slice(ptr noalias noundef nonnull readonly align 4 %x.0, [[USIZE]] noundef %x.1) +// CHECK: noundef [[USIZE]] @takes_slice(ptr {{.*}} %x.0, [[USIZE]] noundef %x.1) #[no_mangle] pub fn takes_slice(x: &[i32]) -> usize { x.len() diff --git a/tests/codegen-llvm/read-only-capture-opt.rs b/tests/codegen-llvm/read-only-capture-opt.rs new file mode 100644 index 00000000000..78d56f8efc2 --- /dev/null +++ b/tests/codegen-llvm/read-only-capture-opt.rs @@ -0,0 +1,18 @@ +//@ compile-flags: -C opt-level=3 -Z mir-opt-level=0 +//@ min-llvm-version: 21 + +#![crate_type = "lib"] + +unsafe extern "C" { + safe fn do_something(p: &i32); +} + +#[unsafe(no_mangle)] +pub fn test() -> i32 { + // CHECK-LABEL: @test( + // CHECK: ret i32 0 + let i = 0; + do_something(&i); + do_something(&i); + i +} diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index 3b29efc8102..8375de3e817 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef, + regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index 2417396de2f..bddf0aea3d7 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef, + regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some( -- cgit 1.4.1-3-g733a5 From b32f4d5792ebff7c734114488184a5b72516a187 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Thu, 21 Aug 2025 11:28:10 +0200 Subject: remove an `as` cast in prefetch codegen --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 06c3d8ed6bc..49d3dedbeab 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -330,7 +330,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); - let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32; + let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_i32(); self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], -- cgit 1.4.1-3-g733a5 From 525c6a356284ea47587f0316cec23cab290b986e Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 24 Aug 2025 10:47:04 +0000 Subject: Directly raise fatal errors inside the codegen backends As opposed to passing it around through Result. --- compiler/rustc_codegen_gcc/src/back/lto.rs | 33 +++++++------- compiler/rustc_codegen_gcc/src/back/write.rs | 7 ++- compiler/rustc_codegen_gcc/src/lib.rs | 12 +++-- compiler/rustc_codegen_llvm/src/back/lto.rs | 62 +++++++++++++------------- compiler/rustc_codegen_llvm/src/back/write.rs | 39 ++++++++-------- compiler/rustc_codegen_llvm/src/lib.rs | 32 ++++++------- compiler/rustc_codegen_ssa/src/back/write.rs | 44 +++++++++--------- compiler/rustc_codegen_ssa/src/traits/write.rs | 12 ++--- 8 files changed, 117 insertions(+), 124 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs index d558dfbc1c4..fcee6b6df62 100644 --- a/compiler/rustc_codegen_gcc/src/back/lto.rs +++ b/compiler/rustc_codegen_gcc/src/back/lto.rs @@ -29,7 +29,7 @@ use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::memmap::Mmap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_session::config::Lto; @@ -51,12 +51,11 @@ fn prepare_lto( cgcx: &CodegenContext, each_linked_rlib_for_lto: &[PathBuf], dcx: DiagCtxtHandle<'_>, -) -> Result { +) -> LtoData { let tmp_path = match tempdir() { Ok(tmp_path) => tmp_path, Err(error) => { - eprintln!("Cannot create temporary directory: {}", error); - return Err(FatalError); + dcx.fatal(format!("Cannot create temporary directory: {}", error)); } }; @@ -91,15 +90,14 @@ fn prepare_lto( upstream_modules.push((module, CString::new(name).unwrap())); } Err(e) => { - dcx.emit_err(e); - return Err(FatalError); + dcx.emit_fatal(e); } } } } } - Ok(LtoData { upstream_modules, tmp_path }) + LtoData { upstream_modules, tmp_path } } fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> { @@ -114,10 +112,10 @@ pub(crate) fn run_fat( cgcx: &CodegenContext, each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, -) -> Result, FatalError> { +) -> ModuleCodegen { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?; + let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); /*let symbols_below_threshold = lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>();*/ fat_lto( @@ -137,7 +135,7 @@ fn fat_lto( mut serialized_modules: Vec<(SerializedModule, CString)>, tmp_path: TempDir, //symbols_below_threshold: &[String], -) -> Result, FatalError> { +) -> ModuleCodegen { let _timer = cgcx.prof.generic_activity("GCC_fat_lto_build_monolithic_module"); info!("going for a fat lto"); @@ -261,7 +259,7 @@ fn fat_lto( // of now. module.module_llvm.temp_dir = Some(tmp_path); - Ok(module) + module } pub struct ModuleBuffer(PathBuf); @@ -286,10 +284,10 @@ pub(crate) fn run_thin( each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?; + let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); if cgcx.opts.cg.linker_plugin_lto.enabled() { unreachable!( "We should never reach this case if the LTO step \ @@ -355,7 +353,7 @@ fn thin_lto( tmp_path: TempDir, cached_modules: Vec<(SerializedModule, WorkProduct)>, //_symbols_below_threshold: &[String], -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); info!("going for that thin, thin LTO"); @@ -518,13 +516,13 @@ fn thin_lto( // TODO: save the directory so that it gets deleted later. std::mem::forget(tmp_path); - Ok((opt_jobs, copy_jobs)) + (opt_jobs, copy_jobs) } pub fn optimize_thin_module( thin_module: ThinModule, _cgcx: &CodegenContext, -) -> Result, FatalError> { +) -> ModuleCodegen { //let dcx = cgcx.create_dcx(); //let module_name = &thin_module.shared.module_names[thin_module.idx]; @@ -634,7 +632,8 @@ pub fn optimize_thin_module( save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } }*/ - Ok(module) + #[allow(clippy::let_and_return)] + module } pub struct ThinBuffer { diff --git a/compiler/rustc_codegen_gcc/src/back/write.rs b/compiler/rustc_codegen_gcc/src/back/write.rs index c1231142c65..84bc7016271 100644 --- a/compiler/rustc_codegen_gcc/src/back/write.rs +++ b/compiler/rustc_codegen_gcc/src/back/write.rs @@ -6,7 +6,6 @@ use rustc_codegen_ssa::back::write::{BitcodeSection, CodegenContext, EmitObj, Mo use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; use rustc_fs_util::link_or_copy; use rustc_session::config::OutputType; -use rustc_span::fatal_error::FatalError; use rustc_target::spec::SplitDebuginfo; use crate::base::add_pic_option; @@ -17,7 +16,7 @@ pub(crate) fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, -) -> Result { +) -> CompiledModule { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -246,7 +245,7 @@ pub(crate) fn codegen( } } - Ok(module.into_compiled_module( + module.into_compiled_module( config.emit_obj != EmitObj::None, cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo == SplitDebuginfo::Unpacked, config.emit_bc, @@ -254,7 +253,7 @@ pub(crate) fn codegen( config.emit_ir, &cgcx.output_filenames, cgcx.invocation_temp.as_deref(), - )) + ) } pub(crate) fn save_temp_bitcode( diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index 4025aba82da..2d7df79ba95 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -110,7 +110,6 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames}; use rustc_span::Symbol; -use rustc_span::fatal_error::FatalError; use rustc_target::spec::RelocModel; use tempfile::TempDir; @@ -362,7 +361,7 @@ impl WriteBackendMethods for GccCodegenBackend { _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - ) -> Result, FatalError> { + ) -> ModuleCodegen { back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules) } @@ -373,7 +372,7 @@ impl WriteBackendMethods for GccCodegenBackend { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError> { + ) -> (Vec>, Vec) { back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules) } @@ -390,15 +389,14 @@ impl WriteBackendMethods for GccCodegenBackend { _dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError> { + ) { module.module_llvm.context.set_optimization_level(to_gcc_opt_level(config.opt_level)); - Ok(()) } fn optimize_thin( cgcx: &CodegenContext, thin: ThinModule, - ) -> Result, FatalError> { + ) -> ModuleCodegen { back::lto::optimize_thin_module(thin, cgcx) } @@ -406,7 +404,7 @@ impl WriteBackendMethods for GccCodegenBackend { cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result { + ) -> CompiledModule { back::write::codegen(cgcx, module, config) } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 853d0295238..d85f432702c 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_session::config::{self, Lto}; @@ -36,7 +36,7 @@ fn prepare_lto( exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], dcx: DiagCtxtHandle<'_>, -) -> Result<(Vec, Vec<(SerializedModule, CString)>), FatalError> { +) -> (Vec, Vec<(SerializedModule, CString)>) { let mut symbols_below_threshold = exported_symbols_for_lto .iter() .map(|symbol| CString::new(symbol.to_owned()).unwrap()) @@ -79,16 +79,13 @@ fn prepare_lto( let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); } - Err(e) => { - dcx.emit_err(e); - return Err(FatalError); - } + Err(e) => dcx.emit_fatal(e), } } } } - Ok((symbols_below_threshold, upstream_modules)) + (symbols_below_threshold, upstream_modules) } fn get_bitcode_slice_from_object_data<'a>( @@ -123,11 +120,11 @@ pub(crate) fn run_fat( exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, -) -> Result, FatalError> { +) -> ModuleCodegen { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = - prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold) @@ -142,11 +139,11 @@ pub(crate) fn run_thin( each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = - prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); if cgcx.opts.cg.linker_plugin_lto.enabled() { @@ -173,7 +170,7 @@ fn fat_lto( modules: Vec>, mut serialized_modules: Vec<(SerializedModule, CString)>, symbols_below_threshold: &[*const libc::c_char], -) -> Result, FatalError> { +) -> ModuleCodegen { let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); info!("going for a fat lto"); @@ -224,7 +221,7 @@ fn fat_lto( assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); let (buffer, name) = serialized_modules.remove(0); info!("no in-memory regular modules to choose from, parsing {:?}", name); - let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?; + let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx); ModuleCodegen::new_regular(name.into_string().unwrap(), llvm_module) } }; @@ -265,7 +262,9 @@ fn fat_lto( }); info!("linking {:?}", name); let data = bc_decoded.data(); - linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?; + linker + .add(data) + .unwrap_or_else(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name })); } drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); @@ -282,7 +281,7 @@ fn fat_lto( save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } - Ok(module) + module } pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); @@ -352,7 +351,7 @@ fn thin_lto( serialized_modules: Vec<(SerializedModule, CString)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, symbols_below_threshold: &[*const libc::c_char], -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); unsafe { info!("going for that thin, thin LTO"); @@ -422,7 +421,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len(), ) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; + .unwrap_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext)); let data = ThinData(data); @@ -492,10 +491,10 @@ fn thin_lto( if let Some(path) = key_map_path && let Err(err) = curr_key_map.save_to_file(&path) { - return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err })); + write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }); } - Ok((opt_jobs, copy_jobs)) + (opt_jobs, copy_jobs) } } @@ -550,7 +549,7 @@ pub(crate) fn run_pass_manager( dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, thin: bool, -) -> Result<(), FatalError> { +) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); let config = cgcx.config(module.kind); @@ -582,7 +581,7 @@ pub(crate) fn run_pass_manager( } unsafe { - write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage); } if enable_gpu && !thin { @@ -596,7 +595,7 @@ pub(crate) fn run_pass_manager( let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { unsafe { - write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage); } } @@ -608,7 +607,6 @@ pub(crate) fn run_pass_manager( } debug!("lto done"); - Ok(()) } pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); @@ -701,7 +699,7 @@ impl Drop for ThinBuffer { pub(crate) fn optimize_thin_module( thin_module: ThinModule, cgcx: &CodegenContext, -) -> Result, FatalError> { +) -> ModuleCodegen { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -712,7 +710,7 @@ pub(crate) fn optimize_thin_module( // into that context. One day, however, we may do this for upstream // crates but for locally codegened modules we may be able to reuse // that LLVM Context and Module. - let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx)?; + let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx); let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. if cgcx.config(ModuleKind::Regular).embed_bitcode() { @@ -746,7 +744,7 @@ pub(crate) fn optimize_thin_module( .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); if unsafe { !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); } @@ -757,7 +755,7 @@ pub(crate) fn optimize_thin_module( .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); if unsafe { !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); } @@ -768,7 +766,7 @@ pub(crate) fn optimize_thin_module( if unsafe { !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target.raw()) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } @@ -780,11 +778,11 @@ pub(crate) fn optimize_thin_module( // little differently. { info!("running thin lto passes over {}", module.name); - run_pass_manager(cgcx, dcx, &mut module, true)?; + run_pass_manager(cgcx, dcx, &mut module, true); save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } - Ok(module) + module } /// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys @@ -850,9 +848,9 @@ pub(crate) fn parse_module<'a>( name: &CStr, data: &[u8], dcx: DiagCtxtHandle<'_>, -) -> Result<&'a llvm::Module, FatalError> { +) -> &'a llvm::Module { unsafe { llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) + .unwrap_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 62998003ca1..31bbf7cc80c 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -20,7 +20,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{DiagCtxtHandle, FatalError, Level}; +use rustc_errors::{DiagCtxtHandle, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -46,10 +46,10 @@ use crate::llvm::{self, DiagnosticInfo}; use crate::type_::Type; use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util}; -pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError { +pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! { match llvm::last_error() { - Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)), - None => dcx.emit_almost_fatal(err), + Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)), + None => dcx.emit_fatal(err), } } @@ -63,7 +63,7 @@ fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, verify_llvm_ir: bool, -) -> Result<(), FatalError> { +) { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); let output_c = path_to_c_string(output); let dwo_output_c; @@ -100,7 +100,7 @@ fn write_output_file<'ll>( } } - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) + result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } pub(crate) fn create_informational_target_machine( @@ -112,7 +112,7 @@ pub(crate) fn create_informational_target_machine( // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false, only_base_features); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise()) + .unwrap_or_else(|err| llvm_err(sess.dcx(), err)) } pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { @@ -139,7 +139,7 @@ pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTar tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.dcx(), err)) } fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { @@ -565,7 +565,7 @@ pub(crate) unsafe fn llvm_optimize( opt_level: config::OptLevel, opt_stage: llvm::OptStage, autodiff_stage: AutodiffStage, -) -> Result<(), FatalError> { +) { // Enzyme: // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized // source code. However, benchmarks show that optimizations increasing the code size @@ -704,7 +704,7 @@ pub(crate) unsafe fn llvm_optimize( llvm_plugins.len(), ) }; - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) + result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. @@ -713,7 +713,7 @@ pub(crate) fn optimize( dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, -) -> Result<(), FatalError> { +) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); let llcx = &*module.module_llvm.llcx; @@ -765,7 +765,7 @@ pub(crate) fn optimize( opt_stage, autodiff_stage, ) - }?; + }; if let Some(thin_lto_buffer) = thin_lto_buffer { let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) }; module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec()); @@ -793,14 +793,13 @@ pub(crate) fn optimize( } } } - Ok(()) } pub(crate) fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, -) -> Result { +) -> CompiledModule { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -909,7 +908,9 @@ pub(crate) fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?; + result + .into_result() + .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out })); } if config.emit_asm { @@ -940,7 +941,7 @@ pub(crate) fn codegen( llvm::FileType::AssemblyFile, &cgcx.prof, config.verify_llvm_ir, - )?; + ); } match config.emit_obj { @@ -976,7 +977,7 @@ pub(crate) fn codegen( llvm::FileType::ObjectFile, &cgcx.prof, config.verify_llvm_ir, - )?; + ); } EmitObj::Bitcode => { @@ -1009,7 +1010,7 @@ pub(crate) fn codegen( && cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo != SplitDebuginfo::Off && cgcx.split_dwarf_kind == SplitDwarfKind::Split; - Ok(module.into_compiled_module( + module.into_compiled_module( config.emit_obj != EmitObj::None, dwarf_object_emitted, config.emit_bc, @@ -1017,7 +1018,7 @@ pub(crate) fn codegen( config.emit_ir, &cgcx.output_filenames, cgcx.invocation_temp.as_deref(), - )) + ) } fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 0fcf31d7993..628cb34fd9e 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -37,7 +37,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetConfig}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -165,15 +165,15 @@ impl WriteBackendMethods for LlvmCodegenBackend { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - ) -> Result, FatalError> { + ) -> ModuleCodegen { let mut module = - back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?; + back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules); let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - back::lto::run_pass_manager(cgcx, dcx, &mut module, false)?; + back::lto::run_pass_manager(cgcx, dcx, &mut module, false); - Ok(module) + module } fn run_thin_lto( cgcx: &CodegenContext, @@ -181,7 +181,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError> { + ) -> (Vec>, Vec) { back::lto::run_thin( cgcx, exported_symbols_for_lto, @@ -195,20 +195,20 @@ impl WriteBackendMethods for LlvmCodegenBackend { dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError> { + ) { back::write::optimize(cgcx, dcx, module, config) } fn optimize_thin( cgcx: &CodegenContext, thin: ThinModule, - ) -> Result, FatalError> { + ) -> ModuleCodegen { back::lto::optimize_thin_module(thin, cgcx) } fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result { + ) -> CompiledModule { back::write::codegen(cgcx, module, config) } fn prepare_thin( @@ -407,12 +407,12 @@ impl ModuleLlvm { cgcx: &CodegenContext, name: &str, dcx: DiagCtxtHandle<'_>, - ) -> Result { + ) -> OwnedTargetMachine { let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name); match (cgcx.tm_factory)(tm_factory_config) { - Ok(m) => Ok(m), + Ok(m) => m, Err(e) => { - return Err(dcx.emit_almost_fatal(ParseTargetMachineConfig(e))); + dcx.emit_fatal(ParseTargetMachineConfig(e)); } } } @@ -422,13 +422,13 @@ impl ModuleLlvm { name: &CStr, buffer: &[u8], dcx: DiagCtxtHandle<'_>, - ) -> Result { + ) -> Self { unsafe { let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx)?; - let tm = ModuleLlvm::tm_from_cgcx(cgcx, name.to_str().unwrap(), dcx)?; + let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx); + let tm = ModuleLlvm::tm_from_cgcx(cgcx, name.to_str().unwrap(), dcx); - Ok(ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) }) + ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) } } } diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 3e590bef617..9f22859ba81 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -15,8 +15,8 @@ use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard}; use rustc_errors::emitter::Emitter; use rustc_errors::translation::Translator; use rustc_errors::{ - Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FatalErrorMarker, Level, - MultiSpan, Style, Suggestions, + Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalErrorMarker, Level, MultiSpan, Style, + Suggestions, }; use rustc_fs_util::link_or_copy; use rustc_incremental::{ @@ -396,8 +396,7 @@ fn generate_thin_lto_work( each_linked_rlib_for_lto, needs_thin_lto, import_only_modules, - ) - .unwrap_or_else(|e| e.raise()); + ); lto_modules .into_iter() .map(|module| { @@ -845,11 +844,11 @@ fn execute_optimize_work_item( cgcx: &CodegenContext, mut module: ModuleCodegen, module_config: &ModuleConfig, -) -> Result, FatalError> { +) -> WorkItemResult { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - B::optimize(cgcx, dcx, &mut module, module_config)?; + B::optimize(cgcx, dcx, &mut module, module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it @@ -869,8 +868,8 @@ fn execute_optimize_work_item( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, module_config)?; - Ok(WorkItemResult::Finished(module)) + let module = B::codegen(cgcx, module, module_config); + WorkItemResult::Finished(module) } ComputedLtoType::Thin => { let (name, thin_buffer) = B::prepare_thin(module, false); @@ -879,7 +878,7 @@ fn execute_optimize_work_item( panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); }); } - Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) + WorkItemResult::NeedsThinLto(name, thin_buffer) } ComputedLtoType::Fat => match bitcode { Some(path) => { @@ -887,12 +886,12 @@ fn execute_optimize_work_item( fs::write(&path, buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); }); - Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { + WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer: SerializedModule::Local(buffer), - })) + }) } - None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))), + None => WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module)), }, } } @@ -988,7 +987,7 @@ fn execute_fat_lto_work_item( mut needs_fat_lto: Vec>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, module_config: &ModuleConfig, -) -> Result, FatalError> { +) -> WorkItemResult { for (module, wp) in import_only_modules { needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module }) } @@ -998,19 +997,19 @@ fn execute_fat_lto_work_item( exported_symbols_for_lto, each_linked_rlib_for_lto, needs_fat_lto, - )?; - let module = B::codegen(cgcx, module, module_config)?; - Ok(WorkItemResult::Finished(module)) + ); + let module = B::codegen(cgcx, module, module_config); + WorkItemResult::Finished(module) } fn execute_thin_lto_work_item( cgcx: &CodegenContext, module: lto::ThinModule, module_config: &ModuleConfig, -) -> Result, FatalError> { - let module = B::optimize_thin(cgcx, module)?; - let module = B::codegen(cgcx, module, module_config)?; - Ok(WorkItemResult::Finished(module)) +) -> WorkItemResult { + let module = B::optimize_thin(cgcx, module); + let module = B::codegen(cgcx, module, module_config); + WorkItemResult::Finished(module) } /// Messages sent to the coordinator. @@ -1737,7 +1736,7 @@ fn spawn_work<'a, B: ExtraBackendMethods>( "codegen_copy_artifacts_from_incr_cache", &*m.name, ); - Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config)) + execute_copy_from_cache_work_item(&cgcx, m, module_config) } WorkItem::FatLto { exported_symbols_for_lto, @@ -1766,12 +1765,11 @@ fn spawn_work<'a, B: ExtraBackendMethods>( })); let msg = match result { - Ok(Ok(result)) => Message::WorkItem:: { result: Ok(result) }, + Ok(result) => Message::WorkItem:: { result: Ok(result) }, // We ignore any `FatalError` coming out of `execute_work_item`, as a // diagnostic was already sent off to the main thread - just surface // that there was an error in this worker. - Ok(Err(FatalError)) => Message::WorkItem:: { result: Err(Some(WorkerFatalError)) }, Err(err) if err.is::() => { Message::WorkItem:: { result: Err(Some(WorkerFatalError)) } } diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index c29ad90735b..cc7c4e46d7b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_middle::dep_graph::WorkProduct; use crate::back::lto::{SerializedModule, ThinModule}; @@ -22,7 +22,7 @@ pub trait WriteBackendMethods: Clone + 'static { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - ) -> Result, FatalError>; + ) -> ModuleCodegen; /// Performs thin LTO by performing necessary global analysis and returning two /// lists, one of the modules that need optimization and another for modules that /// can simply be copied over from the incr. comp. cache. @@ -32,7 +32,7 @@ pub trait WriteBackendMethods: Clone + 'static { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError>; + ) -> (Vec>, Vec); fn print_pass_timings(&self); fn print_statistics(&self); fn optimize( @@ -40,16 +40,16 @@ pub trait WriteBackendMethods: Clone + 'static { dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError>; + ); fn optimize_thin( cgcx: &CodegenContext, thin: ThinModule, - ) -> Result, FatalError>; + ) -> ModuleCodegen; fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result; + ) -> CompiledModule; fn prepare_thin( module: ModuleCodegen, want_summary: bool, -- cgit 1.4.1-3-g733a5 From 455a67bd4f38b10e613fc6e2103598db10ad57fe Mon Sep 17 00:00:00 2001 From: Zalathar Date: Sun, 24 Aug 2025 15:05:26 +1000 Subject: Replace the `llvm::Bool` typedef with a proper newtype --- compiler/rustc_codegen_llvm/src/asm.rs | 11 ++---- compiler/rustc_codegen_llvm/src/builder.rs | 21 ++++------ compiler/rustc_codegen_llvm/src/common.rs | 4 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 57 ++++++++++++++++++++++++++-- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 4 +- compiler/rustc_codegen_llvm/src/llvm_util.rs | 2 +- compiler/rustc_codegen_llvm/src/mono_item.rs | 2 +- compiler/rustc_codegen_llvm/src/type_.rs | 8 ++-- 8 files changed, 75 insertions(+), 34 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a643a91141e..38c1d3b53e8 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -16,6 +16,7 @@ use tracing::debug; use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; +use crate::llvm::ToLlvmBool; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -470,10 +471,6 @@ pub(crate) fn inline_asm_call<'ll>( dest: Option<&'ll llvm::BasicBlock>, catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { - let volatile = if volatile { llvm::True } else { llvm::False }; - let alignstack = if alignstack { llvm::True } else { llvm::False }; - let can_throw = if unwind { llvm::True } else { llvm::False }; - let argtys = inputs .iter() .map(|v| { @@ -500,10 +497,10 @@ pub(crate) fn inline_asm_call<'ll>( asm.len(), cons.as_ptr(), cons.len(), - volatile, - alignstack, + volatile.to_llvm_bool(), + alignstack.to_llvm_bool(), dia, - can_throw, + unwind.to_llvm_bool(), ) }; diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 427c75d40e9..836d227456c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -35,7 +35,7 @@ use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, FullCx, GenericCx, SCx}; use crate::llvm::{ - self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True, + self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, GEPNoWrapFlags, Metadata, ToLlvmBool, True, }; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -717,7 +717,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let mut const_llval = None; let llty = place.layout.llvm_type(self); if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { - if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if llvm::LLVMIsGlobalConstant(global).is_true() { if let Some(init) = llvm::LLVMGetInitializer(global) { if self.val_ty(init) == llty { const_llval = Some(init); @@ -1067,13 +1067,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { unsafe { - llvm::LLVMBuildIntCast2( - self.llbuilder, - val, - dest_ty, - if is_signed { True } else { False }, - UNNAMED, - ) + llvm::LLVMBuildIntCast2(self.llbuilder, val, dest_ty, is_signed.to_llvm_bool(), UNNAMED) } } @@ -1317,7 +1311,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { failure_order: rustc_middle::ty::AtomicOrdering, weak: bool, ) -> (&'ll Value, &'ll Value) { - let weak = if weak { llvm::True } else { llvm::False }; unsafe { let value = llvm::LLVMBuildAtomicCmpXchg( self.llbuilder, @@ -1328,7 +1321,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { AtomicOrdering::from_generic(failure_order), llvm::False, // SingleThreaded ); - llvm::LLVMSetWeak(value, weak); + llvm::LLVMSetWeak(value, weak.to_llvm_bool()); let val = self.extract_value(value, 0); let success = self.extract_value(value, 1); (val, success) @@ -1368,14 +1361,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { scope: SynchronizationScope, ) { let single_threaded = match scope { - SynchronizationScope::SingleThread => llvm::True, - SynchronizationScope::CrossThread => llvm::False, + SynchronizationScope::SingleThread => true, + SynchronizationScope::CrossThread => false, }; unsafe { llvm::LLVMBuildFence( self.llbuilder, AtomicOrdering::from_generic(order), - single_threaded, + single_threaded.to_llvm_bool(), UNNAMED, ); } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index f29fefb66f0..1b341c1f6af 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -20,7 +20,7 @@ use tracing::debug; use crate::consts::const_alloc_to_llvm; pub(crate) use crate::context::CodegenCx; use crate::context::{GenericCx, SCx}; -use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, True}; +use crate::llvm::{self, BasicBlock, ConstantInt, False, Metadata, ToLlvmBool, True}; use crate::type_::Type; use crate::value::Value; @@ -392,7 +392,7 @@ fn struct_in_context<'ll>( packed: bool, ) -> &'ll Value { let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); - unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed as Bool) } + unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed.to_llvm_bool()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 2461f70a86e..912e3d6ff24 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -13,7 +13,7 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] -use std::fmt::Debug; +use std::fmt::{self, Debug}; use std::marker::PhantomData; use std::num::NonZero; use std::ptr; @@ -33,10 +33,59 @@ use crate::llvm; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. -pub(crate) type Bool = c_int; +/// +/// This wrapper does not implement `PartialEq`. +/// To test the underlying boolean value, use [`Self::is_true`]. +#[derive(Clone, Copy)] +#[repr(transparent)] +pub(crate) struct Bool { + value: c_int, +} + +pub(crate) const True: Bool = Bool::TRUE; +pub(crate) const False: Bool = Bool::FALSE; + +impl Bool { + pub(crate) const TRUE: Self = Self { value: 1 }; + pub(crate) const FALSE: Self = Self { value: 0 }; + + pub(crate) const fn from_bool(rust_bool: bool) -> Self { + if rust_bool { Self::TRUE } else { Self::FALSE } + } + + /// Converts this LLVM-C boolean to a Rust `bool` + pub(crate) fn is_true(self) -> bool { + // Since we're interacting with a C API, follow the C convention of + // treating any nonzero value as true. + self.value != Self::FALSE.value + } +} -pub(crate) const True: Bool = 1 as Bool; -pub(crate) const False: Bool = 0 as Bool; +impl Debug for Bool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.value { + 0 => f.write_str("FALSE"), + 1 => f.write_str("TRUE"), + // As with `Self::is_true`, treat any nonzero value as true. + v => write!(f, "TRUE ({v})"), + } + } +} + +/// Convenience trait to convert `bool` to `llvm::Bool` with an explicit method call. +/// +/// Being able to write `b.to_llvm_bool()` is less noisy than `llvm::Bool::from(b)`, +/// while being more explicit and less mistake-prone than something like `b.into()`. +pub(crate) trait ToLlvmBool: Copy { + fn to_llvm_bool(self) -> llvm::Bool; +} + +impl ToLlvmBool for bool { + #[inline(always)] + fn to_llvm_bool(self) -> llvm::Bool { + llvm::Bool::from_bool(self) + } +} /// Wrapper for a raw enum value returned from LLVM's C APIs. /// diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 7fea7b79a8c..6adabe53129 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -215,7 +215,7 @@ pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) { } pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) { - LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); + LLVMSetGlobalConstant(llglobal, is_constant.to_llvm_bool()); } pub(crate) fn get_linkage(llglobal: &Value) -> Linkage { @@ -229,7 +229,7 @@ pub(crate) fn set_linkage(llglobal: &Value, linkage: Linkage) { } pub(crate) fn is_declaration(llglobal: &Value) -> bool { - unsafe { LLVMIsDeclaration(llglobal) == ffi::True } + unsafe { LLVMIsDeclaration(llglobal) }.is_true() } pub(crate) fn get_visibility(llglobal: &Value) -> Visibility { diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 90f7cd43268..d5025bb405c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -26,7 +26,7 @@ static INIT: Once = Once::new(); pub(crate) fn init(sess: &Session) { unsafe { // Before we touch LLVM, make sure that multithreading is enabled. - if llvm::LLVMIsMultithreaded() != 1 { + if !llvm::LLVMIsMultithreaded().is_true() { bug!("LLVM compiled without support for threads"); } INIT.call_once(|| { diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 5075befae8a..52eefe2d4d2 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -133,7 +133,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v).is_true()); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index f02d16baf94..499771aa124 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -15,7 +15,7 @@ use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::{CodegenCx, GenericCx, SCx}; pub(crate) use crate::llvm::Type; -use crate::llvm::{Bool, False, Metadata, True}; +use crate::llvm::{False, Metadata, ToLlvmBool, True}; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use crate::{common, llvm}; @@ -53,7 +53,9 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { - unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } + unsafe { + llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed.to_llvm_bool()) + } } pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) } @@ -152,7 +154,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { self.llcx(), els.as_ptr(), els.len() as c_uint, - packed as Bool, + packed.to_llvm_bool(), ) } } -- cgit 1.4.1-3-g733a5 From b4e97e5d866dd43c110d337affc419d5ac482765 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Sun, 24 Aug 2025 20:49:32 +1000 Subject: Rename `llvm::Bool` aliases to standard const case This avoids the need for `#![allow(non_upper_case_globals)]`. --- compiler/rustc_codegen_llvm/src/allocator.rs | 6 ++--- compiler/rustc_codegen_llvm/src/back/write.rs | 2 +- compiler/rustc_codegen_llvm/src/builder.rs | 30 +++++++++++----------- .../rustc_codegen_llvm/src/builder/autodiff.rs | 4 +-- compiler/rustc_codegen_llvm/src/common.rs | 8 +++--- compiler/rustc_codegen_llvm/src/context.rs | 2 +- compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs | 2 +- .../rustc_codegen_llvm/src/debuginfo/namespace.rs | 2 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 ++-- compiler/rustc_codegen_llvm/src/type_.rs | 6 ++--- 10 files changed, 33 insertions(+), 34 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 23610aa856c..df3e49279d9 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -12,7 +12,7 @@ use smallvec::SmallVec; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; -use crate::llvm::{self, False, True, Type, Value}; +use crate::llvm::{self, FALSE, TRUE, Type, Value}; use crate::{SimpleCx, attributes, debuginfo, llvm_util}; pub(crate) unsafe fn codegen( @@ -80,7 +80,7 @@ pub(crate) unsafe fn codegen( &cx, &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), &i8, - &llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, False), + &llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, FALSE), ); // __rust_no_alloc_shim_is_unstable_v2 @@ -201,7 +201,7 @@ fn create_wrapper_function( .map(|(i, _)| llvm::get_param(llfn, i as c_uint)) .collect::>(); let ret = bx.call(ty, callee, &args, None); - llvm::LLVMSetTailCall(ret, True); + llvm::LLVMSetTailCall(ret, TRUE); if output.is_some() { bx.ret(ret); } else { diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 62998003ca1..7c65d667bda 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1110,7 +1110,7 @@ fn embed_bitcode( llvm::set_section(llglobal, bitcode_section_name(cgcx)); llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::LLVMSetGlobalConstant(llglobal, llvm::True); + llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE); let llconst = common::bytes_in_context(llcx, &[]); let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 836d227456c..37379586d58 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -35,7 +35,7 @@ use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, FullCx, GenericCx, SCx}; use crate::llvm::{ - self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, GEPNoWrapFlags, Metadata, ToLlvmBool, True, + self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, GEPNoWrapFlags, Metadata, TRUE, ToLlvmBool, }; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -493,8 +493,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(add).is_some() { - llvm::LLVMSetNUW(add, True); - llvm::LLVMSetNSW(add, True); + llvm::LLVMSetNUW(add, TRUE); + llvm::LLVMSetNSW(add, TRUE); } add } @@ -503,8 +503,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(sub).is_some() { - llvm::LLVMSetNUW(sub, True); - llvm::LLVMSetNSW(sub, True); + llvm::LLVMSetNUW(sub, TRUE); + llvm::LLVMSetNSW(sub, TRUE); } sub } @@ -513,8 +513,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(mul).is_some() { - llvm::LLVMSetNUW(mul, True); - llvm::LLVMSetNSW(mul, True); + llvm::LLVMSetNUW(mul, TRUE); + llvm::LLVMSetNSW(mul, TRUE); } mul } @@ -528,7 +528,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // an instruction, so we need to check before setting the flag. // (See also `LLVMBuildNUWNeg` which also needs a check.) if llvm::LLVMIsAInstruction(or).is_some() { - llvm::LLVMSetIsDisjoint(or, True); + llvm::LLVMSetIsDisjoint(or, TRUE); } or } @@ -629,7 +629,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); - llvm::LLVMSetVolatile(load, llvm::True); + llvm::LLVMSetVolatile(load, llvm::TRUE); load } } @@ -838,7 +838,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; llvm::LLVMSetAlignment(store, align); if flags.contains(MemFlags::VOLATILE) { - llvm::LLVMSetVolatile(store, llvm::True); + llvm::LLVMSetVolatile(store, llvm::TRUE); } if flags.contains(MemFlags::NONTEMPORAL) { // Make sure that the current target architectures supports "sane" non-temporal @@ -956,7 +956,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let trunc = self.trunc(val, dest_ty); unsafe { if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNUW(trunc, True); + llvm::LLVMSetNUW(trunc, TRUE); } } trunc @@ -968,7 +968,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let trunc = self.trunc(val, dest_ty); unsafe { if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNSW(trunc, True); + llvm::LLVMSetNSW(trunc, TRUE); } } trunc @@ -1223,7 +1223,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false); let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { - llvm::LLVMSetCleanup(landing_pad, llvm::True); + llvm::LLVMSetCleanup(landing_pad, llvm::TRUE); } (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } @@ -1319,7 +1319,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src, AtomicOrdering::from_generic(order), AtomicOrdering::from_generic(failure_order), - llvm::False, // SingleThreaded + llvm::FALSE, // SingleThreaded ); llvm::LLVMSetWeak(value, weak.to_llvm_bool()); let val = self.extract_value(value, 0); @@ -1346,7 +1346,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst, src, AtomicOrdering::from_generic(order), - llvm::False, // SingleThreaded + llvm::FALSE, // SingleThreaded ) }; if ret_ptr && self.val_ty(res) != self.type_ptr() { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index e2df3265f6f..6ddf53cdc87 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -11,7 +11,7 @@ use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; use crate::llvm; -use crate::llvm::{Metadata, True, Type}; +use crate::llvm::{Metadata, TRUE, Type}; use crate::value::Value; pub(crate) fn adjust_activity_to_abi<'tcx>( @@ -293,7 +293,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( // ret double %0 // } // ``` - let enzyme_ty = unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) }; + let enzyme_ty = unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, TRUE) }; // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and // think a bit more about what should go here. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 1b341c1f6af..11b79a7fe68 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -20,7 +20,7 @@ use tracing::debug; use crate::consts::const_alloc_to_llvm; pub(crate) use crate::context::CodegenCx; use crate::context::{GenericCx, SCx}; -use crate::llvm::{self, BasicBlock, ConstantInt, False, Metadata, ToLlvmBool, True}; +use crate::llvm::{self, BasicBlock, ConstantInt, FALSE, Metadata, TRUE, ToLlvmBool}; use crate::type_::Type; use crate::value::Value; @@ -158,7 +158,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.type_kind(t) == TypeKind::Integer, "only allows integer types in const_int" ); - unsafe { llvm::LLVMConstInt(t, i as u64, True) } + unsafe { llvm::LLVMConstInt(t, i as u64, TRUE) } } fn const_u8(&self, i: u8) -> &'ll Value { @@ -192,7 +192,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.type_kind(t) == TypeKind::Integer, "only allows integer types in const_uint" ); - unsafe { llvm::LLVMConstInt(t, i, False) } + unsafe { llvm::LLVMConstInt(t, i, FALSE) } } fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { @@ -377,7 +377,7 @@ pub(crate) fn val_ty(v: &Value) -> &Type { pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), TRUE) } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4a7de7d2e69..4fd6110ac4a 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -701,7 +701,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn get_const_int(&self, ty: &'ll Type, val: u64) -> &'ll Value { - unsafe { llvm::LLVMConstInt(ty, val, llvm::False) } + unsafe { llvm::LLVMConstInt(ty, val, llvm::FALSE) } } pub(crate) fn get_const_i64(&self, n: u64) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 6eb7042da61..7a6dc008c7b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -72,7 +72,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); llvm::set_section(section_var, c".debug_gdb_scripts"); llvm::set_initializer(section_var, cx.const_bytes(section_contents)); - llvm::LLVMSetGlobalConstant(section_var, llvm::True); + llvm::LLVMSetGlobalConstant(section_var, llvm::TRUE); llvm::set_unnamed_address(section_var, llvm::UnnamedAddr::Global); llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index b4d639368b0..1dcf4ff3062 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -38,7 +38,7 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l parent_scope, namespace_name_string.as_ptr(), namespace_name_string.len(), - llvm::False, // ExportSymbols (only relevant for C++ anonymous namespaces) + llvm::FALSE, // ExportSymbols (only relevant for C++ anonymous namespaces) ) }; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 912e3d6ff24..fa2802a891f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -11,7 +11,6 @@ //! the need for an extra cast from `*const u8` on the Rust side. #![allow(non_camel_case_types)] -#![allow(non_upper_case_globals)] use std::fmt::{self, Debug}; use std::marker::PhantomData; @@ -42,8 +41,8 @@ pub(crate) struct Bool { value: c_int, } -pub(crate) const True: Bool = Bool::TRUE; -pub(crate) const False: Bool = Bool::FALSE; +pub(crate) const TRUE: Bool = Bool::TRUE; +pub(crate) const FALSE: Bool = Bool::FALSE; impl Bool { pub(crate) const TRUE: Self = Self { value: 1 }; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 499771aa124..9ecaf5f24fe 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -15,7 +15,7 @@ use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::{CodegenCx, GenericCx, SCx}; pub(crate) use crate::llvm::Type; -use crate::llvm::{False, Metadata, ToLlvmBool, True}; +use crate::llvm::{FALSE, Metadata, TRUE, ToLlvmBool}; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use crate::{common, llvm}; @@ -141,7 +141,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, TRUE) } } pub(crate) fn type_i1(&self) -> &'ll Type { @@ -202,7 +202,7 @@ impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, FALSE) } } fn type_kind(&self, ty: &'ll Type) -> TypeKind { -- cgit 1.4.1-3-g733a5 From fcff8f7f5a0d4add3c05f57de1b34291746c3c08 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Tue, 26 Aug 2025 12:31:33 +1000 Subject: Assert that LLVM range-attribute values don't exceed 128 bits The underlying implementation of `LLVMCreateConstantRangeAttribute` assumes that each of `LowerWords` and `UpperWords` points to enough u64 values to define an integer of the specified bit-length, and will encounter UB if that is not the case. Our safe wrapper function always passes pointers to `[u64; 2]` arrays, regardless of the bit-length specified. That's fine in practice, because scalar primitives never exceed 128 bits, but it is technically a soundness hole in a safe function. We can close the soundness hole by explicitly asserting `size_bits <= 128`. This is effectively just a stricter version of the existing check that the value must be small enough to fit in `c_uint`. --- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 12 ++++++++--- compiler/rustc_codegen_llvm/src/llvm/mod.rs | 26 ++++++++++++++++-------- compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 3 +++ 3 files changed, 30 insertions(+), 11 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index fa2802a891f..7c79cba2273 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1929,11 +1929,17 @@ unsafe extern "C" { C: &Context, effects: MemoryEffects, ) -> &Attribute; + /// ## Safety + /// - Each of `LowerWords` and `UpperWords` must point to an array that is + /// long enough to fully define an integer of size `NumBits`, i.e. each + /// pointer must point to `NumBits.div_ceil(64)` elements or more. + /// - The implementation will make its own copy of the pointed-to `u64` + /// values, so the pointers only need to outlive this function call. pub(crate) fn LLVMRustCreateRangeAttribute( C: &Context, - num_bits: c_uint, - lower_words: *const u64, - upper_words: *const u64, + NumBits: c_uint, + LowerWords: *const u64, + UpperWords: *const u64, ) -> &Attribute; // Operations on functions diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 6adabe53129..d6974e22c85 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -112,16 +112,26 @@ pub(crate) fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> & pub(crate) fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { let lower = range.start; + // LLVM treats the upper bound as exclusive, but allows wrapping. let upper = range.end.wrapping_add(1); - let lower_words = [lower as u64, (lower >> 64) as u64]; - let upper_words = [upper as u64, (upper >> 64) as u64]; + + // Pass each `u128` endpoint value as a `[u64; 2]` array, least-significant part first. + let as_u64_array = |x: u128| [x as u64, (x >> 64) as u64]; + let lower_words: [u64; 2] = as_u64_array(lower); + let upper_words: [u64; 2] = as_u64_array(upper); + + // To ensure that LLVM doesn't try to read beyond the `[u64; 2]` arrays, + // we must explicitly check that `size_bits` does not exceed 128. + let size_bits = size.bits(); + assert!(size_bits <= 128); + // More robust assertions that are redundant with `size_bits <= 128` and + // should be optimized away. + assert!(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()); + assert!(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()); + let size_bits = c_uint::try_from(size_bits).unwrap(); + unsafe { - LLVMRustCreateRangeAttribute( - llcx, - size.bits().try_into().unwrap(), - lower_words.as_ptr(), - upper_words.as_ptr(), - ) + LLVMRustCreateRangeAttribute(llcx, size_bits, lower_words.as_ptr(), upper_words.as_ptr()) } } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index e699e4b9c13..cce40da354d 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -488,6 +488,9 @@ extern "C" LLVMAttributeRef LLVMRustCreateRangeAttribute(LLVMContextRef C, unsigned NumBits, const uint64_t LowerWords[], const uint64_t UpperWords[]) { + // FIXME(Zalathar): There appears to be no stable guarantee that C++ + // `AttrKind` values correspond directly to the `unsigned KindID` values + // accepted by LLVM-C API functions, though in practice they currently do. return LLVMCreateConstantRangeAttribute(C, Attribute::Range, NumBits, LowerWords, UpperWords); } -- cgit 1.4.1-3-g733a5 From c3ab409b4fa3b1fb18f34877d9811bb502ed507f Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 26 Aug 2025 11:13:03 +0200 Subject: Use captures(address) instead of captures(none) for indirect args While provenance cannot be captured through these arguments, the address / object identity can. --- compiler/rustc_codegen_llvm/src/abi.rs | 8 +++++--- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 +- compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 15 +++++---------- compiler/rustc_target/src/callconv/mod.rs | 8 ++++---- tests/codegen-llvm/addr-of-mutate.rs | 6 +++--- tests/codegen-llvm/function-arguments.rs | 4 ++-- tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs | 4 ++-- tests/ui/abi/c-zst.powerpc-linux.stderr | 2 +- tests/ui/abi/c-zst.s390x-linux.stderr | 2 +- tests/ui/abi/c-zst.sparc64-linux.stderr | 2 +- tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr | 2 +- tests/ui/abi/debug.generic.stderr | 4 ++-- tests/ui/abi/debug.loongarch64.stderr | 4 ++-- tests/ui/abi/debug.riscv64.stderr | 4 ++-- tests/ui/codegen/indirect-nocapture.rs | 15 +++++++++++++++ 15 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 tests/ui/codegen/indirect-nocapture.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 399f8b6e762..ac7583f5666 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -44,7 +44,7 @@ const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 6] = [ (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), - (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), + (ArgAttribute::CapturesAddress, llvm::AttributeKind::CapturesAddress), (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), @@ -84,8 +84,10 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' } for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { if regular.contains(attr) { - // captures(address, read_provenance) is only available since LLVM 21. - if attr == ArgAttribute::CapturesReadOnly && llvm_util::get_version() < (21, 0, 0) { + // captures(...) is only available since LLVM 21. + if (attr == ArgAttribute::CapturesReadOnly || attr == ArgAttribute::CapturesAddress) + && llvm_util::get_version() < (21, 0, 0) + { continue; } attrs.push(llattr.create_attr(cx.llcx)); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index fa2802a891f..21e6bbb118e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -263,7 +263,7 @@ pub(crate) enum AttributeKind { MinSize = 4, Naked = 5, NoAlias = 6, - NoCapture = 7, + CapturesAddress = 7, NoInline = 8, NonNull = 9, NoRedZone = 10, diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index e699e4b9c13..2652acb1ec4 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -242,7 +242,7 @@ enum class LLVMRustAttributeKind { MinSize = 4, Naked = 5, NoAlias = 6, - NoCapture = 7, + CapturesAddress = 7, NoInline = 8, NonNull = 9, NoRedZone = 10, @@ -297,12 +297,6 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { return Attribute::Naked; case LLVMRustAttributeKind::NoAlias: return Attribute::NoAlias; - case LLVMRustAttributeKind::NoCapture: -#if LLVM_VERSION_GE(21, 0) - report_fatal_error("NoCapture doesn't exist in LLVM 21"); -#else - return Attribute::NoCapture; -#endif case LLVMRustAttributeKind::NoCfCheck: return Attribute::NoCfCheck; case LLVMRustAttributeKind::NoInline: @@ -377,6 +371,7 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { #else report_fatal_error("DeadOnReturn attribute requires LLVM 21 or later"); #endif + case LLVMRustAttributeKind::CapturesAddress: case LLVMRustAttributeKind::CapturesReadOnly: report_fatal_error("Should be handled separately"); } @@ -429,9 +424,9 @@ extern "C" void LLVMRustEraseInstFromParent(LLVMValueRef Instr) { extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttributeKind RustAttr) { #if LLVM_VERSION_GE(21, 0) - // LLVM 21 replaced the NoCapture attribute with Captures(none). - if (RustAttr == LLVMRustAttributeKind::NoCapture) { - return wrap(Attribute::getWithCaptureInfo(*unwrap(C), CaptureInfo::none())); + if (RustAttr == LLVMRustAttributeKind::CapturesAddress) { + return wrap(Attribute::getWithCaptureInfo( + *unwrap(C), CaptureInfo(CaptureComponents::Address))); } if (RustAttr == LLVMRustAttributeKind::CapturesReadOnly) { return wrap(Attribute::getWithCaptureInfo( diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 5f2a6f7ba38..7a7c63c475b 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -114,7 +114,7 @@ mod attr_impl { bitflags::bitflags! { impl ArgAttribute: u8 { const NoAlias = 1 << 1; - const NoCapture = 1 << 2; + const CapturesAddress = 1 << 2; const NonNull = 1 << 3; const ReadOnly = 1 << 4; const InReg = 1 << 5; @@ -400,11 +400,11 @@ impl<'a, Ty> ArgAbi<'a, Ty> { let mut attrs = ArgAttributes::new(); // For non-immediate arguments the callee gets its own copy of - // the value on the stack, so there are no aliases. It's also - // program-invisible so can't possibly capture + // the value on the stack, so there are no aliases. The function + // can capture the address of the argument, but not the provenance. attrs .set(ArgAttribute::NoAlias) - .set(ArgAttribute::NoCapture) + .set(ArgAttribute::CapturesAddress) .set(ArgAttribute::NonNull) .set(ArgAttribute::NoUndef); attrs.pointee_size = layout.size; diff --git a/tests/codegen-llvm/addr-of-mutate.rs b/tests/codegen-llvm/addr-of-mutate.rs index 71669065289..36d6bf555d1 100644 --- a/tests/codegen-llvm/addr-of-mutate.rs +++ b/tests/codegen-llvm/addr-of-mutate.rs @@ -5,7 +5,7 @@ // Test for the absence of `readonly` on the argument when it is mutated via `&raw const`. // See . -// CHECK: i8 @foo(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align 1{{( captures\(none\))?}} dereferenceable(128) %x) +// CHECK: i8 @foo(ptr{{( dead_on_return)?}} noalias noundef align 1{{( captures\(address\))?}} dereferenceable(128) %x) #[no_mangle] pub fn foo(x: [u8; 128]) -> u8 { let ptr = core::ptr::addr_of!(x).cast_mut(); @@ -15,7 +15,7 @@ pub fn foo(x: [u8; 128]) -> u8 { x[0] } -// CHECK: i1 @second(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) +// CHECK: i1 @second(ptr{{( dead_on_return)?}} noalias noundef align {{[0-9]+}}{{( captures\(address\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) #[no_mangle] pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { let b_bool_ptr = core::ptr::addr_of!(a_ptr_and_b.1.1).cast_mut(); @@ -24,7 +24,7 @@ pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { } // If going through a deref (and there are no other mutating accesses), then `readonly` is fine. -// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) +// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias noundef readonly align {{[0-9]+}}{{( captures\(address\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) #[no_mangle] pub unsafe fn third(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { let b_bool_ptr = core::ptr::addr_of!((*a_ptr_and_b.0).1).cast_mut(); diff --git a/tests/codegen-llvm/function-arguments.rs b/tests/codegen-llvm/function-arguments.rs index db508682862..a0744e44c61 100644 --- a/tests/codegen-llvm/function-arguments.rs +++ b/tests/codegen-llvm/function-arguments.rs @@ -134,7 +134,7 @@ pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {} #[no_mangle] pub fn notunpin_borrow(_: &NotUnpin) {} -// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1) +// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias noundef readonly align 4{{( captures\(address\))?}} dereferenceable(32) %_1) #[no_mangle] pub fn indirect_struct(_: S) {} @@ -197,7 +197,7 @@ pub fn notunpin_box(x: Box) -> Box { x } -// CHECK: @struct_return(ptr{{( dead_on_unwind)?}} noalias{{( nocapture)?}} noundef{{( writable)?}} sret([32 x i8]) align 4{{( captures\(none\))?}} dereferenceable(32){{( %_0)?}}) +// CHECK: @struct_return(ptr{{( dead_on_unwind)?}} noalias noundef{{( writable)?}} sret([32 x i8]) align 4{{( captures\(address\))?}} dereferenceable(32){{( %_0)?}}) #[no_mangle] pub fn struct_return() -> S { S { _field: [0, 0, 0, 0, 0, 0, 0, 0] } diff --git a/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs b/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs index 4666342a16a..7ea08a1a8f7 100644 --- a/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs +++ b/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs @@ -256,11 +256,11 @@ pub struct IntDoubleInt { c: i32, } -// CHECK: define void @f_int_double_int_s_arg(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align 8{{( captures\(none\))?}} dereferenceable(24) %a) +// CHECK: define void @f_int_double_int_s_arg(ptr{{( dead_on_return)?}} noalias noundef align 8{{( captures\(address\))?}} dereferenceable(24) %a) #[no_mangle] pub extern "C" fn f_int_double_int_s_arg(a: IntDoubleInt) {} -// CHECK: define void @f_ret_int_double_int_s(ptr{{( dead_on_unwind)?}} noalias{{( nocapture)?}} noundef{{( writable)?}} sret([24 x i8]) align 8{{( captures\(none\))?}} dereferenceable(24) %_0) +// CHECK: define void @f_ret_int_double_int_s(ptr{{( dead_on_unwind)?}} noalias noundef{{( writable)?}} sret([24 x i8]) align 8{{( captures\(address\))?}} dereferenceable(24) %_0) #[no_mangle] pub extern "C" fn f_ret_int_double_int_s() -> IntDoubleInt { IntDoubleInt { a: 1, b: 2., c: 3 } diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr index b8d6c632b97..ec6b9b8f027 100644 --- a/tests/ui/abi/c-zst.powerpc-linux.stderr +++ b/tests/ui/abi/c-zst.powerpc-linux.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr index b8d6c632b97..ec6b9b8f027 100644 --- a/tests/ui/abi/c-zst.s390x-linux.stderr +++ b/tests/ui/abi/c-zst.s390x-linux.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr index b8d6c632b97..ec6b9b8f027 100644 --- a/tests/ui/abi/c-zst.sparc64-linux.stderr +++ b/tests/ui/abi/c-zst.sparc64-linux.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr index b8d6c632b97..ec6b9b8f027 100644 --- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr +++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index 0fec35dabdd..47341da221f 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -454,7 +454,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(32 bytes), pointee_align: Some( @@ -527,7 +527,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(128 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr index ce8bd41f045..94a01a80873 100644 --- a/tests/ui/abi/debug.loongarch64.stderr +++ b/tests/ui/abi/debug.loongarch64.stderr @@ -454,7 +454,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(32 bytes), pointee_align: Some( @@ -527,7 +527,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(128 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index ce8bd41f045..94a01a80873 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -454,7 +454,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(32 bytes), pointee_align: Some( @@ -527,7 +527,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | NoCapture | NonNull | NoUndef, + regular: NoAlias | CapturesAddress | NonNull | NoUndef, arg_ext: None, pointee_size: Size(128 bytes), pointee_align: Some( diff --git a/tests/ui/codegen/indirect-nocapture.rs b/tests/ui/codegen/indirect-nocapture.rs new file mode 100644 index 00000000000..78024a94c0d --- /dev/null +++ b/tests/ui/codegen/indirect-nocapture.rs @@ -0,0 +1,15 @@ +// Regression test for issue #137668 where an indirect argument have been marked as nocapture +// despite the fact that callee did in fact capture the address. +// +//@ run-pass +//@ compile-flags: -Copt-level=2 + +#[inline(never)] +pub fn f(a: [u32; 64], b: [u32; 64]) -> bool { + &a as *const _ as usize != &b as *const _ as usize +} + +fn main() { + static S: [u32; 64] = [0; 64]; + assert!(f(S, S)); +} -- cgit 1.4.1-3-g733a5