diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
39 files changed, 2215 insertions, 1575 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 009e7e2487b..1703cab942b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -24,6 +24,7 @@ use crate::attributes::{self, llfn_attrs_from_instance}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -41,12 +42,13 @@ trait ArgAttributesExt { const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; -const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 6] = [ (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), - (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), + (ArgAttribute::CapturesAddress, llvm::AttributeKind::CapturesAddress), (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), + (ArgAttribute::CapturesReadOnly, llvm::AttributeKind::CapturesReadOnly), ]; fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { @@ -82,6 +84,12 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' } for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { if regular.contains(attr) { + // captures(...) is only available since LLVM 21. + if (attr == ArgAttribute::CapturesReadOnly || attr == ArgAttribute::CapturesAddress) + && llvm_util::get_version() < (21, 0, 0) + { + continue; + } attrs.push(llattr.create_attr(cx.llcx)); } } @@ -207,9 +215,9 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst); } - // Unsized indirect qrguments + // Unsized indirect arguments cannot be stored PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Cast { cast, pad_i32: _ } => { // The ABI mandates that the value is passed as a different struct representation. @@ -238,6 +246,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); } @@ -264,12 +273,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Pair(next(), next()).store(bx, dst); } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - let place_val = PlaceValue { - llval: next(), - llextra: Some(next()), - align: self.layout.align.abi, - }; - OperandValue::Ref(place_val).store(bx, dst); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Direct(_) | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } @@ -500,7 +504,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { - apply(attrs); + let i = apply(attrs); + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (21, 0, 0) + { + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[llvm::AttributeKind::DeadOnReturn.create_attr(cx.llcx)], + ); + } } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); @@ -526,7 +539,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the declaration has an associated instance, compute extra attributes based on that. if let Some(instance) = instance { - llfn_attrs_from_instance(cx, llfn, instance); + llfn_attrs_from_instance( + cx, + cx.tcx, + llfn, + &cx.tcx.codegen_instance_attrs(instance.def), + Some(instance), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 2b5090ed6db..abd63120397 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -5,13 +5,15 @@ use rustc_ast::expand::allocator::{ }; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; -use crate::llvm::{self, False, True, Type, Value}; +use crate::llvm::{self, FALSE, TRUE, Type, Value}; use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( @@ -79,7 +81,7 @@ pub(crate) unsafe fn codegen( &cx, &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), &i8, - &llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, False), + &llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, FALSE), ); // __rust_no_alloc_shim_is_unstable_v2 @@ -147,6 +149,10 @@ fn create_wrapper_function( llvm::Visibility::from_generic(tcx.sess.default_visibility()), ty, ); + + let attrs = CodegenFnAttrs::new(); + llfn_attrs_from_instance(cx, tcx, llfn, &attrs, None); + let no_return = if no_return { // -> ! DIFlagNoReturn let no_return = llvm::AttributeKind::NoReturn.create_attr(cx.llcx); @@ -156,12 +162,6 @@ fn create_wrapper_function( None }; - if tcx.sess.must_emit_unwind_tables() { - let uwtable = - attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; let mut bx = SBuilder::build(&cx, llbb); @@ -186,7 +186,7 @@ fn create_wrapper_function( .map(|(i, _)| llvm::get_param(llfn, i as c_uint)) .collect::<Vec<_>>(); let ret = bx.call(ty, callee, &args, None); - llvm::LLVMSetTailCall(ret, True); + llvm::LLVMSetTailCall(ret, TRUE); if output.is_some() { bx.ret(ret); } else { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a643a91141e..cc09fa5b69b 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -16,6 +16,7 @@ use tracing::debug; use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; +use crate::llvm::ToLlvmBool; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -239,6 +240,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { constraints.extend_from_slice(&[ + "~{fflags}".to_string(), "~{vtype}".to_string(), "~{vl}".to_string(), "~{vxsat}".to_string(), @@ -338,8 +340,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::NOMEM) { attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); - } else { - // LLVM doesn't have an attribute to represent ReadOnly + SideEffect + } else if options.contains(InlineAsmOptions::READONLY) { + attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx)); } attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); @@ -470,10 +472,6 @@ pub(crate) fn inline_asm_call<'ll>( dest: Option<&'ll llvm::BasicBlock>, catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { - let volatile = if volatile { llvm::True } else { llvm::False }; - let alignstack = if alignstack { llvm::True } else { llvm::False }; - let can_throw = if unwind { llvm::True } else { llvm::False }; - let argtys = inputs .iter() .map(|v| { @@ -500,10 +498,10 @@ pub(crate) fn inline_asm_call<'ll>( asm.len(), cons.as_ptr(), cons.len(), - volatile, - alignstack, + volatile.to_llvm_bool(), + alignstack.to_llvm_bool(), dia, - can_throw, + unwind.to_llvm_bool(), ) }; @@ -664,7 +662,12 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -832,7 +835,12 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4), - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c548f467583..8070ea0b3e9 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,20 +1,21 @@ //! Set and unset common attributes on LLVM values. -use rustc_codegen_ssa::traits::*; use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; +use rustc_middle::middle::codegen_fn_attrs::{ + CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, +}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; -use crate::context::CodegenCx; +use crate::context::SimpleCx; use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; -use crate::{attributes, llvm_util}; +use crate::{Session, attributes, llvm_util}; pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { @@ -28,26 +29,21 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } } -pub(crate) fn has_attr(llfn: &Value, idx: AttributePlace, attr: AttributeKind) -> bool { - llvm::HasAttributeAtIndex(llfn, idx, attr) -} - -pub(crate) fn has_string_attr(llfn: &Value, name: &str) -> bool { - llvm::HasStringAttribute(llfn, name) -} - -pub(crate) fn remove_from_llfn(llfn: &Value, place: AttributePlace, kind: AttributeKind) { - llvm::RemoveRustEnumAttributeAtIndex(llfn, place, kind); -} - -pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: &str) { - llvm::RemoveStringAttrFromFn(llfn, name); -} - /// Get LLVM attribute for the provided inline heuristic. -#[inline] -fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { - if !cx.tcx.sess.opts.unstable_opts.inline_llvm { +pub(crate) fn inline_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, +) -> Option<&'ll Attribute> { + // `optnone` requires `noinline` + let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id()); + let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { + (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, + (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint, + (inline, _) => inline, + }; + + if !tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); } @@ -57,7 +53,7 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) } InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { + if tcx.sess.target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) } else { None @@ -69,12 +65,13 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll #[inline] fn patchable_function_entry_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, attr: Option<PatchableFunctionEntry>, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); let patchable_spec = attr.unwrap_or_else(|| { - PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry) + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry) }); let entry = patchable_spec.entry(); let prefix = patchable_spec.prefix(); @@ -97,12 +94,13 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub(crate) fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn sanitize_attrs<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -120,11 +118,11 @@ pub(crate) fn sanitize_attrs<'ll>( } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); + let features = tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); + tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -145,9 +143,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) llvm::CreateUWTableAttr(llcx, async_unwind) } -pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - let opts = &cx.sess().opts; +pub(crate) fn frame_pointer_type_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { + let mut fp = sess.target.frame_pointer; + let opts = &sess.opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. if opts.unstable_opts.instrument_mcount { @@ -162,8 +163,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&' Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) } -fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let function_return_attr = match cx.sess().opts.unstable_opts.function_return { +fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let function_return_attr = match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => return None, FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern, }; @@ -173,17 +174,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { +fn instrument_function_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - if cx.sess().opts.unstable_opts.instrument_mcount { + if sess.opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + let mcount_name = match &sess.target.llvm_mcount_intrinsic { Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), - None => cx.sess().target.mcount.as_ref(), + None => sess.target.mcount.as_ref(), }; attrs.push(llvm::CreateAttrStringValue( @@ -192,7 +196,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr mcount_name, )); } - if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + if let Some(options) = &sess.opts.unstable_opts.instrument_xray { // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. // Function prologue and epilogue are instrumented with NOP sleds, // a runtime library later replaces them with detours into tracing code. @@ -223,20 +227,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr attrs } -fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if !cx.sess().opts.unstable_opts.no_jump_tables { +fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if !sess.opts.unstable_opts.no_jump_tables { return None; } Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() + if tcx + .sess .opts .unstable_opts .sanitizer @@ -246,22 +250,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { + if tcx.sess.opts.cg.profile_generate.enabled() { return None; } - let attr_value = match cx.sess().target.stack_probes { + let attr_value = match tcx.sess.target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). StackProbeType::Inline => "inline-asm", // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"), + StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - &mangle_internal_symbol(cx.tcx, "__rust_probestack") + &mangle_internal_symbol(tcx, "__rust_probestack") } else { "inline-asm" } @@ -270,8 +274,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) } -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { +fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let sspattr = match sess.stack_protector() { StackProtector::None => return None, StackProtector::All => AttributeKind::StackProtectReq, StackProtector::Strong => AttributeKind::StackProtectStrong, @@ -281,45 +285,59 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(sspattr.create_attr(cx.llcx)) } -fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().target.arch != "s390x" { +fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if sess.target.arch != "s390x" { return None; } - let requested_features = cx.sess().opts.cg.target_feature.split(','); + let requested_features = sess.opts.cg.target_feature.split(','); let found_positive = requested_features.clone().any(|r| r == "+backchain"); if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); +pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) +pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } +/// Get the `target-features` LLVM attribute. +pub(crate) fn target_features_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, + function_features: Vec<String>, +) -> Option<&'ll Attribute> { + let global_features = tcx.global_backend_features(()).iter().map(String::as_str); + let function_features = function_features.iter().map(String::as_str); + let target_features = + global_features.chain(function_features).intersperse(",").collect::<String>(); + (!target_features.is_empty()) + .then(|| llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)) +} + /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } + if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None } } /// Get the default optimizations attrs for a function. #[inline] pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { + match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } @@ -340,17 +358,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, llfn: &'ll Value, - instance: ty::Instance<'tcx>, + codegen_fn_attrs: &CodegenFnAttrs, + instance: Option<ty::Instance<'tcx>>, ) { - let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); - + let sess = tcx.sess; let mut to_add = SmallVec::<[_; 16]>::new(); match codegen_fn_attrs.optimize { OptimizeAttr::Default => { - to_add.extend(default_optimisation_attrs(cx)); + to_add.extend(default_optimisation_attrs(cx, sess)); } OptimizeAttr::DoNotOptimize => { to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx)); @@ -362,29 +381,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - // `optnone` requires `noinline` - let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { - (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, - (inline, _) => inline, - }; - to_add.extend(inline_attr(cx, inline)); - - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); + if sess.must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind)); } - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + if sess.opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } // FIXME: none of these functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(function_return_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(nojumptables_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); + to_add.extend(frame_pointer_type_attr(cx, sess)); + to_add.extend(function_return_attr(cx, sess)); + to_add.extend(instrument_function_attr(cx, sess)); + to_add.extend(nojumptables_attr(cx, sess)); + to_add.extend(probestack_attr(cx, tcx)); + to_add.extend(stackprotector_attr(cx, sess)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); @@ -405,16 +416,19 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // not used. } else { // Do not set sanitizer attributes for naked functions. - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize)); // For non-naked functions, set branch protection attributes on aarch64. - if let Some(BranchProtection { bti, pac_ret }) = - cx.sess().opts.unstable_opts.branch_protection + if let Some(BranchProtection { bti, pac_ret, gcs }) = + sess.opts.unstable_opts.branch_protection { - assert!(cx.sess().target.arch == "aarch64"); + assert!(sess.target.arch == "aarch64"); if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } + if gcs { + to_add.push(llvm::CreateAttrString(cx.llcx, "guarded-control-stack")); + } if let Some(PacRet { leaf, pc, key }) = pac_ret { if pc { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); @@ -436,6 +450,17 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); + if let Some(instance) = instance + && let Some(zv) = + tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + && let Some(name) = zv.value_str() + { + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "alloc-variant-zeroed", + &mangle_internal_symbol(tcx, name.as_str()), + )); + } // apply to argument place instead of function let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); @@ -478,26 +503,40 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align); } - if let Some(backchain) = backchain_attr(cx) { + if let Some(backchain) = backchain_attr(cx, sess) { to_add.push(backchain); } - to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); + to_add.extend(patchable_function_entry_attrs( + cx, + sess, + codegen_fn_attrs.patchable_function_entry, + )); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); + to_add.push(target_cpu_attr(cx, sess)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); + to_add.extend(tune_cpu_attr(cx, sess)); let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); + // Apply function attributes as per usual if there are no user defined + // target features otherwise this will get applied at the callsite. + if function_features.is_empty() { + if let Some(instance) = instance + && let Some(inline_attr) = inline_attr(cx, tcx, instance) + { + to_add.push(inline_attr); + } + } + let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones - .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + .flat_map(|feat| llvm_util::to_llvm_features(sess, feat)) // Convert LLVMFeatures & dependencies to +<feats>s .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { @@ -506,26 +545,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( })) .collect::<Vec<String>>(); - if cx.tcx.sess.target.is_like_wasm { + if sess.target.is_like_wasm { // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + if let Some(instance) = instance + && let Some(module) = wasm_import_module(tcx, instance.def_id()) + { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } } - let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); - let function_features = function_features.iter().map(|s| s.as_str()); - let target_features: String = - global_features.chain(function_features).intersperse(",").collect(); - if !target_features.is_empty() { - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); - } + to_add.extend(target_features_attr(cx, tcx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 0a161442933..f9dc48e3aba 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,104 +1,21 @@ //! A helper class for dealing with static archives -use std::ffi::{CStr, CString, c_char, c_void}; -use std::path::{Path, PathBuf}; -use std::{io, mem, ptr, str}; +use std::ffi::{CStr, c_char, c_void}; +use std::io; use rustc_codegen_ssa::back::archive::{ - ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, - DEFAULT_OBJECT_READER, ObjectReader, UnknownArchiveKind, try_extract_macho_fat_archive, + ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, DEFAULT_OBJECT_READER, ObjectReader, }; use rustc_session::Session; -use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind, last_error}; - -/// Helper for adding many files to an archive. -#[must_use = "must call build() to finish building the archive"] -pub(crate) struct LlvmArchiveBuilder<'a> { - sess: &'a Session, - additions: Vec<Addition>, -} - -enum Addition { - File { path: PathBuf, name_in_archive: String }, - Archive { path: PathBuf, archive: ArchiveRO, skip: Box<dyn FnMut(&str) -> bool> }, -} - -impl Addition { - fn path(&self) -> &Path { - match self { - Addition::File { path, .. } | Addition::Archive { path, .. } => path, - } - } -} - -fn is_relevant_child(c: &Child<'_>) -> bool { - match c.name() { - Some(name) => !name.contains("SYMDEF"), - None => false, - } -} - -impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { - fn add_archive( - &mut self, - archive: &Path, - skip: Box<dyn FnMut(&str) -> bool + 'static>, - ) -> io::Result<()> { - let mut archive = archive.to_path_buf(); - if self.sess.target.llvm_target.contains("-apple-macosx") { - if let Some(new_archive) = try_extract_macho_fat_archive(self.sess, &archive)? { - archive = new_archive - } - } - let archive_ro = match ArchiveRO::open(&archive) { - Ok(ar) => ar, - Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), - }; - if self.additions.iter().any(|ar| ar.path() == archive) { - return Ok(()); - } - self.additions.push(Addition::Archive { - path: archive, - archive: archive_ro, - skip: Box::new(skip), - }); - Ok(()) - } - - /// Adds an arbitrary file to this archive - fn add_file(&mut self, file: &Path) { - let name = file.file_name().unwrap().to_str().unwrap(); - self.additions - .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); - } - - /// Combine the provided files, rlibs, and native libraries into a single - /// `Archive`. - fn build(mut self: Box<Self>, output: &Path) -> bool { - match self.build_with_llvm(output) { - Ok(any_members) => any_members, - Err(error) => { - self.sess.dcx().emit_fatal(ArchiveBuildFailure { path: output.to_owned(), error }) - } - } - } -} +use crate::llvm; pub(crate) struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { - // Keeping LlvmArchiveBuilder around in case of a regression caused by using - // ArArchiveBuilder. - // FIXME(#128955) remove a couple of months after #128936 gets merged in case - // no regression is found. - if false { - Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) - } else { - Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) - } + // Use the `object` crate to build archives, with a little bit of help from LLVM. + Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) } } @@ -109,6 +26,7 @@ static LLVM_OBJECT_READER: ObjectReader = ObjectReader { get_symbols: get_llvm_object_symbols, is_64_bit_object_file: llvm_is_64_bit_object_file, is_ec_object_file: llvm_is_ec_object_file, + is_any_arm64_coff: llvm_is_any_arm64_coff, get_xcoff_member_alignment: DEFAULT_OBJECT_READER.get_xcoff_member_alignment, }; @@ -179,90 +97,6 @@ fn llvm_is_ec_object_file(buf: &[u8]) -> bool { unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } -impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self, output: &Path) -> io::Result<bool> { - let kind = &*self.sess.target.archive_format; - let kind = kind - .parse::<ArchiveKind>() - .map_err(|_| kind) - .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); - - let mut additions = mem::take(&mut self.additions); - // Values in the `members` list below will contain pointers to the strings allocated here. - // So they need to get dropped after all elements of `members` get freed. - let mut strings = Vec::new(); - let mut members = Vec::new(); - - let dst = CString::new(output.to_str().unwrap())?; - - unsafe { - for addition in &mut additions { - match addition { - Addition::File { path, name_in_archive } => { - let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.as_bytes())?; - members.push(llvm::LLVMRustArchiveMemberNew( - path.as_ptr(), - name.as_ptr(), - None, - )); - strings.push(path); - strings.push(name); - } - Addition::Archive { archive, skip, .. } => { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - if !is_relevant_child(&child) { - continue; - } - let child_name = child.name().unwrap(); - if skip(child_name) { - continue; - } - - // It appears that LLVM's archive writer is a little - // buggy if the name we pass down isn't just the - // filename component, so chop that off here and - // pass it in. - // - // See LLVM bug 25877 for more info. - let child_name = - Path::new(child_name).file_name().unwrap().to_str().unwrap(); - let name = CString::new(child_name)?; - let m = llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - ); - members.push(m); - strings.push(name); - } - } - } - } - - let r = llvm::LLVMRustWriteArchive( - dst.as_ptr(), - members.len() as libc::size_t, - members.as_ptr() as *const &_, - true, - kind, - self.sess.target.arch == "arm64ec", - ); - let ret = if r.into_result().is_err() { - let msg = last_error().unwrap_or_else(|| "failed to write archive".into()); - Err(io::Error::new(io::ErrorKind::Other, msg)) - } else { - Ok(!members.is_empty()) - }; - for member in members { - llvm::LLVMRustArchiveMemberFree(member); - } - ret - } - } -} - -fn string_to_io_error(s: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("bad archive: {s}")) +fn llvm_is_any_arm64_coff(buf: &[u8]) -> bool { + unsafe { llvm::LLVMRustIsAnyArm64Coff(buf.as_ptr(), buf.len()) } } diff --git a/compiler/rustc_codegen_llvm/src/back/command_line_args.rs b/compiler/rustc_codegen_llvm/src/back/command_line_args.rs new file mode 100644 index 00000000000..b14713969b3 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/command_line_args.rs @@ -0,0 +1,37 @@ +#[cfg(test)] +mod tests; + +/// Joins command-line arguments into a single space-separated string, quoting +/// and escaping individual arguments as necessary. +/// +/// The result is intended to be informational, for embedding in debug metadata, +/// and might not be properly quoted/escaped for actual command-line use. +pub(crate) fn quote_command_line_args(args: &[String]) -> String { + // Start with a decent-sized buffer, since rustc invocations tend to be long. + let mut buf = String::with_capacity(128); + + for arg in args { + if !buf.is_empty() { + buf.push(' '); + } + + print_arg_quoted(&mut buf, arg); + } + + buf +} + +/// Equivalent to LLVM's `sys::printArg` with quoting always enabled +/// (see llvm/lib/Support/Program.cpp). +fn print_arg_quoted(buf: &mut String, arg: &str) { + buf.reserve(arg.len() + 2); + + buf.push('"'); + for ch in arg.chars() { + if matches!(ch, '"' | '\\' | '$') { + buf.push('\\'); + } + buf.push(ch); + } + buf.push('"'); +} diff --git a/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs b/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs new file mode 100644 index 00000000000..69641fed3bc --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs @@ -0,0 +1,25 @@ +#[test] +fn quote_command_line_args() { + use super::quote_command_line_args; + + struct Case<'a> { + args: &'a [&'a str], + expected: &'a str, + } + + let cases = &[ + Case { args: &[], expected: "" }, + Case { args: &["--hello", "world"], expected: r#""--hello" "world""# }, + Case { args: &["--hello world"], expected: r#""--hello world""# }, + Case { + args: &["plain", "$dollar", "spa ce", r"back\slash", r#""quote""#, "plain"], + expected: r#""plain" "\$dollar" "spa ce" "back\\slash" "\"quote\"" "plain""#, + }, + ]; + + for &Case { args, expected } in cases { + let args = args.iter().copied().map(str::to_owned).collect::<Vec<_>>(); + let actual = quote_command_line_args(&args); + assert_eq!(actual, expected, "args {args:?}"); + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index c269f11e931..5ac3a87c158 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -14,7 +14,8 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; +use rustc_hir::attrs::SanitizerSet; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_session::config::{self, Lto}; @@ -24,9 +25,8 @@ use crate::back::write::{ self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode, }; use crate::errors::{LlvmError, LtoBitcodeFromRlib}; -use crate::llvm::AttributePlace::Function; use crate::llvm::{self, build_string}; -use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes}; +use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx}; /// We keep track of the computed LTO cache keys from the previous /// session to determine which CGUs we can reuse. @@ -37,12 +37,41 @@ fn prepare_lto( exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], dcx: DiagCtxtHandle<'_>, -) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> { +) -> (Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>) { let mut symbols_below_threshold = exported_symbols_for_lto .iter() .map(|symbol| CString::new(symbol.to_owned()).unwrap()) .collect::<Vec<CString>>(); + if cgcx.module_config.instrument_coverage || cgcx.module_config.pgo_gen.enabled() { + // These are weak symbols that point to the profile version and the + // profile name, which need to be treated as exported so LTO doesn't nix + // them. + const PROFILER_WEAK_SYMBOLS: [&CStr; 2] = + [c"__llvm_profile_raw_version", c"__llvm_profile_filename"]; + + symbols_below_threshold.extend(PROFILER_WEAK_SYMBOLS.iter().map(|&sym| sym.to_owned())); + } + + if cgcx.module_config.sanitizer.contains(SanitizerSet::MEMORY) { + let mut msan_weak_symbols = Vec::new(); + + // Similar to profiling, preserve weak msan symbol during LTO. + if cgcx.module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { + msan_weak_symbols.push(c"__msan_keep_going"); + } + + if cgcx.module_config.sanitizer_memory_track_origins != 0 { + msan_weak_symbols.push(c"__msan_track_origins"); + } + + symbols_below_threshold.extend(msan_weak_symbols.into_iter().map(|sym| sym.to_owned())); + } + + // Preserve LLVM-injected, ASAN-related symbols. + // See also https://github.com/rust-lang/rust/issues/113404. + symbols_below_threshold.push(c"___asan_globals_registered".to_owned()); + // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to // __llvm_profile_runtime, therefore we won't know until link time if this symbol // should have default visibility. @@ -80,16 +109,13 @@ fn prepare_lto( let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); } - Err(e) => { - dcx.emit_err(e); - return Err(FatalError); - } + Err(e) => dcx.emit_fatal(e), } } } } - Ok((symbols_below_threshold, upstream_modules)) + (symbols_below_threshold, upstream_modules) } fn get_bitcode_slice_from_object_data<'a>( @@ -124,11 +150,11 @@ pub(crate) fn run_fat( exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<FatLtoInput<LlvmCodegenBackend>>, -) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { +) -> ModuleCodegen<ModuleLlvm> { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = - prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold) @@ -143,11 +169,11 @@ pub(crate) fn run_thin( each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, -) -> Result<(Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { +) -> (Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>) { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = - prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); if cgcx.opts.cg.linker_plugin_lto.enabled() { @@ -159,12 +185,9 @@ pub(crate) fn run_thin( thin_lto(cgcx, dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold) } -pub(crate) fn prepare_thin( - module: ModuleCodegen<ModuleLlvm>, - emit_summary: bool, -) -> (String, ThinBuffer) { +pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { let name = module.name; - let buffer = ThinBuffer::new(module.module_llvm.llmod(), true, emit_summary); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -174,7 +197,7 @@ fn fat_lto( modules: Vec<FatLtoInput<LlvmCodegenBackend>>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, symbols_below_threshold: &[*const libc::c_char], -) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { +) -> ModuleCodegen<ModuleLlvm> { let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); info!("going for a fat lto"); @@ -225,7 +248,7 @@ fn fat_lto( assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); let (buffer, name) = serialized_modules.remove(0); info!("no in-memory regular modules to choose from, parsing {:?}", name); - let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?; + let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx); ModuleCodegen::new_regular(name.into_string().unwrap(), llvm_module) } }; @@ -266,7 +289,9 @@ fn fat_lto( }); info!("linking {:?}", name); let data = bc_decoded.data(); - linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?; + linker + .add(data) + .unwrap_or_else(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name })); } drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); @@ -283,7 +308,7 @@ fn fat_lto( save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } - Ok(module) + module } pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); @@ -353,7 +378,7 @@ fn thin_lto( serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, symbols_below_threshold: &[*const libc::c_char], -) -> Result<(Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { +) -> (Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>) { let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); unsafe { info!("going for that thin, thin LTO"); @@ -423,7 +448,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len(), ) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; + .unwrap_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext)); let data = ThinData(data); @@ -493,10 +518,10 @@ fn thin_lto( if let Some(path) = key_map_path && let Err(err) = curr_key_map.save_to_file(&path) { - return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err })); + write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }); } - Ok((opt_jobs, copy_jobs)) + (opt_jobs, copy_jobs) } } @@ -538,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { config::AutoDiff::Enable => {} // We handle this below config::AutoDiff::NoPostopt => {} + // Disables TypeTree generation + config::AutoDiff::NoTT => {} } } // This helps with handling enums for now. @@ -551,9 +578,9 @@ pub(crate) fn run_pass_manager( dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, -) -> Result<(), FatalError> { +) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. @@ -583,7 +610,7 @@ pub(crate) fn run_pass_manager( } unsafe { - write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage); } if enable_gpu && !thin { @@ -592,37 +619,12 @@ pub(crate) fn run_pass_manager( crate::builder::gpu_offload::handle_gpu_code(cgcx, &cx); } - if cfg!(llvm_enzyme) && enable_ad && !thin { - let cx = - SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); - - for function in cx.get_functions() { - let enzyme_marker = "enzyme_marker"; - if attributes::has_string_attr(function, enzyme_marker) { - // Sanity check: Ensure 'noinline' is present before replacing it. - assert!( - attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), - "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'" - ); - - attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline); - attributes::remove_string_attr_from_llfn(function, enzyme_marker); - - assert!( - !attributes::has_string_attr(function, enzyme_marker), - "Expected function to not have 'enzyme_marker'" - ); - - let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx); - attributes::apply_to_llfn(function, Function, &[always_inline]); - } - } - + if cfg!(feature = "llvm_enzyme") && enable_ad && !thin { let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { unsafe { - write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage); } } @@ -634,7 +636,6 @@ pub(crate) fn run_pass_manager( } debug!("lto done"); - Ok(()) } pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); @@ -685,9 +686,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub(crate) fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { + pub(crate) fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); ThinBuffer(buffer) } } @@ -696,21 +697,21 @@ impl ThinBuffer { let mut ptr = NonNull::new(ptr).unwrap(); ThinBuffer(unsafe { ptr.as_mut() }) } -} -impl ThinBufferMethods for ThinBuffer { - fn data(&self) -> &[u8] { + pub(crate) fn thin_link_data(&self) -> &[u8] { unsafe { - let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferLen(self.0); + let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); slice::from_raw_parts(ptr, len) } } +} - fn thin_link_data(&self) -> &[u8] { +impl ThinBufferMethods for ThinBuffer { + fn data(&self) -> &[u8] { unsafe { - let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); + let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferLen(self.0); slice::from_raw_parts(ptr, len) } } @@ -727,7 +728,7 @@ impl Drop for ThinBuffer { pub(crate) fn optimize_thin_module( thin_module: ThinModule<LlvmCodegenBackend>, cgcx: &CodegenContext<LlvmCodegenBackend>, -) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { +) -> ModuleCodegen<ModuleLlvm> { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -738,10 +739,10 @@ pub(crate) fn optimize_thin_module( // into that context. One day, however, we may do this for upstream // crates but for locally codegened modules we may be able to reuse // that LLVM Context and Module. - let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx)?; + let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx); let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. - if cgcx.config(ModuleKind::Regular).embed_bitcode() { + if cgcx.module_config.embed_bitcode() { module.thin_lto_buffer = Some(thin_module.data().to_vec()); } { @@ -772,7 +773,7 @@ pub(crate) fn optimize_thin_module( .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); if unsafe { !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); } @@ -783,7 +784,7 @@ pub(crate) fn optimize_thin_module( .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); if unsafe { !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); } @@ -794,7 +795,7 @@ pub(crate) fn optimize_thin_module( if unsafe { !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target.raw()) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } @@ -806,11 +807,11 @@ pub(crate) fn optimize_thin_module( // little differently. { info!("running thin lto passes over {}", module.name); - run_pass_manager(cgcx, dcx, &mut module, true)?; + run_pass_manager(cgcx, dcx, &mut module, true); save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } - Ok(module) + module } /// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys @@ -876,9 +877,9 @@ pub(crate) fn parse_module<'a>( name: &CStr, data: &[u8], dcx: DiagCtxtHandle<'_>, -) -> Result<&'a llvm::Module, FatalError> { +) -> &'a llvm::Module { unsafe { llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) + .unwrap_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) } } diff --git a/compiler/rustc_codegen_llvm/src/back/mod.rs b/compiler/rustc_codegen_llvm/src/back/mod.rs new file mode 100644 index 00000000000..fe3883e8c73 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod archive; +mod command_line_args; +pub(crate) mod lto; +pub(crate) mod owned_target_machine; +mod profiling; +pub(crate) mod write; diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 8e82013e94a..903a882916e 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,4 +1,4 @@ -use std::ffi::{CStr, c_char}; +use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -38,15 +38,10 @@ impl OwnedTargetMachine { output_obj_file: &CStr, debug_info_compression: &CStr, use_emulated_tls: bool, - args_cstr_buff: &[u8], + argv0: &str, + command_line_args: &str, use_wasm_eh: bool, ) -> Result<Self, LlvmError<'static>> { - assert!(args_cstr_buff.len() > 0); - assert!( - *args_cstr_buff.last().unwrap() == 0, - "The last character must be a null terminator." - ); - // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { llvm::LLVMRustCreateTargetMachine( @@ -71,8 +66,10 @@ impl OwnedTargetMachine { output_obj_file.as_ptr(), debug_info_compression.as_ptr(), use_emulated_tls, - args_cstr_buff.as_ptr() as *const c_char, - args_cstr_buff.len(), + argv0.as_ptr(), + argv0.len(), + command_line_args.as_ptr(), + command_line_args.len(), use_wasm_eh, ) }; @@ -98,8 +95,6 @@ impl Drop for OwnedTargetMachine { // SAFETY: constructing ensures we have a valid pointer created by // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. - unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut()); - } + unsafe { llvm::LLVMDisposeTargetMachine(self.tm_unique) }; } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 85a06f457eb..1950b8288d1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -20,7 +20,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{DiagCtxtHandle, FatalError, Level}; +use rustc_errors::{DiagCtxtHandle, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -31,6 +31,7 @@ use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym}; use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use tracing::{debug, trace}; +use crate::back::command_line_args::quote_command_line_args; use crate::back::lto::ThinBuffer; use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ @@ -46,10 +47,10 @@ use crate::llvm::{self, DiagnosticInfo}; use crate::type_::Type; use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util}; -pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError { +pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! { match llvm::last_error() { - Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)), - None => dcx.emit_almost_fatal(err), + Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)), + None => dcx.emit_fatal(err), } } @@ -63,7 +64,7 @@ fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, verify_llvm_ir: bool, -) -> Result<(), FatalError> { +) { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); let output_c = path_to_c_string(output); let dwo_output_c; @@ -100,7 +101,7 @@ fn write_output_file<'ll>( } } - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) + result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } pub(crate) fn create_informational_target_machine( @@ -112,7 +113,7 @@ pub(crate) fn create_informational_target_machine( // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false, only_base_features); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise()) + .unwrap_or_else(|err| llvm_err(sess.dcx(), err)) } pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { @@ -139,7 +140,7 @@ pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTar tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.dcx(), err)) } fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { @@ -203,6 +204,9 @@ pub(crate) fn target_machine_factory( optlvl: config::OptLevel, target_features: &[String], ) -> TargetMachineFactoryFn<LlvmCodegenBackend> { + // Self-profile timer for creating a _factory_. + let _prof_timer = sess.prof.generic_activity("target_machine_factory"); + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); @@ -249,23 +253,18 @@ pub(crate) fn target_machine_factory( let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); - // copy the exe path, followed by path all into one buffer - // null terminating them so we can use them as null terminated strings - let args_cstr_buff = { - let mut args_cstr_buff: Vec<u8> = Vec::new(); - let exe_path = std::env::current_exe().unwrap_or_default(); - let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default(); - - args_cstr_buff.extend_from_slice(exe_path_str.as_bytes()); - args_cstr_buff.push(0); - - for arg in sess.expanded_args.iter() { - args_cstr_buff.extend_from_slice(arg.as_bytes()); - args_cstr_buff.push(0); - } - - args_cstr_buff - }; + // Command-line information to be included in the target machine. + // This seems to only be used for embedding in PDB debuginfo files. + // FIXME(Zalathar): Maybe skip this for non-PDB targets? + let argv0 = std::env::current_exe() + .unwrap_or_default() + .into_os_string() + .into_string() + .unwrap_or_default(); + let command_line_args = quote_command_line_args(&sess.expanded_args); + // Self-profile counter for the number of bytes produced by command-line quoting. + // Values are summed, so the summary result is cumulative across all TM factories. + sess.prof.artifact_size("quoted_command_line_args", "-", command_line_args.len() as u64); let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); match sess.opts.debuginfo_compression { @@ -288,7 +287,11 @@ pub(crate) fn target_machine_factory( let use_wasm_eh = wants_wasm_eh(sess); + let prof = SelfProfilerRef::clone(&sess.prof); Arc::new(move |config: TargetMachineFactoryConfig| { + // Self-profile timer for invoking a factory to create a target machine. + let _prof_timer = prof.generic_activity("target_machine_factory_inner"); + let path_to_cstring_helper = |path: Option<PathBuf>| -> CString { let path = path.unwrap_or_default(); let path = path_mapping @@ -323,7 +326,8 @@ pub(crate) fn target_machine_factory( &output_obj_file, &debuginfo_compression, use_emulated_tls, - &args_cstr_buff, + &argv0, + &command_line_args, use_wasm_eh, ) }) @@ -565,7 +569,7 @@ pub(crate) unsafe fn llvm_optimize( opt_level: config::OptLevel, opt_stage: llvm::OptStage, autodiff_stage: AutodiffStage, -) -> Result<(), FatalError> { +) { // Enzyme: // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized // source code. However, benchmarks show that optimizations increasing the code size @@ -574,7 +578,8 @@ pub(crate) unsafe fn llvm_optimize( // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting // differentiated. - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let run_enzyme = autodiff_stage == AutodiffStage::DuringAD; let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore); let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter); @@ -704,7 +709,7 @@ pub(crate) unsafe fn llvm_optimize( llvm_plugins.len(), ) }; - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) + result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. @@ -713,7 +718,7 @@ pub(crate) fn optimize( dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, -) -> Result<(), FatalError> { +) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); let llcx = &*module.module_llvm.llcx; @@ -740,7 +745,8 @@ pub(crate) fn optimize( // If we know that we will later run AD, then we disable vectorization and loop unrolling. // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; // The embedded bitcode is used to run LTO/ThinLTO. // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. @@ -765,7 +771,7 @@ pub(crate) fn optimize( opt_stage, autodiff_stage, ) - }?; + }; if let Some(thin_lto_buffer) = thin_lto_buffer { let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) }; module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec()); @@ -793,14 +799,13 @@ pub(crate) fn optimize( } } } - Ok(()) } pub(crate) fn codegen( cgcx: &CodegenContext<LlvmCodegenBackend>, module: ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, -) -> Result<CompiledModule, FatalError> { +) -> CompiledModule { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -838,7 +843,7 @@ pub(crate) fn codegen( "LLVM_module_codegen_make_bitcode", &*module.name, ); - ThinBuffer::new(llmod, config.emit_thin_lto, false) + ThinBuffer::new(llmod, config.emit_thin_lto) }; let data = thin.data(); let _timer = cgcx @@ -862,7 +867,7 @@ pub(crate) fn codegen( .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); let thin_bc = module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode"); - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); + embed_bitcode(cgcx, llcx, llmod, &thin_bc); } } @@ -909,7 +914,9 @@ pub(crate) fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?; + result + .into_result() + .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out })); } if config.emit_asm { @@ -940,7 +947,7 @@ pub(crate) fn codegen( llvm::FileType::AssemblyFile, &cgcx.prof, config.verify_llvm_ir, - )?; + ); } match config.emit_obj { @@ -976,7 +983,7 @@ pub(crate) fn codegen( llvm::FileType::ObjectFile, &cgcx.prof, config.verify_llvm_ir, - )?; + ); } EmitObj::Bitcode => { @@ -1009,7 +1016,7 @@ pub(crate) fn codegen( && cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo != SplitDebuginfo::Off && cgcx.split_dwarf_kind == SplitDwarfKind::Split; - Ok(module.into_compiled_module( + module.into_compiled_module( config.emit_obj != EmitObj::None, dwarf_object_emitted, config.emit_bc, @@ -1017,7 +1024,7 @@ pub(crate) fn codegen( config.emit_ir, &cgcx.output_filenames, cgcx.invocation_temp.as_deref(), - )) + ) } fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> { @@ -1058,7 +1065,6 @@ fn embed_bitcode( cgcx: &CodegenContext<LlvmCodegenBackend>, llcx: &llvm::Context, llmod: &llvm::Module, - cmdline: &str, bitcode: &[u8], ) { // We're adding custom sections to the output object file, but we definitely @@ -1074,7 +1080,9 @@ fn embed_bitcode( // * Mach-O - this is for macOS. Inspecting the source code for the native // linker here shows that the `.llvmbc` and `.llvmcmd` sections are // automatically skipped by the linker. In that case there's nothing extra - // that we need to do here. + // that we need to do here. We do need to make sure that the + // `__LLVM,__cmdline` section exists even though it is empty as otherwise + // ld64 rejects the object file. // // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and // `.llvmcmd` sections, so there's nothing extra we need to do. @@ -1109,9 +1117,9 @@ fn embed_bitcode( llvm::set_section(llglobal, bitcode_section_name(cgcx)); llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::LLVMSetGlobalConstant(llglobal, llvm::True); + llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE); - let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llconst = common::bytes_in_context(llcx, &[]); let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); llvm::set_initializer(llglobal, llconst); let section = if cgcx.target_is_like_darwin { @@ -1128,7 +1136,7 @@ fn embed_bitcode( let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); llvm::append_module_inline_asm(llmod, &asm); - let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]); llvm::append_module_inline_asm(llmod, &asm); } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5dda836988c..6d12b511e9c 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -18,9 +18,10 @@ use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; use rustc_session::config::DebugInfo; use rustc_span::Symbol; @@ -104,22 +105,40 @@ pub(crate) fn compile_codegen_unit( if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty()); attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } + // Define Objective-C module info and module flags. Note, the module info will + // also be added to the `llvm.compiler.used` variable, created later. + // + // These are only necessary when we need the linker to do its Objective-C-specific + // magic. We could theoretically do it unconditionally, but at a slight cost to linker + // performance in the common case where it's unnecessary. + if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() { + if cx.objc_abi_version() == 1 { + cx.define_objc_module_info(); + } + cx.add_objc_module_flags(); + } + // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.compiler.used` variable, created next. if cx.sess().instrument_coverage() { cx.coverageinfo_finalize(); } - // Create the llvm.used and llvm.compiler.used variables. + // Create the llvm.used variable. if !cx.used_statics.is_empty() { cx.create_used_variable_impl(c"llvm.used", &cx.used_statics); } - if !cx.compiler_used_statics.is_empty() { - cx.create_used_variable_impl(c"llvm.compiler.used", &cx.compiler_used_statics); + + // Create the llvm.compiler.used variable. + { + let compiler_used_statics = cx.compiler_used_statics.borrow(); + if !compiler_used_statics.is_empty() { + cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics); + } } // Run replace-all-uses-with for statics that need it. This must diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 917d07e3c61..5271d0b4bb8 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,6 +2,7 @@ use std::borrow::{Borrow, Cow}; use std::ops::Deref; use std::{iter, ptr}; +use rustc_ast::expand::typetree::FncTree; pub(crate) mod autodiff; pub(crate) mod gpu_offload; @@ -35,7 +36,7 @@ use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, FullCx, GenericCx, SCx}; use crate::llvm::{ - self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True, + self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, GEPNoWrapFlags, Metadata, TRUE, ToLlvmBool, }; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -493,8 +494,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(add).is_some() { - llvm::LLVMSetNUW(add, True); - llvm::LLVMSetNSW(add, True); + llvm::LLVMSetNUW(add, TRUE); + llvm::LLVMSetNSW(add, TRUE); } add } @@ -503,8 +504,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(sub).is_some() { - llvm::LLVMSetNUW(sub, True); - llvm::LLVMSetNSW(sub, True); + llvm::LLVMSetNUW(sub, TRUE); + llvm::LLVMSetNSW(sub, TRUE); } sub } @@ -513,8 +514,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(mul).is_some() { - llvm::LLVMSetNUW(mul, True); - llvm::LLVMSetNSW(mul, True); + llvm::LLVMSetNUW(mul, TRUE); + llvm::LLVMSetNSW(mul, TRUE); } mul } @@ -528,7 +529,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // an instruction, so we need to check before setting the flag. // (See also `LLVMBuildNUWNeg` which also needs a check.) if llvm::LLVMIsAInstruction(or).is_some() { - llvm::LLVMSetIsDisjoint(or, True); + llvm::LLVMSetIsDisjoint(or, TRUE); } or } @@ -629,7 +630,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); - llvm::LLVMSetVolatile(load, llvm::True); + llvm::LLVMSetVolatile(load, llvm::TRUE); load } } @@ -717,7 +718,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let mut const_llval = None; let llty = place.layout.llvm_type(self); if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { - if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if llvm::LLVMIsGlobalConstant(global).is_true() { if let Some(init) = llvm::LLVMGetInitializer(global) { if self.val_ty(init) == llty { const_llval = Some(init); @@ -838,7 +839,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; llvm::LLVMSetAlignment(store, align); if flags.contains(MemFlags::VOLATILE) { - llvm::LLVMSetVolatile(store, llvm::True); + llvm::LLVMSetVolatile(store, llvm::TRUE); } if flags.contains(MemFlags::NONTEMPORAL) { // Make sure that the current target architectures supports "sane" non-temporal @@ -956,7 +957,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let trunc = self.trunc(val, dest_ty); unsafe { if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNUW(trunc, True); + llvm::LLVMSetNUW(trunc, TRUE); } } trunc @@ -968,7 +969,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let trunc = self.trunc(val, dest_ty); unsafe { if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNSW(trunc, True); + llvm::LLVMSetNSW(trunc, TRUE); } } trunc @@ -1067,13 +1068,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { unsafe { - llvm::LLVMBuildIntCast2( - self.llbuilder, - val, - dest_ty, - if is_signed { True } else { False }, - UNNAMED, - ) + llvm::LLVMBuildIntCast2(self.llbuilder, val, dest_ty, is_signed.to_llvm_bool(), UNNAMED) } } @@ -1097,16 +1092,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ty: Ty<'tcx>, lhs: Self::Value, rhs: Self::Value, - ) -> Option<Self::Value> { - // FIXME: See comment on the definition of `three_way_compare`. - if crate::llvm_util::get_version() < (20, 0, 0) { - return None; - } - + ) -> Self::Value { let size = ty.primitive_size(self.tcx); let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" }; - Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])) + self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]) } /* Miscellaneous instructions */ @@ -1118,11 +1108,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + tt: Option<FncTree>, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - unsafe { + let memcpy = unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, dst, @@ -1131,7 +1122,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align.bytes() as c_uint, size, is_volatile, - ); + ) + }; + + // TypeTree metadata for memcpy is especially important: when Enzyme encounters + // a memcpy during autodiff, it needs to know the structure of the data being + // copied to properly track derivatives. For example, copying an array of floats + // vs. copying a struct with mixed types requires different derivative handling. + // The TypeTree tells Enzyme exactly what memory layout to expect. + if let Some(tt) = tt { + crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt); } } @@ -1229,7 +1229,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false); let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { - llvm::LLVMSetCleanup(landing_pad, llvm::True); + llvm::LLVMSetCleanup(landing_pad, llvm::TRUE); } (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } @@ -1317,7 +1317,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { failure_order: rustc_middle::ty::AtomicOrdering, weak: bool, ) -> (&'ll Value, &'ll Value) { - let weak = if weak { llvm::True } else { llvm::False }; unsafe { let value = llvm::LLVMBuildAtomicCmpXchg( self.llbuilder, @@ -1326,9 +1325,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src, AtomicOrdering::from_generic(order), AtomicOrdering::from_generic(failure_order), - llvm::False, // SingleThreaded + llvm::FALSE, // SingleThreaded ); - llvm::LLVMSetWeak(value, weak); + llvm::LLVMSetWeak(value, weak.to_llvm_bool()); let val = self.extract_value(value, 0); let success = self.extract_value(value, 1); (val, success) @@ -1353,7 +1352,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst, src, AtomicOrdering::from_generic(order), - llvm::False, // SingleThreaded + llvm::FALSE, // SingleThreaded ) }; if ret_ptr && self.val_ty(res) != self.type_ptr() { @@ -1368,14 +1367,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { scope: SynchronizationScope, ) { let single_threaded = match scope { - SynchronizationScope::SingleThread => llvm::True, - SynchronizationScope::CrossThread => llvm::False, + SynchronizationScope::SingleThread => true, + SynchronizationScope::CrossThread => false, }; unsafe { llvm::LLVMBuildFence( self.llbuilder, AtomicOrdering::from_generic(order), - single_threaded, + single_threaded.to_llvm_bool(), UNNAMED, ); } @@ -1399,7 +1398,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, - fn_attrs: Option<&CodegenFnAttrs>, + fn_call_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1416,10 +1415,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } // Emit CFI pointer type membership test - self.cfi_type_test(fn_attrs, fn_abi, instance, llfn); + self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn); // Emit KCFI operand bundle - let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); + let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn); if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1436,6 +1435,29 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { c"".as_ptr(), ) }; + + if let Some(instance) = instance { + // Attributes on the function definition being called + let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id()); + if let Some(fn_call_attrs) = fn_call_attrs + && !fn_call_attrs.target_features.is_empty() + // If there is an inline attribute and a target feature that matches + // we will add the attribute to the callsite otherwise we'll omit + // this and not add the attribute to prevent soundness issues. + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance) + && self.cx.tcx.is_target_feature_call_safe( + &fn_call_attrs.target_features, + &fn_defn_attrs.target_features, + ) + { + attributes::apply_to_callsite( + call, + llvm::AttributePlace::Function, + &[inlining_rule], + ); + } + } + if let Some(fn_abi) = fn_abi { fn_abi.apply_attrs_callsite(self, call); } @@ -1453,7 +1475,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { instance: Option<Instance<'tcx>>, ) { let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance); - llvm::LLVMRustSetTailCallKind(call, llvm::TailCallKind::MustTail); + llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); match &fn_abi.ret.mode { PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(), @@ -1696,7 +1718,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + if crate::llvm_util::get_version() >= (22, 0, 0) { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]); + } else { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + } } } impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 829b3c513c2..c3485f56391 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,40 +1,133 @@ use std::ptr; -use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode}; -use rustc_codegen_ssa::ModuleCodegen; +use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; +use rustc_ast::expand::typetree::FncTree; use rustc_codegen_ssa::common::TypeKind; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; -use rustc_errors::FatalError; -use rustc_middle::bug; -use tracing::{debug, trace}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv}; +use rustc_middle::{bug, ty}; +use rustc_target::callconv::PassMode; +use tracing::debug; -use crate::back::write::llvm_err; -use crate::builder::{SBuilder, UNNAMED}; +use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; -use crate::errors::{AutoDiffWithoutEnable, LlvmError}; -use crate::llvm::AttributePlace::Function; -use crate::llvm::{Metadata, True}; +use crate::llvm; +use crate::llvm::{Metadata, TRUE, Type}; use crate::value::Value; -use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm}; -fn get_params(fnc: &Value) -> Vec<&Value> { - let param_num = llvm::LLVMCountParams(fnc) as usize; - let mut fnc_args: Vec<&Value> = vec![]; - fnc_args.reserve(param_num); - unsafe { - llvm::LLVMGetParams(fnc, fnc_args.as_mut_ptr()); - fnc_args.set_len(param_num); +pub(crate) fn adjust_activity_to_abi<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + typing_env: TypingEnv<'tcx>, + da: &mut Vec<DiffActivity>, +) { + let fn_ty = instance.ty(tcx, typing_env); + + if !matches!(fn_ty.kind(), ty::FnDef(..)) { + bug!("expected fn def for autodiff, got {:?}", fn_ty); } - fnc_args -} -fn has_sret(fnc: &Value) -> bool { - let num_args = llvm::LLVMCountParams(fnc) as usize; - if num_args == 0 { - false - } else { - unsafe { llvm::LLVMRustHasAttributeAtIndex(fnc, 0, llvm::AttributeKind::StructRet) } + // We don't actually pass the types back into the type system. + // All we do is decide how to handle the arguments. + let sig = fn_ty.fn_sig(tcx).skip_binder(); + + // FIXME(Sa4dUs): pass proper varargs once we have support for differentiating variadic functions + let Ok(fn_abi) = + tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) + else { + bug!("failed to get fn_abi of instance with empty varargs"); + }; + + let mut new_activities = vec![]; + let mut new_positions = vec![]; + let mut del_activities = 0; + for (i, ty) in sig.inputs().iter().enumerate() { + if let Some(inner_ty) = ty.builtin_deref(true) { + if inner_ty.is_slice() { + // Now we need to figure out the size of each slice element in memory to allow + // safety checks and usability improvements in the backend. + let sty = match inner_ty.builtin_index() { + Some(sty) => sty, + None => { + panic!("slice element type unknown"); + } + }; + let pci = PseudoCanonicalInput { + typing_env: TypingEnv::fully_monomorphized(), + value: sty, + }; + + let layout = tcx.layout_of(pci); + let elem_size = match layout { + Ok(layout) => layout.size, + Err(_) => { + bug!("autodiff failed to compute slice element size"); + } + }; + let elem_size: u32 = elem_size.bytes() as u32; + + // We know that the length will be passed as extra arg. + if !da.is_empty() { + // We are looking at a slice. The length of that slice will become an + // extra integer on llvm level. Integers are always const. + // However, if the slice get's duplicated, we want to know to later check the + // size. So we mark the new size argument as FakeActivitySize. + // There is one FakeActivitySize per slice, so for convenience we store the + // slice element size in bytes in it. We will use the size in the backend. + let activity = match da[i] { + DiffActivity::DualOnly + | DiffActivity::Dual + | DiffActivity::Dualv + | DiffActivity::DuplicatedOnly + | DiffActivity::Duplicated => { + DiffActivity::FakeActivitySize(Some(elem_size)) + } + DiffActivity::Const => DiffActivity::Const, + _ => bug!("unexpected activity for ptr/ref"), + }; + new_activities.push(activity); + new_positions.push(i + 1); + } + + continue; + } + } + + let pci = PseudoCanonicalInput { typing_env: TypingEnv::fully_monomorphized(), value: *ty }; + + let layout = match tcx.layout_of(pci) { + Ok(layout) => layout.layout, + Err(_) => { + bug!("failed to compute layout for type {:?}", ty); + } + }; + + let pass_mode = &fn_abi.args[i].mode; + + // For ZST, just ignore and don't add its activity, as this arg won't be present + // in the LLVM passed to Enzyme. + // Some targets pass ZST indirectly in the C ABI, in that case, handle it as a normal arg + // FIXME(Sa4dUs): Enforce ZST corresponding diff activity be `Const` + if *pass_mode == PassMode::Ignore { + del_activities += 1; + da.remove(i); + } + + // If the argument is lowered as a `ScalarPair`, we need to duplicate its activity. + // Otherwise, the number of activities won't match the number of LLVM arguments and + // this will lead to errors when verifying the Enzyme call. + if let rustc_abi::BackendRepr::ScalarPair(_, _) = layout.backend_repr() { + new_activities.push(da[i].clone()); + new_positions.push(i + 1 - del_activities); + } + } + // now add the extra activities coming from slices + // Reverse order to not invalidate the indices + for _ in 0..new_activities.len() { + let pos = new_positions.pop().unwrap(); + let activity = new_activities.pop().unwrap(); + da.insert(pos, activity); } } @@ -48,14 +141,13 @@ fn has_sret(fnc: &Value) -> bool { // need to match those. // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it // using iterators and peek()? -fn match_args_from_caller_to_enzyme<'ll>( +fn match_args_from_caller_to_enzyme<'ll, 'tcx>( cx: &SimpleCx<'ll>, - builder: &SBuilder<'ll, 'll>, + builder: &mut Builder<'_, 'll, 'tcx>, width: u32, args: &mut Vec<&'ll llvm::Value>, inputs: &[DiffActivity], outer_args: &[&'ll llvm::Value], - has_sret: bool, ) { debug!("matching autodiff arguments"); // We now handle the issue that Rust level arguments not always match the llvm-ir level @@ -67,14 +159,6 @@ fn match_args_from_caller_to_enzyme<'ll>( let mut outer_pos: usize = 0; let mut activity_pos = 0; - if has_sret { - // Then the first outer arg is the sret pointer. Enzyme doesn't know about sret, so the - // inner function will still return something. We increase our outer_pos by one, - // and once we're done with all other args we will take the return of the inner call and - // update the sret pointer with it - outer_pos = 1; - } - let enzyme_const = cx.create_metadata(b"enzyme_const"); let enzyme_out = cx.create_metadata(b"enzyme_out"); let enzyme_dup = cx.create_metadata(b"enzyme_dup"); @@ -193,92 +277,6 @@ fn match_args_from_caller_to_enzyme<'ll>( } } -// On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input -// arguments. We do however need to declare them with their correct return type. -// We already figured the correct return type out in our frontend, when generating the outer_fn, -// so we can now just go ahead and use that. This is not always trivial, e.g. because sret. -// Beyond sret, this article describes our challenges nicely: -// <https://yorickpeterse.com/articles/the-mess-that-is-handling-structure-arguments-and-returns-in-llvm/> -// I.e. (i32, f32) will get merged into i64, but we don't handle that yet. -fn compute_enzyme_fn_ty<'ll>( - cx: &SimpleCx<'ll>, - attrs: &AutoDiffAttrs, - fn_to_diff: &'ll Value, - outer_fn: &'ll Value, -) -> &'ll llvm::Type { - let fn_ty = cx.get_type_of_global(outer_fn); - let mut ret_ty = cx.get_return_type(fn_ty); - - let has_sret = has_sret(outer_fn); - - if has_sret { - // Now we don't just forward the return type, so we have to figure it out based on the - // primal return type, in combination with the autodiff settings. - let fn_ty = cx.get_type_of_global(fn_to_diff); - let inner_ret_ty = cx.get_return_type(fn_ty); - - let void_ty = unsafe { llvm::LLVMVoidTypeInContext(cx.llcx) }; - if inner_ret_ty == void_ty { - // This indicates that even the inner function has an sret. - // Right now I only look for an sret in the outer function. - // This *probably* needs some extra handling, but I never ran - // into such a case. So I'll wait for user reports to have a test case. - bug!("sret in inner function"); - } - - if attrs.width == 1 { - // Enzyme returns a struct of style: - // `{ original_ret(if requested), float, float, ... }` - let mut struct_elements = vec![]; - if attrs.has_primal_ret() { - struct_elements.push(inner_ret_ty); - } - // Next, we push the list of active floats, since they will be lowered to `enzyme_out`, - // and therefore part of the return struct. - let param_tys = cx.func_params_types(fn_ty); - for (act, param_ty) in attrs.input_activity.iter().zip(param_tys) { - if matches!(act, DiffActivity::Active) { - // Now find the float type at position i based on the fn_ty, - // to know what (f16/f32/f64/...) to add to the struct. - struct_elements.push(param_ty); - } - } - ret_ty = cx.type_struct(&struct_elements, false); - } else { - // First we check if we also have to deal with the primal return. - match attrs.mode { - DiffMode::Forward => match attrs.ret_activity { - DiffActivity::Dual => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64 + 1) }; - ret_ty = arr_ty; - } - DiffActivity::DualOnly => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64) }; - ret_ty = arr_ty; - } - DiffActivity::Const => { - todo!("Not sure, do we need to do something here?"); - } - _ => { - bug!("unreachable"); - } - }, - DiffMode::Reverse => { - todo!("Handle sret for reverse mode"); - } - _ => { - bug!("unreachable"); - } - } - } - } - - // LLVM can figure out the input types on it's own, so we take a shortcut here. - unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) } -} - /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another /// function with expected naming and calling conventions[^1] which will be /// discovered by the enzyme LLVM pass and its body populated with the differentiated @@ -288,11 +286,16 @@ fn compute_enzyme_fn_ty<'ll>( /// [^1]: <https://enzyme.mit.edu/getting_started/CallingConvention/> // FIXME(ZuseZ4): `outer_fn` should include upstream safety checks to // cover some assumptions of enzyme/autodiff, which could lead to UB otherwise. -fn generate_enzyme_call<'ll>( +pub(crate) fn generate_enzyme_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, cx: &SimpleCx<'ll>, fn_to_diff: &'ll Value, - outer_fn: &'ll Value, + outer_name: &str, + ret_ty: &'ll Type, + fn_args: &[&'ll Value], attrs: AutoDiffAttrs, + dest: PlaceRef<'tcx, &'ll Value>, + fnc_tree: FncTree, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -302,11 +305,9 @@ fn generate_enzyme_call<'ll>( } .to_string(); - // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple + // add outer_name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. - let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::str::from_utf8(&name).unwrap(); - ad_name.push_str(outer_fn_name); + ad_name.push_str(outer_name); // Let us assume the user wrote the following function square: // @@ -316,14 +317,8 @@ fn generate_enzyme_call<'ll>( // %0 = fmul double %x, %x // ret double %0 // } - // ``` // - // The user now applies autodiff to the function square, in which case fn_to_diff will be `square`. - // Our macro generates the following placeholder code (slightly simplified): - // - // ```llvm // define double @dsquare(double %x) { - // ; placeholder code // return 0.0; // } // ``` @@ -340,175 +335,48 @@ fn generate_enzyme_call<'ll>( // ret double %0 // } // ``` - unsafe { - let enzyme_ty = compute_enzyme_fn_ty(cx, &attrs, fn_to_diff, outer_fn); - - // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and - // think a bit more about what should go here. - let cc = llvm::LLVMGetFunctionCallConv(outer_fn); - let ad_fn = declare_simple_fn( - cx, - &ad_name, - llvm::CallConv::try_from(cc).expect("invalid callconv"), - llvm::UnnamedAddr::No, - llvm::Visibility::Default, - enzyme_ty, - ); - - // Otherwise LLVM might inline our temporary code before the enzyme pass has a chance to - // do it's work. - let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); - attributes::apply_to_llfn(ad_fn, Function, &[attr]); - - // We add a made-up attribute just such that we can recognize it after AD to update - // (no)-inline attributes. We'll then also remove this attribute. - let enzyme_marker_attr = llvm::CreateAttrString(cx.llcx, "enzyme_marker"); - attributes::apply_to_llfn(outer_fn, Function, &[enzyme_marker_attr]); - - // first, remove all calls from fnc - let entry = llvm::LLVMGetFirstBasicBlock(outer_fn); - let br = llvm::LLVMRustGetTerminator(entry); - llvm::LLVMRustEraseInstFromParent(br); - - let last_inst = llvm::LLVMRustGetLastInstruction(entry).unwrap(); - let mut builder = SBuilder::build(cx, entry); - - let num_args = llvm::LLVMCountParams(&fn_to_diff); - let mut args = Vec::with_capacity(num_args as usize + 1); - args.push(fn_to_diff); - - let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); - if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { - args.push(cx.get_metadata_value(enzyme_primal_ret)); - } - if attrs.width > 1 { - let enzyme_width = cx.create_metadata(b"enzyme_width"); - args.push(cx.get_metadata_value(enzyme_width)); - args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); - } - - let has_sret = has_sret(outer_fn); - let outer_args: Vec<&llvm::Value> = get_params(outer_fn); - match_args_from_caller_to_enzyme( - &cx, - &builder, - attrs.width, - &mut args, - &attrs.input_activity, - &outer_args, - has_sret, - ); - - let call = builder.call(enzyme_ty, ad_fn, &args, None); - - // This part is a bit iffy. LLVM requires that a call to an inlineable function has some - // metadata attached to it, but we just created this code oota. Given that the - // differentiated function already has partly confusing metadata, and given that this - // affects nothing but the auttodiff IR, we take a shortcut and just steal metadata from the - // dummy code which we inserted at a higher level. - // FIXME(ZuseZ4): Work with Enzyme core devs to clarify what debug metadata issues we have, - // and how to best improve it for enzyme core and rust-enzyme. - let md_ty = cx.get_md_kind_id("dbg"); - if llvm::LLVMRustHasMetadata(last_inst, md_ty) { - let md = llvm::LLVMRustDIGetInstMetadata(last_inst) - .expect("failed to get instruction metadata"); - let md_todiff = cx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, md_ty, md_todiff); - } else { - // We don't panic, since depending on whether we are in debug or release mode, we might - // have no debug info to copy, which would then be ok. - trace!("no dbg info"); - } - - // Now that we copied the metadata, get rid of dummy code. - llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst); - - if cx.val_ty(call) == cx.type_void() || has_sret { - if has_sret { - // This is what we already have in our outer_fn (shortened): - // define void @_foo(ptr <..> sret([32 x i8]) initializes((0, 32)) %0, <...>) { - // %7 = call [4 x double] (...) @__enzyme_fwddiff_foo(ptr @square, metadata !"enzyme_width", i64 4, <...>) - // <Here we are, we want to add the following two lines> - // store [4 x double] %7, ptr %0, align 8 - // ret void - // } - - // now store the result of the enzyme call into the sret pointer. - let sret_ptr = outer_args[0]; - let call_ty = cx.val_ty(call); - if attrs.width == 1 { - assert_eq!(cx.type_kind(call_ty), TypeKind::Struct); - } else { - assert_eq!(cx.type_kind(call_ty), TypeKind::Array); - } - llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr); - } - builder.ret_void(); - } else { - builder.ret(call); - } - - // Let's crash in case that we messed something up above and generated invalid IR. - llvm::LLVMRustVerifyFunction( - outer_fn, - llvm::LLVMRustVerifierFailureAction::LLVMAbortProcessAction, - ); + let enzyme_ty = unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, TRUE) }; + + // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and + // think a bit more about what should go here. + let cc = unsafe { llvm::LLVMGetFunctionCallConv(fn_to_diff) }; + let ad_fn = declare_simple_fn( + cx, + &ad_name, + llvm::CallConv::try_from(cc).expect("invalid callconv"), + llvm::UnnamedAddr::No, + llvm::Visibility::Default, + enzyme_ty, + ); + + let num_args = llvm::LLVMCountParams(&fn_to_diff); + let mut args = Vec::with_capacity(num_args as usize + 1); + args.push(fn_to_diff); + + let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); + if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { + args.push(cx.get_metadata_value(enzyme_primal_ret)); } -} - -pub(crate) fn differentiate<'ll>( - module: &'ll ModuleCodegen<ModuleLlvm>, - cgcx: &CodegenContext<LlvmCodegenBackend>, - diff_items: Vec<AutoDiffItem>, -) -> Result<(), FatalError> { - for item in &diff_items { - trace!("{}", item); - } - - let diag_handler = cgcx.create_dcx(); - - let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - - // First of all, did the user try to use autodiff without using the -Zautodiff=Enable flag? - if !diff_items.is_empty() - && !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) - { - return Err(diag_handler.handle().emit_almost_fatal(AutoDiffWithoutEnable)); + if attrs.width > 1 { + let enzyme_width = cx.create_metadata(b"enzyme_width"); + args.push(cx.get_metadata_value(enzyme_width)); + args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); } - // Here we replace the placeholder code with the actual autodiff code, which calls Enzyme. - for item in diff_items.iter() { - let name = item.source.clone(); - let fn_def: Option<&llvm::Value> = cx.get_function(&name); - let Some(fn_def) = fn_def else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find source function".to_owned(), - }, - )); - }; - debug!(?item.target); - let fn_target: Option<&llvm::Value> = cx.get_function(&item.target); - let Some(fn_target) = fn_target else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find target function".to_owned(), - }, - )); - }; - - generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); + match_args_from_caller_to_enzyme( + &cx, + builder, + attrs.width, + &mut args, + &attrs.input_activity, + fn_args, + ); + + if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() { + crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree); } - // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts - - trace!("done with differentiate()"); + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); - Ok(()) + builder.store_to_place(call, dest.val); } diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 1280ab1442a..0737a18384b 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -193,7 +193,7 @@ fn gen_define_handling<'ll>( // reference) types. let num_ptr_types = types .iter() - .map(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)) + .filter(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)) .count(); // We do not know their size anymore at this level, so hardcode a placeholder. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index f29fefb66f0..aa2df46329f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -20,7 +20,7 @@ use tracing::debug; use crate::consts::const_alloc_to_llvm; pub(crate) use crate::context::CodegenCx; use crate::context::{GenericCx, SCx}; -use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, True}; +use crate::llvm::{self, BasicBlock, ConstantInt, FALSE, Metadata, TRUE, ToLlvmBool}; use crate::type_::Type; use crate::value::Value; @@ -108,6 +108,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { bytes_in_context(self.llcx(), bytes) } + pub(crate) fn null_terminate_const_bytes(&self, bytes: &[u8]) -> &'ll Value { + null_terminate_bytes_in_context(self.llcx(), bytes) + } + pub(crate) fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); @@ -158,7 +162,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.type_kind(t) == TypeKind::Integer, "only allows integer types in const_int" ); - unsafe { llvm::LLVMConstInt(t, i as u64, True) } + unsafe { llvm::LLVMConstInt(t, i as u64, TRUE) } } fn const_u8(&self, i: u8) -> &'ll Value { @@ -192,7 +196,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.type_kind(t) == TypeKind::Integer, "only allows integer types in const_uint" ); - unsafe { llvm::LLVMConstInt(t, i, False) } + unsafe { llvm::LLVMConstInt(t, i, FALSE) } } fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { @@ -377,7 +381,17 @@ pub(crate) fn val_ty(v: &Value) -> &Type { pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), TRUE) + } +} + +pub(crate) fn null_terminate_bytes_in_context<'ll>( + llcx: &'ll llvm::Context, + bytes: &[u8], +) -> &'ll Value { + unsafe { + let ptr = bytes.as_ptr() as *const c_char; + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), FALSE) } } @@ -392,7 +406,7 @@ fn struct_in_context<'ll>( packed: bool, ) -> &'ll Value { let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); - unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed as Bool) } + unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed.to_llvm_bool()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 6b06daf3477..a110ecbb75d 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -4,6 +4,7 @@ use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -11,10 +12,11 @@ use rustc_middle::mir::interpret::{ Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar, read_target_uint, }; -use rustc_middle::mir::mono::{Linkage, MonoItem}; +use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; +use rustc_span::Symbol; use tracing::{debug, instrument, trace}; use crate::common::CodegenCx; @@ -330,6 +332,10 @@ impl<'ll> CodegenCx<'ll, '_> { } g + } else if let Some(classname) = fn_attrs.objc_class { + self.get_objc_classref(classname) + } else if let Some(methname) = fn_attrs.objc_selector { + self.get_objc_selref(methname) } else { check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) }; @@ -451,6 +457,8 @@ impl<'ll> CodegenCx<'ll, '_> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; + + // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, g, alloc.align); llvm::set_initializer(g, v); @@ -540,8 +548,225 @@ impl<'ll> CodegenCx<'ll, '_> { /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, /// an array of ptr. - pub(crate) fn add_compiler_used_global(&mut self, global: &'ll Value) { - self.compiler_used_statics.push(global); + pub(crate) fn add_compiler_used_global(&self, global: &'ll Value) { + self.compiler_used_statics.borrow_mut().push(global); + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `CGObjCCommonMac::CreateCStringLiteral`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134 + fn define_objc_classname(&self, classname: &str) -> &'ll Value { + assert_eq!(self.objc_abi_version(), 1); + + let llval = self.null_terminate_const_bytes(classname.as_bytes()); + let llty = self.val_ty(llval); + let sym = self.generate_local_symbol_name("OBJC_CLASS_NAME_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.i8_align.abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__TEXT,__cstring,cstring_literals"); + llvm::LLVMSetGlobalConstant(g, llvm::TRUE); + llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); + self.add_compiler_used_global(g); + + g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `ObjCNonFragileABITypesHelper`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L6052 + fn get_objc_class_t(&self) -> &'ll Type { + if let Some(class_t) = self.objc_class_t.get() { + return class_t; + } + + assert_eq!(self.objc_abi_version(), 2); + + // struct _class_t { + // struct _class_t* isa; + // struct _class_t* const superclass; + // void* cache; + // IMP* vtable; + // struct class_ro_t* ro; + // } + + let class_t = self.type_named_struct("struct._class_t"); + let els = [self.type_ptr(); 5]; + let packed = false; + self.set_struct_body(class_t, &els, packed); + + self.objc_class_t.set(Some(class_t)); + class_t + } + + // We do our best here to match what Clang does when compiling Objective-C natively. We + // deduplicate references within a CGU, but we need a reference definition in each referencing + // CGU. All attempts at using external references to a single reference definition result in + // linker errors. + fn get_objc_classref(&self, classname: Symbol) -> &'ll Value { + let mut classrefs = self.objc_classrefs.borrow_mut(); + if let Some(classref) = classrefs.get(&classname).copied() { + return classref; + } + + let g = match self.objc_abi_version() { + 1 => { + // See Clang's `CGObjCMac::EmitClassRefFromId`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5205 + let llval = self.define_objc_classname(classname.as_str()); + let llty = self.type_ptr(); + let sym = self.generate_local_symbol_name("OBJC_CLASS_REFERENCES_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__OBJC,__cls_refs,literal_pointers,no_dead_strip"); + self.add_compiler_used_global(g); + g + } + 2 => { + // See Clang's `CGObjCNonFragileABIMac::EmitClassRefFromId`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7423 + let llval = { + let extern_sym = format!("OBJC_CLASS_$_{}", classname.as_str()); + let extern_llty = self.get_objc_class_t(); + self.declare_global(&extern_sym, extern_llty) + }; + let llty = self.type_ptr(); + let sym = self.generate_local_symbol_name("OBJC_CLASSLIST_REFERENCES_$_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); + llvm::set_section(g, c"__DATA,__objc_classrefs,regular,no_dead_strip"); + self.add_compiler_used_global(g); + g + } + _ => unreachable!(), + }; + + classrefs.insert(classname, g); + g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. We + // deduplicate references within a CGU, but we need a reference definition in each referencing + // CGU. All attempts at using external references to a single reference definition result in + // linker errors. + // + // Newer versions of Apple Clang generate calls to `@"objc_msgSend$methname"` selector stub + // functions. We don't currently do that. The code we generate is closer to what Apple Clang + // generates with the `-fno-objc-msgsend-selector-stubs` option. + fn get_objc_selref(&self, methname: Symbol) -> &'ll Value { + let mut selrefs = self.objc_selrefs.borrow_mut(); + if let Some(selref) = selrefs.get(&methname).copied() { + return selref; + } + + let abi_version = self.objc_abi_version(); + + // See Clang's `CGObjCCommonMac::CreateCStringLiteral`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134 + let methname_llval = self.null_terminate_const_bytes(methname.as_str().as_bytes()); + let methname_llty = self.val_ty(methname_llval); + let methname_sym = self.generate_local_symbol_name("OBJC_METH_VAR_NAME_"); + let methname_g = self.define_global(&methname_sym, methname_llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", methname_sym); + }); + set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align.abi); + llvm::set_initializer(methname_g, methname_llval); + llvm::set_linkage(methname_g, llvm::Linkage::PrivateLinkage); + llvm::set_section( + methname_g, + match abi_version { + 1 => c"__TEXT,__cstring,cstring_literals", + 2 => c"__TEXT,__objc_methname,cstring_literals", + _ => unreachable!(), + }, + ); + llvm::LLVMSetGlobalConstant(methname_g, llvm::TRUE); + llvm::LLVMSetUnnamedAddress(methname_g, llvm::UnnamedAddr::Global); + self.add_compiler_used_global(methname_g); + + // See Clang's `CGObjCMac::EmitSelectorAddr`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5243 + // And Clang's `CGObjCNonFragileABIMac::EmitSelectorAddr`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7586 + let selref_llval = methname_g; + let selref_llty = self.type_ptr(); + let selref_sym = self.generate_local_symbol_name("OBJC_SELECTOR_REFERENCES_"); + let selref_g = self.define_global(&selref_sym, selref_llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", selref_sym); + }); + set_global_alignment(self, selref_g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(selref_g, selref_llval); + llvm::set_externally_initialized(selref_g, true); + llvm::set_linkage( + selref_g, + match abi_version { + 1 => llvm::Linkage::PrivateLinkage, + 2 => llvm::Linkage::InternalLinkage, + _ => unreachable!(), + }, + ); + llvm::set_section( + selref_g, + match abi_version { + 1 => c"__OBJC,__message_refs,literal_pointers,no_dead_strip", + 2 => c"__DATA,__objc_selrefs,literal_pointers,no_dead_strip", + _ => unreachable!(), + }, + ); + self.add_compiler_used_global(selref_g); + + selrefs.insert(methname, selref_g); + selref_g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `ObjCTypesHelper`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5936 + // And Clang's `CGObjCMac::EmitModuleInfo`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5151 + pub(crate) fn define_objc_module_info(&mut self) { + assert_eq!(self.objc_abi_version(), 1); + + // struct _objc_module { + // long version; // Hardcoded to 7 in Clang. + // long size; // sizeof(struct _objc_module) + // char* name; // Hardcoded to classname "" in Clang. + // struct _objc_symtab* symtab; // Null without class or category definitions. + // } + + let llty = self.type_named_struct("struct._objc_module"); + let i32_llty = self.type_i32(); + let ptr_llty = self.type_ptr(); + let packed = false; + self.set_struct_body(llty, &[i32_llty, i32_llty, ptr_llty, ptr_llty], packed); + + let version = self.const_uint(i32_llty, 7); + let size = self.const_uint(i32_llty, 16); + let name = self.define_objc_classname(""); + let symtab = self.const_null(ptr_llty); + let llval = crate::common::named_struct(llty, &[version, size, name, symtab]); + + let sym = "OBJC_MODULES"; + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__OBJC,__module_info,regular,no_dead_strip"); + + self.add_compiler_used_global(g); } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ee77774c688..aa5c17269fb 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use std::str; use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx}; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; -use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN}; @@ -27,7 +26,7 @@ use rustc_session::config::{ BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet, }; use rustc_span::source_map::Spanned; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; @@ -120,7 +119,7 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Statics that will be placed in the llvm.compiler.used variable /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details - pub compiler_used_statics: Vec<&'ll Value>, + pub compiler_used_statics: RefCell<Vec<&'ll Value>>, /// Mapping of non-scalar types to llvm types. pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>, @@ -147,6 +146,15 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// `global_asm!` needs to be able to find this new global so that it can /// compute the correct mangled symbol name to insert into the asm. pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, + + /// Cached Objective-C class type + pub objc_class_t: Cell<Option<&'ll Type>>, + + /// Cache of Objective-C class references + pub objc_classrefs: RefCell<FxHashMap<Symbol, &'ll Value>>, + + /// Cache of Objective-C selector references + pub objc_selrefs: RefCell<FxHashMap<Symbol, &'ll Value>>, } fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { @@ -173,35 +181,6 @@ pub(crate) unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (20, 0, 0) { - if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { - // LLVM 20 defines three additional address spaces for alternate - // pointer kinds used in Windows. - // See https://github.com/llvm/llvm-project/pull/111879 - target_data_layout = - target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); - } - if sess.target.arch.starts_with("sparc") { - // LLVM 20 updates the sparc layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/106951 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("mips64") { - // LLVM 20 updates the mips64 layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/112084 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("powerpc64") { - // LLVM 20 updates the powerpc64 layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/118004 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("wasm32") || sess.target.arch.starts_with("wasm64") { - // LLVM 20 updates the wasm(32|64) layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/119204 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - } if llvm_version < (21, 0, 0) { if sess.target.arch == "nvptx64" { // LLVM 21 updated the default layout on nvptx: https://github.com/llvm/llvm-project/pull/124961 @@ -213,6 +192,16 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("p8:128:128:128:48", "p8:128:128") } } + if llvm_version < (22, 0, 0) { + if sess.target.arch == "avr" { + // LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010 + target_data_layout = target_data_layout.replace("n8:16", "n8") + } + if sess.target.arch == "nvptx64" { + // LLVM 22 updated the NVPTX layout to indicate 256-bit vector load/store: https://github.com/llvm/llvm-project/pull/155198 + target_data_layout = target_data_layout.replace("-i256:256", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. { @@ -372,7 +361,17 @@ pub(crate) unsafe fn create_module<'ll>( } } - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + if let Some(regparm_count) = sess.opts.unstable_opts.regparm { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "NumRegisterParameters", + regparm_count, + ); + } + + if let Some(BranchProtection { bti, pac_ret, gcs }) = sess.opts.unstable_opts.branch_protection + { if sess.target.arch == "aarch64" { llvm::add_module_flag_u32( llmod, @@ -405,6 +404,12 @@ pub(crate) unsafe fn create_module<'ll>( "sign-return-address-with-bkey", u32::from(pac_opts.key == PAuthKey::B), ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "guarded-control-stack", + gcs.into(), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -457,6 +462,15 @@ pub(crate) unsafe fn create_module<'ll>( } } + if sess.opts.unstable_opts.indirect_branch_cs_prefix { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "indirect_branch_cs_prefix", + 1, + ); + } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) { // Set up the small-data optimization limit for architectures that use @@ -617,7 +631,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { const_globals: Default::default(), statics_to_rauw: RefCell::new(Vec::new()), used_statics: Vec::new(), - compiler_used_statics: Vec::new(), + compiler_used_statics: Default::default(), type_lowering: Default::default(), scalar_lltypes: Default::default(), coverage_cx, @@ -628,6 +642,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { intrinsics: Default::default(), local_gen_sym_counter: Cell::new(0), renamed_statics: Default::default(), + objc_class_t: Cell::new(None), + objc_classrefs: Default::default(), + objc_selrefs: Default::default(), }, PhantomData, ) @@ -652,12 +669,71 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); llvm::set_section(g, c"llvm.metadata"); } + + /// The Objective-C ABI that is used. + /// + /// This corresponds to the `-fobjc-abi-version=` flag in Clang / GCC. + pub(crate) fn objc_abi_version(&self) -> u32 { + assert!(self.tcx.sess.target.is_like_darwin); + if self.tcx.sess.target.arch == "x86" && self.tcx.sess.target.os == "macos" { + // 32-bit x86 macOS uses ABI version 1 (a.k.a. the "fragile ABI"). + 1 + } else { + // All other Darwin-like targets we support use ABI version 2 + // (a.k.a the "non-fragile ABI"). + 2 + } + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `CGObjCCommonMac::EmitImageInfo`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5085 + pub(crate) fn add_objc_module_flags(&self) { + let abi_version = self.objc_abi_version(); + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Version", + abi_version, + ); + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Image Info Version", + 0, + ); + + llvm::add_module_flag_str( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Image Info Section", + match abi_version { + 1 => "__OBJC,__image_info,regular", + 2 => "__DATA,__objc_imageinfo,regular,no_dead_strip", + _ => unreachable!(), + }, + ); + + if self.tcx.sess.target.env == "sim" { + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Is Simulated", + 1 << 5, + ); + } + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Class Properties", + 1 << 6, + ); + } } impl<'ll> SimpleCx<'ll> { - pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type { - assert_eq!(self.type_kind(ty), TypeKind::Function); - unsafe { llvm::LLVMGetReturnType(ty) } - } pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type { unsafe { llvm::LLVMGlobalGetValueType(val) } } @@ -682,7 +758,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { } pub(crate) fn get_const_int(&self, ty: &'ll Type, val: u64) -> &'ll Value { - unsafe { llvm::LLVMConstInt(ty, val, llvm::False) } + unsafe { llvm::LLVMConstInt(ty, val, llvm::FALSE) } } pub(crate) fn get_const_i64(&self, n: u64) -> &'ll Value { @@ -721,16 +797,6 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) } } - - pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { - let mut functions = vec![]; - let mut func = unsafe { llvm::LLVMGetFirstFunction(self.llmod()) }; - while let Some(f) = func { - functions.push(f); - func = unsafe { llvm::LLVMGetNextFunction(f) } - } - functions - } } impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { @@ -810,7 +876,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); + let target_cpu = attributes::target_cpu_attr(self, self.sess()); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } @@ -825,22 +891,22 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { + if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); + attrs.push(attributes::target_cpu_attr(self, self.sess())); + attrs.extend(attributes::tune_cpu_attr(self, self.sess())); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { let entry_name = self.sess().target.entry_name.as_ref(); if self.get_declared_value(entry_name).is_none() { - Some(self.declare_entry_fn( + let llfn = self.declare_entry_fn( entry_name, llvm::CallConv::from_conv( self.sess().target.entry_abi, @@ -848,7 +914,13 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { ), llvm::UnnamedAddr::Global, fn_type, - )) + ); + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Function, + attributes::target_features_attr(self, self.tcx, vec![]).as_slice(), + ); + Some(llfn) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} @@ -972,7 +1044,10 @@ impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::ReferencesError(_) + | LayoutError::InvalidSimd { .. } = err + { self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) @@ -989,7 +1064,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { match err { - FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + FnAbiError::Layout( + LayoutError::SizeOverflow(_) + | LayoutError::Cycle(_) + | LayoutError::InvalidSimd { .. }, + ) => { self.tcx.dcx().emit_fatal(Spanned { span, node: err }); } _ => match fn_abi_request { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 6eb7042da61..3081badb821 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -2,9 +2,9 @@ use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::DebuggerVisualizerType; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; use crate::builder::Builder; @@ -72,7 +72,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); llvm::set_section(section_var, c".debug_gdb_scripts"); llvm::set_initializer(section_var, cx.const_bytes(section_contents)); - llvm::LLVMSetGlobalConstant(section_var, llvm::True); + llvm::LLVMSetGlobalConstant(section_var, llvm::TRUE); llvm::set_unnamed_address(section_var, llvm::UnnamedAddr::Global); llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 0e9dbfba658..4b74c04ed7a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -19,7 +19,9 @@ use rustc_middle::ty::{ self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, }; use rustc_session::config::{self, DebugInfo, Lto}; -use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Symbol, hygiene}; +use rustc_span::{ + DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Span, Symbol, hygiene, +}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; @@ -30,9 +32,7 @@ use self::type_map::{DINodeCreationResult, Stub, UniqueTypeId}; use super::CodegenUnitDebugContext; use super::namespace::mangled_name_of_instance; use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; -use super::utils::{ - DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, -}; +use super::utils::{DIB, debug_context, get_namespace_for_item, is_node_local_to_unit}; use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::dwarf_const; use crate::debuginfo::metadata::type_map::build_type_with_children; @@ -101,32 +101,33 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, array_type: Ty<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let ty::Array(element_type, len) = array_type.kind() else { bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) }; - let element_type_di_node = type_di_node(cx, *element_type); + let element_type_di_node = spanned_type_di_node(cx, *element_type, span); return_if_di_node_created_in_meantime!(cx, unique_type_id); - let (size, align) = cx.size_and_align_of(array_type); + let (size, align) = cx.spanned_size_and_align_of(array_type, span); let upper_bound = len .try_to_target_usize(cx.tcx) .expect("expected monomorphic const in codegen") as c_longlong; - let subrange = - unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; + let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; + let subscripts = &[subrange]; - let subscripts = create_DIArray(DIB(cx), &[subrange]); let di_node = unsafe { - llvm::LLVMRustDIBuilderCreateArrayType( + llvm::LLVMDIBuilderCreateArrayType( DIB(cx), size.bits(), align.bits() as u32, element_type_di_node, - subscripts, + subscripts.as_ptr(), + subscripts.len() as c_uint, ) }; @@ -172,17 +173,13 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( "ptr_type={ptr_type}, pointee_type={pointee_type}", ); - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - pointer_size.bits(), - pointer_align.abi.bits() as u32, - 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_c_char_ptr(), - ptr_type_debuginfo_name.len(), - ) - }; + let di_node = create_pointer_type( + cx, + pointee_type_di_node, + pointer_size, + pointer_align.abi, + &ptr_type_debuginfo_name, + ); DINodeCreationResult { di_node, already_stored_in_typemap: false } } @@ -230,17 +227,13 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // The data pointer type is a regular, thin pointer, regardless of whether this // is a slice or a trait object. - let data_ptr_type_di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - addr_field.size.bits(), - addr_field.align.abi.bits() as u32, - 0, // Ignore DWARF address space. - std::ptr::null(), - 0, - ) - }; + let data_ptr_type_di_node = create_pointer_type( + cx, + pointee_type_di_node, + addr_field.size, + addr_field.align.abi, + "", + ); smallvec![ build_field_di_node( @@ -315,7 +308,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); - let fn_di_node = create_subroutine_type(cx, create_DIArray(DIB(cx), &signature_di_nodes[..])); + let fn_di_node = create_subroutine_type(cx, &signature_di_nodes[..]); // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); @@ -326,26 +319,44 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( } _ => unreachable!(), }; - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - fn_di_node, - size.bits(), - align.bits() as u32, - 0, // Ignore DWARF address space. - name.as_c_char_ptr(), - name.len(), - ) - }; + let di_node = create_pointer_type(cx, fn_di_node, size, align, &name); DINodeCreationResult::new(di_node, false) } pub(super) fn create_subroutine_type<'ll>( cx: &CodegenCx<'ll, '_>, - signature: &'ll DICompositeType, + signature: &[Option<&'ll llvm::Metadata>], ) -> &'ll DICompositeType { - unsafe { llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(cx), signature) } + unsafe { + llvm::LLVMDIBuilderCreateSubroutineType( + DIB(cx), + None, // ("File" is ignored and has no effect) + signature.as_ptr(), + signature.len() as c_uint, + DIFlags::FlagZero, // (default value) + ) + } +} + +fn create_pointer_type<'ll>( + cx: &CodegenCx<'ll, '_>, + pointee_ty: &'ll llvm::Metadata, + size: Size, + align: Align, + name: &str, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreatePointerType( + DIB(cx), + pointee_ty, + size.bits(), + align.bits() as u32, + 0, // Ignore DWARF address space. + name.as_ptr(), + name.len(), + ) + } } /// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs @@ -423,6 +434,14 @@ fn build_slice_type_di_node<'ll, 'tcx>( /// This function will look up the debuginfo node in the TypeMap. If it can't find it, it /// will create the node by dispatching to the corresponding `build_*_di_node()` function. pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { + spanned_type_di_node(cx, t, DUMMY_SP) +} + +pub(crate) fn spanned_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, + span: Span, +) -> &'ll DIType { let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) @@ -437,7 +456,7 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> build_basic_type_di_node(cx, t) } ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), - ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t, span), ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), @@ -458,9 +477,9 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), - AdtKind::Union => build_union_type_di_node(cx, unique_type_id), - AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id, span), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id, span), + AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), @@ -802,14 +821,15 @@ fn build_basic_type_di_node<'ll, 'tcx>( }; let typedef_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateTypedef( + llvm::LLVMDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_c_char_ptr(), + typedef_name.as_ptr(), typedef_name.len(), unknown_file_metadata(cx), - 0, - None, + 0, // (no line number) + None, // (no scope) + 0u32, // (no alignment specified) ) }; @@ -823,12 +843,13 @@ fn create_basic_type<'ll, 'tcx>( encoding: u32, ) -> &'ll DIBasicType { unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( + llvm::LLVMDIBuilderCreateBasicType( DIB(cx), - name.as_c_char_ptr(), + name.as_ptr(), name.len(), size.bits(), encoding, + DIFlags::FlagZero, ) } } @@ -1014,10 +1035,10 @@ fn create_member_type<'ll, 'tcx>( type_di_node: &'ll DIType, ) -> &'ll DIType { unsafe { - llvm::LLVMRustDIBuilderCreateMemberType( + llvm::LLVMDIBuilderCreateMemberType( DIB(cx), owner, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, @@ -1055,6 +1076,7 @@ fn visibility_di_flags<'ll, 'tcx>( fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let struct_type = unique_type_id.expect_ty(); let ty::Adt(adt_def, _) = struct_type.kind() else { @@ -1062,7 +1084,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( }; assert!(adt_def.is_struct()); let containing_scope = get_namespace_for_item(cx, adt_def.did()); - let struct_type_and_layout = cx.layout_of(struct_type); + let struct_type_and_layout = cx.spanned_layout_of(struct_type, span); let variant_def = adt_def.non_enum_variant(); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) @@ -1255,6 +1277,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( fn build_union_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let union_type = unique_type_id.expect_ty(); let (union_def_id, variant_def) = match union_type.kind() { @@ -1262,7 +1285,7 @@ fn build_union_type_di_node<'ll, 'tcx>( _ => bug!("build_union_type_di_node on a non-ADT"), }; let containing_scope = get_namespace_for_item(cx, union_def_id); - let union_ty_and_layout = cx.layout_of(union_type); + let union_ty_and_layout = cx.spanned_layout_of(union_type, span); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(union_def_id))) @@ -1425,7 +1448,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); - let trait_ref = tcx.erase_regions(trait_ref); + let trait_ref = tcx.erase_and_anonymize_regions(trait_ref); tcx.vtable_entries(trait_ref) } else { @@ -1552,7 +1575,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( // Unwrap potential addrspacecast let vtable = find_vtable_behind_cast(vtable); let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); - let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_ref_self = cx.tcx.erase_and_anonymize_regions(trait_ref_self); let trait_def_id = trait_ref_self.def_id; let trait_vis = cx.tcx.visibility(trait_def_id); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index a5c80895741..4ecc3086e1b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; use smallvec::smallvec; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::dwarf_const::DW_TAG_const_type; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; @@ -378,20 +378,17 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_struct_type_wrapper_di_node, None, ), - unsafe { - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), - enum_type_di_node, - TAG_FIELD_NAME.as_c_char_ptr(), - TAG_FIELD_NAME.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - variant_names_type_di_node, - visibility_flags, - Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), - tag_base_type_align.bits() as u32, - ) - } + create_static_member_type( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + visibility_flags, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align, + ), ] } @@ -570,27 +567,28 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( let build_assoc_const = |name: &str, type_di_node_: &'ll DIType, value: u64, - align: Align| unsafe { + align: Align| + -> &'ll llvm::Metadata { // FIXME: Currently we force all DISCR_* values to be u64's as LLDB seems to have // problems inspecting other value types. Since DISCR_* is typically only going to be // directly inspected via the debugger visualizer - which compares it to the `tag` value // (whose type is not modified at all) it shouldn't cause any real problems. let (t_di, align) = if name == ASSOC_CONST_DISCR_NAME { - (type_di_node_, align.bits() as u32) + (type_di_node_, align) } else { let ty_u64 = Ty::new_uint(cx.tcx, ty::UintTy::U64); - (type_di_node(cx, ty_u64), Align::EIGHT.bits() as u32) + (type_di_node(cx, ty_u64), Align::EIGHT) }; // must wrap type in a `const` modifier for LLDB to be able to inspect the value of the member - let field_type = - llvm::LLVMRustDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di); + let field_type = unsafe { + llvm::LLVMDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di) + }; - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), + create_static_member_type( + cx, wrapper_struct_type_di_node, - name.as_c_char_ptr(), - name.len(), + name, unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, field_type, @@ -975,3 +973,30 @@ fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, s .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) } + +fn create_static_member_type<'ll>( + cx: &CodegenCx<'ll, '_>, + scope: &'ll llvm::Metadata, + name: &str, + file: &'ll llvm::Metadata, + line_number: c_uint, + ty: &'ll llvm::Metadata, + flags: DIFlags, + value: Option<&'ll llvm::Value>, + align: Align, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreateStaticMemberType( + DIB(cx), + scope, + name.as_ptr(), + name.len(), + file, + line_number, + ty, + flags, + value, + align.bits() as c_uint, + ) + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 7c701926d2c..caff3586079 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -10,7 +10,7 @@ use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::Symbol; +use rustc_span::{Span, Symbol}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -30,13 +30,14 @@ mod native; pub(super) fn build_enum_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) }; - let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_and_layout = cx.spanned_layout_of(enum_type, span); if wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout) { return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index d1502d2b1e6..37200fdc41a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; +use libc::c_uint; use rustc_abi::{Align, Size, VariantIdx}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; @@ -9,7 +10,7 @@ use rustc_middle::bug; use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt}; use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; use crate::llvm::{self}; @@ -191,7 +192,7 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope: Option<&'ll DIScope>, flags: DIFlags, ) -> StubInfo<'ll, 'tcx> { - let empty_array = create_DIArray(DIB(cx), &[]); + let no_elements: &[Option<&llvm::Metadata>] = &[]; let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); let (file_metadata, line_number) = if let Some(def_location) = def_location { @@ -207,10 +208,10 @@ pub(super) fn stub<'ll, 'tcx>( _ => None, }; unsafe { - llvm::LLVMRustDIBuilderCreateStructType( + llvm::LLVMDIBuilderCreateStructType( DIB(cx), containing_scope, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, @@ -218,28 +219,30 @@ pub(super) fn stub<'ll, 'tcx>( align.bits() as u32, flags, None, - empty_array, - 0, + no_elements.as_ptr(), + no_elements.len() as c_uint, + 0u32, // (Objective-C runtime version; default is 0) vtable_holder, - unique_type_id_str.as_c_char_ptr(), + unique_type_id_str.as_ptr(), unique_type_id_str.len(), ) } } Stub::Union => unsafe { - llvm::LLVMRustDIBuilderCreateUnionType( + llvm::LLVMDIBuilderCreateUnionType( DIB(cx), containing_scope, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, size.bits(), align.bits() as u32, flags, - Some(empty_array), - 0, - unique_type_id_str.as_c_char_ptr(), + no_elements.as_ptr(), + no_elements.len() as c_uint, + 0u32, // (Objective-C runtime version; default is 0) + unique_type_id_str.as_ptr(), unique_type_id_str.len(), ) }, @@ -276,7 +279,7 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( && let ty::Adt(adt_def, args) = ty.kind() { let def_id = adt_def.did(); - // If any sub type reference the original type definition and the sub type has a type + // If any child type references the original type definition and the child type has a type // parameter that strictly contains the original parameter, the original type is a recursive // type that can expanding indefinitely. Example, // ``` @@ -285,21 +288,43 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( // Item(T), // } // ``` - let is_expanding_recursive = adt_def.is_enum() - && debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| { - if def_id == *parent_def_id { - args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| { - if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type()) - { - arg != parent_arg && arg.contains(parent_arg) - } else { - false - } - }) - } else { - false - } - }); + let is_expanding_recursive = { + let stack = debug_context(cx).adt_stack.borrow(); + stack + .iter() + .enumerate() + .rev() + .skip(1) + .filter(|(_, (ancestor_def_id, _))| def_id == *ancestor_def_id) + .any(|(ancestor_index, (_, ancestor_args))| { + args.iter() + .zip(ancestor_args.iter()) + .filter_map(|(arg, ancestor_arg)| arg.as_type().zip(ancestor_arg.as_type())) + .any(|(arg, ancestor_arg)| + // Strictly contains. + (arg != ancestor_arg && arg.contains(ancestor_arg)) + // Check all types between current and ancestor use the + // ancestor_arg. + // Otherwise, duplicate wrappers in normal recursive type may be + // regarded as expanding. + // ``` + // struct Recursive { + // a: Box<Box<Recursive>>, + // } + // ``` + // It can produce an ADT stack like this, + // - Box<Recursive> + // - Recursive + // - Box<Box<Recursive>> + && stack[ancestor_index + 1..stack.len()].iter().all( + |(_, intermediate_args)| + intermediate_args + .iter() + .filter_map(|arg| arg.as_type()) + .any(|mid_arg| mid_arg.contains(ancestor_arg)) + )) + }) + }; if is_expanding_recursive { // FIXME: indicate that this is an expanding recursive type in stub metadata? return DINodeCreationResult::new(stub_info.metadata, false); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 6cbf2dbf7d3..af64e4ebed0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -28,7 +28,9 @@ use rustc_target::spec::DebuginfoKind; use smallvec::SmallVec; use tracing::debug; -use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, type_di_node}; +use self::metadata::{ + UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, spanned_type_di_node, type_di_node, +}; use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; use crate::builder::Builder; @@ -50,15 +52,6 @@ mod utils; use self::create_scope_map::compute_mir_scopes; pub(crate) use self::metadata::build_global_var_di_node; -// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were -// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp` -// to decide which C++ API to call. Instead, we should just have two separate -// FFI functions and choose the correct one on the Rust side. -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, @@ -172,35 +165,38 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { if direct_offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(direct_offset.bytes()); } for &offset in indirect_offsets { addr_ops.push(DW_OP_deref); if offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(offset.bytes() as u64); + addr_ops.push(offset.bytes()); } } if let Some(fragment) = fragment { // `DW_OP_LLVM_fragment` takes as arguments the fragment's // offset and size, both of them in bits. addr_ops.push(DW_OP_LLVM_fragment); - addr_ops.push(fragment.start.bits() as u64); - addr_ops.push((fragment.end - fragment.start).bits() as u64); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); } + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), + llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( + di_builder, variable_alloca, dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, + addr_expr, dbg_loc, self.llbb(), - ); - } + ) + }; } fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { @@ -347,7 +343,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let file_metadata = file_metadata(self, &loc.file); let function_type_metadata = - create_subroutine_type(self, get_function_signature(self, fn_abi)); + create_subroutine_type(self, &get_function_signature(self, fn_abi)); let mut name = String::with_capacity(64); type_names::push_item_name(tcx, def_id, false, &mut name); @@ -439,9 +435,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - ) -> &'ll DIArray { + ) -> Vec<Option<&'ll llvm::Metadata>> { if cx.sess().opts.debuginfo != DebugInfo::Full { - return create_DIArray(DIB(cx), &[]); + return vec![]; } let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); @@ -482,7 +478,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); } - create_DIArray(DIB(cx), &signature[..]) + signature } fn get_template_parameters<'ll, 'tcx>( @@ -533,31 +529,26 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - if let Some(impl_def_id) = cx.tcx.impl_of_assoc(instance.def_id()) { - // If the method does *not* belong to a trait, proceed - if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( - instance.args, - cx.typing_env(), - cx.tcx.type_of(impl_def_id), - ); - - // Only "class" methods are generally understood by LLVM, - // so avoid methods on other types (e.g., `<*mut T>::null`). - if let ty::Adt(def, ..) = impl_self_ty.kind() - && !def.is_box() - { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() - { - return (type_di_node(cx, impl_self_ty), true); - } else { - return (namespace::item_namespace(cx, def.did()), false); - } + // For trait method impls we still use the "parallel namespace" + // strategy + if let Some(imp_def_id) = cx.tcx.inherent_impl_of_assoc(instance.def_id()) { + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + cx.tcx.type_of(imp_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - } else { - // For trait method impls we still use the "parallel namespace" - // strategy } } @@ -631,30 +622,41 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); - let type_metadata = type_di_node(self, variable_type); + let type_metadata = spanned_type_di_node(self, variable_type, span); - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; let align = self.align_of(variable_type); let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_c_char_ptr(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bits() as u32, - ) + + match variable_kind { + ArgumentVariable(arg_index) => unsafe { + llvm::LLVMDIBuilderCreateParameterVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + arg_index as c_uint, + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + ) + }, + LocalVariable => unsafe { + llvm::LLVMDIBuilderCreateAutoVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + align.bits() as u32, + ) + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index b4d639368b0..1dcf4ff3062 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -38,7 +38,7 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l parent_scope, namespace_name_string.as_ptr(), namespace_name_string.len(), - llvm::False, // ExportSymbols (only relevant for C++ anonymous namespaces) + llvm::FALSE, // ExportSymbols (only relevant for C++ anonymous namespaces) ) }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index cc1d504b430..7e1e49310f6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } + unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 960a895a203..36cdb498839 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -76,7 +76,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>( attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attrs.extend(attributes::non_lazy_bind_attr(cx)); + attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess)); attributes::apply_to_llfn(llfn, Function, &attrs); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7b27e496986..013108d1286 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -3,24 +3,28 @@ use std::cmp::Ordering; use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; +use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; -use rustc_hir as hir; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_hir::{self as hir}; use rustc_middle::mir::BinOp; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; -use rustc_middle::ty::{self, GenericArgsRef, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty, TyCtxt, TypingEnv}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; -use rustc_symbol_mangling::mangle_internal_symbol; -use rustc_target::spec::PanicStrategy; +use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; +use rustc_target::callconv::PassMode; use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; +use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::context::CodegenCx; +use crate::errors::AutoDiffWithoutEnable; use crate::llvm::{self, Metadata}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -189,6 +193,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &[ptr, args[1].immediate()], ) } + sym::autodiff => { + codegen_autodiff(self, tcx, instance, args, result); + return Ok(()); + } sym::is_val_statically_known => { if let OperandValue::Immediate(imm) = args[0].val { self.call_intrinsic( @@ -321,10 +329,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); + let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_i32(); self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], - &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)], + &[ + ptr, + self.const_i32(rw), + self.const_i32(locality), + self.const_i32(cache_type), + ], ) } sym::carrying_mul_add => { @@ -368,7 +382,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::rotate_left | sym::rotate_right | sym::saturating_add - | sym::saturating_sub => { + | sym::saturating_sub + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { let ty = args[0].layout.ty; if !ty.is_integral() { tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { @@ -409,18 +425,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::bitreverse => { self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()]) } - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same + sym::rotate_left + | sym::rotate_right + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { + let is_left = name == sym::rotate_left || name == sym::unchecked_funnel_shl; + let lhs = args[0].immediate(); + let (rhs, raw_shift) = + if name == sym::rotate_left || name == sym::rotate_right { + // rotate = funnel shift with first two args the same + (lhs, args[1].immediate()) + } else { + (args[1].immediate(), args[2].immediate()) + }; let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' }); // llvm expects shift to be the same type as the values, but rust // always uses `u32`. - let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false); - self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift]) + self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; @@ -649,7 +673,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( catch_func: &'ll Value, dest: PlaceRef<'tcx, &'ll Value>, ) { - if bx.sess().panic_strategy() == PanicStrategy::Abort { + if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.call(try_func_ty, None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; @@ -1113,6 +1137,148 @@ fn get_rust_try_fn<'a, 'll, 'tcx>( rust_try } +fn codegen_autodiff<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + result: PlaceRef<'tcx, &'ll Value>, +) { + if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) { + let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable); + } + + let fn_args = instance.args; + let callee_ty = instance.ty(tcx, bx.typing_env()); + + let sig = callee_ty.fn_sig(tcx).skip_binder(); + + let ret_ty = sig.output(); + let llret_ty = bx.layout_of(ret_ty).llvm_type(bx); + + // Get source, diff, and attrs + let (source_id, source_args) = match fn_args.into_type_list(tcx)[0].kind() { + ty::FnDef(def_id, source_params) => (def_id, source_params), + _ => bug!("invalid autodiff intrinsic args"), + }; + + let fn_source = match Instance::try_resolve(tcx, bx.cx.typing_env(), *source_id, source_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + source_id, + source_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let source_symbol = symbol_name_for_instance_in_crate(tcx, fn_source.clone(), LOCAL_CRATE); + let Some(fn_to_diff) = bx.cx.get_function(&source_symbol) else { + bug!("could not find source function") + }; + + let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() { + ty::FnDef(def_id, diff_args) => (def_id, diff_args), + _ => bug!("invalid args"), + }; + + let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + diff_id, + diff_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let val_arr = get_args_from_tuple(bx, args[2], fn_diff); + let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE); + + let Some(mut diff_attrs) = autodiff_attrs(tcx, fn_diff.def_id()) else { + bug!("could not find autodiff attrs") + }; + + adjust_activity_to_abi( + tcx, + fn_source, + TypingEnv::fully_monomorphized(), + &mut diff_attrs.input_activity, + ); + + let fnc_tree = + rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized())); + + // Build body + generate_enzyme_call( + bx, + bx.cx, + fn_to_diff, + &diff_symbol, + llret_ty, + &val_arr, + diff_attrs.clone(), + result, + fnc_tree, + ); +} + +fn get_args_from_tuple<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tuple_op: OperandRef<'tcx, &'ll Value>, + fn_instance: Instance<'tcx>, +) -> Vec<&'ll Value> { + let cx = bx.cx; + let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty()); + + match tuple_op.val { + OperandValue::Immediate(val) => vec![val], + OperandValue::Pair(v1, v2) => vec![v1, v2], + OperandValue::Ref(ptr) => { + let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout }; + + let mut result = Vec::with_capacity(fn_abi.args.len()); + let mut tuple_index = 0; + + for arg in &fn_abi.args { + match arg.mode { + PassMode::Ignore => {} + PassMode::Direct(_) | PassMode::Cast { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(val); + tuple_index += 1; + } + PassMode::Pair(_, _) => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(bx.extract_value(pair_val, 0)); + result.push(bx.extract_value(pair_val, 1)); + tuple_index += 1; + } + PassMode::Indirect { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + result.push(field.val.llval); + tuple_index += 1; + } + } + } + + result + } + + OperandValue::ZeroSized => vec![], + } +} + fn generic_simd_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ca84b6de8b1..b9584a9117f 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -30,7 +30,6 @@ use context::SimpleCx; use errors::ParseTargetMachineConfig; use llvm_util::target_config; use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -38,7 +37,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetConfig}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -46,19 +45,13 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_span::Symbol; - -mod back { - pub(crate) mod archive; - pub(crate) mod lto; - pub(crate) mod owned_target_machine; - mod profiling; - pub(crate) mod write; -} +use rustc_target::spec::{RelocModel, TlsModel}; mod abi; mod allocator; mod asm; mod attributes; +mod back; mod base; mod builder; mod callee; @@ -75,6 +68,7 @@ mod llvm_util; mod mono_item; mod type_; mod type_of; +mod typetree; mod va_arg; mod value; @@ -173,20 +167,15 @@ impl WriteBackendMethods for LlvmCodegenBackend { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec<FatLtoInput<Self>>, - diff_fncs: Vec<AutoDiffItem>, - ) -> Result<ModuleCodegen<Self::Module>, FatalError> { + ) -> ModuleCodegen<Self::Module> { let mut module = - back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?; - - if !diff_fncs.is_empty() { - builder::autodiff::differentiate(&module, cgcx, diff_fncs)?; - } + back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules); let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - back::lto::run_pass_manager(cgcx, dcx, &mut module, false)?; + back::lto::run_pass_manager(cgcx, dcx, &mut module, false); - Ok(module) + module } fn run_thin_lto( cgcx: &CodegenContext<Self>, @@ -194,7 +183,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>, - ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> { + ) -> (Vec<ThinModule<Self>>, Vec<WorkProduct>) { back::lto::run_thin( cgcx, exported_symbols_for_lto, @@ -208,27 +197,24 @@ impl WriteBackendMethods for LlvmCodegenBackend { dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen<Self::Module>, config: &ModuleConfig, - ) -> Result<(), FatalError> { + ) { back::write::optimize(cgcx, dcx, module, config) } fn optimize_thin( cgcx: &CodegenContext<Self>, thin: ThinModule<Self>, - ) -> Result<ModuleCodegen<Self::Module>, FatalError> { + ) -> ModuleCodegen<Self::Module> { back::lto::optimize_thin_module(thin, cgcx) } fn codegen( cgcx: &CodegenContext<Self>, module: ModuleCodegen<Self::Module>, config: &ModuleConfig, - ) -> Result<CompiledModule, FatalError> { + ) -> CompiledModule { back::write::codegen(cgcx, module, config) } - fn prepare_thin( - module: ModuleCodegen<Self::Module>, - emit_summary: bool, - ) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module, emit_summary) + fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module) } fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer) { (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) @@ -260,16 +246,7 @@ impl CodegenBackend for LlvmCodegenBackend { match req.kind { PrintKind::RelocationModels => { writeln!(out, "Available relocation models:").unwrap(); - for name in &[ - "static", - "pic", - "pie", - "dynamic-no-pic", - "ropi", - "rwpi", - "ropi-rwpi", - "default", - ] { + for name in RelocModel::ALL.iter().map(RelocModel::desc).chain(["default"]) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -283,9 +260,7 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:").unwrap(); - for name in - &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] - { + for name in TlsModel::ALL.iter().map(TlsModel::desc) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -420,12 +395,12 @@ impl ModuleLlvm { cgcx: &CodegenContext<LlvmCodegenBackend>, name: &str, dcx: DiagCtxtHandle<'_>, - ) -> Result<OwnedTargetMachine, FatalError> { + ) -> OwnedTargetMachine { let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name); match (cgcx.tm_factory)(tm_factory_config) { - Ok(m) => Ok(m), + Ok(m) => m, Err(e) => { - return Err(dcx.emit_almost_fatal(ParseTargetMachineConfig(e))); + dcx.emit_fatal(ParseTargetMachineConfig(e)); } } } @@ -435,13 +410,13 @@ impl ModuleLlvm { name: &CStr, buffer: &[u8], dcx: DiagCtxtHandle<'_>, - ) -> Result<Self, FatalError> { + ) -> Self { unsafe { let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx)?; - let tm = ModuleLlvm::tm_from_cgcx(cgcx, name.to_str().unwrap(), dcx)?; + let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx); + let tm = ModuleLlvm::tm_from_cgcx(cgcx, name.to_str().unwrap(), dcx); - Ok(ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) }) + ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) } } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs deleted file mode 100644 index 51bcc4d123d..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! A wrapper around LLVM's archive (.a) code - -use std::path::Path; -use std::{slice, str}; - -use rustc_fs_util::path_to_c_string; - -pub(crate) struct ArchiveRO { - pub raw: &'static mut super::Archive, -} - -unsafe impl Send for ArchiveRO {} - -pub(crate) struct Iter<'a> { - raw: &'a mut super::ArchiveIterator<'a>, -} - -pub(crate) struct Child<'a> { - pub raw: &'a mut super::ArchiveChild<'a>, -} - -impl ArchiveRO { - /// Opens a static archive for read-only purposes. This is more optimized - /// than the `open` method because it uses LLVM's internal `Archive` class - /// rather than shelling out to `ar` for everything. - /// - /// If this archive is used with a mutable method, then an error will be - /// raised. - pub(crate) fn open(dst: &Path) -> Result<ArchiveRO, String> { - unsafe { - let s = path_to_c_string(dst); - let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { - super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) - })?; - Ok(ArchiveRO { raw: ar }) - } - } - - pub(crate) fn iter(&self) -> Iter<'_> { - unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } - } -} - -impl Drop for ArchiveRO { - fn drop(&mut self) { - unsafe { - super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Iterator for Iter<'a> { - type Item = Result<Child<'a>, String>; - - fn next(&mut self) -> Option<Result<Child<'a>, String>> { - unsafe { - match super::LLVMRustArchiveIteratorNext(self.raw) { - Some(raw) => Some(Ok(Child { raw })), - None => super::last_error().map(Err), - } - } - } -} - -impl<'a> Drop for Iter<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Child<'a> { - pub(crate) fn name(&self) -> Option<&'a str> { - unsafe { - let mut name_len = 0; - let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); - if name_ptr.is_null() { - None - } else { - let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); - str::from_utf8(name).ok().map(|s| s.trim()) - } - } - } -} - -impl<'a> Drop for Child<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 56d756e52cc..e63043b2122 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -3,9 +3,36 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; -use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; +use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value}; use crate::llvm::{Bool, Builder}; +// TypeTree types +pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct EnzymeTypeTree { + _unused: [u8; 0], +} + +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub(crate) enum CConcreteType { + DT_Anything = 0, + DT_Integer = 1, + DT_Pointer = 2, + DT_Half = 3, + DT_Float = 4, + DT_Double = 5, + DT_Unknown = 6, + DT_FP128 = 9, +} + +pub(crate) struct TypeTree { + pub(crate) inner: CTypeTreeRef, +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { // Enzyme @@ -59,19 +86,49 @@ pub(crate) enum LLVMRustVerifierFailureAction { LLVMReturnStatusAction = 2, } -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) use self::Enzyme_AD::*; -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) mod Enzyme_AD { use std::ffi::{CString, c_char}; use libc::c_void; + use super::{CConcreteType, CTypeTreeRef, Context}; + unsafe extern "C" { pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char); } + + // TypeTree functions + unsafe extern "C" { + pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef; + pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef); + pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool; + pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64); + pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef); + pub(crate) fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ); + pub(crate) fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ); + pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char; + pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char); + } + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; @@ -134,13 +191,74 @@ pub(crate) mod Enzyme_AD { } } -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) use self::Fallback_AD::*; -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) mod Fallback_AD { #![allow(unused_variables)] + use libc::c_char; + + use super::{CConcreteType, CTypeTreeRef, Context}; + + // TypeTree function fallbacks + pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) { + unimplemented!() + } + pub(crate) fn set_inline(val: bool) { unimplemented!() } @@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD { unimplemented!() } } + +impl TypeTree { + pub(crate) fn new() -> TypeTree { + let inner = unsafe { EnzymeNewTypeTree() }; + TypeTree { inner } + } + + pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree { + let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) }; + TypeTree { inner } + } + + pub(crate) fn merge(self, other: Self) -> Self { + unsafe { + EnzymeMergeTypeTree(self.inner, other.inner); + } + drop(other); + self + } + + #[must_use] + pub(crate) fn shift( + self, + layout: &str, + offset: isize, + max_size: isize, + add_offset: usize, + ) -> Self { + let layout = std::ffi::CString::new(layout).unwrap(); + + unsafe { + EnzymeTypeTreeShiftIndiciesEq( + self.inner, + layout.as_ptr(), + offset as i64, + max_size as i64, + add_offset as u64, + ); + } + + self + } + + pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) { + unsafe { + EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx); + } + } +} + +impl Clone for TypeTree { + fn clone(&self) -> Self { + let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) }; + TypeTree { inner } + } +} + +impl std::fmt::Display for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ptr = unsafe { EnzymeTypeTreeToString(self.inner) }; + let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) }; + match cstr.to_str() { + Ok(x) => write!(f, "{}", x)?, + Err(err) => write!(f, "could not parse: {}", err)?, + } + + // delete C string pointer + unsafe { + EnzymeTypeTreeToStringFree(ptr); + } + + Ok(()) + } +} + +impl std::fmt::Debug for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + <Self as std::fmt::Display>::fmt(self, f) + } +} + +impl Drop for TypeTree { + fn drop(&mut self) { + unsafe { EnzymeFreeTypeTree(self.inner) } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 75d3d27f74e..afd2991a09c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -11,9 +11,8 @@ //! the need for an extra cast from `*const u8` on the Rust side. #![allow(non_camel_case_types)] -#![allow(non_upper_case_globals)] -use std::fmt::Debug; +use std::fmt::{self, Debug}; use std::marker::PhantomData; use std::num::NonZero; use std::ptr; @@ -25,18 +24,67 @@ use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ - DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, - DIFile, DIFlags, DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, - DISubrange, DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, + DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, + DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, + DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind, }; use crate::llvm; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. -pub(crate) type Bool = c_int; +/// +/// This wrapper does not implement `PartialEq`. +/// To test the underlying boolean value, use [`Self::is_true`]. +#[derive(Clone, Copy)] +#[repr(transparent)] +pub(crate) struct Bool { + value: c_int, +} + +pub(crate) const TRUE: Bool = Bool::TRUE; +pub(crate) const FALSE: Bool = Bool::FALSE; + +impl Bool { + pub(crate) const TRUE: Self = Self { value: 1 }; + pub(crate) const FALSE: Self = Self { value: 0 }; + + pub(crate) const fn from_bool(rust_bool: bool) -> Self { + if rust_bool { Self::TRUE } else { Self::FALSE } + } + + /// Converts this LLVM-C boolean to a Rust `bool` + pub(crate) fn is_true(self) -> bool { + // Since we're interacting with a C API, follow the C convention of + // treating any nonzero value as true. + self.value != Self::FALSE.value + } +} + +impl Debug for Bool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.value { + 0 => f.write_str("FALSE"), + 1 => f.write_str("TRUE"), + // As with `Self::is_true`, treat any nonzero value as true. + v => write!(f, "TRUE ({v})"), + } + } +} -pub(crate) const True: Bool = 1 as Bool; -pub(crate) const False: Bool = 0 as Bool; +/// Convenience trait to convert `bool` to `llvm::Bool` with an explicit method call. +/// +/// Being able to write `b.to_llvm_bool()` is less noisy than `llvm::Bool::from(b)`, +/// while being more explicit and less mistake-prone than something like `b.into()`. +pub(crate) trait ToLlvmBool: Copy { + fn to_llvm_bool(self) -> llvm::Bool; +} + +impl ToLlvmBool for bool { + #[inline(always)] + fn to_llvm_bool(self) -> llvm::Bool { + llvm::Bool::from_bool(self) + } +} /// Wrapper for a raw enum value returned from LLVM's C APIs. /// @@ -97,6 +145,7 @@ pub(crate) enum ModuleFlagMergeBehavior { // Consts for the LLVM CallConv type, pre-cast to usize. +/// Must match the layout of `LLVMTailCallKind`. #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] #[allow(dead_code)] @@ -214,7 +263,7 @@ pub(crate) enum AttributeKind { MinSize = 4, Naked = 5, NoAlias = 6, - NoCapture = 7, + CapturesAddress = 7, NoInline = 8, NonNull = 9, NoRedZone = 10, @@ -249,6 +298,8 @@ pub(crate) enum AttributeKind { FnRetThunkExtern = 41, Writable = 42, DeadOnUnwind = 43, + DeadOnReturn = 44, + CapturesReadOnly = 45, } /// LLVMIntPredicate @@ -331,10 +382,15 @@ impl RealPredicate { } } -/// LLVMTypeKind -#[derive(Copy, Clone, PartialEq, Debug)] +/// Must match the layout of `LLVMTypeKind`. +/// +/// Use [`RawEnum<TypeKind>`] for values of `LLVMTypeKind` returned from LLVM, +/// to avoid risk of UB if LLVM adds new enum values. +/// +/// All of LLVM's variants should be declared here, even if no Rust-side code refers +/// to them, because unknown variants will cause [`RawEnum::to_rust`] to panic. +#[derive(Copy, Clone, PartialEq, Debug, TryFromU32)] #[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] pub(crate) enum TypeKind { Void = 0, Half = 1, @@ -609,17 +665,6 @@ pub(crate) enum DiagnosticLevel { Remark, } -/// LLVMRustArchiveKind -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) enum ArchiveKind { - K_GNU, - K_BSD, - K_DARWIN, - K_COFF, - K_AIXBIG, -} - unsafe extern "C" { // LLVMRustThinLTOData pub(crate) type ThinLTOData; @@ -665,6 +710,7 @@ pub(crate) enum MemoryEffects { None, ReadOnly, InaccessibleMemOnly, + ReadOnlyNotPure, } /// LLVMOpcode @@ -761,6 +807,8 @@ unsafe extern "C" { pub(crate) type Metadata; pub(crate) type BasicBlock; pub(crate) type Comdat; + /// `&'ll DbgRecord` represents `LLVMDbgRecordRef`. + pub(crate) type DbgRecord; } #[repr(C)] pub(crate) struct Builder<'a>(InvariantOpaque<'a>); @@ -768,19 +816,12 @@ pub(crate) struct Builder<'a>(InvariantOpaque<'a>); pub(crate) struct PassManager<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub type TargetMachine; - pub(crate) type Archive; } -#[repr(C)] -pub(crate) struct ArchiveIterator<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub(crate) struct ArchiveChild<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub(crate) type Twine; pub(crate) type DiagnosticInfo; pub(crate) type SMDiagnostic; } -#[repr(C)] -pub(crate) struct RustArchiveMember<'a>(InvariantOpaque<'a>); /// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); @@ -852,7 +893,6 @@ pub(crate) mod debuginfo { pub(crate) type DIVariable = DIDescriptor; pub(crate) type DIGlobalVariableExpression = DIDescriptor; pub(crate) type DIArray = DIDescriptor; - pub(crate) type DISubrange = DIDescriptor; pub(crate) type DIEnumerator = DIDescriptor; pub(crate) type DITemplateTypeParameter = DIDescriptor; @@ -1014,6 +1054,8 @@ unsafe extern "C" { SLen: c_uint, ) -> MetadataKindId; + pub(crate) fn LLVMDisposeTargetMachine(T: ptr::NonNull<TargetMachine>); + // Create modules. pub(crate) fn LLVMModuleCreateWithNameInContext( ModuleID: *const c_char, @@ -1045,6 +1087,8 @@ unsafe extern "C" { CanThrow: llvm::Bool, ) -> &'ll Value; + pub(crate) safe fn LLVMGetTypeKind(Ty: &Type) -> RawEnum<TypeKind>; + // Operations on integer types pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; @@ -1196,7 +1240,8 @@ unsafe extern "C" { pub(crate) safe fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub(crate) safe fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); - pub(crate) safe fn LLVMRustSetTailCallKind(CallInst: &Value, Kind: TailCallKind); + pub(crate) safe fn LLVMSetTailCallKind(CallInst: &Value, kind: TailCallKind); + pub(crate) safe fn LLVMSetExternallyInitialized(GlobalVar: &Value, IsExtInit: Bool); // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( @@ -1830,6 +1875,177 @@ unsafe extern "C" { Scope: &'ll Metadata, InlinedAt: Option<&'ll Metadata>, ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateSubroutineType<'ll>( + Builder: &DIBuilder<'ll>, + File: Option<&'ll Metadata>, // (ignored and has no effect) + ParameterTypes: *const Option<&'ll Metadata>, + NumParameterTypes: c_uint, + Flags: DIFlags, // (default is `DIFlags::DIFlagZero`) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateUnionType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Elements: *const Option<&'ll Metadata>, + NumElements: c_uint, + RunTimeLang: c_uint, // (optional Objective-C runtime version; default is 0) + UniqueId: *const c_uchar, // See "PTR_LEN_STR". + UniqueIdLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateArrayType<'ll>( + Builder: &DIBuilder<'ll>, + Size: u64, + Align: u32, + Ty: &'ll Metadata, + Subscripts: *const &'ll Metadata, + NumSubscripts: c_uint, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateBasicType<'ll>( + Builder: &DIBuilder<'ll>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + SizeInBits: u64, + Encoding: c_uint, // (`LLVMDWARFTypeEncoding`) + Flags: DIFlags, // (default is `DIFlags::DIFlagZero`) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreatePointerType<'ll>( + Builder: &DIBuilder<'ll>, + PointeeTy: &'ll Metadata, + SizeInBits: u64, + AlignInBits: u32, + AddressSpace: c_uint, // (optional DWARF address space; default is 0) + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStructType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + DerivedFrom: Option<&'ll Metadata>, + Elements: *const Option<&'ll Metadata>, + NumElements: c_uint, + RunTimeLang: c_uint, // (optional Objective-C runtime version; default is 0) + VTableHolder: Option<&'ll Metadata>, + UniqueId: *const c_uchar, // See "PTR_LEN_STR". + UniqueIdLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStaticMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + Type: &'ll Metadata, + Flags: DIFlags, + ConstantVal: Option<&'ll Value>, + AlignInBits: u32, + ) -> &'ll Metadata; + + /// Creates a "qualified type" in the C/C++ sense, by adding modifiers + /// like `const` or `volatile`. + pub(crate) fn LLVMDIBuilderCreateQualifiedType<'ll>( + Builder: &DIBuilder<'ll>, + Tag: c_uint, // (DWARF tag, e.g. `DW_TAG_const_type`) + Type: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateTypedef<'ll>( + Builder: &DIBuilder<'ll>, + Type: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Scope: Option<&'ll Metadata>, + AlignInBits: u32, // (optional; default is 0) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>( + Builder: &DIBuilder<'ll>, + LowerBound: i64, + Count: i64, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>( + Builder: &DIBuilder<'ll>, + Data: *const Option<&'ll Metadata>, + NumElements: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateExpression<'ll>( + Builder: &DIBuilder<'ll>, + Addr: *const u64, + Length: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Storage: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ArgNo: c_uint, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + ) -> &'ll Metadata; } #[link(name = "llvm-wrapper", kind = "static")] @@ -1840,9 +2056,6 @@ unsafe extern "C" { // Create and destroy contexts. pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; - /// See llvm::LLVMTypeKind::getTypeID. - pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on all values pub(crate) fn LLVMRustGlobalAddMetadata<'a>( Val: &'a Value, @@ -1892,11 +2105,17 @@ unsafe extern "C" { C: &Context, effects: MemoryEffects, ) -> &Attribute; + /// ## Safety + /// - Each of `LowerWords` and `UpperWords` must point to an array that is + /// long enough to fully define an integer of size `NumBits`, i.e. each + /// pointer must point to `NumBits.div_ceil(64)` elements or more. + /// - The implementation will make its own copy of the pointed-to `u64` + /// values, so the pointers only need to outlive this function call. pub(crate) fn LLVMRustCreateRangeAttribute( C: &Context, - num_bits: c_uint, - lower_words: *const u64, - upper_words: *const u64, + NumBits: c_uint, + LowerWords: *const u64, + UpperWords: *const u64, ) -> &Attribute; // Operations on functions @@ -2129,11 +2348,6 @@ unsafe extern "C" { SourceLen: size_t, ) -> &'a DIFile; - pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>( - Builder: &DIBuilder<'a>, - ParameterTypes: &'a DIArray, - ) -> &'a DICompositeType; - pub(crate) fn LLVMRustDIBuilderCreateFunction<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, @@ -2167,66 +2381,6 @@ unsafe extern "C" { TParam: &'a DIArray, ) -> &'a DISubprogram; - pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>( - Builder: &DIBuilder<'a>, - Name: *const c_char, - NameLen: size_t, - SizeInBits: u64, - Encoding: c_uint, - ) -> &'a DIBasicType; - - pub(crate) fn LLVMRustDIBuilderCreateTypedef<'a>( - Builder: &DIBuilder<'a>, - Type: &'a DIBasicType, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Scope: Option<&'a DIScope>, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>( - Builder: &DIBuilder<'a>, - PointeeTy: &'a DIType, - SizeInBits: u64, - AlignInBits: u32, - AddressSpace: c_uint, - Name: *const c_char, - NameLen: size_t, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateStructType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIDescriptor>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - DerivedFrom: Option<&'a DIType>, - Elements: &'a DIArray, - RunTimeLang: c_uint, - VTableHolder: Option<&'a DIType>, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DICompositeType; - - pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - SizeInBits: u64, - AlignInBits: u32, - OffsetInBits: u64, - Flags: DIFlags, - Ty: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2242,25 +2396,6 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateStaticMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - Flags: DIFlags, - val: Option<&'a Value>, - AlignInBits: u32, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateQualifiedType<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Type: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( Builder: &DIBuilder<'a>, Context: Option<&'a DIScope>, @@ -2277,51 +2412,6 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub(crate) fn LLVMRustDIBuilderCreateArrayType<'a>( - Builder: &DIBuilder<'a>, - Size: u64, - AlignInBits: u32, - Ty: &'a DIType, - Subscripts: &'a DIArray, - ) -> &'a DIType; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ); - pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2345,22 +2435,6 @@ unsafe extern "C" { IsScoped: bool, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateUnionType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - Elements: Option<&'a DIArray>, - RunTimeLang: c_uint, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2437,12 +2511,13 @@ unsafe extern "C" { OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, UseEmulatedTls: bool, - ArgsCstrBuff: *const c_char, - ArgsCstrBuffLen: usize, + Argv0: *const c_uchar, // See "PTR_LEN_STR". + Argv0Len: size_t, + CommandLineArgs: *const c_uchar, // See "PTR_LEN_STR". + CommandLineArgsLen: size_t, UseWasmEH: bool, ) -> *mut TargetMachine; - pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, @@ -2504,19 +2579,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub(crate) fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - pub(crate) fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub(crate) fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub(crate) fn LLVMRustArchiveIteratorNext<'a>( - AIR: &ArchiveIterator<'a>, - ) -> Option<&'a mut ArchiveChild<'a>>; - pub(crate) fn LLVMRustArchiveChildName( - ACR: &ArchiveChild<'_>, - size: &mut size_t, - ) -> *const c_char; - pub(crate) fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub(crate) fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub(crate) fn LLVMRustDestroyArchive(AR: &'static mut Archive); - pub(crate) fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); pub(crate) fn LLVMRustUnpackOptimizationDiagnostic<'a>( @@ -2554,21 +2616,6 @@ unsafe extern "C" { num_ranges: &mut usize, ) -> bool; - pub(crate) fn LLVMRustWriteArchive( - Dst: *const c_char, - NumMembers: size_t, - Members: *const &RustArchiveMember<'_>, - WriteSymbtab: bool, - Kind: ArchiveKind, - isEC: bool, - ) -> LLVMRustResult; - pub(crate) fn LLVMRustArchiveMemberNew<'a>( - Filename: *const c_char, - Name: *const c_char, - Child: Option<&ArchiveChild<'a>>, - ) -> &'a mut RustArchiveMember<'a>; - pub(crate) fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); pub(crate) fn LLVMRustPositionBuilderPastAllocas<'a>(B: &Builder<'a>, Fn: &'a Value); @@ -2587,7 +2634,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustThinLTOBufferCreate( M: &Module, is_thin: bool, - emit_summary: bool, ) -> &'static mut ThinLTOBuffer; pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; @@ -2671,6 +2717,8 @@ unsafe extern "C" { pub(crate) fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIsAnyArm64Coff(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustSetNoSanitizeAddress(Global: &Value); pub(crate) fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 154ba4fd690..1115d82fa85 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -3,7 +3,6 @@ use std::ffi::{CStr, CString}; use std::num::NonZero; use std::ptr; -use std::str::FromStr; use std::string::FromUtf8Error; use libc::c_uint; @@ -16,7 +15,6 @@ pub(crate) use self::MetadataType::*; pub(crate) use self::ffi::*; use crate::common::AsCCharPtr; -pub(crate) mod archive_ro; pub(crate) mod diagnostic; pub(crate) mod enzyme_ffi; mod ffi; @@ -42,32 +40,6 @@ pub(crate) fn AddFunctionAttributes<'ll>( } } -pub(crate) fn HasAttributeAtIndex<'ll>( - llfn: &'ll Value, - idx: AttributePlace, - kind: AttributeKind, -) -> bool { - unsafe { LLVMRustHasAttributeAtIndex(llfn, idx.as_uint(), kind) } -} - -pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: &str) -> bool { - unsafe { LLVMRustHasFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) { - unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveRustEnumAttributeAtIndex( - llfn: &Value, - place: AttributePlace, - kind: AttributeKind, -) { - unsafe { - LLVMRustRemoveEnumAttributeAtIndex(llfn, place.as_uint(), kind); - } -} - pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, @@ -140,16 +112,26 @@ pub(crate) fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> & pub(crate) fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { let lower = range.start; + // LLVM treats the upper bound as exclusive, but allows wrapping. let upper = range.end.wrapping_add(1); - let lower_words = [lower as u64, (lower >> 64) as u64]; - let upper_words = [upper as u64, (upper >> 64) as u64]; + + // Pass each `u128` endpoint value as a `[u64; 2]` array, least-significant part first. + let as_u64_array = |x: u128| [x as u64, (x >> 64) as u64]; + let lower_words: [u64; 2] = as_u64_array(lower); + let upper_words: [u64; 2] = as_u64_array(upper); + + // To ensure that LLVM doesn't try to read beyond the `[u64; 2]` arrays, + // we must explicitly check that `size_bits` does not exceed 128. + let size_bits = size.bits(); + assert!(size_bits <= 128); + // More robust assertions that are redundant with `size_bits <= 128` and + // should be optimized away. + assert!(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()); + assert!(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()); + let size_bits = c_uint::try_from(size_bits).unwrap(); + unsafe { - LLVMRustCreateRangeAttribute( - llcx, - size.bits().try_into().unwrap(), - lower_words.as_ptr(), - upper_words.as_ptr(), - ) + LLVMRustCreateRangeAttribute(llcx, size_bits, lower_words.as_ptr(), upper_words.as_ptr()) } } @@ -178,21 +160,6 @@ pub(crate) enum CodeGenOptSize { CodeGenOptSizeAggressive = 2, } -impl FromStr for ArchiveKind { - type Err = (); - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "gnu" => Ok(ArchiveKind::K_GNU), - "bsd" => Ok(ArchiveKind::K_BSD), - "darwin" => Ok(ArchiveKind::K_DARWIN), - "coff" => Ok(ArchiveKind::K_COFF), - "aix_big" => Ok(ArchiveKind::K_AIXBIG), - _ => Err(()), - } - } -} - pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); @@ -258,7 +225,7 @@ pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) { } pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) { - LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); + LLVMSetGlobalConstant(llglobal, is_constant.to_llvm_bool()); } pub(crate) fn get_linkage(llglobal: &Value) -> Linkage { @@ -272,7 +239,7 @@ pub(crate) fn set_linkage(llglobal: &Value, linkage: Linkage) { } pub(crate) fn is_declaration(llglobal: &Value) -> bool { - unsafe { LLVMIsDeclaration(llglobal) == ffi::True } + unsafe { LLVMIsDeclaration(llglobal) }.is_true() } pub(crate) fn get_visibility(llglobal: &Value) -> Visibility { @@ -291,6 +258,10 @@ pub(crate) fn set_alignment(llglobal: &Value, align: Align) { } } +pub(crate) fn set_externally_initialized(llglobal: &Value, is_ext_init: bool) { + LLVMSetExternallyInitialized(llglobal, is_ext_init.to_llvm_bool()); +} + /// Get the `name`d comdat from `llmod` and assign it to `llglobal`. /// /// Inserts the comdat into `llmod` if it does not exist. diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 28d2100f478..3b920168e06 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -26,7 +26,7 @@ static INIT: Once = Once::new(); pub(crate) fn init(sess: &Session) { unsafe { // Before we touch LLVM, make sure that multithreading is enabled. - if llvm::LLVMIsMultithreaded() != 1 { + if !llvm::LLVMIsMultithreaded().is_true() { bug!("LLVM compiled without support for threads"); } INIT.call_once(|| { @@ -106,7 +106,7 @@ unsafe fn configure_llvm(sess: &Session) { if sess.target.os == "emscripten" && !sess.opts.unstable_opts.emscripten_wasm_eh - && sess.panic_strategy() == PanicStrategy::Unwind + && sess.panic_strategy().unwinds() { add("-enable-emscripten-cxx-exceptions", false); } @@ -217,27 +217,16 @@ impl<'a> IntoIterator for LLVMFeature<'a> { /// Rust can also be build with an external precompiled version of LLVM which might lead to failures /// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics. pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> { - let arch = if sess.target.arch == "x86_64" { - "x86" - } else if sess.target.arch == "arm64ec" { - "aarch64" - } else if sess.target.arch == "sparc64" { - "sparc" - } else if sess.target.arch == "powerpc64" { - "powerpc" - } else { - &*sess.target.arch + let raw_arch = &*sess.target.arch; + let arch = match raw_arch { + "x86_64" => "x86", + "arm64ec" => "aarch64", + "sparc64" => "sparc", + "powerpc64" => "powerpc", + _ => raw_arch, }; + let (major, _, _) = get_version(); match (arch, s) { - ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( - "sse4.2", - smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], - )), - ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), - ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), - ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), - ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), - ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")), ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")), ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")), @@ -246,9 +235,6 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - // Before LLVM 20 those two features were packaged together as b16b16 - ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), - ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => Some(LLVMFeature::with_dependencies( @@ -262,56 +248,26 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea // Filter out features that are not supported by the current LLVM version ("aarch64", "fpmr") => None, // only existed in 18 ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), - // NVPTX targets added in LLVM 20 - ("nvptx64", "sm_100") if get_version().0 < 20 => None, - ("nvptx64", "sm_100a") if get_version().0 < 20 => None, - ("nvptx64", "sm_101") if get_version().0 < 20 => None, - ("nvptx64", "sm_101a") if get_version().0 < 20 => None, - ("nvptx64", "sm_120") if get_version().0 < 20 => None, - ("nvptx64", "sm_120a") if get_version().0 < 20 => None, - ("nvptx64", "ptx86") if get_version().0 < 20 => None, - ("nvptx64", "ptx87") if get_version().0 < 20 => None, // Filter out features that are not supported by the current LLVM version - ("loongarch64", "div32" | "lam-bh" | "lamcas" | "ld-seq-sa" | "scq") - if get_version().0 < 20 => - { - None - } - // Filter out features that are not supported by the current LLVM version - ("riscv32" | "riscv64", "zacas") if get_version().0 < 20 => None, - ( - "s390x", - "message-security-assist-extension12" - | "concurrent-functions" - | "miscellaneous-extensions-4" - | "vector-enhancements-3" - | "vector-packed-decimal-enhancement-3", - ) if get_version().0 < 20 => None, + ("loongarch32" | "loongarch64", "32s") if major < 21 => None, + ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), + ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), + ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( + "sse4.2", + smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], + )), + ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), + ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), + ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), + ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), + ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies( s, smallvec![TargetFeatureFoldStrength::EnableOnly("evex512")], )), - // Support for `wide-arithmetic` will first land in LLVM 20 as part of - // llvm/llvm-project#111598 - ("wasm32" | "wasm64", "wide-arithmetic") if get_version() < (20, 0, 0) => None, - ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), - // In LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available and SPARC-V8+ ABI used". - // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L27-L28 - // Before LLVM 19, there was no `v8plus` feature and `v9` means "SPARC-V9 instruction available". - // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 - ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), - ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), - // These new `amx` variants and `movrs` were introduced in LLVM20 - ("x86", "amx-avx512" | "amx-fp8" | "amx-movrs" | "amx-tf32" | "amx-transpose") - if get_version().0 < 20 => - { - None - } - ("x86", "movrs") if get_version().0 < 20 => None, ("x86", "avx10.1") => Some(LLVMFeature::new("avx10.1-512")), - ("x86", "avx10.2") if get_version().0 < 20 => None, - ("x86", "avx10.2") if get_version().0 >= 20 => Some(LLVMFeature::new("avx10.2-512")), + ("x86", "avx10.2") => Some(LLVMFeature::new("avx10.2-512")), ("x86", "apxf") => Some(LLVMFeature::with_dependencies( "egpr", smallvec![ @@ -715,17 +671,7 @@ pub(crate) fn global_llvm_features( }; // Features implied by an implicit or explicit `--target`. - features.extend( - sess.target - .features - .split(',') - .filter(|v| !v.is_empty()) - // Drop +v8plus feature introduced in LLVM 20. - // (Hard-coded target features do not go through `to_llvm_feature` since they already - // are LLVM feature names, hence we need a special case here.) - .filter(|v| *v != "+v8plus" || get_version() >= (20, 0, 0)) - .map(String::from), - ); + features.extend(sess.target.features.split(',').filter(|v| !v.is_empty()).map(String::from)); if wants_wasm_eh(sess) && sess.panic_strategy() == PanicStrategy::Unwind { features.push("+exception-handling".into()); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f9edaded60d..52eefe2d4d2 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,8 +1,9 @@ use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; @@ -132,7 +133,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v).is_true()); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 89365503138..9ecaf5f24fe 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -15,7 +15,7 @@ use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::{CodegenCx, GenericCx, SCx}; pub(crate) use crate::llvm::Type; -use crate::llvm::{Bool, False, Metadata, True}; +use crate::llvm::{FALSE, Metadata, TRUE, ToLlvmBool}; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use crate::{common, llvm}; @@ -53,7 +53,9 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { } pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { - unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } + unsafe { + llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed.to_llvm_bool()) + } } pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) } @@ -139,7 +141,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { } pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, TRUE) } } pub(crate) fn type_i1(&self) -> &'ll Type { @@ -152,7 +154,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { self.llcx(), els.as_ptr(), els.len() as c_uint, - packed as Bool, + packed.to_llvm_bool(), ) } } @@ -200,11 +202,11 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, FALSE) } } fn type_kind(&self, ty: &'ll Type) -> TypeKind { - unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } + llvm::LLVMGetTypeKind(ty).to_rust().to_generic() } fn type_ptr(&self) -> &'ll Type { diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 4e7096da502..84998b5499b 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -7,6 +7,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; +use rustc_span::{DUMMY_SP, Span}; use tracing::debug; use crate::common::*; @@ -149,7 +150,11 @@ impl<'a, 'tcx> CodegenCx<'a, 'tcx> { } pub(crate) fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { - let layout = self.layout_of(ty); + self.spanned_size_and_align_of(ty, DUMMY_SP) + } + + pub(crate) fn spanned_size_and_align_of(&self, ty: Ty<'tcx>, span: Span) -> (Size, Align) { + let layout = self.spanned_layout_of(ty, span); (layout.size, layout.align.abi) } } @@ -226,7 +231,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // Make sure lifetimes are erased, to avoid generating distinct LLVM // types for Rust types that only differ in the choice of lifetimes. - let normal_ty = cx.tcx.erase_regions(self.ty); + let normal_ty = cx.tcx.erase_and_anonymize_regions(self.ty); let mut defer = None; let llty = if self.ty != normal_ty { diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs new file mode 100644 index 00000000000..7e263503700 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -0,0 +1,122 @@ +use rustc_ast::expand::typetree::FncTree; +#[cfg(feature = "llvm_enzyme")] +use { + crate::attributes, + rustc_ast::expand::typetree::TypeTree as RustTypeTree, + std::ffi::{CString, c_char, c_uint}, +}; + +use crate::llvm::{self, Value}; + +#[cfg(feature = "llvm_enzyme")] +fn to_enzyme_typetree( + rust_typetree: RustTypeTree, + _data_layout: &str, + llcx: &llvm::Context, +) -> llvm::TypeTree { + let mut enzyme_tt = llvm::TypeTree::new(); + process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); + enzyme_tt +} +#[cfg(feature = "llvm_enzyme")] +fn process_typetree_recursive( + enzyme_tt: &mut llvm::TypeTree, + rust_typetree: &RustTypeTree, + parent_indices: &[i64], + llcx: &llvm::Context, +) { + for rust_type in &rust_typetree.0 { + let concrete_type = match rust_type.kind { + rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything, + rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer, + rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer, + rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half, + rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float, + rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double, + rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128, + rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown, + }; + + let mut indices = parent_indices.to_vec(); + if !parent_indices.is_empty() { + indices.push(rust_type.offset as i64); + } else if rust_type.offset == -1 { + indices.push(-1); + } else { + indices.push(rust_type.offset as i64); + } + + enzyme_tt.insert(&indices, concrete_type, llcx); + + if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer + && !rust_type.child.0.is_empty() + { + process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx); + } + } +} + +#[cfg(feature = "llvm_enzyme")] +pub(crate) fn add_tt<'ll>( + llmod: &'ll llvm::Module, + llcx: &'ll llvm::Context, + fn_def: &'ll Value, + tt: FncTree, +) { + let inputs = tt.args; + let ret_tt: RustTypeTree = tt.ret; + + let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) }; + let llvm_data_layout = + std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + let attr_name = "enzyme_type"; + let c_attr_name = CString::new(attr_name).unwrap(); + + for (i, input) in inputs.iter().enumerate() { + unsafe { + let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } + } + + unsafe { + let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let ret_attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } +} + +#[cfg(not(feature = "llvm_enzyme"))] +pub(crate) fn add_tt<'ll>( + _llmod: &'ll llvm::Module, + _llcx: &'ll llvm::Context, + _fn_def: &'ll Value, + _tt: FncTree, +) { + unimplemented!() +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ce079f3cb0a..d48c7cf874a 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -28,9 +28,12 @@ fn round_pointer_up_to_alignment<'ll>( align: Align, ptr_ty: &'ll Type, ) -> &'ll Value { - let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); - ptr_as_int = round_up_to_alignment(bx, ptr_as_int, align); - bx.inttoptr(ptr_as_int, ptr_ty) + let ptr = bx.inbounds_ptradd(addr, bx.const_i32(align.bytes() as i32 - 1)); + bx.call_intrinsic( + "llvm.ptrmask", + &[ptr_ty, bx.type_i32()], + &[ptr, bx.const_int(bx.isize_ty, -(align.bytes() as isize) as i64)], + ) } fn emit_direct_ptr_va_arg<'ll, 'tcx>( @@ -735,6 +738,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>( src_align, bx.const_u32(layout.layout.size().bytes() as u32), MemFlags::empty(), + None, ); tmp } else { @@ -905,6 +909,21 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( ) } "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + "arm" => { + // Types wider than 16 bytes are not currently supported. Clang has special logic for + // such types, but `VaArgSafe` is not implemented for any type that is this large. + assert!(bx.cx.size_of(target_ty).bytes() <= 16); + + emit_ptr_va_arg( + bx, + addr, + target_ty, + PassMode::Direct, + SlotSize::Bytes4, + AllowHigherAlign::Yes, + ForceRightAdjust::No, + ) + } "s390x" => emit_s390x_va_arg(bx, addr, target_ty), "powerpc" => emit_powerpc_va_arg(bx, addr, target_ty), "powerpc64" | "powerpc64le" => emit_ptr_va_arg( |