diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
37 files changed, 1714 insertions, 780 deletions
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 2d11628250c..67bd1e59bb0 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -46,5 +46,6 @@ tracing = "0.1" [features] # tidy-alphabetical-start check_only = ["rustc_llvm/check_only"] +llvm_enzyme = [] # tidy-alphabetical-end diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index ac7583f5666..1703cab942b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -215,9 +215,9 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst); } - // Unsized indirect qrguments + // Unsized indirect arguments cannot be stored PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Cast { cast, pad_i32: _ } => { // The ABI mandates that the value is passed as a different struct representation. @@ -246,6 +246,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); } @@ -272,12 +273,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Pair(next(), next()).store(bx, dst); } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - let place_val = PlaceValue { - llval: next(), - llextra: Some(next()), - align: self.layout.align.abi, - }; - OperandValue::Ref(place_val).store(bx, dst); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Direct(_) | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } @@ -543,7 +539,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the declaration has an associated instance, compute extra attributes based on that. if let Some(instance) = instance { - llfn_attrs_from_instance(cx, llfn, instance); + llfn_attrs_from_instance( + cx, + cx.tcx, + llfn, + &cx.tcx.codegen_instance_attrs(instance.def), + Some(instance), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index df3e49279d9..896d6755c75 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -5,15 +5,17 @@ use rustc_ast::expand::allocator::{ }; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; +use rustc_span::sym; use rustc_symbol_mangling::mangle_internal_symbol; -use smallvec::SmallVec; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; use crate::llvm::{self, FALSE, TRUE, Type, Value}; -use crate::{SimpleCx, attributes, debuginfo, llvm_util}; +use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, @@ -58,7 +60,26 @@ pub(crate) unsafe fn codegen( let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); - create_wrapper_function(tcx, &cx, &from_name, Some(&to_name), &args, output, false); + let alloc_attr_flag = match method.name { + sym::alloc => CodegenFnAttrFlags::ALLOCATOR, + sym::dealloc => CodegenFnAttrFlags::DEALLOCATOR, + sym::realloc => CodegenFnAttrFlags::REALLOCATOR, + sym::alloc_zeroed => CodegenFnAttrFlags::ALLOCATOR_ZEROED, + _ => unreachable!("Unknown allocator method!"), + }; + + let mut attrs = CodegenFnAttrs::new(); + attrs.flags |= alloc_attr_flag; + create_wrapper_function( + tcx, + &cx, + &from_name, + Some(&to_name), + &args, + output, + false, + &attrs, + ); } } @@ -71,6 +92,7 @@ pub(crate) unsafe fn codegen( &[usize, usize], // size, align None, true, + &CodegenFnAttrs::new(), ); unsafe { @@ -92,6 +114,7 @@ pub(crate) unsafe fn codegen( &[], None, false, + &CodegenFnAttrs::new(), ); } @@ -138,6 +161,7 @@ fn create_wrapper_function( args: &[&Type], output: Option<&Type>, no_return: bool, + attrs: &CodegenFnAttrs, ) { let ty = cx.type_func(args, output.unwrap_or_else(|| cx.type_void())); let llfn = declare_simple_fn( @@ -149,18 +173,7 @@ fn create_wrapper_function( ty, ); - let mut attrs = SmallVec::<[_; 2]>::new(); - - let target_cpu = llvm_util::target_cpu(tcx.sess); - let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu); - - let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess) - .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)); - - attrs.push(target_cpu_attr); - attrs.extend(tune_cpu_attr); - - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + llfn_attrs_from_instance(cx, tcx, llfn, attrs, None); let no_return = if no_return { // -> ! DIFlagNoReturn @@ -171,12 +184,6 @@ fn create_wrapper_function( None }; - if tcx.sess.must_emit_unwind_tables() { - let uwtable = - attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; let mut bx = SBuilder::build(&cx, llbb); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 38c1d3b53e8..93b1cf272ab 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -240,6 +240,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { constraints.extend_from_slice(&[ + "~{fflags}".to_string(), "~{vtype}".to_string(), "~{vl}".to_string(), "~{vxsat}".to_string(), @@ -339,8 +340,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::NOMEM) { attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); - } else { - // LLVM doesn't have an attribute to represent ReadOnly + SideEffect + } else if options.contains(InlineAsmOptions::READONLY) { + attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx)); } attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); @@ -537,9 +538,7 @@ pub(crate) fn inline_asm_call<'ll>( bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)), ) })); - let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) }; - let md = bx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, kind, md); + bx.cx.set_metadata_node(call, kind, &srcloc); Some(call) } @@ -661,7 +660,12 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -829,7 +833,12 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4), - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 573c51a9539..8070ea0b3e9 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,20 +1,21 @@ //! Set and unset common attributes on LLVM values. -use rustc_codegen_ssa::traits::*; use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; +use rustc_middle::middle::codegen_fn_attrs::{ + CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, +}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; -use crate::context::CodegenCx; +use crate::context::SimpleCx; use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; -use crate::{attributes, llvm_util}; +use crate::{Session, attributes, llvm_util}; pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { @@ -30,18 +31,19 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ /// Get LLVM attribute for the provided inline heuristic. pub(crate) fn inline_attr<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, instance: ty::Instance<'tcx>, ) -> Option<&'ll Attribute> { // `optnone` requires `noinline` - let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id()); let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, + (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint, (inline, _) => inline, }; - if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + if !tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); } @@ -51,7 +53,7 @@ pub(crate) fn inline_attr<'ll, 'tcx>( Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) } InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { + if tcx.sess.target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) } else { None @@ -63,12 +65,13 @@ pub(crate) fn inline_attr<'ll, 'tcx>( #[inline] fn patchable_function_entry_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, attr: Option<PatchableFunctionEntry>, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); let patchable_spec = attr.unwrap_or_else(|| { - PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry) + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry) }); let entry = patchable_spec.entry(); let prefix = patchable_spec.prefix(); @@ -91,12 +94,13 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub(crate) fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn sanitize_attrs<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -114,11 +118,11 @@ pub(crate) fn sanitize_attrs<'ll>( } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); + let features = tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); + tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -139,9 +143,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) llvm::CreateUWTableAttr(llcx, async_unwind) } -pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - let opts = &cx.sess().opts; +pub(crate) fn frame_pointer_type_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { + let mut fp = sess.target.frame_pointer; + let opts = &sess.opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. if opts.unstable_opts.instrument_mcount { @@ -156,8 +163,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&' Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) } -fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let function_return_attr = match cx.sess().opts.unstable_opts.function_return { +fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let function_return_attr = match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => return None, FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern, }; @@ -167,17 +174,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { +fn instrument_function_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - if cx.sess().opts.unstable_opts.instrument_mcount { + if sess.opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + let mcount_name = match &sess.target.llvm_mcount_intrinsic { Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), - None => cx.sess().target.mcount.as_ref(), + None => sess.target.mcount.as_ref(), }; attrs.push(llvm::CreateAttrStringValue( @@ -186,7 +196,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr mcount_name, )); } - if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + if let Some(options) = &sess.opts.unstable_opts.instrument_xray { // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. // Function prologue and epilogue are instrumented with NOP sleds, // a runtime library later replaces them with detours into tracing code. @@ -217,20 +227,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr attrs } -fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if !cx.sess().opts.unstable_opts.no_jump_tables { +fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if !sess.opts.unstable_opts.no_jump_tables { return None; } Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() + if tcx + .sess .opts .unstable_opts .sanitizer @@ -240,22 +250,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { + if tcx.sess.opts.cg.profile_generate.enabled() { return None; } - let attr_value = match cx.sess().target.stack_probes { + let attr_value = match tcx.sess.target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). StackProbeType::Inline => "inline-asm", // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"), + StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - &mangle_internal_symbol(cx.tcx, "__rust_probestack") + &mangle_internal_symbol(tcx, "__rust_probestack") } else { "inline-asm" } @@ -264,8 +274,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) } -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { +fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let sspattr = match sess.stack_protector() { StackProtector::None => return None, StackProtector::All => AttributeKind::StackProtectReq, StackProtector::Strong => AttributeKind::StackProtectStrong, @@ -275,33 +285,34 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(sspattr.create_attr(cx.llcx)) } -fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().target.arch != "s390x" { +fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if sess.target.arch != "s390x" { return None; } - let requested_features = cx.sess().opts.cg.target_feature.split(','); + let requested_features = sess.opts.cg.target_feature.split(','); let found_positive = requested_features.clone().any(|r| r == "+backchain"); if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); +pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) +pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } /// Get the `target-features` LLVM attribute. -pub(crate) fn target_features_attr<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn target_features_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, function_features: Vec<String>, ) -> Option<&'ll Attribute> { - let global_features = cx.tcx.global_backend_features(()).iter().map(String::as_str); + let global_features = tcx.global_backend_features(()).iter().map(String::as_str); let function_features = function_features.iter().map(String::as_str); let target_features = global_features.chain(function_features).intersperse(",").collect::<String>(); @@ -311,22 +322,22 @@ pub(crate) fn target_features_attr<'ll>( /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } + if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None } } /// Get the default optimizations attrs for a function. #[inline] pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { + match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } @@ -347,17 +358,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, llfn: &'ll Value, - instance: ty::Instance<'tcx>, + codegen_fn_attrs: &CodegenFnAttrs, + instance: Option<ty::Instance<'tcx>>, ) { - let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); - + let sess = tcx.sess; let mut to_add = SmallVec::<[_; 16]>::new(); match codegen_fn_attrs.optimize { OptimizeAttr::Default => { - to_add.extend(default_optimisation_attrs(cx)); + to_add.extend(default_optimisation_attrs(cx, sess)); } OptimizeAttr::DoNotOptimize => { to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx)); @@ -369,21 +381,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); + if sess.must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind)); } - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + if sess.opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } // FIXME: none of these functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(function_return_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(nojumptables_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); + to_add.extend(frame_pointer_type_attr(cx, sess)); + to_add.extend(function_return_attr(cx, sess)); + to_add.extend(instrument_function_attr(cx, sess)); + to_add.extend(nojumptables_attr(cx, sess)); + to_add.extend(probestack_attr(cx, tcx)); + to_add.extend(stackprotector_attr(cx, sess)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); @@ -404,16 +416,19 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // not used. } else { // Do not set sanitizer attributes for naked functions. - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize)); // For non-naked functions, set branch protection attributes on aarch64. - if let Some(BranchProtection { bti, pac_ret }) = - cx.sess().opts.unstable_opts.branch_protection + if let Some(BranchProtection { bti, pac_ret, gcs }) = + sess.opts.unstable_opts.branch_protection { - assert!(cx.sess().target.arch == "aarch64"); + assert!(sess.target.arch == "aarch64"); if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } + if gcs { + to_add.push(llvm::CreateAttrString(cx.llcx, "guarded-control-stack")); + } if let Some(PacRet { leaf, pc, key }) = pac_ret { if pc { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); @@ -435,14 +450,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); - if let Some(zv) = - cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + if let Some(instance) = instance + && let Some(zv) = + tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) && let Some(name) = zv.value_str() { to_add.push(llvm::CreateAttrStringValue( cx.llcx, "alloc-variant-zeroed", - &mangle_internal_symbol(cx.tcx, name.as_str()), + &mangle_internal_symbol(tcx, name.as_str()), )); } // apply to argument place instead of function @@ -487,18 +503,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align); } - if let Some(backchain) = backchain_attr(cx) { + if let Some(backchain) = backchain_attr(cx, sess) { to_add.push(backchain); } - to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); + to_add.extend(patchable_function_entry_attrs( + cx, + sess, + codegen_fn_attrs.patchable_function_entry, + )); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); + to_add.push(target_cpu_attr(cx, sess)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); + to_add.extend(tune_cpu_attr(cx, sess)); let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); @@ -506,7 +526,9 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // Apply function attributes as per usual if there are no user defined // target features otherwise this will get applied at the callsite. if function_features.is_empty() { - if let Some(inline_attr) = inline_attr(cx, instance) { + if let Some(instance) = instance + && let Some(inline_attr) = inline_attr(cx, tcx, instance) + { to_add.push(inline_attr); } } @@ -514,7 +536,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones - .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + .flat_map(|feat| llvm_util::to_llvm_features(sess, feat)) // Convert LLVMFeatures & dependencies to +<feats>s .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { @@ -523,20 +545,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( })) .collect::<Vec<String>>(); - if cx.tcx.sess.target.is_like_wasm { + if sess.target.is_like_wasm { // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + if let Some(instance) = instance + && let Some(module) = wasm_import_module(tcx, instance.def_id()) + { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.symbol_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } } - to_add.extend(target_features_attr(cx, function_features)); + to_add.extend(target_features_attr(cx, tcx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/command_line_args.rs b/compiler/rustc_codegen_llvm/src/back/command_line_args.rs new file mode 100644 index 00000000000..b14713969b3 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/command_line_args.rs @@ -0,0 +1,37 @@ +#[cfg(test)] +mod tests; + +/// Joins command-line arguments into a single space-separated string, quoting +/// and escaping individual arguments as necessary. +/// +/// The result is intended to be informational, for embedding in debug metadata, +/// and might not be properly quoted/escaped for actual command-line use. +pub(crate) fn quote_command_line_args(args: &[String]) -> String { + // Start with a decent-sized buffer, since rustc invocations tend to be long. + let mut buf = String::with_capacity(128); + + for arg in args { + if !buf.is_empty() { + buf.push(' '); + } + + print_arg_quoted(&mut buf, arg); + } + + buf +} + +/// Equivalent to LLVM's `sys::printArg` with quoting always enabled +/// (see llvm/lib/Support/Program.cpp). +fn print_arg_quoted(buf: &mut String, arg: &str) { + buf.reserve(arg.len() + 2); + + buf.push('"'); + for ch in arg.chars() { + if matches!(ch, '"' | '\\' | '$') { + buf.push('\\'); + } + buf.push(ch); + } + buf.push('"'); +} diff --git a/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs b/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs new file mode 100644 index 00000000000..69641fed3bc --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs @@ -0,0 +1,25 @@ +#[test] +fn quote_command_line_args() { + use super::quote_command_line_args; + + struct Case<'a> { + args: &'a [&'a str], + expected: &'a str, + } + + let cases = &[ + Case { args: &[], expected: "" }, + Case { args: &["--hello", "world"], expected: r#""--hello" "world""# }, + Case { args: &["--hello world"], expected: r#""--hello world""# }, + Case { + args: &["plain", "$dollar", "spa ce", r"back\slash", r#""quote""#, "plain"], + expected: r#""plain" "\$dollar" "spa ce" "back\\slash" "\"quote\"" "plain""#, + }, + ]; + + for &Case { args, expected } in cases { + let args = args.iter().copied().map(str::to_owned).collect::<Vec<_>>(); + let actual = quote_command_line_args(&args); + assert_eq!(actual, expected, "args {args:?}"); + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index f571716d9dd..5ac3a87c158 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -563,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { config::AutoDiff::Enable => {} // We handle this below config::AutoDiff::NoPostopt => {} + // Disables TypeTree generation + config::AutoDiff::NoTT => {} } } // This helps with handling enums for now. @@ -617,7 +619,7 @@ pub(crate) fn run_pass_manager( crate::builder::gpu_offload::handle_gpu_code(cgcx, &cx); } - if cfg!(llvm_enzyme) && enable_ad && !thin { + if cfg!(feature = "llvm_enzyme") && enable_ad && !thin { let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { diff --git a/compiler/rustc_codegen_llvm/src/back/mod.rs b/compiler/rustc_codegen_llvm/src/back/mod.rs index 6cb89f80ab8..fe3883e8c73 100644 --- a/compiler/rustc_codegen_llvm/src/back/mod.rs +++ b/compiler/rustc_codegen_llvm/src/back/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod archive; +mod command_line_args; pub(crate) mod lto; pub(crate) mod owned_target_machine; mod profiling; diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 6d8178320fe..903a882916e 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,4 +1,3 @@ -use std::assert_matches::assert_matches; use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -39,13 +38,10 @@ impl OwnedTargetMachine { output_obj_file: &CStr, debug_info_compression: &CStr, use_emulated_tls: bool, - args_cstr_buff: &[u8], + argv0: &str, + command_line_args: &str, use_wasm_eh: bool, ) -> Result<Self, LlvmError<'static>> { - // The argument list is passed as the concatenation of one or more C strings. - // This implies that there must be a last byte, and it must be 0. - assert_matches!(args_cstr_buff, [.., b'\0'], "the last byte must be a NUL terminator"); - // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { llvm::LLVMRustCreateTargetMachine( @@ -70,8 +66,10 @@ impl OwnedTargetMachine { output_obj_file.as_ptr(), debug_info_compression.as_ptr(), use_emulated_tls, - args_cstr_buff.as_ptr(), - args_cstr_buff.len(), + argv0.as_ptr(), + argv0.len(), + command_line_args.as_ptr(), + command_line_args.len(), use_wasm_eh, ) }; @@ -97,8 +95,6 @@ impl Drop for OwnedTargetMachine { // SAFETY: constructing ensures we have a valid pointer created by // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. - unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_ptr()); - } + unsafe { llvm::LLVMDisposeTargetMachine(self.tm_unique) }; } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 423f0da4878..1950b8288d1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -31,6 +31,7 @@ use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym}; use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use tracing::{debug, trace}; +use crate::back::command_line_args::quote_command_line_args; use crate::back::lto::ThinBuffer; use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ @@ -203,6 +204,9 @@ pub(crate) fn target_machine_factory( optlvl: config::OptLevel, target_features: &[String], ) -> TargetMachineFactoryFn<LlvmCodegenBackend> { + // Self-profile timer for creating a _factory_. + let _prof_timer = sess.prof.generic_activity("target_machine_factory"); + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); @@ -249,23 +253,18 @@ pub(crate) fn target_machine_factory( let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); - // copy the exe path, followed by path all into one buffer - // null terminating them so we can use them as null terminated strings - let args_cstr_buff = { - let mut args_cstr_buff: Vec<u8> = Vec::new(); - let exe_path = std::env::current_exe().unwrap_or_default(); - let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default(); - - args_cstr_buff.extend_from_slice(exe_path_str.as_bytes()); - args_cstr_buff.push(0); - - for arg in sess.expanded_args.iter() { - args_cstr_buff.extend_from_slice(arg.as_bytes()); - args_cstr_buff.push(0); - } - - args_cstr_buff - }; + // Command-line information to be included in the target machine. + // This seems to only be used for embedding in PDB debuginfo files. + // FIXME(Zalathar): Maybe skip this for non-PDB targets? + let argv0 = std::env::current_exe() + .unwrap_or_default() + .into_os_string() + .into_string() + .unwrap_or_default(); + let command_line_args = quote_command_line_args(&sess.expanded_args); + // Self-profile counter for the number of bytes produced by command-line quoting. + // Values are summed, so the summary result is cumulative across all TM factories. + sess.prof.artifact_size("quoted_command_line_args", "-", command_line_args.len() as u64); let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); match sess.opts.debuginfo_compression { @@ -288,7 +287,11 @@ pub(crate) fn target_machine_factory( let use_wasm_eh = wants_wasm_eh(sess); + let prof = SelfProfilerRef::clone(&sess.prof); Arc::new(move |config: TargetMachineFactoryConfig| { + // Self-profile timer for invoking a factory to create a target machine. + let _prof_timer = prof.generic_activity("target_machine_factory_inner"); + let path_to_cstring_helper = |path: Option<PathBuf>| -> CString { let path = path.unwrap_or_default(); let path = path_mapping @@ -323,7 +326,8 @@ pub(crate) fn target_machine_factory( &output_obj_file, &debuginfo_compression, use_emulated_tls, - &args_cstr_buff, + &argv0, + &command_line_args, use_wasm_eh, ) }) @@ -574,7 +578,8 @@ pub(crate) unsafe fn llvm_optimize( // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting // differentiated. - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let run_enzyme = autodiff_stage == AutodiffStage::DuringAD; let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore); let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter); @@ -740,7 +745,8 @@ pub(crate) fn optimize( // If we know that we will later run AD, then we disable vectorization and loop unrolling. // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; // The embedded bitcode is used to run LTO/ThinLTO. // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 9cc5d8dbc21..6d12b511e9c 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -105,22 +105,40 @@ pub(crate) fn compile_codegen_unit( if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty()); attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } + // Define Objective-C module info and module flags. Note, the module info will + // also be added to the `llvm.compiler.used` variable, created later. + // + // These are only necessary when we need the linker to do its Objective-C-specific + // magic. We could theoretically do it unconditionally, but at a slight cost to linker + // performance in the common case where it's unnecessary. + if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() { + if cx.objc_abi_version() == 1 { + cx.define_objc_module_info(); + } + cx.add_objc_module_flags(); + } + // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.compiler.used` variable, created next. if cx.sess().instrument_coverage() { cx.coverageinfo_finalize(); } - // Create the llvm.used and llvm.compiler.used variables. + // Create the llvm.used variable. if !cx.used_statics.is_empty() { cx.create_used_variable_impl(c"llvm.used", &cx.used_statics); } - if !cx.compiler_used_statics.is_empty() { - cx.create_used_variable_impl(c"llvm.compiler.used", &cx.compiler_used_statics); + + // Create the llvm.compiler.used variable. + { + let compiler_used_statics = cx.compiler_used_statics.borrow(); + if !compiler_used_statics.is_empty() { + cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics); + } } // Run replace-all-uses-with for statics that need it. This must diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7d0691366e6..e7cb18ab22d 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1,11 +1,12 @@ use std::borrow::{Borrow, Cow}; +use std::iter; use std::ops::Deref; -use std::{iter, ptr}; +use rustc_ast::expand::typetree::FncTree; pub(crate) mod autodiff; pub(crate) mod gpu_offload; -use libc::{c_char, c_uint, size_t}; +use libc::{c_char, c_uint}; use rustc_abi as abi; use rustc_abi::{Align, Size, WrappingRange}; use rustc_codegen_ssa::MemFlags; @@ -395,10 +396,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { md.push(weight(is_cold)); } - unsafe { - let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t); - self.cx.set_metadata(switch, llvm::MD_prof, md_node); - } + self.cx.set_metadata_node(switch, llvm::MD_prof, &md); } fn invoke( @@ -800,22 +798,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { return; } - unsafe { - let llty = self.cx.val_ty(load); - let md = [ - llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)), - llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))), - ]; - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len()); - self.set_metadata(load, llvm::MD_range, md); - } + let llty = self.cx.val_ty(load); + let md = [ + llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)), + llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))), + ]; + self.set_metadata_node(load, llvm::MD_range, &md); } fn nonnull_metadata(&mut self, load: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(load, llvm::MD_nonnull, md); - } + self.set_metadata_node(load, llvm::MD_nonnull, &[]); } fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { @@ -864,8 +856,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // // [1]: https://llvm.org/docs/LangRef.html#store-instruction let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1)); - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1); - self.set_metadata(store, llvm::MD_nontemporal, md); + self.set_metadata_node(store, llvm::MD_nontemporal, &[one]); } } store @@ -1091,16 +1082,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ty: Ty<'tcx>, lhs: Self::Value, rhs: Self::Value, - ) -> Option<Self::Value> { - // FIXME: See comment on the definition of `three_way_compare`. - if crate::llvm_util::get_version() < (20, 0, 0) { - return None; - } - + ) -> Self::Value { let size = ty.primitive_size(self.tcx); let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" }; - Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])) + self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]) } /* Miscellaneous instructions */ @@ -1112,11 +1098,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + tt: Option<FncTree>, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - unsafe { + let memcpy = unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, dst, @@ -1125,7 +1112,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align.bytes() as c_uint, size, is_volatile, - ); + ) + }; + + // TypeTree metadata for memcpy is especially important: when Enzyme encounters + // a memcpy during autodiff, it needs to know the structure of the data being + // copied to properly track derivatives. For example, copying an array of floats + // vs. copying a struct with mixed types requires different derivative handling. + // The TypeTree tells Enzyme exactly what memory layout to expect. + if let Some(tt) = tt { + crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt); } } @@ -1375,10 +1371,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn set_invariant_load(&mut self, load: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(load, llvm::MD_invariant_load, md); - } + self.set_metadata_node(load, llvm::MD_invariant_load, &[]); } fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { @@ -1438,7 +1431,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // If there is an inline attribute and a target feature that matches // we will add the attribute to the callsite otherwise we'll omit // this and not add the attribute to prevent soundness issues. - && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance) + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance) && self.cx.tcx.is_target_feature_call_safe( &fn_call_attrs.target_features, &fn_defn_attrs.target_features, @@ -1522,25 +1515,16 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> { } impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn align_metadata(&mut self, load: &'ll Value, align: Align) { - unsafe { - let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))]; - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len()); - self.set_metadata(load, llvm::MD_align, md); - } + let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))]; + self.set_metadata_node(load, llvm::MD_align, &md); } fn noundef_metadata(&mut self, load: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(load, llvm::MD_noundef, md); - } + self.set_metadata_node(load, llvm::MD_noundef, &[]); } pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(inst, llvm::MD_unpredictable, md); - } + self.set_metadata_node(inst, llvm::MD_unpredictable, &[]); } } impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 6ddf53cdc87..4a749642265 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,10 +1,12 @@ use std::ptr; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; +use rustc_ast::expand::typetree::FncTree; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; -use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; +use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv}; use rustc_middle::{bug, ty}; +use rustc_target::callconv::PassMode; use tracing::debug; use crate::builder::{Builder, PlaceRef, UNNAMED}; @@ -16,9 +18,12 @@ use crate::value::Value; pub(crate) fn adjust_activity_to_abi<'tcx>( tcx: TyCtxt<'tcx>, - fn_ty: Ty<'tcx>, + instance: Instance<'tcx>, + typing_env: TypingEnv<'tcx>, da: &mut Vec<DiffActivity>, ) { + let fn_ty = instance.ty(tcx, typing_env); + if !matches!(fn_ty.kind(), ty::FnDef(..)) { bug!("expected fn def for autodiff, got {:?}", fn_ty); } @@ -27,8 +32,16 @@ pub(crate) fn adjust_activity_to_abi<'tcx>( // All we do is decide how to handle the arguments. let sig = fn_ty.fn_sig(tcx).skip_binder(); + // FIXME(Sa4dUs): pass proper varargs once we have support for differentiating variadic functions + let Ok(fn_abi) = + tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) + else { + bug!("failed to get fn_abi of instance with empty varargs"); + }; + let mut new_activities = vec![]; let mut new_positions = vec![]; + let mut del_activities = 0; for (i, ty) in sig.inputs().iter().enumerate() { if let Some(inner_ty) = ty.builtin_deref(true) { if inner_ty.is_slice() { @@ -80,6 +93,34 @@ pub(crate) fn adjust_activity_to_abi<'tcx>( continue; } } + + let pci = PseudoCanonicalInput { typing_env: TypingEnv::fully_monomorphized(), value: *ty }; + + let layout = match tcx.layout_of(pci) { + Ok(layout) => layout.layout, + Err(_) => { + bug!("failed to compute layout for type {:?}", ty); + } + }; + + let pass_mode = &fn_abi.args[i].mode; + + // For ZST, just ignore and don't add its activity, as this arg won't be present + // in the LLVM passed to Enzyme. + // Some targets pass ZST indirectly in the C ABI, in that case, handle it as a normal arg + // FIXME(Sa4dUs): Enforce ZST corresponding diff activity be `Const` + if *pass_mode == PassMode::Ignore { + del_activities += 1; + da.remove(i); + } + + // If the argument is lowered as a `ScalarPair`, we need to duplicate its activity. + // Otherwise, the number of activities won't match the number of LLVM arguments and + // this will lead to errors when verifying the Enzyme call. + if let rustc_abi::BackendRepr::ScalarPair(_, _) = layout.backend_repr() { + new_activities.push(da[i].clone()); + new_positions.push(i + 1 - del_activities); + } } // now add the extra activities coming from slices // Reverse order to not invalidate the indices @@ -254,6 +295,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args: &[&'ll Value], attrs: AutoDiffAttrs, dest: PlaceRef<'tcx, &'ll Value>, + fnc_tree: FncTree, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -330,7 +372,18 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args, ); + if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() { + crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree); + } + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); - builder.store_to_place(call, dest.val); + let fn_ret_ty = builder.cx.val_ty(call); + if fn_ret_ty != builder.cx.type_void() && fn_ret_ty != builder.cx.type_struct(&[], false) { + // If we return void or an empty struct, then our caller (due to how we generated it) + // does not expect a return value. As such, we have no pointer (or place) into which + // we could store our value, and would store into an undef, which would cause UB. + // As such, we just ignore the return value in those cases. + builder.store_to_place(call, dest.val); + } } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 11b79a7fe68..aa2df46329f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -108,6 +108,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { bytes_in_context(self.llcx(), bytes) } + pub(crate) fn null_terminate_const_bytes(&self, bytes: &[u8]) -> &'ll Value { + null_terminate_bytes_in_context(self.llcx(), bytes) + } + pub(crate) fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); @@ -381,6 +385,16 @@ pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> & } } +pub(crate) fn null_terminate_bytes_in_context<'ll>( + llcx: &'ll llvm::Context, + bytes: &[u8], +) -> &'ll Value { + unsafe { + let ptr = bytes.as_ptr() as *const c_char; + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), FALSE) + } +} + pub(crate) fn named_struct<'ll>(ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); unsafe { llvm::LLVMConstNamedStruct(ty, elts.as_ptr(), len) } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index dc9bb743560..b4ca85a26c8 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -16,6 +16,7 @@ use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; +use rustc_span::Symbol; use tracing::{debug, instrument, trace}; use crate::common::CodegenCx; @@ -331,6 +332,10 @@ impl<'ll> CodegenCx<'ll, '_> { } g + } else if let Some(classname) = fn_attrs.objc_class { + self.get_objc_classref(classname) + } else if let Some(methname) = fn_attrs.objc_selector { + self.get_objc_selref(methname) } else { check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) }; @@ -489,16 +494,7 @@ impl<'ll> CodegenCx<'ll, '_> { let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); let alloc = self.create_metadata(bytes); let data = [section, alloc]; - let meta = - unsafe { llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()) }; - let val = self.get_metadata_value(meta); - unsafe { - llvm::LLVMAddNamedMetadataOperand( - self.llmod, - c"wasm.custom_sections".as_ptr(), - val, - ) - }; + self.module_add_named_metadata_node(self.llmod(), c"wasm.custom_sections", &data); } } else { base::set_link_section(g, attrs); @@ -543,8 +539,225 @@ impl<'ll> CodegenCx<'ll, '_> { /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, /// an array of ptr. - pub(crate) fn add_compiler_used_global(&mut self, global: &'ll Value) { - self.compiler_used_statics.push(global); + pub(crate) fn add_compiler_used_global(&self, global: &'ll Value) { + self.compiler_used_statics.borrow_mut().push(global); + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `CGObjCCommonMac::CreateCStringLiteral`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134 + fn define_objc_classname(&self, classname: &str) -> &'ll Value { + assert_eq!(self.objc_abi_version(), 1); + + let llval = self.null_terminate_const_bytes(classname.as_bytes()); + let llty = self.val_ty(llval); + let sym = self.generate_local_symbol_name("OBJC_CLASS_NAME_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.i8_align); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__TEXT,__cstring,cstring_literals"); + llvm::LLVMSetGlobalConstant(g, llvm::TRUE); + llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); + self.add_compiler_used_global(g); + + g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `ObjCNonFragileABITypesHelper`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L6052 + fn get_objc_class_t(&self) -> &'ll Type { + if let Some(class_t) = self.objc_class_t.get() { + return class_t; + } + + assert_eq!(self.objc_abi_version(), 2); + + // struct _class_t { + // struct _class_t* isa; + // struct _class_t* const superclass; + // void* cache; + // IMP* vtable; + // struct class_ro_t* ro; + // } + + let class_t = self.type_named_struct("struct._class_t"); + let els = [self.type_ptr(); 5]; + let packed = false; + self.set_struct_body(class_t, &els, packed); + + self.objc_class_t.set(Some(class_t)); + class_t + } + + // We do our best here to match what Clang does when compiling Objective-C natively. We + // deduplicate references within a CGU, but we need a reference definition in each referencing + // CGU. All attempts at using external references to a single reference definition result in + // linker errors. + fn get_objc_classref(&self, classname: Symbol) -> &'ll Value { + let mut classrefs = self.objc_classrefs.borrow_mut(); + if let Some(classref) = classrefs.get(&classname).copied() { + return classref; + } + + let g = match self.objc_abi_version() { + 1 => { + // See Clang's `CGObjCMac::EmitClassRefFromId`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5205 + let llval = self.define_objc_classname(classname.as_str()); + let llty = self.type_ptr(); + let sym = self.generate_local_symbol_name("OBJC_CLASS_REFERENCES_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__OBJC,__cls_refs,literal_pointers,no_dead_strip"); + self.add_compiler_used_global(g); + g + } + 2 => { + // See Clang's `CGObjCNonFragileABIMac::EmitClassRefFromId`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7423 + let llval = { + let extern_sym = format!("OBJC_CLASS_$_{}", classname.as_str()); + let extern_llty = self.get_objc_class_t(); + self.declare_global(&extern_sym, extern_llty) + }; + let llty = self.type_ptr(); + let sym = self.generate_local_symbol_name("OBJC_CLASSLIST_REFERENCES_$_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); + llvm::set_section(g, c"__DATA,__objc_classrefs,regular,no_dead_strip"); + self.add_compiler_used_global(g); + g + } + _ => unreachable!(), + }; + + classrefs.insert(classname, g); + g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. We + // deduplicate references within a CGU, but we need a reference definition in each referencing + // CGU. All attempts at using external references to a single reference definition result in + // linker errors. + // + // Newer versions of Apple Clang generate calls to `@"objc_msgSend$methname"` selector stub + // functions. We don't currently do that. The code we generate is closer to what Apple Clang + // generates with the `-fno-objc-msgsend-selector-stubs` option. + fn get_objc_selref(&self, methname: Symbol) -> &'ll Value { + let mut selrefs = self.objc_selrefs.borrow_mut(); + if let Some(selref) = selrefs.get(&methname).copied() { + return selref; + } + + let abi_version = self.objc_abi_version(); + + // See Clang's `CGObjCCommonMac::CreateCStringLiteral`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134 + let methname_llval = self.null_terminate_const_bytes(methname.as_str().as_bytes()); + let methname_llty = self.val_ty(methname_llval); + let methname_sym = self.generate_local_symbol_name("OBJC_METH_VAR_NAME_"); + let methname_g = self.define_global(&methname_sym, methname_llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", methname_sym); + }); + set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align); + llvm::set_initializer(methname_g, methname_llval); + llvm::set_linkage(methname_g, llvm::Linkage::PrivateLinkage); + llvm::set_section( + methname_g, + match abi_version { + 1 => c"__TEXT,__cstring,cstring_literals", + 2 => c"__TEXT,__objc_methname,cstring_literals", + _ => unreachable!(), + }, + ); + llvm::LLVMSetGlobalConstant(methname_g, llvm::TRUE); + llvm::LLVMSetUnnamedAddress(methname_g, llvm::UnnamedAddr::Global); + self.add_compiler_used_global(methname_g); + + // See Clang's `CGObjCMac::EmitSelectorAddr`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5243 + // And Clang's `CGObjCNonFragileABIMac::EmitSelectorAddr`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7586 + let selref_llval = methname_g; + let selref_llty = self.type_ptr(); + let selref_sym = self.generate_local_symbol_name("OBJC_SELECTOR_REFERENCES_"); + let selref_g = self.define_global(&selref_sym, selref_llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", selref_sym); + }); + set_global_alignment(self, selref_g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(selref_g, selref_llval); + llvm::set_externally_initialized(selref_g, true); + llvm::set_linkage( + selref_g, + match abi_version { + 1 => llvm::Linkage::PrivateLinkage, + 2 => llvm::Linkage::InternalLinkage, + _ => unreachable!(), + }, + ); + llvm::set_section( + selref_g, + match abi_version { + 1 => c"__OBJC,__message_refs,literal_pointers,no_dead_strip", + 2 => c"__DATA,__objc_selrefs,literal_pointers,no_dead_strip", + _ => unreachable!(), + }, + ); + self.add_compiler_used_global(selref_g); + + selrefs.insert(methname, selref_g); + selref_g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `ObjCTypesHelper`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5936 + // And Clang's `CGObjCMac::EmitModuleInfo`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5151 + pub(crate) fn define_objc_module_info(&mut self) { + assert_eq!(self.objc_abi_version(), 1); + + // struct _objc_module { + // long version; // Hardcoded to 7 in Clang. + // long size; // sizeof(struct _objc_module) + // char* name; // Hardcoded to classname "" in Clang. + // struct _objc_symtab* symtab; // Null without class or category definitions. + // } + + let llty = self.type_named_struct("struct._objc_module"); + let i32_llty = self.type_i32(); + let ptr_llty = self.type_ptr(); + let packed = false; + self.set_struct_body(llty, &[i32_llty, i32_llty, ptr_llty, ptr_llty], packed); + + let version = self.const_uint(i32_llty, 7); + let size = self.const_uint(i32_llty, 16); + let name = self.define_objc_classname(""); + let symtab = self.const_null(ptr_llty); + let llval = crate::common::named_struct(llty, &[version, size, name, symtab]); + + let sym = "OBJC_MODULES"; + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__OBJC,__module_info,regular,no_dead_strip"); + + self.add_compiler_used_global(g); } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index a69fa54a54a..922575dd63c 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -26,7 +26,7 @@ use rustc_session::config::{ BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet, }; use rustc_span::source_map::Spanned; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; @@ -34,7 +34,7 @@ use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; -use crate::llvm::Metadata; +use crate::llvm::{Metadata, MetadataKindId, Module}; use crate::type_::Type; use crate::value::Value; use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util}; @@ -119,7 +119,7 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Statics that will be placed in the llvm.compiler.used variable /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details - pub compiler_used_statics: Vec<&'ll Value>, + pub compiler_used_statics: RefCell<Vec<&'ll Value>>, /// Mapping of non-scalar types to llvm types. pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>, @@ -146,6 +146,15 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// `global_asm!` needs to be able to find this new global so that it can /// compute the correct mangled symbol name to insert into the asm. pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, + + /// Cached Objective-C class type + pub objc_class_t: Cell<Option<&'ll Type>>, + + /// Cache of Objective-C class references + pub objc_classrefs: RefCell<FxHashMap<Symbol, &'ll Value>>, + + /// Cache of Objective-C selector references + pub objc_selrefs: RefCell<FxHashMap<Symbol, &'ll Value>>, } fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { @@ -172,35 +181,6 @@ pub(crate) unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (20, 0, 0) { - if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { - // LLVM 20 defines three additional address spaces for alternate - // pointer kinds used in Windows. - // See https://github.com/llvm/llvm-project/pull/111879 - target_data_layout = - target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); - } - if sess.target.arch.starts_with("sparc") { - // LLVM 20 updates the sparc layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/106951 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("mips64") { - // LLVM 20 updates the mips64 layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/112084 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("powerpc64") { - // LLVM 20 updates the powerpc64 layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/118004 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("wasm32") || sess.target.arch.starts_with("wasm64") { - // LLVM 20 updates the wasm(32|64) layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/119204 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - } if llvm_version < (21, 0, 0) { if sess.target.arch == "nvptx64" { // LLVM 21 updated the default layout on nvptx: https://github.com/llvm/llvm-project/pull/124961 @@ -390,7 +370,8 @@ pub(crate) unsafe fn create_module<'ll>( ); } - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + if let Some(BranchProtection { bti, pac_ret, gcs }) = sess.opts.unstable_opts.branch_protection + { if sess.target.arch == "aarch64" { llvm::add_module_flag_u32( llmod, @@ -423,6 +404,12 @@ pub(crate) unsafe fn create_module<'ll>( "sign-return-address-with-bkey", u32::from(pac_opts.key == PAuthKey::B), ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "guarded-control-stack", + gcs.into(), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -508,14 +495,7 @@ pub(crate) unsafe fn create_module<'ll>( format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION")); let name_metadata = cx.create_metadata(rustc_producer.as_bytes()); - - unsafe { - llvm::LLVMAddNamedMetadataOperand( - llmod, - c"llvm.ident".as_ptr(), - &cx.get_metadata_value(llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)), - ); - } + cx.module_add_named_metadata_node(llmod, c"llvm.ident", &[name_metadata]); // Emit RISC-V specific target-abi metadata // to workaround lld as the LTO plugin not @@ -644,7 +624,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { const_globals: Default::default(), statics_to_rauw: RefCell::new(Vec::new()), used_statics: Vec::new(), - compiler_used_statics: Vec::new(), + compiler_used_statics: Default::default(), type_lowering: Default::default(), scalar_lltypes: Default::default(), coverage_cx, @@ -655,6 +635,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { intrinsics: Default::default(), local_gen_sym_counter: Cell::new(0), renamed_statics: Default::default(), + objc_class_t: Cell::new(None), + objc_classrefs: Default::default(), + objc_selrefs: Default::default(), }, PhantomData, ) @@ -679,6 +662,69 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); llvm::set_section(g, c"llvm.metadata"); } + + /// The Objective-C ABI that is used. + /// + /// This corresponds to the `-fobjc-abi-version=` flag in Clang / GCC. + pub(crate) fn objc_abi_version(&self) -> u32 { + assert!(self.tcx.sess.target.is_like_darwin); + if self.tcx.sess.target.arch == "x86" && self.tcx.sess.target.os == "macos" { + // 32-bit x86 macOS uses ABI version 1 (a.k.a. the "fragile ABI"). + 1 + } else { + // All other Darwin-like targets we support use ABI version 2 + // (a.k.a the "non-fragile ABI"). + 2 + } + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `CGObjCCommonMac::EmitImageInfo`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5085 + pub(crate) fn add_objc_module_flags(&self) { + let abi_version = self.objc_abi_version(); + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Version", + abi_version, + ); + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Image Info Version", + 0, + ); + + llvm::add_module_flag_str( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Image Info Section", + match abi_version { + 1 => "__OBJC,__image_info,regular", + 2 => "__DATA,__objc_imageinfo,regular,no_dead_strip", + _ => unreachable!(), + }, + ); + + if self.tcx.sess.target.env == "sim" { + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Is Simulated", + 1 << 5, + ); + } + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Class Properties", + 1 << 6, + ); + } } impl<'ll> SimpleCx<'ll> { pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type { @@ -823,7 +869,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); + let target_cpu = attributes::target_cpu_attr(self, self.sess()); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } @@ -838,15 +884,15 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { + if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); + attrs.push(attributes::target_cpu_attr(self, self.sess())); + attrs.extend(attributes::tune_cpu_attr(self, self.sess())); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } @@ -865,7 +911,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn( llfn, llvm::AttributePlace::Function, - attributes::target_features_attr(self, vec![]).as_slice(), + attributes::target_features_attr(self, self.tcx, vec![]).as_slice(), ); Some(llfn) } else { @@ -949,15 +995,75 @@ impl CodegenCx<'_, '_> { } impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { + /// Wrapper for `LLVMMDNodeInContext2`, i.e. `llvm::MDNode::get`. + pub(crate) fn md_node_in_context(&self, md_list: &[&'ll Metadata]) -> &'ll Metadata { + unsafe { llvm::LLVMMDNodeInContext2(self.llcx(), md_list.as_ptr(), md_list.len()) } + } + /// A wrapper for [`llvm::LLVMSetMetadata`], but it takes `Metadata` as a parameter instead of `Value`. pub(crate) fn set_metadata<'a>( &self, val: &'a Value, - kind_id: impl Into<llvm::MetadataKindId>, + kind_id: MetadataKindId, md: &'ll Metadata, ) { let node = self.get_metadata_value(md); - llvm::LLVMSetMetadata(val, kind_id.into(), node); + llvm::LLVMSetMetadata(val, kind_id, node); + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMMetadataAsValue` (to adapt that node to an `llvm::Value`) + /// - `LLVMSetMetadata` (to set that node as metadata of `kind_id` for `instruction`) + pub(crate) fn set_metadata_node( + &self, + instruction: &'ll Value, + kind_id: MetadataKindId, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + self.set_metadata(instruction, kind_id, md); + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMMetadataAsValue` (to adapt that node to an `llvm::Value`) + /// - `LLVMAddNamedMetadataOperand` (to set that node as metadata of `kind_name` for `module`) + pub(crate) fn module_add_named_metadata_node( + &self, + module: &'ll Module, + kind_name: &CStr, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + let md_as_val = self.get_metadata_value(md); + unsafe { llvm::LLVMAddNamedMetadataOperand(module, kind_name.as_ptr(), md_as_val) }; + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMRustGlobalAddMetadata` (to set that node as metadata of `kind_id` for `global`) + pub(crate) fn global_add_metadata_node( + &self, + global: &'ll Value, + kind_id: MetadataKindId, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + unsafe { llvm::LLVMRustGlobalAddMetadata(global, kind_id, md) }; + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMGlobalSetMetadata` (to set that node as metadata of `kind_id` for `global`) + pub(crate) fn global_set_metadata_node( + &self, + global: &'ll Value, + kind_id: MetadataKindId, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + unsafe { llvm::LLVMGlobalSetMetadata(global, kind_id, md) }; } } @@ -991,7 +1097,10 @@ impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::ReferencesError(_) + | LayoutError::InvalidSimd { .. } = err + { self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) @@ -1008,7 +1117,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { match err { - FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + FnAbiError::Layout( + LayoutError::SizeOverflow(_) + | LayoutError::Cycle(_) + | LayoutError::InvalidSimd { .. }, + ) => { self.tcx.dcx().emit_fatal(Spanned { span, node: err }); } _ => match fn_abi_request { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index bc4f6bb6a82..d50eb533ffd 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -2,26 +2,25 @@ use std::ffi::CString; -use crate::common::AsCCharPtr; use crate::coverageinfo::ffi; use crate::llvm; pub(crate) fn covmap_var_name() -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovmapVarNameToString(s); })) .expect("covmap variable name should not contain NUL") } pub(crate) fn covmap_section_name(llmod: &llvm::Module) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovmapSectionNameToString(llmod, s); })) .expect("covmap section name should not contain NUL") } pub(crate) fn covfun_section_name(llmod: &llvm::Module) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovfunSectionNameToString(llmod, s); })) .expect("covfun section name should not contain NUL") @@ -34,7 +33,7 @@ pub(crate) fn create_pgo_func_name_var<'ll>( unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar( llfn, - mangled_fn_name.as_c_char_ptr(), + mangled_fn_name.as_ptr(), mangled_fn_name.len(), ) } @@ -44,7 +43,7 @@ pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef<str>]) -> Vec<u8 let (pointers, lengths) = filenames .into_iter() .map(AsRef::as_ref) - .map(|s: &str| (s.as_c_char_ptr(), s.len())) + .map(|s: &str| (s.as_ptr(), s.len())) .unzip::<_, _, Vec<_>, Vec<_>>(); llvm::build_byte_buffer(|buffer| unsafe { @@ -89,12 +88,12 @@ pub(crate) fn write_function_mappings_to_buffer( /// Hashes some bytes into a 64-bit hash, via LLVM's `IndexedInstrProf::ComputeHash`, /// as required for parts of the LLVM coverage mapping format. pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_c_char_ptr(), bytes.len()) } + unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_ptr(), bytes.len()) } } /// Returns LLVM's `coverage::CovMapVersion::CurrentVersion` (CoverageMapping.h) /// as a raw numeric value. For historical reasons, the numeric value is 1 less /// than the number in the version's name, so `Version7` is actually `6u32`. pub(crate) fn mapping_version() -> u32 { - unsafe { llvm::LLVMRustCoverageMappingVersion() } + llvm::LLVMRustCoverageMappingVersion() } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs b/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs index 40842915222..52d04625749 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs @@ -35,3 +35,6 @@ declare_constant!(DW_OP_plus_uconst: u64); /// Double-checked by a static assertion in `RustWrapper.cpp`. #[allow(non_upper_case_globals)] pub(crate) const DW_OP_LLVM_fragment: u64 = 0x1000; +// It describes the actual value of a source variable which might not exist in registers or in memory. +#[allow(non_upper_case_globals)] +pub(crate) const DW_OP_stack_value: u64 = 0x9f; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 7a6dc008c7b..3081badb821 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -2,9 +2,9 @@ use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::DebuggerVisualizerType; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; use crate::builder::Builder; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index dc3a84b6a15..2f9e7cae54f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -32,9 +32,7 @@ use self::type_map::{DINodeCreationResult, Stub, UniqueTypeId}; use super::CodegenUnitDebugContext; use super::namespace::mangled_name_of_instance; use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; -use super::utils::{ - DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, -}; +use super::utils::{DIB, debug_context, get_namespace_for_item, is_node_local_to_unit}; use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::dwarf_const; use crate::debuginfo::metadata::type_map::build_type_with_children; @@ -119,17 +117,17 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( .try_to_target_usize(cx.tcx) .expect("expected monomorphic const in codegen") as c_longlong; - let subrange = - unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; + let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; + let subscripts = &[subrange]; - let subscripts = create_DIArray(DIB(cx), &[subrange]); let di_node = unsafe { - llvm::LLVMRustDIBuilderCreateArrayType( + llvm::LLVMDIBuilderCreateArrayType( DIB(cx), size.bits(), align.bits() as u32, element_type_di_node, - subscripts, + subscripts.as_ptr(), + subscripts.len() as c_uint, ) }; @@ -175,17 +173,13 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( "ptr_type={ptr_type}, pointee_type={pointee_type}", ); - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - pointer_size.bits(), - pointer_align.abi.bits() as u32, - 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_c_char_ptr(), - ptr_type_debuginfo_name.len(), - ) - }; + let di_node = create_pointer_type( + cx, + pointee_type_di_node, + pointer_size, + pointer_align.abi, + &ptr_type_debuginfo_name, + ); DINodeCreationResult { di_node, already_stored_in_typemap: false } } @@ -233,17 +227,13 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // The data pointer type is a regular, thin pointer, regardless of whether this // is a slice or a trait object. - let data_ptr_type_di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - addr_field.size.bits(), - addr_field.align.abi.bits() as u32, - 0, // Ignore DWARF address space. - std::ptr::null(), - 0, - ) - }; + let data_ptr_type_di_node = create_pointer_type( + cx, + pointee_type_di_node, + addr_field.size, + addr_field.align.abi, + "", + ); smallvec![ build_field_di_node( @@ -318,7 +308,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); - let fn_di_node = create_subroutine_type(cx, create_DIArray(DIB(cx), &signature_di_nodes[..])); + let fn_di_node = create_subroutine_type(cx, &signature_di_nodes[..]); // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); @@ -329,26 +319,44 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( } _ => unreachable!(), }; - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - fn_di_node, - size.bits(), - align.bits() as u32, - 0, // Ignore DWARF address space. - name.as_c_char_ptr(), - name.len(), - ) - }; + let di_node = create_pointer_type(cx, fn_di_node, size, align, &name); DINodeCreationResult::new(di_node, false) } pub(super) fn create_subroutine_type<'ll>( cx: &CodegenCx<'ll, '_>, - signature: &'ll DICompositeType, + signature: &[Option<&'ll llvm::Metadata>], ) -> &'ll DICompositeType { - unsafe { llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(cx), signature) } + unsafe { + llvm::LLVMDIBuilderCreateSubroutineType( + DIB(cx), + None, // ("File" is ignored and has no effect) + signature.as_ptr(), + signature.len() as c_uint, + DIFlags::FlagZero, // (default value) + ) + } +} + +fn create_pointer_type<'ll>( + cx: &CodegenCx<'ll, '_>, + pointee_ty: &'ll llvm::Metadata, + size: Size, + align: Align, + name: &str, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreatePointerType( + DIB(cx), + pointee_ty, + size.bits(), + align.bits() as u32, + 0, // Ignore DWARF address space. + name.as_ptr(), + name.len(), + ) + } } /// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs @@ -469,8 +477,8 @@ pub(crate) fn spanned_type_di_node<'ll, 'tcx>( ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), - AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id, span), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id, span), AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), @@ -813,14 +821,15 @@ fn build_basic_type_di_node<'ll, 'tcx>( }; let typedef_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateTypedef( + llvm::LLVMDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_c_char_ptr(), + typedef_name.as_ptr(), typedef_name.len(), unknown_file_metadata(cx), - 0, - None, + 0, // (no line number) + None, // (no scope) + 0u32, // (no alignment specified) ) }; @@ -834,12 +843,13 @@ fn create_basic_type<'ll, 'tcx>( encoding: u32, ) -> &'ll DIBasicType { unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( + llvm::LLVMDIBuilderCreateBasicType( DIB(cx), - name.as_c_char_ptr(), + name.as_ptr(), name.len(), size.bits(), encoding, + DIFlags::FlagZero, ) } } @@ -1025,15 +1035,15 @@ fn create_member_type<'ll, 'tcx>( type_di_node: &'ll DIType, ) -> &'ll DIType { unsafe { - llvm::LLVMRustDIBuilderCreateMemberType( + llvm::LLVMDIBuilderCreateMemberType( DIB(cx), owner, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, layout.size.bits(), - layout.align.abi.bits() as u32, + layout.align.bits() as u32, offset.bits(), flags, type_di_node, @@ -1066,6 +1076,7 @@ fn visibility_di_flags<'ll, 'tcx>( fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let struct_type = unique_type_id.expect_ty(); let ty::Adt(adt_def, _) = struct_type.kind() else { @@ -1073,7 +1084,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( }; assert!(adt_def.is_struct()); let containing_scope = get_namespace_for_item(cx, adt_def.did()); - let struct_type_and_layout = cx.layout_of(struct_type); + let struct_type_and_layout = cx.spanned_layout_of(struct_type, span); let variant_def = adt_def.non_enum_variant(); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) @@ -1266,6 +1277,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( fn build_union_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let union_type = unique_type_id.expect_ty(); let (union_def_id, variant_def) = match union_type.kind() { @@ -1273,7 +1285,7 @@ fn build_union_type_di_node<'ll, 'tcx>( _ => bug!("build_union_type_di_node on a non-ADT"), }; let containing_scope = get_namespace_for_item(cx, union_def_id); - let union_ty_and_layout = cx.layout_of(union_type); + let union_ty_and_layout = cx.spanned_layout_of(union_type, span); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(union_def_id))) @@ -1595,21 +1607,11 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); - unsafe { - let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; - llvm::LLVMRustGlobalAddMetadata( - vtable, - llvm::MD_type as c_uint, - llvm::LLVMMDNodeInContext2(cx.llcx, v.as_ptr(), v.len()), - ); - let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); - let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); - llvm::LLVMGlobalSetMetadata( - vtable, - llvm::MetadataType::MD_vcall_visibility as c_uint, - vcall_visibility_metadata, - ); - } + let type_ = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; + cx.global_add_metadata_node(vtable, llvm::MD_type, &type_); + + let vcall_visibility = [llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64))]; + cx.global_set_metadata_node(vtable, llvm::MD_vcall_visibility, &vcall_visibility); } /// Creates debug information for the given vtable, which is for the diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index a5c80895741..4ecc3086e1b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; use smallvec::smallvec; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::dwarf_const::DW_TAG_const_type; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; @@ -378,20 +378,17 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_struct_type_wrapper_di_node, None, ), - unsafe { - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), - enum_type_di_node, - TAG_FIELD_NAME.as_c_char_ptr(), - TAG_FIELD_NAME.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - variant_names_type_di_node, - visibility_flags, - Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), - tag_base_type_align.bits() as u32, - ) - } + create_static_member_type( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + visibility_flags, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align, + ), ] } @@ -570,27 +567,28 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( let build_assoc_const = |name: &str, type_di_node_: &'ll DIType, value: u64, - align: Align| unsafe { + align: Align| + -> &'ll llvm::Metadata { // FIXME: Currently we force all DISCR_* values to be u64's as LLDB seems to have // problems inspecting other value types. Since DISCR_* is typically only going to be // directly inspected via the debugger visualizer - which compares it to the `tag` value // (whose type is not modified at all) it shouldn't cause any real problems. let (t_di, align) = if name == ASSOC_CONST_DISCR_NAME { - (type_di_node_, align.bits() as u32) + (type_di_node_, align) } else { let ty_u64 = Ty::new_uint(cx.tcx, ty::UintTy::U64); - (type_di_node(cx, ty_u64), Align::EIGHT.bits() as u32) + (type_di_node(cx, ty_u64), Align::EIGHT) }; // must wrap type in a `const` modifier for LLDB to be able to inspect the value of the member - let field_type = - llvm::LLVMRustDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di); + let field_type = unsafe { + llvm::LLVMDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di) + }; - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), + create_static_member_type( + cx, wrapper_struct_type_di_node, - name.as_c_char_ptr(), - name.len(), + name, unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, field_type, @@ -975,3 +973,30 @@ fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, s .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) } + +fn create_static_member_type<'ll>( + cx: &CodegenCx<'ll, '_>, + scope: &'ll llvm::Metadata, + name: &str, + file: &'ll llvm::Metadata, + line_number: c_uint, + ty: &'ll llvm::Metadata, + flags: DIFlags, + value: Option<&'ll llvm::Value>, + align: Align, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreateStaticMemberType( + DIB(cx), + scope, + name.as_ptr(), + name.len(), + file, + line_number, + ty, + flags, + value, + align.bits() as c_uint, + ) + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 62d38d463ab..1ae6e6e5eec 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -289,7 +289,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( file_metadata, line_number, enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, + enum_type_and_layout.align.bits() as u32, DIFlags::FlagZero, tag_member_di_node, create_DIArray(DIB(cx), &[]), @@ -449,7 +449,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( file_di_node, line_number, enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, + enum_type_and_layout.align.bits() as u32, Size::ZERO.bits(), discr, DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 18a783a348a..37200fdc41a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; +use libc::c_uint; use rustc_abi::{Align, Size, VariantIdx}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; @@ -9,7 +10,7 @@ use rustc_middle::bug; use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt}; use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; use crate::llvm::{self}; @@ -191,7 +192,7 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope: Option<&'ll DIScope>, flags: DIFlags, ) -> StubInfo<'ll, 'tcx> { - let empty_array = create_DIArray(DIB(cx), &[]); + let no_elements: &[Option<&llvm::Metadata>] = &[]; let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); let (file_metadata, line_number) = if let Some(def_location) = def_location { @@ -207,10 +208,10 @@ pub(super) fn stub<'ll, 'tcx>( _ => None, }; unsafe { - llvm::LLVMRustDIBuilderCreateStructType( + llvm::LLVMDIBuilderCreateStructType( DIB(cx), containing_scope, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, @@ -218,28 +219,30 @@ pub(super) fn stub<'ll, 'tcx>( align.bits() as u32, flags, None, - empty_array, - 0, + no_elements.as_ptr(), + no_elements.len() as c_uint, + 0u32, // (Objective-C runtime version; default is 0) vtable_holder, - unique_type_id_str.as_c_char_ptr(), + unique_type_id_str.as_ptr(), unique_type_id_str.len(), ) } } Stub::Union => unsafe { - llvm::LLVMRustDIBuilderCreateUnionType( + llvm::LLVMDIBuilderCreateUnionType( DIB(cx), containing_scope, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, size.bits(), align.bits() as u32, flags, - Some(empty_array), - 0, - unique_type_id_str.as_c_char_ptr(), + no_elements.as_ptr(), + no_elements.len() as c_uint, + 0u32, // (Objective-C runtime version; default is 0) + unique_type_id_str.as_ptr(), unique_type_id_str.len(), ) }, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 79334f7f9fe..c6ad1c2e18e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -52,15 +52,6 @@ mod utils; use self::create_scope_map::compute_mir_scopes; pub(crate) use self::metadata::build_global_var_di_node; -// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were -// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp` -// to decide which C++ API to call. Instead, we should just have two separate -// FFI functions and choose the correct one on the Rust side. -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, @@ -165,7 +156,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { variable_alloca: Self::Value, direct_offset: Size, indirect_offsets: &[Size], - fragment: Option<Range<Size>>, + fragment: &Option<Range<Size>>, ) { use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst}; @@ -174,7 +165,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { if direct_offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); + addr_ops.push(direct_offset.bytes()); + } + for &offset in indirect_offsets { + addr_ops.push(DW_OP_deref); + if offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst); + addr_ops.push(offset.bytes()); + } + } + if let Some(fragment) = fragment { + // `DW_OP_LLVM_fragment` takes as arguments the fragment's + // offset and size, both of them in bits. + addr_ops.push(DW_OP_LLVM_fragment); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); + } + + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; + unsafe { + llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( + di_builder, + variable_alloca, + dbg_var, + addr_expr, + dbg_loc, + self.llbb(), + ) + }; + } + + fn dbg_var_value( + &mut self, + dbg_var: &'ll DIVariable, + dbg_loc: &'ll DILocation, + value: Self::Value, + direct_offset: Size, + indirect_offsets: &[Size], + fragment: &Option<Range<Size>>, + ) { + use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst, DW_OP_stack_value}; + + // Convert the direct and indirect offsets and fragment byte range to address ops. + let mut addr_ops = SmallVec::<[u64; 8]>::new(); + + if direct_offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst); addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(DW_OP_stack_value); } for &offset in indirect_offsets { addr_ops.push(DW_OP_deref); @@ -191,14 +232,16 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { addr_ops.push((fragment.end - fragment.start).bits() as u64); } + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; unsafe { - // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), - variable_alloca, + llvm::LLVMDIBuilderInsertDbgValueRecordAtEnd( + di_builder, + value, dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, + addr_expr, dbg_loc, self.llbb(), ); @@ -349,7 +392,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let file_metadata = file_metadata(self, &loc.file); let function_type_metadata = - create_subroutine_type(self, get_function_signature(self, fn_abi)); + create_subroutine_type(self, &get_function_signature(self, fn_abi)); let mut name = String::with_capacity(64); type_names::push_item_name(tcx, def_id, false, &mut name); @@ -441,9 +484,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - ) -> &'ll DIArray { + ) -> Vec<Option<&'ll llvm::Metadata>> { if cx.sess().opts.debuginfo != DebugInfo::Full { - return create_DIArray(DIB(cx), &[]); + return vec![]; } let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); @@ -484,7 +527,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); } - create_DIArray(DIB(cx), &signature[..]) + signature } fn get_template_parameters<'ll, 'tcx>( @@ -630,28 +673,39 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let type_metadata = spanned_type_di_node(self, variable_type, span); - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; let align = self.align_of(variable_type); let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_c_char_ptr(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bits() as u32, - ) + + match variable_kind { + ArgumentVariable(arg_index) => unsafe { + llvm::LLVMDIBuilderCreateParameterVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + arg_index as c_uint, + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + ) + }, + LocalVariable => unsafe { + llvm::LLVMDIBuilderCreateAutoVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + align.bits() as u32, + ) + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index cc1d504b430..7e1e49310f6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } + unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 960a895a203..36cdb498839 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -76,7 +76,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>( attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attrs.extend(attributes::non_lazy_bind_attr(cx)); + attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess)); attributes::apply_to_llfn(llfn, Function, &attrs); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 85f71f331a4..467655b0bfc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -18,7 +18,6 @@ use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; use rustc_target::callconv::PassMode; -use rustc_target::spec::PanicStrategy; use tracing::debug; use crate::abi::FnAbiLlvmExt; @@ -298,7 +297,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let align = if name == sym::unaligned_volatile_load { 1 } else { - result.layout.align.abi.bytes() as u32 + result.layout.align.bytes() as u32 }; unsafe { llvm::LLVMSetAlignment(load, align); @@ -674,7 +673,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( catch_func: &'ll Value, dest: PlaceRef<'tcx, &'ll Value>, ) { - if bx.sess().panic_strategy() == PanicStrategy::Abort { + if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.call(try_func_ty, None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; @@ -1048,7 +1047,7 @@ fn codegen_emcc_try<'ll, 'tcx>( // create an alloca and pass a pointer to that. let ptr_size = bx.tcx().data_layout.pointer_size(); let ptr_align = bx.tcx().data_layout.pointer_align().abi; - let i8_align = bx.tcx().data_layout.i8_align.abi; + let i8_align = bx.tcx().data_layout.i8_align; // Required in order for there to be no padding between the fields. assert!(i8_align <= ptr_align); let catch_data = bx.alloca(2 * ptr_size, ptr_align); @@ -1208,10 +1207,14 @@ fn codegen_autodiff<'ll, 'tcx>( adjust_activity_to_abi( tcx, - fn_source.ty(tcx, TypingEnv::fully_monomorphized()), + fn_source, + TypingEnv::fully_monomorphized(), &mut diff_attrs.input_activity, ); + let fnc_tree = + rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized())); + // Build body generate_enzyme_call( bx, @@ -1222,6 +1225,7 @@ fn codegen_autodiff<'ll, 'tcx>( &val_arr, diff_attrs.clone(), result, + fnc_tree, ); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 6fb23d09843..2405a25c702 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -45,6 +45,7 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_span::Symbol; +use rustc_target::spec::{RelocModel, TlsModel}; mod abi; mod allocator; @@ -67,6 +68,7 @@ mod llvm_util; mod mono_item; mod type_; mod type_of; +mod typetree; mod va_arg; mod value; @@ -230,6 +232,10 @@ impl CodegenBackend for LlvmCodegenBackend { crate::DEFAULT_LOCALE_RESOURCE } + fn name(&self) -> &'static str { + "llvm" + } + fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited } @@ -244,16 +250,7 @@ impl CodegenBackend for LlvmCodegenBackend { match req.kind { PrintKind::RelocationModels => { writeln!(out, "Available relocation models:").unwrap(); - for name in &[ - "static", - "pic", - "pie", - "dynamic-no-pic", - "ropi", - "rwpi", - "ropi-rwpi", - "default", - ] { + for name in RelocModel::ALL.iter().map(RelocModel::desc).chain(["default"]) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -267,9 +264,7 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:").unwrap(); - for name in - &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] - { + for name in TlsModel::ALL.iter().map(TlsModel::desc) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -359,7 +354,14 @@ impl CodegenBackend for LlvmCodegenBackend { // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, metadata, outputs); + link_binary( + sess, + &LlvmArchiveBuilderBuilder, + codegen_results, + metadata, + outputs, + self.name(), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 56d756e52cc..e63043b2122 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -3,9 +3,36 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; -use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; +use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value}; use crate::llvm::{Bool, Builder}; +// TypeTree types +pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct EnzymeTypeTree { + _unused: [u8; 0], +} + +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub(crate) enum CConcreteType { + DT_Anything = 0, + DT_Integer = 1, + DT_Pointer = 2, + DT_Half = 3, + DT_Float = 4, + DT_Double = 5, + DT_Unknown = 6, + DT_FP128 = 9, +} + +pub(crate) struct TypeTree { + pub(crate) inner: CTypeTreeRef, +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { // Enzyme @@ -59,19 +86,49 @@ pub(crate) enum LLVMRustVerifierFailureAction { LLVMReturnStatusAction = 2, } -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) use self::Enzyme_AD::*; -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) mod Enzyme_AD { use std::ffi::{CString, c_char}; use libc::c_void; + use super::{CConcreteType, CTypeTreeRef, Context}; + unsafe extern "C" { pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char); } + + // TypeTree functions + unsafe extern "C" { + pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef; + pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef); + pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool; + pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64); + pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef); + pub(crate) fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ); + pub(crate) fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ); + pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char; + pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char); + } + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; @@ -134,13 +191,74 @@ pub(crate) mod Enzyme_AD { } } -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) use self::Fallback_AD::*; -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) mod Fallback_AD { #![allow(unused_variables)] + use libc::c_char; + + use super::{CConcreteType, CTypeTreeRef, Context}; + + // TypeTree function fallbacks + pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) { + unimplemented!() + } + pub(crate) fn set_inline(val: bool) { unimplemented!() } @@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD { unimplemented!() } } + +impl TypeTree { + pub(crate) fn new() -> TypeTree { + let inner = unsafe { EnzymeNewTypeTree() }; + TypeTree { inner } + } + + pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree { + let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) }; + TypeTree { inner } + } + + pub(crate) fn merge(self, other: Self) -> Self { + unsafe { + EnzymeMergeTypeTree(self.inner, other.inner); + } + drop(other); + self + } + + #[must_use] + pub(crate) fn shift( + self, + layout: &str, + offset: isize, + max_size: isize, + add_offset: usize, + ) -> Self { + let layout = std::ffi::CString::new(layout).unwrap(); + + unsafe { + EnzymeTypeTreeShiftIndiciesEq( + self.inner, + layout.as_ptr(), + offset as i64, + max_size as i64, + add_offset as u64, + ); + } + + self + } + + pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) { + unsafe { + EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx); + } + } +} + +impl Clone for TypeTree { + fn clone(&self) -> Self { + let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) }; + TypeTree { inner } + } +} + +impl std::fmt::Display for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ptr = unsafe { EnzymeTypeTreeToString(self.inner) }; + let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) }; + match cstr.to_str() { + Ok(x) => write!(f, "{}", x)?, + Err(err) => write!(f, "could not parse: {}", err)?, + } + + // delete C string pointer + unsafe { + EnzymeTypeTreeToStringFree(ptr); + } + + Ok(()) + } +} + +impl std::fmt::Debug for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + <Self as std::fmt::Display>::fmt(self, f) + } +} + +impl Drop for TypeTree { + fn drop(&mut self) { + unsafe { EnzymeFreeTypeTree(self.inner) } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 0679f55ab7f..7fbba029407 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -24,11 +24,12 @@ use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ - DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, - DIFile, DIFlags, DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, - DISubrange, DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, + DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, + DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, + DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind, }; use crate::llvm; +use crate::llvm::MetadataKindId; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. @@ -513,31 +514,6 @@ pub(crate) enum FileType { ObjectFile, } -/// LLVMMetadataType -#[derive(Copy, Clone)] -#[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] -pub(crate) enum MetadataType { - MD_dbg = 0, - MD_tbaa = 1, - MD_prof = 2, - MD_fpmath = 3, - MD_range = 4, - MD_tbaa_struct = 5, - MD_invariant_load = 6, - MD_alias_scope = 7, - MD_noalias = 8, - MD_nontemporal = 9, - MD_mem_parallel_loop_access = 10, - MD_nonnull = 11, - MD_unpredictable = 15, - MD_align = 17, - MD_type = 19, - MD_vcall_visibility = 28, - MD_noundef = 29, - MD_kcfi_type = 36, -} - /// Must match the layout of `LLVMInlineAsmDialect`. #[derive(Copy, Clone, PartialEq)] #[repr(C)] @@ -710,6 +686,7 @@ pub(crate) enum MemoryEffects { None, ReadOnly, InaccessibleMemOnly, + ReadOnlyNotPure, } /// LLVMOpcode @@ -806,6 +783,8 @@ unsafe extern "C" { pub(crate) type Metadata; pub(crate) type BasicBlock; pub(crate) type Comdat; + /// `&'ll DbgRecord` represents `LLVMDbgRecordRef`. + pub(crate) type DbgRecord; } #[repr(C)] pub(crate) struct Builder<'a>(InvariantOpaque<'a>); @@ -890,7 +869,6 @@ pub(crate) mod debuginfo { pub(crate) type DIVariable = DIDescriptor; pub(crate) type DIGlobalVariableExpression = DIDescriptor; pub(crate) type DIArray = DIDescriptor; - pub(crate) type DISubrange = DIDescriptor; pub(crate) type DIEnumerator = DIDescriptor; pub(crate) type DITemplateTypeParameter = DIDescriptor; @@ -1033,16 +1011,6 @@ pub(crate) type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; pub(crate) type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; -#[derive(Copy, Clone)] -#[repr(transparent)] -pub(crate) struct MetadataKindId(c_uint); - -impl From<MetadataType> for MetadataKindId { - fn from(value: MetadataType) -> Self { - Self(value as c_uint) - } -} - unsafe extern "C" { // Create and destroy contexts. pub(crate) fn LLVMContextDispose(C: &'static mut Context); @@ -1052,6 +1020,8 @@ unsafe extern "C" { SLen: c_uint, ) -> MetadataKindId; + pub(crate) fn LLVMDisposeTargetMachine(T: ptr::NonNull<TargetMachine>); + // Create modules. pub(crate) fn LLVMModuleCreateWithNameInContext( ModuleID: *const c_char, @@ -1135,7 +1105,11 @@ unsafe extern "C" { pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub(crate) safe fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: MetadataKindId, Node: &'a Value); - pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub(crate) fn LLVMGlobalSetMetadata<'a>( + Val: &'a Value, + KindID: MetadataKindId, + Metadata: &'a Metadata, + ); pub(crate) safe fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1237,6 +1211,7 @@ unsafe extern "C" { pub(crate) safe fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); pub(crate) safe fn LLVMSetTailCallKind(CallInst: &Value, kind: TailCallKind); + pub(crate) safe fn LLVMSetExternallyInitialized(GlobalVar: &Value, IsExtInit: Bool); // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( @@ -1870,6 +1845,186 @@ unsafe extern "C" { Scope: &'ll Metadata, InlinedAt: Option<&'ll Metadata>, ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateSubroutineType<'ll>( + Builder: &DIBuilder<'ll>, + File: Option<&'ll Metadata>, // (ignored and has no effect) + ParameterTypes: *const Option<&'ll Metadata>, + NumParameterTypes: c_uint, + Flags: DIFlags, // (default is `DIFlags::DIFlagZero`) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateUnionType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Elements: *const Option<&'ll Metadata>, + NumElements: c_uint, + RunTimeLang: c_uint, // (optional Objective-C runtime version; default is 0) + UniqueId: *const c_uchar, // See "PTR_LEN_STR". + UniqueIdLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateArrayType<'ll>( + Builder: &DIBuilder<'ll>, + Size: u64, + Align: u32, + Ty: &'ll Metadata, + Subscripts: *const &'ll Metadata, + NumSubscripts: c_uint, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateBasicType<'ll>( + Builder: &DIBuilder<'ll>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + SizeInBits: u64, + Encoding: c_uint, // (`LLVMDWARFTypeEncoding`) + Flags: DIFlags, // (default is `DIFlags::DIFlagZero`) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreatePointerType<'ll>( + Builder: &DIBuilder<'ll>, + PointeeTy: &'ll Metadata, + SizeInBits: u64, + AlignInBits: u32, + AddressSpace: c_uint, // (optional DWARF address space; default is 0) + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStructType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + DerivedFrom: Option<&'ll Metadata>, + Elements: *const Option<&'ll Metadata>, + NumElements: c_uint, + RunTimeLang: c_uint, // (optional Objective-C runtime version; default is 0) + VTableHolder: Option<&'ll Metadata>, + UniqueId: *const c_uchar, // See "PTR_LEN_STR". + UniqueIdLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStaticMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + Type: &'ll Metadata, + Flags: DIFlags, + ConstantVal: Option<&'ll Value>, + AlignInBits: u32, + ) -> &'ll Metadata; + + /// Creates a "qualified type" in the C/C++ sense, by adding modifiers + /// like `const` or `volatile`. + pub(crate) fn LLVMDIBuilderCreateQualifiedType<'ll>( + Builder: &DIBuilder<'ll>, + Tag: c_uint, // (DWARF tag, e.g. `DW_TAG_const_type`) + Type: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateTypedef<'ll>( + Builder: &DIBuilder<'ll>, + Type: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Scope: Option<&'ll Metadata>, + AlignInBits: u32, // (optional; default is 0) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>( + Builder: &DIBuilder<'ll>, + LowerBound: i64, + Count: i64, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>( + Builder: &DIBuilder<'ll>, + Data: *const Option<&'ll Metadata>, + NumElements: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateExpression<'ll>( + Builder: &DIBuilder<'ll>, + Addr: *const u64, + Length: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Storage: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderInsertDbgValueRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Val: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ArgNo: c_uint, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + ) -> &'ll Metadata; } #[link(name = "llvm-wrapper", kind = "static")] @@ -1883,7 +2038,7 @@ unsafe extern "C" { // Operations on all values pub(crate) fn LLVMRustGlobalAddMetadata<'a>( Val: &'a Value, - KindID: c_uint, + KindID: MetadataKindId, Metadata: &'a Metadata, ); pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; @@ -2080,8 +2235,11 @@ unsafe extern "C" { ConstraintsLen: size_t, ) -> bool; + /// A list of pointer-length strings is passed as two pointer-length slices, + /// one slice containing pointers and one slice containing their corresponding + /// lengths. The implementation will check that both slices have the same length. pub(crate) fn LLVMRustCoverageWriteFilenamesToBuffer( - Filenames: *const *const c_char, + Filenames: *const *const c_uchar, // See "PTR_LEN_STR". FilenamesLen: size_t, Lengths: *const size_t, LengthsLen: size_t, @@ -2104,18 +2262,25 @@ unsafe extern "C" { pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, - FuncName: *const c_char, + FuncName: *const c_uchar, // See "PTR_LEN_STR". FuncNameLen: size_t, ) -> &Value; - pub(crate) fn LLVMRustCoverageHashBytes(Bytes: *const c_char, NumBytes: size_t) -> u64; - - pub(crate) fn LLVMRustCoverageWriteCovmapSectionNameToString(M: &Module, OutStr: &RustString); + pub(crate) fn LLVMRustCoverageHashBytes( + Bytes: *const c_uchar, // See "PTR_LEN_STR". + NumBytes: size_t, + ) -> u64; - pub(crate) fn LLVMRustCoverageWriteCovfunSectionNameToString(M: &Module, OutStr: &RustString); - - pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); + pub(crate) safe fn LLVMRustCoverageWriteCovmapSectionNameToString( + M: &Module, + OutStr: &RustString, + ); + pub(crate) safe fn LLVMRustCoverageWriteCovfunSectionNameToString( + M: &Module, + OutStr: &RustString, + ); + pub(crate) safe fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); - pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) safe fn LLVMRustCoverageMappingVersion() -> u32; pub(crate) fn LLVMRustDebugMetadataVersion() -> u32; pub(crate) fn LLVMRustVersionMajor() -> u32; pub(crate) fn LLVMRustVersionMinor() -> u32; @@ -2172,11 +2337,6 @@ unsafe extern "C" { SourceLen: size_t, ) -> &'a DIFile; - pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>( - Builder: &DIBuilder<'a>, - ParameterTypes: &'a DIArray, - ) -> &'a DICompositeType; - pub(crate) fn LLVMRustDIBuilderCreateFunction<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, @@ -2210,66 +2370,6 @@ unsafe extern "C" { TParam: &'a DIArray, ) -> &'a DISubprogram; - pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>( - Builder: &DIBuilder<'a>, - Name: *const c_char, - NameLen: size_t, - SizeInBits: u64, - Encoding: c_uint, - ) -> &'a DIBasicType; - - pub(crate) fn LLVMRustDIBuilderCreateTypedef<'a>( - Builder: &DIBuilder<'a>, - Type: &'a DIBasicType, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Scope: Option<&'a DIScope>, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>( - Builder: &DIBuilder<'a>, - PointeeTy: &'a DIType, - SizeInBits: u64, - AlignInBits: u32, - AddressSpace: c_uint, - Name: *const c_char, - NameLen: size_t, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateStructType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIDescriptor>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - DerivedFrom: Option<&'a DIType>, - Elements: &'a DIArray, - RunTimeLang: c_uint, - VTableHolder: Option<&'a DIType>, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DICompositeType; - - pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - SizeInBits: u64, - AlignInBits: u32, - OffsetInBits: u64, - Flags: DIFlags, - Ty: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2285,25 +2385,6 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateStaticMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - Flags: DIFlags, - val: Option<&'a Value>, - AlignInBits: u32, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateQualifiedType<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Type: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( Builder: &DIBuilder<'a>, Context: Option<&'a DIScope>, @@ -2320,51 +2401,6 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub(crate) fn LLVMRustDIBuilderCreateArrayType<'a>( - Builder: &DIBuilder<'a>, - Size: u64, - AlignInBits: u32, - Ty: &'a DIType, - Subscripts: &'a DIArray, - ) -> &'a DIType; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ); - pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2388,22 +2424,6 @@ unsafe extern "C" { IsScoped: bool, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateUnionType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - Elements: Option<&'a DIArray>, - RunTimeLang: c_uint, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2480,12 +2500,13 @@ unsafe extern "C" { OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, UseEmulatedTls: bool, - ArgsCstrBuff: *const c_uchar, // See "PTR_LEN_STR". - ArgsCstrBuffLen: usize, + Argv0: *const c_uchar, // See "PTR_LEN_STR". + Argv0Len: size_t, + CommandLineArgs: *const c_uchar, // See "PTR_LEN_STR". + CommandLineArgsLen: size_t, UseWasmEH: bool, ) -> *mut TargetMachine; - pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs b/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs new file mode 100644 index 00000000000..a8a671b5c85 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs @@ -0,0 +1,71 @@ +use libc::c_uint; + +pub(crate) use self::fixed_kinds::*; + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub(crate) struct MetadataKindId(c_uint); + +macro_rules! declare_fixed_metadata_kinds { + ( + $( + FIXED_MD_KIND($variant:ident, $value:literal) + )* + ) => { + // Use a submodule to group all declarations into one `#[expect(..)]`. + #[expect(dead_code)] + mod fixed_kinds { + use super::MetadataKindId; + $( + #[expect(non_upper_case_globals)] + pub(crate) const $variant: MetadataKindId = MetadataKindId($value); + )* + } + }; +} + +// Must be kept in sync with the corresponding static assertions in `RustWrapper.cpp`. +declare_fixed_metadata_kinds! { + FIXED_MD_KIND(MD_dbg, 0) + FIXED_MD_KIND(MD_tbaa, 1) + FIXED_MD_KIND(MD_prof, 2) + FIXED_MD_KIND(MD_fpmath, 3) + FIXED_MD_KIND(MD_range, 4) + FIXED_MD_KIND(MD_tbaa_struct, 5) + FIXED_MD_KIND(MD_invariant_load, 6) + FIXED_MD_KIND(MD_alias_scope, 7) + FIXED_MD_KIND(MD_noalias, 8) + FIXED_MD_KIND(MD_nontemporal, 9) + FIXED_MD_KIND(MD_mem_parallel_loop_access, 10) + FIXED_MD_KIND(MD_nonnull, 11) + FIXED_MD_KIND(MD_dereferenceable, 12) + FIXED_MD_KIND(MD_dereferenceable_or_null, 13) + FIXED_MD_KIND(MD_make_implicit, 14) + FIXED_MD_KIND(MD_unpredictable, 15) + FIXED_MD_KIND(MD_invariant_group, 16) + FIXED_MD_KIND(MD_align, 17) + FIXED_MD_KIND(MD_loop, 18) + FIXED_MD_KIND(MD_type, 19) + FIXED_MD_KIND(MD_section_prefix, 20) + FIXED_MD_KIND(MD_absolute_symbol, 21) + FIXED_MD_KIND(MD_associated, 22) + FIXED_MD_KIND(MD_callees, 23) + FIXED_MD_KIND(MD_irr_loop, 24) + FIXED_MD_KIND(MD_access_group, 25) + FIXED_MD_KIND(MD_callback, 26) + FIXED_MD_KIND(MD_preserve_access_index, 27) + FIXED_MD_KIND(MD_vcall_visibility, 28) + FIXED_MD_KIND(MD_noundef, 29) + FIXED_MD_KIND(MD_annotation, 30) + FIXED_MD_KIND(MD_nosanitize, 31) + FIXED_MD_KIND(MD_func_sanitize, 32) + FIXED_MD_KIND(MD_exclude, 33) + FIXED_MD_KIND(MD_memprof, 34) + FIXED_MD_KIND(MD_callsite, 35) + FIXED_MD_KIND(MD_kcfi_type, 36) + FIXED_MD_KIND(MD_pcsections, 37) + FIXED_MD_KIND(MD_DIAssignID, 38) + FIXED_MD_KIND(MD_coro_outside_frame, 39) + FIXED_MD_KIND(MD_mmra, 40) + FIXED_MD_KIND(MD_noalias_addrspace, 41) +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index d6974e22c85..9a53dacb1df 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -11,13 +11,14 @@ use rustc_llvm::RustString; pub(crate) use self::CallConv::*; pub(crate) use self::CodeGenOptSize::*; -pub(crate) use self::MetadataType::*; pub(crate) use self::ffi::*; +pub(crate) use self::metadata_kind::*; use crate::common::AsCCharPtr; pub(crate) mod diagnostic; pub(crate) mod enzyme_ffi; mod ffi; +mod metadata_kind; pub(crate) use self::enzyme_ffi::*; @@ -258,6 +259,10 @@ pub(crate) fn set_alignment(llglobal: &Value, align: Align) { } } +pub(crate) fn set_externally_initialized(llglobal: &Value, is_ext_init: bool) { + LLVMSetExternallyInitialized(llglobal, is_ext_init.to_llvm_bool()); +} + /// Get the `name`d comdat from `llmod` and assign it to `llglobal`. /// /// Inserts the comdat into `llmod` if it does not exist. diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index d927ffd78c2..3b920168e06 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -106,7 +106,7 @@ unsafe fn configure_llvm(sess: &Session) { if sess.target.os == "emscripten" && !sess.opts.unstable_opts.emscripten_wasm_eh - && sess.panic_strategy() == PanicStrategy::Unwind + && sess.panic_strategy().unwinds() { add("-enable-emscripten-cxx-exceptions", false); } @@ -217,27 +217,16 @@ impl<'a> IntoIterator for LLVMFeature<'a> { /// Rust can also be build with an external precompiled version of LLVM which might lead to failures /// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics. pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> { - let arch = if sess.target.arch == "x86_64" { - "x86" - } else if sess.target.arch == "arm64ec" { - "aarch64" - } else if sess.target.arch == "sparc64" { - "sparc" - } else if sess.target.arch == "powerpc64" { - "powerpc" - } else { - &*sess.target.arch + let raw_arch = &*sess.target.arch; + let arch = match raw_arch { + "x86_64" => "x86", + "arm64ec" => "aarch64", + "sparc64" => "sparc", + "powerpc64" => "powerpc", + _ => raw_arch, }; + let (major, _, _) = get_version(); match (arch, s) { - ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( - "sse4.2", - smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], - )), - ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), - ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), - ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), - ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), - ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")), ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")), ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")), @@ -246,9 +235,6 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - // Before LLVM 20 those two features were packaged together as b16b16 - ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), - ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => Some(LLVMFeature::with_dependencies( @@ -262,57 +248,26 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea // Filter out features that are not supported by the current LLVM version ("aarch64", "fpmr") => None, // only existed in 18 ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), - // NVPTX targets added in LLVM 20 - ("nvptx64", "sm_100") if get_version().0 < 20 => None, - ("nvptx64", "sm_100a") if get_version().0 < 20 => None, - ("nvptx64", "sm_101") if get_version().0 < 20 => None, - ("nvptx64", "sm_101a") if get_version().0 < 20 => None, - ("nvptx64", "sm_120") if get_version().0 < 20 => None, - ("nvptx64", "sm_120a") if get_version().0 < 20 => None, - ("nvptx64", "ptx86") if get_version().0 < 20 => None, - ("nvptx64", "ptx87") if get_version().0 < 20 => None, // Filter out features that are not supported by the current LLVM version - ("loongarch64", "div32" | "lam-bh" | "lamcas" | "ld-seq-sa" | "scq") - if get_version().0 < 20 => - { - None - } - ("loongarch32" | "loongarch64", "32s") if get_version().0 < 21 => None, - // Filter out features that are not supported by the current LLVM version - ("riscv32" | "riscv64", "zacas" | "rva23u64" | "supm") if get_version().0 < 20 => None, - ( - "s390x", - "message-security-assist-extension12" - | "concurrent-functions" - | "miscellaneous-extensions-4" - | "vector-enhancements-3" - | "vector-packed-decimal-enhancement-3", - ) if get_version().0 < 20 => None, + ("loongarch32" | "loongarch64", "32s") if major < 21 => None, + ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), + ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), + ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( + "sse4.2", + smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], + )), + ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), + ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), + ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), + ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), + ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies( s, smallvec![TargetFeatureFoldStrength::EnableOnly("evex512")], )), - // Support for `wide-arithmetic` will first land in LLVM 20 as part of - // llvm/llvm-project#111598 - ("wasm32" | "wasm64", "wide-arithmetic") if get_version() < (20, 0, 0) => None, - ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), - // In LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available and SPARC-V8+ ABI used". - // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L27-L28 - // Before LLVM 19, there was no `v8plus` feature and `v9` means "SPARC-V9 instruction available". - // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 - ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), - ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), - // These new `amx` variants and `movrs` were introduced in LLVM20 - ("x86", "amx-avx512" | "amx-fp8" | "amx-movrs" | "amx-tf32" | "amx-transpose") - if get_version().0 < 20 => - { - None - } - ("x86", "movrs") if get_version().0 < 20 => None, ("x86", "avx10.1") => Some(LLVMFeature::new("avx10.1-512")), - ("x86", "avx10.2") if get_version().0 < 20 => None, - ("x86", "avx10.2") if get_version().0 >= 20 => Some(LLVMFeature::new("avx10.2-512")), + ("x86", "avx10.2") => Some(LLVMFeature::new("avx10.2-512")), ("x86", "apxf") => Some(LLVMFeature::with_dependencies( "egpr", smallvec![ @@ -716,17 +671,7 @@ pub(crate) fn global_llvm_features( }; // Features implied by an implicit or explicit `--target`. - features.extend( - sess.target - .features - .split(',') - .filter(|v| !v.is_empty()) - // Drop +v8plus feature introduced in LLVM 20. - // (Hard-coded target features do not go through `to_llvm_feature` since they already - // are LLVM feature names, hence we need a special case here.) - .filter(|v| *v != "+v8plus" || get_version() >= (20, 0, 0)) - .map(String::from), - ); + features.extend(sess.target.features.split(',').filter(|v| !v.is_empty()).map(String::from)); if wants_wasm_eh(sess) && sess.panic_strategy() == PanicStrategy::Unwind { features.push("+exception-handling".into()); diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 9ecaf5f24fe..1e5cf8374e3 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -302,26 +302,14 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn add_type_metadata(&self, function: &'ll Value, typeid: &[u8]) { let typeid_metadata = self.create_metadata(typeid); - unsafe { - let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; - llvm::LLVMRustGlobalAddMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()), - ) - } + let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; + self.global_add_metadata_node(function, llvm::MD_type, &v); } fn set_type_metadata(&self, function: &'ll Value, typeid: &[u8]) { let typeid_metadata = self.create_metadata(typeid); - unsafe { - let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()), - ) - } + let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; + self.global_set_metadata_node(function, llvm::MD_type, &v); } fn typeid_metadata(&self, typeid: &[u8]) -> Option<&'ll Metadata> { @@ -329,32 +317,12 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { - let kcfi_type_metadata = self.const_u32(kcfi_typeid); - unsafe { - llvm::LLVMRustGlobalAddMetadata( - function, - llvm::MD_kcfi_type as c_uint, - llvm::LLVMMDNodeInContext2( - self.llcx, - &llvm::LLVMValueAsMetadata(kcfi_type_metadata), - 1, - ), - ) - } + let kcfi_type_metadata = [llvm::LLVMValueAsMetadata(self.const_u32(kcfi_typeid))]; + self.global_add_metadata_node(function, llvm::MD_kcfi_type, &kcfi_type_metadata); } fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { - let kcfi_type_metadata = self.const_u32(kcfi_typeid); - unsafe { - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_kcfi_type as c_uint, - llvm::LLVMMDNodeInContext2( - self.llcx, - &llvm::LLVMValueAsMetadata(kcfi_type_metadata), - 1, - ), - ) - } + let kcfi_type_metadata = [llvm::LLVMValueAsMetadata(self.const_u32(kcfi_typeid))]; + self.global_set_metadata_node(function, llvm::MD_kcfi_type, &kcfi_type_metadata); } } diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs new file mode 100644 index 00000000000..7e263503700 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -0,0 +1,122 @@ +use rustc_ast::expand::typetree::FncTree; +#[cfg(feature = "llvm_enzyme")] +use { + crate::attributes, + rustc_ast::expand::typetree::TypeTree as RustTypeTree, + std::ffi::{CString, c_char, c_uint}, +}; + +use crate::llvm::{self, Value}; + +#[cfg(feature = "llvm_enzyme")] +fn to_enzyme_typetree( + rust_typetree: RustTypeTree, + _data_layout: &str, + llcx: &llvm::Context, +) -> llvm::TypeTree { + let mut enzyme_tt = llvm::TypeTree::new(); + process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); + enzyme_tt +} +#[cfg(feature = "llvm_enzyme")] +fn process_typetree_recursive( + enzyme_tt: &mut llvm::TypeTree, + rust_typetree: &RustTypeTree, + parent_indices: &[i64], + llcx: &llvm::Context, +) { + for rust_type in &rust_typetree.0 { + let concrete_type = match rust_type.kind { + rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything, + rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer, + rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer, + rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half, + rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float, + rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double, + rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128, + rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown, + }; + + let mut indices = parent_indices.to_vec(); + if !parent_indices.is_empty() { + indices.push(rust_type.offset as i64); + } else if rust_type.offset == -1 { + indices.push(-1); + } else { + indices.push(rust_type.offset as i64); + } + + enzyme_tt.insert(&indices, concrete_type, llcx); + + if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer + && !rust_type.child.0.is_empty() + { + process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx); + } + } +} + +#[cfg(feature = "llvm_enzyme")] +pub(crate) fn add_tt<'ll>( + llmod: &'ll llvm::Module, + llcx: &'ll llvm::Context, + fn_def: &'ll Value, + tt: FncTree, +) { + let inputs = tt.args; + let ret_tt: RustTypeTree = tt.ret; + + let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) }; + let llvm_data_layout = + std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + let attr_name = "enzyme_type"; + let c_attr_name = CString::new(attr_name).unwrap(); + + for (i, input) in inputs.iter().enumerate() { + unsafe { + let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } + } + + unsafe { + let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let ret_attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } +} + +#[cfg(not(feature = "llvm_enzyme"))] +pub(crate) fn add_tt<'ll>( + _llmod: &'ll llvm::Module, + _llcx: &'ll llvm::Context, + _fn_def: &'ll Value, + _tt: FncTree, +) { + unimplemented!() +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ab08125217f..234366e491c 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -193,7 +193,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( // the offset again. bx.switch_to_block(maybe_reg); - if gr_type && layout.align.abi.bytes() > 8 { + if gr_type && layout.align.bytes() > 8 { reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); } @@ -291,7 +291,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( bx.inbounds_ptradd(va_list_addr, bx.const_usize(1)) // fpr }; - let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align.abi); + let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align); // "Align" the register count when the type is passed as `i64`. if is_i64 || (is_f64 && is_soft_float_abi) { @@ -329,7 +329,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( // Increase the used-register count. let reg_incr = if is_i64 || (is_f64 && is_soft_float_abi) { 2 } else { 1 }; let new_num_regs = bx.add(num_regs, bx.cx.const_u8(reg_incr)); - bx.store(new_num_regs, num_regs_addr, dl.i8_align.abi); + bx.store(new_num_regs, num_regs_addr, dl.i8_align); bx.br(end); @@ -339,7 +339,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( let mem_addr = { bx.switch_to_block(in_mem); - bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align.abi); + bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align); // Everything in the overflow area is rounded up to a size of at least 4. let overflow_area_align = Align::from_bytes(4).unwrap(); @@ -738,6 +738,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>( src_align, bx.const_u32(layout.layout.size().bytes() as u32), MemFlags::empty(), + None, ); tmp } else { @@ -760,7 +761,7 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>( // byte boundary if alignment needed by type exceeds 8 byte boundary. // It isn't stated explicitly in the standard, but in practice we use // alignment greater than 16 where necessary. - if layout.layout.align.abi.bytes() > 8 { + if layout.layout.align.bytes() > 8 { unreachable!("all instances of VaArgSafe have an alignment <= 8"); } @@ -813,7 +814,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( let va_ndx_offset = va_reg_offset + 4; let offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_ndx_offset)); - let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi); + let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align); let offset = round_up_to_alignment(bx, offset, layout.align.abi); let slot_size = layout.size.align_to(Align::from_bytes(4).unwrap()).bytes() as i32; |