diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
26 files changed, 773 insertions, 292 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 11be7041167..1703cab942b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -246,6 +246,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); } @@ -538,7 +539,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the declaration has an associated instance, compute extra attributes based on that. if let Some(instance) = instance { - llfn_attrs_from_instance(cx, llfn, instance); + llfn_attrs_from_instance( + cx, + cx.tcx, + llfn, + &cx.tcx.codegen_instance_attrs(instance.def), + Some(instance), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index df3e49279d9..896d6755c75 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -5,15 +5,17 @@ use rustc_ast::expand::allocator::{ }; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; +use rustc_span::sym; use rustc_symbol_mangling::mangle_internal_symbol; -use smallvec::SmallVec; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; use crate::llvm::{self, FALSE, TRUE, Type, Value}; -use crate::{SimpleCx, attributes, debuginfo, llvm_util}; +use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, @@ -58,7 +60,26 @@ pub(crate) unsafe fn codegen( let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); - create_wrapper_function(tcx, &cx, &from_name, Some(&to_name), &args, output, false); + let alloc_attr_flag = match method.name { + sym::alloc => CodegenFnAttrFlags::ALLOCATOR, + sym::dealloc => CodegenFnAttrFlags::DEALLOCATOR, + sym::realloc => CodegenFnAttrFlags::REALLOCATOR, + sym::alloc_zeroed => CodegenFnAttrFlags::ALLOCATOR_ZEROED, + _ => unreachable!("Unknown allocator method!"), + }; + + let mut attrs = CodegenFnAttrs::new(); + attrs.flags |= alloc_attr_flag; + create_wrapper_function( + tcx, + &cx, + &from_name, + Some(&to_name), + &args, + output, + false, + &attrs, + ); } } @@ -71,6 +92,7 @@ pub(crate) unsafe fn codegen( &[usize, usize], // size, align None, true, + &CodegenFnAttrs::new(), ); unsafe { @@ -92,6 +114,7 @@ pub(crate) unsafe fn codegen( &[], None, false, + &CodegenFnAttrs::new(), ); } @@ -138,6 +161,7 @@ fn create_wrapper_function( args: &[&Type], output: Option<&Type>, no_return: bool, + attrs: &CodegenFnAttrs, ) { let ty = cx.type_func(args, output.unwrap_or_else(|| cx.type_void())); let llfn = declare_simple_fn( @@ -149,18 +173,7 @@ fn create_wrapper_function( ty, ); - let mut attrs = SmallVec::<[_; 2]>::new(); - - let target_cpu = llvm_util::target_cpu(tcx.sess); - let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu); - - let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess) - .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)); - - attrs.push(target_cpu_attr); - attrs.extend(tune_cpu_attr); - - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + llfn_attrs_from_instance(cx, tcx, llfn, attrs, None); let no_return = if no_return { // -> ! DIFlagNoReturn @@ -171,12 +184,6 @@ fn create_wrapper_function( None }; - if tcx.sess.must_emit_unwind_tables() { - let uwtable = - attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; let mut bx = SBuilder::build(&cx, llbb); diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index dcf6b945497..8070ea0b3e9 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,20 +1,21 @@ //! Set and unset common attributes on LLVM values. -use rustc_codegen_ssa::traits::*; use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; +use rustc_middle::middle::codegen_fn_attrs::{ + CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, +}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; -use crate::context::CodegenCx; +use crate::context::SimpleCx; use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; -use crate::{attributes, llvm_util}; +use crate::{Session, attributes, llvm_util}; pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { @@ -30,18 +31,19 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ /// Get LLVM attribute for the provided inline heuristic. pub(crate) fn inline_attr<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, instance: ty::Instance<'tcx>, ) -> Option<&'ll Attribute> { // `optnone` requires `noinline` - let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id()); let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, + (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint, (inline, _) => inline, }; - if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + if !tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); } @@ -51,7 +53,7 @@ pub(crate) fn inline_attr<'ll, 'tcx>( Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) } InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { + if tcx.sess.target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) } else { None @@ -63,12 +65,13 @@ pub(crate) fn inline_attr<'ll, 'tcx>( #[inline] fn patchable_function_entry_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, attr: Option<PatchableFunctionEntry>, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); let patchable_spec = attr.unwrap_or_else(|| { - PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry) + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry) }); let entry = patchable_spec.entry(); let prefix = patchable_spec.prefix(); @@ -91,12 +94,13 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub(crate) fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn sanitize_attrs<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -114,11 +118,11 @@ pub(crate) fn sanitize_attrs<'ll>( } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); + let features = tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); + tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -139,9 +143,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) llvm::CreateUWTableAttr(llcx, async_unwind) } -pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - let opts = &cx.sess().opts; +pub(crate) fn frame_pointer_type_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { + let mut fp = sess.target.frame_pointer; + let opts = &sess.opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. if opts.unstable_opts.instrument_mcount { @@ -156,8 +163,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&' Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) } -fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let function_return_attr = match cx.sess().opts.unstable_opts.function_return { +fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let function_return_attr = match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => return None, FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern, }; @@ -167,17 +174,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { +fn instrument_function_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - if cx.sess().opts.unstable_opts.instrument_mcount { + if sess.opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + let mcount_name = match &sess.target.llvm_mcount_intrinsic { Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), - None => cx.sess().target.mcount.as_ref(), + None => sess.target.mcount.as_ref(), }; attrs.push(llvm::CreateAttrStringValue( @@ -186,7 +196,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr mcount_name, )); } - if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + if let Some(options) = &sess.opts.unstable_opts.instrument_xray { // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. // Function prologue and epilogue are instrumented with NOP sleds, // a runtime library later replaces them with detours into tracing code. @@ -217,20 +227,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr attrs } -fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if !cx.sess().opts.unstable_opts.no_jump_tables { +fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if !sess.opts.unstable_opts.no_jump_tables { return None; } Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() + if tcx + .sess .opts .unstable_opts .sanitizer @@ -240,22 +250,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { + if tcx.sess.opts.cg.profile_generate.enabled() { return None; } - let attr_value = match cx.sess().target.stack_probes { + let attr_value = match tcx.sess.target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). StackProbeType::Inline => "inline-asm", // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"), + StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - &mangle_internal_symbol(cx.tcx, "__rust_probestack") + &mangle_internal_symbol(tcx, "__rust_probestack") } else { "inline-asm" } @@ -264,8 +274,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) } -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { +fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let sspattr = match sess.stack_protector() { StackProtector::None => return None, StackProtector::All => AttributeKind::StackProtectReq, StackProtector::Strong => AttributeKind::StackProtectStrong, @@ -275,33 +285,34 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(sspattr.create_attr(cx.llcx)) } -fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().target.arch != "s390x" { +fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if sess.target.arch != "s390x" { return None; } - let requested_features = cx.sess().opts.cg.target_feature.split(','); + let requested_features = sess.opts.cg.target_feature.split(','); let found_positive = requested_features.clone().any(|r| r == "+backchain"); if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); +pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) +pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } /// Get the `target-features` LLVM attribute. -pub(crate) fn target_features_attr<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn target_features_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, function_features: Vec<String>, ) -> Option<&'ll Attribute> { - let global_features = cx.tcx.global_backend_features(()).iter().map(String::as_str); + let global_features = tcx.global_backend_features(()).iter().map(String::as_str); let function_features = function_features.iter().map(String::as_str); let target_features = global_features.chain(function_features).intersperse(",").collect::<String>(); @@ -311,22 +322,22 @@ pub(crate) fn target_features_attr<'ll>( /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } + if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None } } /// Get the default optimizations attrs for a function. #[inline] pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { + match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } @@ -347,17 +358,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, llfn: &'ll Value, - instance: ty::Instance<'tcx>, + codegen_fn_attrs: &CodegenFnAttrs, + instance: Option<ty::Instance<'tcx>>, ) { - let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); - + let sess = tcx.sess; let mut to_add = SmallVec::<[_; 16]>::new(); match codegen_fn_attrs.optimize { OptimizeAttr::Default => { - to_add.extend(default_optimisation_attrs(cx)); + to_add.extend(default_optimisation_attrs(cx, sess)); } OptimizeAttr::DoNotOptimize => { to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx)); @@ -369,21 +381,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); + if sess.must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind)); } - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + if sess.opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } // FIXME: none of these functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(function_return_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(nojumptables_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); + to_add.extend(frame_pointer_type_attr(cx, sess)); + to_add.extend(function_return_attr(cx, sess)); + to_add.extend(instrument_function_attr(cx, sess)); + to_add.extend(nojumptables_attr(cx, sess)); + to_add.extend(probestack_attr(cx, tcx)); + to_add.extend(stackprotector_attr(cx, sess)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); @@ -404,13 +416,13 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // not used. } else { // Do not set sanitizer attributes for naked functions. - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize)); // For non-naked functions, set branch protection attributes on aarch64. if let Some(BranchProtection { bti, pac_ret, gcs }) = - cx.sess().opts.unstable_opts.branch_protection + sess.opts.unstable_opts.branch_protection { - assert!(cx.sess().target.arch == "aarch64"); + assert!(sess.target.arch == "aarch64"); if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } @@ -438,14 +450,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); - if let Some(zv) = - cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + if let Some(instance) = instance + && let Some(zv) = + tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) && let Some(name) = zv.value_str() { to_add.push(llvm::CreateAttrStringValue( cx.llcx, "alloc-variant-zeroed", - &mangle_internal_symbol(cx.tcx, name.as_str()), + &mangle_internal_symbol(tcx, name.as_str()), )); } // apply to argument place instead of function @@ -490,18 +503,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align); } - if let Some(backchain) = backchain_attr(cx) { + if let Some(backchain) = backchain_attr(cx, sess) { to_add.push(backchain); } - to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); + to_add.extend(patchable_function_entry_attrs( + cx, + sess, + codegen_fn_attrs.patchable_function_entry, + )); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); + to_add.push(target_cpu_attr(cx, sess)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); + to_add.extend(tune_cpu_attr(cx, sess)); let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); @@ -509,7 +526,9 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // Apply function attributes as per usual if there are no user defined // target features otherwise this will get applied at the callsite. if function_features.is_empty() { - if let Some(inline_attr) = inline_attr(cx, instance) { + if let Some(instance) = instance + && let Some(inline_attr) = inline_attr(cx, tcx, instance) + { to_add.push(inline_attr); } } @@ -517,7 +536,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones - .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + .flat_map(|feat| llvm_util::to_llvm_features(sess, feat)) // Convert LLVMFeatures & dependencies to +<feats>s .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { @@ -526,20 +545,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( })) .collect::<Vec<String>>(); - if cx.tcx.sess.target.is_like_wasm { + if sess.target.is_like_wasm { // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + if let Some(instance) = instance + && let Some(module) = wasm_import_module(tcx, instance.def_id()) + { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.symbol_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } } - to_add.extend(target_features_attr(cx, function_features)); + to_add.extend(target_features_attr(cx, tcx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 78107d95e5a..5ac3a87c158 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -563,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { config::AutoDiff::Enable => {} // We handle this below config::AutoDiff::NoPostopt => {} + // Disables TypeTree generation + config::AutoDiff::NoTT => {} } } // This helps with handling enums for now. diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index d5228f0e0de..903a882916e 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -95,8 +95,6 @@ impl Drop for OwnedTargetMachine { // SAFETY: constructing ensures we have a valid pointer created by // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. - unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_ptr()); - } + unsafe { llvm::LLVMDisposeTargetMachine(self.tm_unique) }; } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 978134cc32b..6d12b511e9c 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -105,7 +105,7 @@ pub(crate) fn compile_codegen_unit( if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty()); attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0f17cc9063a..5271d0b4bb8 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,6 +2,7 @@ use std::borrow::{Borrow, Cow}; use std::ops::Deref; use std::{iter, ptr}; +use rustc_ast::expand::typetree::FncTree; pub(crate) mod autodiff; pub(crate) mod gpu_offload; @@ -1107,11 +1108,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + tt: Option<FncTree>, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - unsafe { + let memcpy = unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, dst, @@ -1120,7 +1122,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align.bytes() as c_uint, size, is_volatile, - ); + ) + }; + + // TypeTree metadata for memcpy is especially important: when Enzyme encounters + // a memcpy during autodiff, it needs to know the structure of the data being + // copied to properly track derivatives. For example, copying an array of floats + // vs. copying a struct with mixed types requires different derivative handling. + // The TypeTree tells Enzyme exactly what memory layout to expect. + if let Some(tt) = tt { + crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt); } } @@ -1433,7 +1444,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // If there is an inline attribute and a target feature that matches // we will add the attribute to the callsite otherwise we'll omit // this and not add the attribute to prevent soundness issues. - && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance) + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance) && self.cx.tcx.is_target_feature_call_safe( &fn_call_attrs.target_features, &fn_defn_attrs.target_features, diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index b66e3dfdeec..4a749642265 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,6 +1,7 @@ use std::ptr; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; +use rustc_ast::expand::typetree::FncTree; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv}; @@ -294,6 +295,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args: &[&'ll Value], attrs: AutoDiffAttrs, dest: PlaceRef<'tcx, &'ll Value>, + fnc_tree: FncTree, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -370,7 +372,18 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args, ); + if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() { + crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree); + } + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); - builder.store_to_place(call, dest.val); + let fn_ret_ty = builder.cx.val_ty(call); + if fn_ret_ty != builder.cx.type_void() && fn_ret_ty != builder.cx.type_struct(&[], false) { + // If we return void or an empty struct, then our caller (due to how we generated it) + // does not expect a return value. As such, we have no pointer (or place) into which + // we could store our value, and would store into an undef, which would cause UB. + // As such, we just ignore the return value in those cases. + builder.store_to_place(call, dest.val); + } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index a110ecbb75d..40375ef6510 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -564,7 +564,7 @@ impl<'ll> CodegenCx<'ll, '_> { let g = self.define_global(&sym, llty).unwrap_or_else(|| { bug!("symbol `{}` is already defined", sym); }); - set_global_alignment(self, g, self.tcx.data_layout.i8_align.abi); + set_global_alignment(self, g, self.tcx.data_layout.i8_align); llvm::set_initializer(g, llval); llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); llvm::set_section(g, c"__TEXT,__cstring,cstring_literals"); @@ -680,7 +680,7 @@ impl<'ll> CodegenCx<'ll, '_> { let methname_g = self.define_global(&methname_sym, methname_llty).unwrap_or_else(|| { bug!("symbol `{}` is already defined", methname_sym); }); - set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align.abi); + set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align); llvm::set_initializer(methname_g, methname_llval); llvm::set_linkage(methname_g, llvm::Linkage::PrivateLinkage); llvm::set_section( diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 057f525c76f..b1da6f7c740 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -34,7 +34,7 @@ use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; -use crate::llvm::Metadata; +use crate::llvm::{Metadata, MetadataKindId}; use crate::type_::Type; use crate::value::Value; use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util}; @@ -876,7 +876,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); + let target_cpu = attributes::target_cpu_attr(self, self.sess()); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } @@ -891,15 +891,15 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { + if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); + attrs.push(attributes::target_cpu_attr(self, self.sess())); + attrs.extend(attributes::tune_cpu_attr(self, self.sess())); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } @@ -918,7 +918,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn( llfn, llvm::AttributePlace::Function, - attributes::target_features_attr(self, vec![]).as_slice(), + attributes::target_features_attr(self, self.tcx, vec![]).as_slice(), ); Some(llfn) } else { @@ -1006,11 +1006,11 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { pub(crate) fn set_metadata<'a>( &self, val: &'a Value, - kind_id: impl Into<llvm::MetadataKindId>, + kind_id: MetadataKindId, md: &'ll Metadata, ) { let node = self.get_metadata_value(md); - llvm::LLVMSetMetadata(val, kind_id.into(), node); + llvm::LLVMSetMetadata(val, kind_id, node); } } @@ -1044,7 +1044,10 @@ impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::ReferencesError(_) + | LayoutError::InvalidSimd { .. } = err + { self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) @@ -1061,7 +1064,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { match err { - FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + FnAbiError::Layout( + LayoutError::SizeOverflow(_) + | LayoutError::Cycle(_) + | LayoutError::InvalidSimd { .. }, + ) => { self.tcx.dcx().emit_fatal(Spanned { span, node: err }); } _ => match fn_abi_request { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index bc4f6bb6a82..d50eb533ffd 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -2,26 +2,25 @@ use std::ffi::CString; -use crate::common::AsCCharPtr; use crate::coverageinfo::ffi; use crate::llvm; pub(crate) fn covmap_var_name() -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovmapVarNameToString(s); })) .expect("covmap variable name should not contain NUL") } pub(crate) fn covmap_section_name(llmod: &llvm::Module) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovmapSectionNameToString(llmod, s); })) .expect("covmap section name should not contain NUL") } pub(crate) fn covfun_section_name(llmod: &llvm::Module) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovfunSectionNameToString(llmod, s); })) .expect("covfun section name should not contain NUL") @@ -34,7 +33,7 @@ pub(crate) fn create_pgo_func_name_var<'ll>( unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar( llfn, - mangled_fn_name.as_c_char_ptr(), + mangled_fn_name.as_ptr(), mangled_fn_name.len(), ) } @@ -44,7 +43,7 @@ pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef<str>]) -> Vec<u8 let (pointers, lengths) = filenames .into_iter() .map(AsRef::as_ref) - .map(|s: &str| (s.as_c_char_ptr(), s.len())) + .map(|s: &str| (s.as_ptr(), s.len())) .unzip::<_, _, Vec<_>, Vec<_>>(); llvm::build_byte_buffer(|buffer| unsafe { @@ -89,12 +88,12 @@ pub(crate) fn write_function_mappings_to_buffer( /// Hashes some bytes into a 64-bit hash, via LLVM's `IndexedInstrProf::ComputeHash`, /// as required for parts of the LLVM coverage mapping format. pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_c_char_ptr(), bytes.len()) } + unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_ptr(), bytes.len()) } } /// Returns LLVM's `coverage::CovMapVersion::CurrentVersion` (CoverageMapping.h) /// as a raw numeric value. For historical reasons, the numeric value is 1 less /// than the number in the version's name, so `Version7` is actually `6u32`. pub(crate) fn mapping_version() -> u32 { - unsafe { llvm::LLVMRustCoverageMappingVersion() } + llvm::LLVMRustCoverageMappingVersion() } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 7a6dc008c7b..3081badb821 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -2,9 +2,9 @@ use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::DebuggerVisualizerType; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; use crate::builder::Builder; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index aa8b8bd152d..bc20c759413 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -117,7 +117,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( .try_to_target_usize(cx.tcx) .expect("expected monomorphic const in codegen") as c_longlong; - let subrange = unsafe { llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; + let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; let subscripts = &[subrange]; let di_node = unsafe { @@ -477,8 +477,8 @@ pub(crate) fn spanned_type_di_node<'ll, 'tcx>( ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), - AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id, span), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id, span), AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), @@ -1043,7 +1043,7 @@ fn create_member_type<'ll, 'tcx>( file_metadata, line_number, layout.size.bits(), - layout.align.abi.bits() as u32, + layout.align.bits() as u32, offset.bits(), flags, type_di_node, @@ -1076,6 +1076,7 @@ fn visibility_di_flags<'ll, 'tcx>( fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let struct_type = unique_type_id.expect_ty(); let ty::Adt(adt_def, _) = struct_type.kind() else { @@ -1083,7 +1084,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( }; assert!(adt_def.is_struct()); let containing_scope = get_namespace_for_item(cx, adt_def.did()); - let struct_type_and_layout = cx.layout_of(struct_type); + let struct_type_and_layout = cx.spanned_layout_of(struct_type, span); let variant_def = adt_def.non_enum_variant(); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) @@ -1276,6 +1277,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( fn build_union_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let union_type = unique_type_id.expect_ty(); let (union_def_id, variant_def) = match union_type.kind() { @@ -1283,7 +1285,7 @@ fn build_union_type_di_node<'ll, 'tcx>( _ => bug!("build_union_type_di_node on a non-ADT"), }; let containing_scope = get_namespace_for_item(cx, union_def_id); - let union_ty_and_layout = cx.layout_of(union_type); + let union_ty_and_layout = cx.spanned_layout_of(union_type, span); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(union_def_id))) @@ -1609,16 +1611,12 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; llvm::LLVMRustGlobalAddMetadata( vtable, - llvm::MD_type as c_uint, + llvm::MD_type, llvm::LLVMMDNodeInContext2(cx.llcx, v.as_ptr(), v.len()), ); let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); - llvm::LLVMGlobalSetMetadata( - vtable, - llvm::MetadataType::MD_vcall_visibility as c_uint, - vcall_visibility_metadata, - ); + llvm::LLVMGlobalSetMetadata(vtable, llvm::MD_vcall_visibility, vcall_visibility_metadata); } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 62d38d463ab..1ae6e6e5eec 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -289,7 +289,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( file_metadata, line_number, enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, + enum_type_and_layout.align.bits() as u32, DIFlags::FlagZero, tag_member_di_node, create_DIArray(DIB(cx), &[]), @@ -449,7 +449,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( file_di_node, line_number, enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, + enum_type_and_layout.align.bits() as u32, Size::ZERO.bits(), discr, DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 126082aa3aa..af64e4ebed0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -52,15 +52,6 @@ mod utils; use self::create_scope_map::compute_mir_scopes; pub(crate) use self::metadata::build_global_var_di_node; -// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were -// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp` -// to decide which C++ API to call. Instead, we should just have two separate -// FFI functions and choose the correct one on the Rust side. -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, @@ -174,35 +165,38 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { if direct_offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(direct_offset.bytes()); } for &offset in indirect_offsets { addr_ops.push(DW_OP_deref); if offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(offset.bytes() as u64); + addr_ops.push(offset.bytes()); } } if let Some(fragment) = fragment { // `DW_OP_LLVM_fragment` takes as arguments the fragment's // offset and size, both of them in bits. addr_ops.push(DW_OP_LLVM_fragment); - addr_ops.push(fragment.start.bits() as u64); - addr_ops.push((fragment.end - fragment.start).bits() as u64); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); } + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), + llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( + di_builder, variable_alloca, dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, + addr_expr, dbg_loc, self.llbb(), - ); - } + ) + }; } fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { @@ -630,28 +624,39 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let type_metadata = spanned_type_di_node(self, variable_type, span); - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; let align = self.align_of(variable_type); let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_c_char_ptr(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bits() as u32, - ) + + match variable_kind { + ArgumentVariable(arg_index) => unsafe { + llvm::LLVMDIBuilderCreateParameterVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + arg_index as c_uint, + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + ) + }, + LocalVariable => unsafe { + llvm::LLVMDIBuilderCreateAutoVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + align.bits() as u32, + ) + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index cc1d504b430..7e1e49310f6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } + unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 960a895a203..36cdb498839 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -76,7 +76,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>( attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attrs.extend(attributes::non_lazy_bind_attr(cx)); + attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess)); attributes::apply_to_llfn(llfn, Function, &attrs); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 50398a32142..467655b0bfc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -297,7 +297,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let align = if name == sym::unaligned_volatile_load { 1 } else { - result.layout.align.abi.bytes() as u32 + result.layout.align.bytes() as u32 }; unsafe { llvm::LLVMSetAlignment(load, align); @@ -1047,7 +1047,7 @@ fn codegen_emcc_try<'ll, 'tcx>( // create an alloca and pass a pointer to that. let ptr_size = bx.tcx().data_layout.pointer_size(); let ptr_align = bx.tcx().data_layout.pointer_align().abi; - let i8_align = bx.tcx().data_layout.i8_align.abi; + let i8_align = bx.tcx().data_layout.i8_align; // Required in order for there to be no padding between the fields. assert!(i8_align <= ptr_align); let catch_data = bx.alloca(2 * ptr_size, ptr_align); @@ -1212,6 +1212,9 @@ fn codegen_autodiff<'ll, 'tcx>( &mut diff_attrs.input_activity, ); + let fnc_tree = + rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized())); + // Build body generate_enzyme_call( bx, @@ -1222,6 +1225,7 @@ fn codegen_autodiff<'ll, 'tcx>( &val_arr, diff_attrs.clone(), result, + fnc_tree, ); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 13bdb7cb1a2..2405a25c702 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -68,6 +68,7 @@ mod llvm_util; mod mono_item; mod type_; mod type_of; +mod typetree; mod va_arg; mod value; @@ -231,6 +232,10 @@ impl CodegenBackend for LlvmCodegenBackend { crate::DEFAULT_LOCALE_RESOURCE } + fn name(&self) -> &'static str { + "llvm" + } + fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited } @@ -349,7 +354,14 @@ impl CodegenBackend for LlvmCodegenBackend { // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, metadata, outputs); + link_binary( + sess, + &LlvmArchiveBuilderBuilder, + codegen_results, + metadata, + outputs, + self.name(), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 695435eb6da..e63043b2122 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -3,9 +3,36 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; -use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; +use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value}; use crate::llvm::{Bool, Builder}; +// TypeTree types +pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct EnzymeTypeTree { + _unused: [u8; 0], +} + +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub(crate) enum CConcreteType { + DT_Anything = 0, + DT_Integer = 1, + DT_Pointer = 2, + DT_Half = 3, + DT_Float = 4, + DT_Double = 5, + DT_Unknown = 6, + DT_FP128 = 9, +} + +pub(crate) struct TypeTree { + pub(crate) inner: CTypeTreeRef, +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { // Enzyme @@ -68,10 +95,40 @@ pub(crate) mod Enzyme_AD { use libc::c_void; + use super::{CConcreteType, CTypeTreeRef, Context}; + unsafe extern "C" { pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char); } + + // TypeTree functions + unsafe extern "C" { + pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef; + pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef); + pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool; + pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64); + pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef); + pub(crate) fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ); + pub(crate) fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ); + pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char; + pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char); + } + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; @@ -141,6 +198,67 @@ pub(crate) use self::Fallback_AD::*; pub(crate) mod Fallback_AD { #![allow(unused_variables)] + use libc::c_char; + + use super::{CConcreteType, CTypeTreeRef, Context}; + + // TypeTree function fallbacks + pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) { + unimplemented!() + } + pub(crate) fn set_inline(val: bool) { unimplemented!() } @@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD { unimplemented!() } } + +impl TypeTree { + pub(crate) fn new() -> TypeTree { + let inner = unsafe { EnzymeNewTypeTree() }; + TypeTree { inner } + } + + pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree { + let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) }; + TypeTree { inner } + } + + pub(crate) fn merge(self, other: Self) -> Self { + unsafe { + EnzymeMergeTypeTree(self.inner, other.inner); + } + drop(other); + self + } + + #[must_use] + pub(crate) fn shift( + self, + layout: &str, + offset: isize, + max_size: isize, + add_offset: usize, + ) -> Self { + let layout = std::ffi::CString::new(layout).unwrap(); + + unsafe { + EnzymeTypeTreeShiftIndiciesEq( + self.inner, + layout.as_ptr(), + offset as i64, + max_size as i64, + add_offset as u64, + ); + } + + self + } + + pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) { + unsafe { + EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx); + } + } +} + +impl Clone for TypeTree { + fn clone(&self) -> Self { + let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) }; + TypeTree { inner } + } +} + +impl std::fmt::Display for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ptr = unsafe { EnzymeTypeTreeToString(self.inner) }; + let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) }; + match cstr.to_str() { + Ok(x) => write!(f, "{}", x)?, + Err(err) => write!(f, "could not parse: {}", err)?, + } + + // delete C string pointer + unsafe { + EnzymeTypeTreeToStringFree(ptr); + } + + Ok(()) + } +} + +impl std::fmt::Debug for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + <Self as std::fmt::Display>::fmt(self, f) + } +} + +impl Drop for TypeTree { + fn drop(&mut self) { + unsafe { EnzymeFreeTypeTree(self.inner) } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index fd972f371df..e9f92267a7d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -25,10 +25,11 @@ use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, - DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, DISubrange, - DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, + DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, + DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind, }; use crate::llvm; +use crate::llvm::MetadataKindId; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. @@ -513,31 +514,6 @@ pub(crate) enum FileType { ObjectFile, } -/// LLVMMetadataType -#[derive(Copy, Clone)] -#[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] -pub(crate) enum MetadataType { - MD_dbg = 0, - MD_tbaa = 1, - MD_prof = 2, - MD_fpmath = 3, - MD_range = 4, - MD_tbaa_struct = 5, - MD_invariant_load = 6, - MD_alias_scope = 7, - MD_noalias = 8, - MD_nontemporal = 9, - MD_mem_parallel_loop_access = 10, - MD_nonnull = 11, - MD_unpredictable = 15, - MD_align = 17, - MD_type = 19, - MD_vcall_visibility = 28, - MD_noundef = 29, - MD_kcfi_type = 36, -} - /// Must match the layout of `LLVMInlineAsmDialect`. #[derive(Copy, Clone, PartialEq)] #[repr(C)] @@ -807,6 +783,8 @@ unsafe extern "C" { pub(crate) type Metadata; pub(crate) type BasicBlock; pub(crate) type Comdat; + /// `&'ll DbgRecord` represents `LLVMDbgRecordRef`. + pub(crate) type DbgRecord; } #[repr(C)] pub(crate) struct Builder<'a>(InvariantOpaque<'a>); @@ -891,7 +869,6 @@ pub(crate) mod debuginfo { pub(crate) type DIVariable = DIDescriptor; pub(crate) type DIGlobalVariableExpression = DIDescriptor; pub(crate) type DIArray = DIDescriptor; - pub(crate) type DISubrange = DIDescriptor; pub(crate) type DIEnumerator = DIDescriptor; pub(crate) type DITemplateTypeParameter = DIDescriptor; @@ -1034,16 +1011,6 @@ pub(crate) type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; pub(crate) type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; -#[derive(Copy, Clone)] -#[repr(transparent)] -pub(crate) struct MetadataKindId(c_uint); - -impl From<MetadataType> for MetadataKindId { - fn from(value: MetadataType) -> Self { - Self(value as c_uint) - } -} - unsafe extern "C" { // Create and destroy contexts. pub(crate) fn LLVMContextDispose(C: &'static mut Context); @@ -1053,6 +1020,8 @@ unsafe extern "C" { SLen: c_uint, ) -> MetadataKindId; + pub(crate) fn LLVMDisposeTargetMachine(T: ptr::NonNull<TargetMachine>); + // Create modules. pub(crate) fn LLVMModuleCreateWithNameInContext( ModuleID: *const c_char, @@ -1136,7 +1105,11 @@ unsafe extern "C" { pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub(crate) safe fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: MetadataKindId, Node: &'a Value); - pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub(crate) fn LLVMGlobalSetMetadata<'a>( + Val: &'a Value, + KindID: MetadataKindId, + Metadata: &'a Metadata, + ); pub(crate) safe fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1990,6 +1963,59 @@ unsafe extern "C" { Scope: Option<&'ll Metadata>, AlignInBits: u32, // (optional; default is 0) ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>( + Builder: &DIBuilder<'ll>, + LowerBound: i64, + Count: i64, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>( + Builder: &DIBuilder<'ll>, + Data: *const Option<&'ll Metadata>, + NumElements: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateExpression<'ll>( + Builder: &DIBuilder<'ll>, + Addr: *const u64, + Length: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Storage: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ArgNo: c_uint, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + ) -> &'ll Metadata; } #[link(name = "llvm-wrapper", kind = "static")] @@ -2003,7 +2029,7 @@ unsafe extern "C" { // Operations on all values pub(crate) fn LLVMRustGlobalAddMetadata<'a>( Val: &'a Value, - KindID: c_uint, + KindID: MetadataKindId, Metadata: &'a Metadata, ); pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; @@ -2200,8 +2226,11 @@ unsafe extern "C" { ConstraintsLen: size_t, ) -> bool; + /// A list of pointer-length strings is passed as two pointer-length slices, + /// one slice containing pointers and one slice containing their corresponding + /// lengths. The implementation will check that both slices have the same length. pub(crate) fn LLVMRustCoverageWriteFilenamesToBuffer( - Filenames: *const *const c_char, + Filenames: *const *const c_uchar, // See "PTR_LEN_STR". FilenamesLen: size_t, Lengths: *const size_t, LengthsLen: size_t, @@ -2224,18 +2253,25 @@ unsafe extern "C" { pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, - FuncName: *const c_char, + FuncName: *const c_uchar, // See "PTR_LEN_STR". FuncNameLen: size_t, ) -> &Value; - pub(crate) fn LLVMRustCoverageHashBytes(Bytes: *const c_char, NumBytes: size_t) -> u64; - - pub(crate) fn LLVMRustCoverageWriteCovmapSectionNameToString(M: &Module, OutStr: &RustString); - - pub(crate) fn LLVMRustCoverageWriteCovfunSectionNameToString(M: &Module, OutStr: &RustString); + pub(crate) fn LLVMRustCoverageHashBytes( + Bytes: *const c_uchar, // See "PTR_LEN_STR". + NumBytes: size_t, + ) -> u64; - pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); + pub(crate) safe fn LLVMRustCoverageWriteCovmapSectionNameToString( + M: &Module, + OutStr: &RustString, + ); + pub(crate) safe fn LLVMRustCoverageWriteCovfunSectionNameToString( + M: &Module, + OutStr: &RustString, + ); + pub(crate) safe fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); - pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) safe fn LLVMRustCoverageMappingVersion() -> u32; pub(crate) fn LLVMRustDebugMetadataVersion() -> u32; pub(crate) fn LLVMRustVersionMajor() -> u32; pub(crate) fn LLVMRustVersionMinor() -> u32; @@ -2356,43 +2392,6 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ); - pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2499,7 +2498,6 @@ unsafe extern "C" { UseWasmEH: bool, ) -> *mut TargetMachine; - pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs b/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs new file mode 100644 index 00000000000..a8a671b5c85 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs @@ -0,0 +1,71 @@ +use libc::c_uint; + +pub(crate) use self::fixed_kinds::*; + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub(crate) struct MetadataKindId(c_uint); + +macro_rules! declare_fixed_metadata_kinds { + ( + $( + FIXED_MD_KIND($variant:ident, $value:literal) + )* + ) => { + // Use a submodule to group all declarations into one `#[expect(..)]`. + #[expect(dead_code)] + mod fixed_kinds { + use super::MetadataKindId; + $( + #[expect(non_upper_case_globals)] + pub(crate) const $variant: MetadataKindId = MetadataKindId($value); + )* + } + }; +} + +// Must be kept in sync with the corresponding static assertions in `RustWrapper.cpp`. +declare_fixed_metadata_kinds! { + FIXED_MD_KIND(MD_dbg, 0) + FIXED_MD_KIND(MD_tbaa, 1) + FIXED_MD_KIND(MD_prof, 2) + FIXED_MD_KIND(MD_fpmath, 3) + FIXED_MD_KIND(MD_range, 4) + FIXED_MD_KIND(MD_tbaa_struct, 5) + FIXED_MD_KIND(MD_invariant_load, 6) + FIXED_MD_KIND(MD_alias_scope, 7) + FIXED_MD_KIND(MD_noalias, 8) + FIXED_MD_KIND(MD_nontemporal, 9) + FIXED_MD_KIND(MD_mem_parallel_loop_access, 10) + FIXED_MD_KIND(MD_nonnull, 11) + FIXED_MD_KIND(MD_dereferenceable, 12) + FIXED_MD_KIND(MD_dereferenceable_or_null, 13) + FIXED_MD_KIND(MD_make_implicit, 14) + FIXED_MD_KIND(MD_unpredictable, 15) + FIXED_MD_KIND(MD_invariant_group, 16) + FIXED_MD_KIND(MD_align, 17) + FIXED_MD_KIND(MD_loop, 18) + FIXED_MD_KIND(MD_type, 19) + FIXED_MD_KIND(MD_section_prefix, 20) + FIXED_MD_KIND(MD_absolute_symbol, 21) + FIXED_MD_KIND(MD_associated, 22) + FIXED_MD_KIND(MD_callees, 23) + FIXED_MD_KIND(MD_irr_loop, 24) + FIXED_MD_KIND(MD_access_group, 25) + FIXED_MD_KIND(MD_callback, 26) + FIXED_MD_KIND(MD_preserve_access_index, 27) + FIXED_MD_KIND(MD_vcall_visibility, 28) + FIXED_MD_KIND(MD_noundef, 29) + FIXED_MD_KIND(MD_annotation, 30) + FIXED_MD_KIND(MD_nosanitize, 31) + FIXED_MD_KIND(MD_func_sanitize, 32) + FIXED_MD_KIND(MD_exclude, 33) + FIXED_MD_KIND(MD_memprof, 34) + FIXED_MD_KIND(MD_callsite, 35) + FIXED_MD_KIND(MD_kcfi_type, 36) + FIXED_MD_KIND(MD_pcsections, 37) + FIXED_MD_KIND(MD_DIAssignID, 38) + FIXED_MD_KIND(MD_coro_outside_frame, 39) + FIXED_MD_KIND(MD_mmra, 40) + FIXED_MD_KIND(MD_noalias_addrspace, 41) +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 1115d82fa85..9a53dacb1df 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -11,13 +11,14 @@ use rustc_llvm::RustString; pub(crate) use self::CallConv::*; pub(crate) use self::CodeGenOptSize::*; -pub(crate) use self::MetadataType::*; pub(crate) use self::ffi::*; +pub(crate) use self::metadata_kind::*; use crate::common::AsCCharPtr; pub(crate) mod diagnostic; pub(crate) mod enzyme_ffi; mod ffi; +mod metadata_kind; pub(crate) use self::enzyme_ffi::*; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 9ecaf5f24fe..5b97898a4b8 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -306,7 +306,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; llvm::LLVMRustGlobalAddMetadata( function, - llvm::MD_type as c_uint, + llvm::MD_type, llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()), ) } @@ -318,7 +318,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; llvm::LLVMGlobalSetMetadata( function, - llvm::MD_type as c_uint, + llvm::MD_type, llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()), ) } @@ -333,7 +333,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMRustGlobalAddMetadata( function, - llvm::MD_kcfi_type as c_uint, + llvm::MD_kcfi_type, llvm::LLVMMDNodeInContext2( self.llcx, &llvm::LLVMValueAsMetadata(kcfi_type_metadata), @@ -348,7 +348,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMGlobalSetMetadata( function, - llvm::MD_kcfi_type as c_uint, + llvm::MD_kcfi_type, llvm::LLVMMDNodeInContext2( self.llcx, &llvm::LLVMValueAsMetadata(kcfi_type_metadata), diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs new file mode 100644 index 00000000000..7e263503700 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -0,0 +1,122 @@ +use rustc_ast::expand::typetree::FncTree; +#[cfg(feature = "llvm_enzyme")] +use { + crate::attributes, + rustc_ast::expand::typetree::TypeTree as RustTypeTree, + std::ffi::{CString, c_char, c_uint}, +}; + +use crate::llvm::{self, Value}; + +#[cfg(feature = "llvm_enzyme")] +fn to_enzyme_typetree( + rust_typetree: RustTypeTree, + _data_layout: &str, + llcx: &llvm::Context, +) -> llvm::TypeTree { + let mut enzyme_tt = llvm::TypeTree::new(); + process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); + enzyme_tt +} +#[cfg(feature = "llvm_enzyme")] +fn process_typetree_recursive( + enzyme_tt: &mut llvm::TypeTree, + rust_typetree: &RustTypeTree, + parent_indices: &[i64], + llcx: &llvm::Context, +) { + for rust_type in &rust_typetree.0 { + let concrete_type = match rust_type.kind { + rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything, + rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer, + rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer, + rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half, + rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float, + rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double, + rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128, + rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown, + }; + + let mut indices = parent_indices.to_vec(); + if !parent_indices.is_empty() { + indices.push(rust_type.offset as i64); + } else if rust_type.offset == -1 { + indices.push(-1); + } else { + indices.push(rust_type.offset as i64); + } + + enzyme_tt.insert(&indices, concrete_type, llcx); + + if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer + && !rust_type.child.0.is_empty() + { + process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx); + } + } +} + +#[cfg(feature = "llvm_enzyme")] +pub(crate) fn add_tt<'ll>( + llmod: &'ll llvm::Module, + llcx: &'ll llvm::Context, + fn_def: &'ll Value, + tt: FncTree, +) { + let inputs = tt.args; + let ret_tt: RustTypeTree = tt.ret; + + let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) }; + let llvm_data_layout = + std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + let attr_name = "enzyme_type"; + let c_attr_name = CString::new(attr_name).unwrap(); + + for (i, input) in inputs.iter().enumerate() { + unsafe { + let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } + } + + unsafe { + let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let ret_attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } +} + +#[cfg(not(feature = "llvm_enzyme"))] +pub(crate) fn add_tt<'ll>( + _llmod: &'ll llvm::Module, + _llcx: &'ll llvm::Context, + _fn_def: &'ll Value, + _tt: FncTree, +) { + unimplemented!() +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ab08125217f..234366e491c 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -193,7 +193,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( // the offset again. bx.switch_to_block(maybe_reg); - if gr_type && layout.align.abi.bytes() > 8 { + if gr_type && layout.align.bytes() > 8 { reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); } @@ -291,7 +291,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( bx.inbounds_ptradd(va_list_addr, bx.const_usize(1)) // fpr }; - let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align.abi); + let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align); // "Align" the register count when the type is passed as `i64`. if is_i64 || (is_f64 && is_soft_float_abi) { @@ -329,7 +329,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( // Increase the used-register count. let reg_incr = if is_i64 || (is_f64 && is_soft_float_abi) { 2 } else { 1 }; let new_num_regs = bx.add(num_regs, bx.cx.const_u8(reg_incr)); - bx.store(new_num_regs, num_regs_addr, dl.i8_align.abi); + bx.store(new_num_regs, num_regs_addr, dl.i8_align); bx.br(end); @@ -339,7 +339,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( let mem_addr = { bx.switch_to_block(in_mem); - bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align.abi); + bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align); // Everything in the overflow area is rounded up to a size of at least 4. let overflow_area_align = Align::from_bytes(4).unwrap(); @@ -738,6 +738,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>( src_align, bx.const_u32(layout.layout.size().bytes() as u32), MemFlags::empty(), + None, ); tmp } else { @@ -760,7 +761,7 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>( // byte boundary if alignment needed by type exceeds 8 byte boundary. // It isn't stated explicitly in the standard, but in practice we use // alignment greater than 16 where necessary. - if layout.layout.align.abi.bytes() > 8 { + if layout.layout.align.bytes() > 8 { unreachable!("all instances of VaArgSafe have an alignment <= 8"); } @@ -813,7 +814,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( let va_ndx_offset = va_reg_offset + 4; let offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_ndx_offset)); - let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi); + let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align); let offset = round_up_to_alignment(bx, offset, layout.align.abi); let slot_size = layout.size.align_to(Align::from_bytes(4).unwrap()).bytes() as i32; |