diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
19 files changed, 584 insertions, 198 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 11be7041167..1703cab942b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -246,6 +246,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); } @@ -538,7 +539,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the declaration has an associated instance, compute extra attributes based on that. if let Some(instance) = instance { - llfn_attrs_from_instance(cx, llfn, instance); + llfn_attrs_from_instance( + cx, + cx.tcx, + llfn, + &cx.tcx.codegen_instance_attrs(instance.def), + Some(instance), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index df3e49279d9..abd63120397 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -5,15 +5,16 @@ use rustc_ast::expand::allocator::{ }; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; -use smallvec::SmallVec; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; use crate::llvm::{self, FALSE, TRUE, Type, Value}; -use crate::{SimpleCx, attributes, debuginfo, llvm_util}; +use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, @@ -149,18 +150,8 @@ fn create_wrapper_function( ty, ); - let mut attrs = SmallVec::<[_; 2]>::new(); - - let target_cpu = llvm_util::target_cpu(tcx.sess); - let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu); - - let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess) - .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)); - - attrs.push(target_cpu_attr); - attrs.extend(tune_cpu_attr); - - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + let attrs = CodegenFnAttrs::new(); + llfn_attrs_from_instance(cx, tcx, llfn, &attrs, None); let no_return = if no_return { // -> ! DIFlagNoReturn @@ -171,12 +162,6 @@ fn create_wrapper_function( None }; - if tcx.sess.must_emit_unwind_tables() { - let uwtable = - attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; let mut bx = SBuilder::build(&cx, llbb); diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index dcf6b945497..8070ea0b3e9 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,20 +1,21 @@ //! Set and unset common attributes on LLVM values. -use rustc_codegen_ssa::traits::*; use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; +use rustc_middle::middle::codegen_fn_attrs::{ + CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, +}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; -use crate::context::CodegenCx; +use crate::context::SimpleCx; use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; -use crate::{attributes, llvm_util}; +use crate::{Session, attributes, llvm_util}; pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { @@ -30,18 +31,19 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ /// Get LLVM attribute for the provided inline heuristic. pub(crate) fn inline_attr<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, instance: ty::Instance<'tcx>, ) -> Option<&'ll Attribute> { // `optnone` requires `noinline` - let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id()); let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, + (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint, (inline, _) => inline, }; - if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + if !tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); } @@ -51,7 +53,7 @@ pub(crate) fn inline_attr<'ll, 'tcx>( Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) } InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { + if tcx.sess.target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) } else { None @@ -63,12 +65,13 @@ pub(crate) fn inline_attr<'ll, 'tcx>( #[inline] fn patchable_function_entry_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, attr: Option<PatchableFunctionEntry>, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); let patchable_spec = attr.unwrap_or_else(|| { - PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry) + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry) }); let entry = patchable_spec.entry(); let prefix = patchable_spec.prefix(); @@ -91,12 +94,13 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub(crate) fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn sanitize_attrs<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -114,11 +118,11 @@ pub(crate) fn sanitize_attrs<'ll>( } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); + let features = tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); + tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -139,9 +143,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) llvm::CreateUWTableAttr(llcx, async_unwind) } -pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - let opts = &cx.sess().opts; +pub(crate) fn frame_pointer_type_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { + let mut fp = sess.target.frame_pointer; + let opts = &sess.opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. if opts.unstable_opts.instrument_mcount { @@ -156,8 +163,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&' Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) } -fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let function_return_attr = match cx.sess().opts.unstable_opts.function_return { +fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let function_return_attr = match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => return None, FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern, }; @@ -167,17 +174,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { +fn instrument_function_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - if cx.sess().opts.unstable_opts.instrument_mcount { + if sess.opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + let mcount_name = match &sess.target.llvm_mcount_intrinsic { Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), - None => cx.sess().target.mcount.as_ref(), + None => sess.target.mcount.as_ref(), }; attrs.push(llvm::CreateAttrStringValue( @@ -186,7 +196,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr mcount_name, )); } - if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + if let Some(options) = &sess.opts.unstable_opts.instrument_xray { // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. // Function prologue and epilogue are instrumented with NOP sleds, // a runtime library later replaces them with detours into tracing code. @@ -217,20 +227,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr attrs } -fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if !cx.sess().opts.unstable_opts.no_jump_tables { +fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if !sess.opts.unstable_opts.no_jump_tables { return None; } Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() + if tcx + .sess .opts .unstable_opts .sanitizer @@ -240,22 +250,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { + if tcx.sess.opts.cg.profile_generate.enabled() { return None; } - let attr_value = match cx.sess().target.stack_probes { + let attr_value = match tcx.sess.target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). StackProbeType::Inline => "inline-asm", // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"), + StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - &mangle_internal_symbol(cx.tcx, "__rust_probestack") + &mangle_internal_symbol(tcx, "__rust_probestack") } else { "inline-asm" } @@ -264,8 +274,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) } -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { +fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let sspattr = match sess.stack_protector() { StackProtector::None => return None, StackProtector::All => AttributeKind::StackProtectReq, StackProtector::Strong => AttributeKind::StackProtectStrong, @@ -275,33 +285,34 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(sspattr.create_attr(cx.llcx)) } -fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().target.arch != "s390x" { +fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if sess.target.arch != "s390x" { return None; } - let requested_features = cx.sess().opts.cg.target_feature.split(','); + let requested_features = sess.opts.cg.target_feature.split(','); let found_positive = requested_features.clone().any(|r| r == "+backchain"); if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); +pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) +pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } /// Get the `target-features` LLVM attribute. -pub(crate) fn target_features_attr<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn target_features_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, function_features: Vec<String>, ) -> Option<&'ll Attribute> { - let global_features = cx.tcx.global_backend_features(()).iter().map(String::as_str); + let global_features = tcx.global_backend_features(()).iter().map(String::as_str); let function_features = function_features.iter().map(String::as_str); let target_features = global_features.chain(function_features).intersperse(",").collect::<String>(); @@ -311,22 +322,22 @@ pub(crate) fn target_features_attr<'ll>( /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } + if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None } } /// Get the default optimizations attrs for a function. #[inline] pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { + match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } @@ -347,17 +358,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, llfn: &'ll Value, - instance: ty::Instance<'tcx>, + codegen_fn_attrs: &CodegenFnAttrs, + instance: Option<ty::Instance<'tcx>>, ) { - let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); - + let sess = tcx.sess; let mut to_add = SmallVec::<[_; 16]>::new(); match codegen_fn_attrs.optimize { OptimizeAttr::Default => { - to_add.extend(default_optimisation_attrs(cx)); + to_add.extend(default_optimisation_attrs(cx, sess)); } OptimizeAttr::DoNotOptimize => { to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx)); @@ -369,21 +381,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); + if sess.must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind)); } - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + if sess.opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } // FIXME: none of these functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(function_return_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(nojumptables_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); + to_add.extend(frame_pointer_type_attr(cx, sess)); + to_add.extend(function_return_attr(cx, sess)); + to_add.extend(instrument_function_attr(cx, sess)); + to_add.extend(nojumptables_attr(cx, sess)); + to_add.extend(probestack_attr(cx, tcx)); + to_add.extend(stackprotector_attr(cx, sess)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); @@ -404,13 +416,13 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // not used. } else { // Do not set sanitizer attributes for naked functions. - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize)); // For non-naked functions, set branch protection attributes on aarch64. if let Some(BranchProtection { bti, pac_ret, gcs }) = - cx.sess().opts.unstable_opts.branch_protection + sess.opts.unstable_opts.branch_protection { - assert!(cx.sess().target.arch == "aarch64"); + assert!(sess.target.arch == "aarch64"); if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } @@ -438,14 +450,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); - if let Some(zv) = - cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + if let Some(instance) = instance + && let Some(zv) = + tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) && let Some(name) = zv.value_str() { to_add.push(llvm::CreateAttrStringValue( cx.llcx, "alloc-variant-zeroed", - &mangle_internal_symbol(cx.tcx, name.as_str()), + &mangle_internal_symbol(tcx, name.as_str()), )); } // apply to argument place instead of function @@ -490,18 +503,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align); } - if let Some(backchain) = backchain_attr(cx) { + if let Some(backchain) = backchain_attr(cx, sess) { to_add.push(backchain); } - to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); + to_add.extend(patchable_function_entry_attrs( + cx, + sess, + codegen_fn_attrs.patchable_function_entry, + )); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); + to_add.push(target_cpu_attr(cx, sess)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); + to_add.extend(tune_cpu_attr(cx, sess)); let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); @@ -509,7 +526,9 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // Apply function attributes as per usual if there are no user defined // target features otherwise this will get applied at the callsite. if function_features.is_empty() { - if let Some(inline_attr) = inline_attr(cx, instance) { + if let Some(instance) = instance + && let Some(inline_attr) = inline_attr(cx, tcx, instance) + { to_add.push(inline_attr); } } @@ -517,7 +536,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones - .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + .flat_map(|feat| llvm_util::to_llvm_features(sess, feat)) // Convert LLVMFeatures & dependencies to +<feats>s .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { @@ -526,20 +545,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( })) .collect::<Vec<String>>(); - if cx.tcx.sess.target.is_like_wasm { + if sess.target.is_like_wasm { // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + if let Some(instance) = instance + && let Some(module) = wasm_import_module(tcx, instance.def_id()) + { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.symbol_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } } - to_add.extend(target_features_attr(cx, function_features)); + to_add.extend(target_features_attr(cx, tcx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 78107d95e5a..5ac3a87c158 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -563,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { config::AutoDiff::Enable => {} // We handle this below config::AutoDiff::NoPostopt => {} + // Disables TypeTree generation + config::AutoDiff::NoTT => {} } } // This helps with handling enums for now. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 978134cc32b..6d12b511e9c 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -105,7 +105,7 @@ pub(crate) fn compile_codegen_unit( if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty()); attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0f17cc9063a..5271d0b4bb8 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,6 +2,7 @@ use std::borrow::{Borrow, Cow}; use std::ops::Deref; use std::{iter, ptr}; +use rustc_ast::expand::typetree::FncTree; pub(crate) mod autodiff; pub(crate) mod gpu_offload; @@ -1107,11 +1108,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + tt: Option<FncTree>, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - unsafe { + let memcpy = unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, dst, @@ -1120,7 +1122,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align.bytes() as c_uint, size, is_volatile, - ); + ) + }; + + // TypeTree metadata for memcpy is especially important: when Enzyme encounters + // a memcpy during autodiff, it needs to know the structure of the data being + // copied to properly track derivatives. For example, copying an array of floats + // vs. copying a struct with mixed types requires different derivative handling. + // The TypeTree tells Enzyme exactly what memory layout to expect. + if let Some(tt) = tt { + crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt); } } @@ -1433,7 +1444,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // If there is an inline attribute and a target feature that matches // we will add the attribute to the callsite otherwise we'll omit // this and not add the attribute to prevent soundness issues. - && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance) + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance) && self.cx.tcx.is_target_feature_call_safe( &fn_call_attrs.target_features, &fn_defn_attrs.target_features, diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index b66e3dfdeec..c3485f56391 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,6 +1,7 @@ use std::ptr; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; +use rustc_ast::expand::typetree::FncTree; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv}; @@ -294,6 +295,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args: &[&'ll Value], attrs: AutoDiffAttrs, dest: PlaceRef<'tcx, &'ll Value>, + fnc_tree: FncTree, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -370,6 +372,10 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args, ); + if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() { + crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree); + } + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); builder.store_to_place(call, dest.val); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index b7d2241da47..aa5c17269fb 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -876,7 +876,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); + let target_cpu = attributes::target_cpu_attr(self, self.sess()); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } @@ -891,15 +891,15 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { + if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); + attrs.push(attributes::target_cpu_attr(self, self.sess())); + attrs.extend(attributes::tune_cpu_attr(self, self.sess())); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } @@ -918,7 +918,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn( llfn, llvm::AttributePlace::Function, - attributes::target_features_attr(self, vec![]).as_slice(), + attributes::target_features_attr(self, self.tcx, vec![]).as_slice(), ); Some(llfn) } else { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 7a6dc008c7b..3081badb821 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -2,9 +2,9 @@ use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::DebuggerVisualizerType; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; use crate::builder::Builder; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 4ba72cd61a0..4b74c04ed7a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -117,7 +117,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( .try_to_target_usize(cx.tcx) .expect("expected monomorphic const in codegen") as c_longlong; - let subrange = unsafe { llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; + let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; let subscripts = &[subrange]; let di_node = unsafe { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 126082aa3aa..af64e4ebed0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -52,15 +52,6 @@ mod utils; use self::create_scope_map::compute_mir_scopes; pub(crate) use self::metadata::build_global_var_di_node; -// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were -// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp` -// to decide which C++ API to call. Instead, we should just have two separate -// FFI functions and choose the correct one on the Rust side. -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, @@ -174,35 +165,38 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { if direct_offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(direct_offset.bytes()); } for &offset in indirect_offsets { addr_ops.push(DW_OP_deref); if offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(offset.bytes() as u64); + addr_ops.push(offset.bytes()); } } if let Some(fragment) = fragment { // `DW_OP_LLVM_fragment` takes as arguments the fragment's // offset and size, both of them in bits. addr_ops.push(DW_OP_LLVM_fragment); - addr_ops.push(fragment.start.bits() as u64); - addr_ops.push((fragment.end - fragment.start).bits() as u64); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); } + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), + llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( + di_builder, variable_alloca, dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, + addr_expr, dbg_loc, self.llbb(), - ); - } + ) + }; } fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { @@ -630,28 +624,39 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let type_metadata = spanned_type_di_node(self, variable_type, span); - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; let align = self.align_of(variable_type); let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_c_char_ptr(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bits() as u32, - ) + + match variable_kind { + ArgumentVariable(arg_index) => unsafe { + llvm::LLVMDIBuilderCreateParameterVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + arg_index as c_uint, + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + ) + }, + LocalVariable => unsafe { + llvm::LLVMDIBuilderCreateAutoVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + align.bits() as u32, + ) + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index cc1d504b430..7e1e49310f6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } + unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 960a895a203..36cdb498839 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -76,7 +76,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>( attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attrs.extend(attributes::non_lazy_bind_attr(cx)); + attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess)); attributes::apply_to_llfn(llfn, Function, &attrs); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 50398a32142..013108d1286 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1212,6 +1212,9 @@ fn codegen_autodiff<'ll, 'tcx>( &mut diff_attrs.input_activity, ); + let fnc_tree = + rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized())); + // Build body generate_enzyme_call( bx, @@ -1222,6 +1225,7 @@ fn codegen_autodiff<'ll, 'tcx>( &val_arr, diff_attrs.clone(), result, + fnc_tree, ); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 13bdb7cb1a2..b9584a9117f 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -68,6 +68,7 @@ mod llvm_util; mod mono_item; mod type_; mod type_of; +mod typetree; mod va_arg; mod value; diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 695435eb6da..e63043b2122 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -3,9 +3,36 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; -use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; +use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value}; use crate::llvm::{Bool, Builder}; +// TypeTree types +pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct EnzymeTypeTree { + _unused: [u8; 0], +} + +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub(crate) enum CConcreteType { + DT_Anything = 0, + DT_Integer = 1, + DT_Pointer = 2, + DT_Half = 3, + DT_Float = 4, + DT_Double = 5, + DT_Unknown = 6, + DT_FP128 = 9, +} + +pub(crate) struct TypeTree { + pub(crate) inner: CTypeTreeRef, +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { // Enzyme @@ -68,10 +95,40 @@ pub(crate) mod Enzyme_AD { use libc::c_void; + use super::{CConcreteType, CTypeTreeRef, Context}; + unsafe extern "C" { pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char); } + + // TypeTree functions + unsafe extern "C" { + pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef; + pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef); + pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool; + pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64); + pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef); + pub(crate) fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ); + pub(crate) fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ); + pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char; + pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char); + } + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; @@ -141,6 +198,67 @@ pub(crate) use self::Fallback_AD::*; pub(crate) mod Fallback_AD { #![allow(unused_variables)] + use libc::c_char; + + use super::{CConcreteType, CTypeTreeRef, Context}; + + // TypeTree function fallbacks + pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) { + unimplemented!() + } + pub(crate) fn set_inline(val: bool) { unimplemented!() } @@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD { unimplemented!() } } + +impl TypeTree { + pub(crate) fn new() -> TypeTree { + let inner = unsafe { EnzymeNewTypeTree() }; + TypeTree { inner } + } + + pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree { + let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) }; + TypeTree { inner } + } + + pub(crate) fn merge(self, other: Self) -> Self { + unsafe { + EnzymeMergeTypeTree(self.inner, other.inner); + } + drop(other); + self + } + + #[must_use] + pub(crate) fn shift( + self, + layout: &str, + offset: isize, + max_size: isize, + add_offset: usize, + ) -> Self { + let layout = std::ffi::CString::new(layout).unwrap(); + + unsafe { + EnzymeTypeTreeShiftIndiciesEq( + self.inner, + layout.as_ptr(), + offset as i64, + max_size as i64, + add_offset as u64, + ); + } + + self + } + + pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) { + unsafe { + EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx); + } + } +} + +impl Clone for TypeTree { + fn clone(&self) -> Self { + let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) }; + TypeTree { inner } + } +} + +impl std::fmt::Display for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ptr = unsafe { EnzymeTypeTreeToString(self.inner) }; + let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) }; + match cstr.to_str() { + Ok(x) => write!(f, "{}", x)?, + Err(err) => write!(f, "could not parse: {}", err)?, + } + + // delete C string pointer + unsafe { + EnzymeTypeTreeToStringFree(ptr); + } + + Ok(()) + } +} + +impl std::fmt::Debug for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + <Self as std::fmt::Display>::fmt(self, f) + } +} + +impl Drop for TypeTree { + fn drop(&mut self) { + unsafe { EnzymeFreeTypeTree(self.inner) } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 38a6a311954..afd2991a09c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -25,8 +25,8 @@ use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, - DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, DISubrange, - DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, + DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, + DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind, }; use crate::llvm; @@ -807,6 +807,8 @@ unsafe extern "C" { pub(crate) type Metadata; pub(crate) type BasicBlock; pub(crate) type Comdat; + /// `&'ll DbgRecord` represents `LLVMDbgRecordRef`. + pub(crate) type DbgRecord; } #[repr(C)] pub(crate) struct Builder<'a>(InvariantOpaque<'a>); @@ -891,7 +893,6 @@ pub(crate) mod debuginfo { pub(crate) type DIVariable = DIDescriptor; pub(crate) type DIGlobalVariableExpression = DIDescriptor; pub(crate) type DIArray = DIDescriptor; - pub(crate) type DISubrange = DIDescriptor; pub(crate) type DIEnumerator = DIDescriptor; pub(crate) type DITemplateTypeParameter = DIDescriptor; @@ -1992,6 +1993,59 @@ unsafe extern "C" { Scope: Option<&'ll Metadata>, AlignInBits: u32, // (optional; default is 0) ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>( + Builder: &DIBuilder<'ll>, + LowerBound: i64, + Count: i64, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>( + Builder: &DIBuilder<'ll>, + Data: *const Option<&'ll Metadata>, + NumElements: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateExpression<'ll>( + Builder: &DIBuilder<'ll>, + Addr: *const u64, + Length: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Storage: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ArgNo: c_uint, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + ) -> &'ll Metadata; } #[link(name = "llvm-wrapper", kind = "static")] @@ -2358,43 +2412,6 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ); - pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs new file mode 100644 index 00000000000..7e263503700 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -0,0 +1,122 @@ +use rustc_ast::expand::typetree::FncTree; +#[cfg(feature = "llvm_enzyme")] +use { + crate::attributes, + rustc_ast::expand::typetree::TypeTree as RustTypeTree, + std::ffi::{CString, c_char, c_uint}, +}; + +use crate::llvm::{self, Value}; + +#[cfg(feature = "llvm_enzyme")] +fn to_enzyme_typetree( + rust_typetree: RustTypeTree, + _data_layout: &str, + llcx: &llvm::Context, +) -> llvm::TypeTree { + let mut enzyme_tt = llvm::TypeTree::new(); + process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); + enzyme_tt +} +#[cfg(feature = "llvm_enzyme")] +fn process_typetree_recursive( + enzyme_tt: &mut llvm::TypeTree, + rust_typetree: &RustTypeTree, + parent_indices: &[i64], + llcx: &llvm::Context, +) { + for rust_type in &rust_typetree.0 { + let concrete_type = match rust_type.kind { + rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything, + rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer, + rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer, + rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half, + rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float, + rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double, + rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128, + rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown, + }; + + let mut indices = parent_indices.to_vec(); + if !parent_indices.is_empty() { + indices.push(rust_type.offset as i64); + } else if rust_type.offset == -1 { + indices.push(-1); + } else { + indices.push(rust_type.offset as i64); + } + + enzyme_tt.insert(&indices, concrete_type, llcx); + + if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer + && !rust_type.child.0.is_empty() + { + process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx); + } + } +} + +#[cfg(feature = "llvm_enzyme")] +pub(crate) fn add_tt<'ll>( + llmod: &'ll llvm::Module, + llcx: &'ll llvm::Context, + fn_def: &'ll Value, + tt: FncTree, +) { + let inputs = tt.args; + let ret_tt: RustTypeTree = tt.ret; + + let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) }; + let llvm_data_layout = + std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + let attr_name = "enzyme_type"; + let c_attr_name = CString::new(attr_name).unwrap(); + + for (i, input) in inputs.iter().enumerate() { + unsafe { + let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } + } + + unsafe { + let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let ret_attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } +} + +#[cfg(not(feature = "llvm_enzyme"))] +pub(crate) fn add_tt<'ll>( + _llmod: &'ll llvm::Module, + _llcx: &'ll llvm::Context, + _fn_def: &'ll Value, + _tt: FncTree, +) { + unimplemented!() +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ab08125217f..d48c7cf874a 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -738,6 +738,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>( src_align, bx.const_u32(layout.layout.size().bytes() as u32), MemFlags::empty(), + None, ); tmp } else { |