diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
28 files changed, 909 insertions, 372 deletions
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 2d11628250c..67bd1e59bb0 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -46,5 +46,6 @@ tracing = "0.1" [features] # tidy-alphabetical-start check_only = ["rustc_llvm/check_only"] +llvm_enzyme = [] # tidy-alphabetical-end diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index ac7583f5666..1703cab942b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -215,9 +215,9 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst); } - // Unsized indirect qrguments + // Unsized indirect arguments cannot be stored PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Cast { cast, pad_i32: _ } => { // The ABI mandates that the value is passed as a different struct representation. @@ -246,6 +246,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); } @@ -272,12 +273,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Pair(next(), next()).store(bx, dst); } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - let place_val = PlaceValue { - llval: next(), - llextra: Some(next()), - align: self.layout.align.abi, - }; - OperandValue::Ref(place_val).store(bx, dst); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Direct(_) | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } @@ -543,7 +539,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the declaration has an associated instance, compute extra attributes based on that. if let Some(instance) = instance { - llfn_attrs_from_instance(cx, llfn, instance); + llfn_attrs_from_instance( + cx, + cx.tcx, + llfn, + &cx.tcx.codegen_instance_attrs(instance.def), + Some(instance), + ); } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index df3e49279d9..abd63120397 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -5,15 +5,16 @@ use rustc_ast::expand::allocator::{ }; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; -use smallvec::SmallVec; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; use crate::llvm::{self, FALSE, TRUE, Type, Value}; -use crate::{SimpleCx, attributes, debuginfo, llvm_util}; +use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, @@ -149,18 +150,8 @@ fn create_wrapper_function( ty, ); - let mut attrs = SmallVec::<[_; 2]>::new(); - - let target_cpu = llvm_util::target_cpu(tcx.sess); - let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu); - - let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess) - .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)); - - attrs.push(target_cpu_attr); - attrs.extend(tune_cpu_attr); - - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + let attrs = CodegenFnAttrs::new(); + llfn_attrs_from_instance(cx, tcx, llfn, &attrs, None); let no_return = if no_return { // -> ! DIFlagNoReturn @@ -171,12 +162,6 @@ fn create_wrapper_function( None }; - if tcx.sess.must_emit_unwind_tables() { - let uwtable = - attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; let mut bx = SBuilder::build(&cx, llbb); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index b79176e9098..cc09fa5b69b 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -340,8 +340,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::NOMEM) { attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); - } else { - // LLVM doesn't have an attribute to represent ReadOnly + SideEffect + } else if options.contains(InlineAsmOptions::READONLY) { + attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx)); } attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); @@ -662,7 +662,12 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -830,7 +835,12 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4), - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 573c51a9539..8070ea0b3e9 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,20 +1,21 @@ //! Set and unset common attributes on LLVM values. -use rustc_codegen_ssa::traits::*; use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; +use rustc_middle::middle::codegen_fn_attrs::{ + CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, +}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; -use crate::context::CodegenCx; +use crate::context::SimpleCx; use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; -use crate::{attributes, llvm_util}; +use crate::{Session, attributes, llvm_util}; pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { @@ -30,18 +31,19 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ /// Get LLVM attribute for the provided inline heuristic. pub(crate) fn inline_attr<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, instance: ty::Instance<'tcx>, ) -> Option<&'ll Attribute> { // `optnone` requires `noinline` - let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id()); let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, + (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint, (inline, _) => inline, }; - if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + if !tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); } @@ -51,7 +53,7 @@ pub(crate) fn inline_attr<'ll, 'tcx>( Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) } InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { + if tcx.sess.target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) } else { None @@ -63,12 +65,13 @@ pub(crate) fn inline_attr<'ll, 'tcx>( #[inline] fn patchable_function_entry_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, attr: Option<PatchableFunctionEntry>, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); let patchable_spec = attr.unwrap_or_else(|| { - PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry) + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry) }); let entry = patchable_spec.entry(); let prefix = patchable_spec.prefix(); @@ -91,12 +94,13 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub(crate) fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn sanitize_attrs<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -114,11 +118,11 @@ pub(crate) fn sanitize_attrs<'ll>( } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); + let features = tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); + tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -139,9 +143,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) llvm::CreateUWTableAttr(llcx, async_unwind) } -pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - let opts = &cx.sess().opts; +pub(crate) fn frame_pointer_type_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { + let mut fp = sess.target.frame_pointer; + let opts = &sess.opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. if opts.unstable_opts.instrument_mcount { @@ -156,8 +163,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&' Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) } -fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let function_return_attr = match cx.sess().opts.unstable_opts.function_return { +fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let function_return_attr = match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => return None, FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern, }; @@ -167,17 +174,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { +fn instrument_function_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - if cx.sess().opts.unstable_opts.instrument_mcount { + if sess.opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + let mcount_name = match &sess.target.llvm_mcount_intrinsic { Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), - None => cx.sess().target.mcount.as_ref(), + None => sess.target.mcount.as_ref(), }; attrs.push(llvm::CreateAttrStringValue( @@ -186,7 +196,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr mcount_name, )); } - if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + if let Some(options) = &sess.opts.unstable_opts.instrument_xray { // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. // Function prologue and epilogue are instrumented with NOP sleds, // a runtime library later replaces them with detours into tracing code. @@ -217,20 +227,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr attrs } -fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if !cx.sess().opts.unstable_opts.no_jump_tables { +fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if !sess.opts.unstable_opts.no_jump_tables { return None; } Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() + if tcx + .sess .opts .unstable_opts .sanitizer @@ -240,22 +250,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { + if tcx.sess.opts.cg.profile_generate.enabled() { return None; } - let attr_value = match cx.sess().target.stack_probes { + let attr_value = match tcx.sess.target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). StackProbeType::Inline => "inline-asm", // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"), + StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - &mangle_internal_symbol(cx.tcx, "__rust_probestack") + &mangle_internal_symbol(tcx, "__rust_probestack") } else { "inline-asm" } @@ -264,8 +274,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) } -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { +fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let sspattr = match sess.stack_protector() { StackProtector::None => return None, StackProtector::All => AttributeKind::StackProtectReq, StackProtector::Strong => AttributeKind::StackProtectStrong, @@ -275,33 +285,34 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(sspattr.create_attr(cx.llcx)) } -fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().target.arch != "s390x" { +fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if sess.target.arch != "s390x" { return None; } - let requested_features = cx.sess().opts.cg.target_feature.split(','); + let requested_features = sess.opts.cg.target_feature.split(','); let found_positive = requested_features.clone().any(|r| r == "+backchain"); if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); +pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) +pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } /// Get the `target-features` LLVM attribute. -pub(crate) fn target_features_attr<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn target_features_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, function_features: Vec<String>, ) -> Option<&'ll Attribute> { - let global_features = cx.tcx.global_backend_features(()).iter().map(String::as_str); + let global_features = tcx.global_backend_features(()).iter().map(String::as_str); let function_features = function_features.iter().map(String::as_str); let target_features = global_features.chain(function_features).intersperse(",").collect::<String>(); @@ -311,22 +322,22 @@ pub(crate) fn target_features_attr<'ll>( /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } + if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None } } /// Get the default optimizations attrs for a function. #[inline] pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { + match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } @@ -347,17 +358,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, llfn: &'ll Value, - instance: ty::Instance<'tcx>, + codegen_fn_attrs: &CodegenFnAttrs, + instance: Option<ty::Instance<'tcx>>, ) { - let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); - + let sess = tcx.sess; let mut to_add = SmallVec::<[_; 16]>::new(); match codegen_fn_attrs.optimize { OptimizeAttr::Default => { - to_add.extend(default_optimisation_attrs(cx)); + to_add.extend(default_optimisation_attrs(cx, sess)); } OptimizeAttr::DoNotOptimize => { to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx)); @@ -369,21 +381,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); + if sess.must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind)); } - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + if sess.opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } // FIXME: none of these functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(function_return_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(nojumptables_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); + to_add.extend(frame_pointer_type_attr(cx, sess)); + to_add.extend(function_return_attr(cx, sess)); + to_add.extend(instrument_function_attr(cx, sess)); + to_add.extend(nojumptables_attr(cx, sess)); + to_add.extend(probestack_attr(cx, tcx)); + to_add.extend(stackprotector_attr(cx, sess)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); @@ -404,16 +416,19 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // not used. } else { // Do not set sanitizer attributes for naked functions. - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize)); // For non-naked functions, set branch protection attributes on aarch64. - if let Some(BranchProtection { bti, pac_ret }) = - cx.sess().opts.unstable_opts.branch_protection + if let Some(BranchProtection { bti, pac_ret, gcs }) = + sess.opts.unstable_opts.branch_protection { - assert!(cx.sess().target.arch == "aarch64"); + assert!(sess.target.arch == "aarch64"); if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } + if gcs { + to_add.push(llvm::CreateAttrString(cx.llcx, "guarded-control-stack")); + } if let Some(PacRet { leaf, pc, key }) = pac_ret { if pc { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); @@ -435,14 +450,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); - if let Some(zv) = - cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + if let Some(instance) = instance + && let Some(zv) = + tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) && let Some(name) = zv.value_str() { to_add.push(llvm::CreateAttrStringValue( cx.llcx, "alloc-variant-zeroed", - &mangle_internal_symbol(cx.tcx, name.as_str()), + &mangle_internal_symbol(tcx, name.as_str()), )); } // apply to argument place instead of function @@ -487,18 +503,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align); } - if let Some(backchain) = backchain_attr(cx) { + if let Some(backchain) = backchain_attr(cx, sess) { to_add.push(backchain); } - to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); + to_add.extend(patchable_function_entry_attrs( + cx, + sess, + codegen_fn_attrs.patchable_function_entry, + )); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); + to_add.push(target_cpu_attr(cx, sess)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); + to_add.extend(tune_cpu_attr(cx, sess)); let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); @@ -506,7 +526,9 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // Apply function attributes as per usual if there are no user defined // target features otherwise this will get applied at the callsite. if function_features.is_empty() { - if let Some(inline_attr) = inline_attr(cx, instance) { + if let Some(instance) = instance + && let Some(inline_attr) = inline_attr(cx, tcx, instance) + { to_add.push(inline_attr); } } @@ -514,7 +536,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones - .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + .flat_map(|feat| llvm_util::to_llvm_features(sess, feat)) // Convert LLVMFeatures & dependencies to +<feats>s .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { @@ -523,20 +545,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( })) .collect::<Vec<String>>(); - if cx.tcx.sess.target.is_like_wasm { + if sess.target.is_like_wasm { // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + if let Some(instance) = instance + && let Some(module) = wasm_import_module(tcx, instance.def_id()) + { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.symbol_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } } - to_add.extend(target_features_attr(cx, function_features)); + to_add.extend(target_features_attr(cx, tcx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/command_line_args.rs b/compiler/rustc_codegen_llvm/src/back/command_line_args.rs new file mode 100644 index 00000000000..b14713969b3 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/command_line_args.rs @@ -0,0 +1,37 @@ +#[cfg(test)] +mod tests; + +/// Joins command-line arguments into a single space-separated string, quoting +/// and escaping individual arguments as necessary. +/// +/// The result is intended to be informational, for embedding in debug metadata, +/// and might not be properly quoted/escaped for actual command-line use. +pub(crate) fn quote_command_line_args(args: &[String]) -> String { + // Start with a decent-sized buffer, since rustc invocations tend to be long. + let mut buf = String::with_capacity(128); + + for arg in args { + if !buf.is_empty() { + buf.push(' '); + } + + print_arg_quoted(&mut buf, arg); + } + + buf +} + +/// Equivalent to LLVM's `sys::printArg` with quoting always enabled +/// (see llvm/lib/Support/Program.cpp). +fn print_arg_quoted(buf: &mut String, arg: &str) { + buf.reserve(arg.len() + 2); + + buf.push('"'); + for ch in arg.chars() { + if matches!(ch, '"' | '\\' | '$') { + buf.push('\\'); + } + buf.push(ch); + } + buf.push('"'); +} diff --git a/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs b/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs new file mode 100644 index 00000000000..69641fed3bc --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/command_line_args/tests.rs @@ -0,0 +1,25 @@ +#[test] +fn quote_command_line_args() { + use super::quote_command_line_args; + + struct Case<'a> { + args: &'a [&'a str], + expected: &'a str, + } + + let cases = &[ + Case { args: &[], expected: "" }, + Case { args: &["--hello", "world"], expected: r#""--hello" "world""# }, + Case { args: &["--hello world"], expected: r#""--hello world""# }, + Case { + args: &["plain", "$dollar", "spa ce", r"back\slash", r#""quote""#, "plain"], + expected: r#""plain" "\$dollar" "spa ce" "back\\slash" "\"quote\"" "plain""#, + }, + ]; + + for &Case { args, expected } in cases { + let args = args.iter().copied().map(str::to_owned).collect::<Vec<_>>(); + let actual = quote_command_line_args(&args); + assert_eq!(actual, expected, "args {args:?}"); + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index f571716d9dd..5ac3a87c158 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -563,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { config::AutoDiff::Enable => {} // We handle this below config::AutoDiff::NoPostopt => {} + // Disables TypeTree generation + config::AutoDiff::NoTT => {} } } // This helps with handling enums for now. @@ -617,7 +619,7 @@ pub(crate) fn run_pass_manager( crate::builder::gpu_offload::handle_gpu_code(cgcx, &cx); } - if cfg!(llvm_enzyme) && enable_ad && !thin { + if cfg!(feature = "llvm_enzyme") && enable_ad && !thin { let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { diff --git a/compiler/rustc_codegen_llvm/src/back/mod.rs b/compiler/rustc_codegen_llvm/src/back/mod.rs index 6cb89f80ab8..fe3883e8c73 100644 --- a/compiler/rustc_codegen_llvm/src/back/mod.rs +++ b/compiler/rustc_codegen_llvm/src/back/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod archive; +mod command_line_args; pub(crate) mod lto; pub(crate) mod owned_target_machine; mod profiling; diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 6d8178320fe..903a882916e 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,4 +1,3 @@ -use std::assert_matches::assert_matches; use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -39,13 +38,10 @@ impl OwnedTargetMachine { output_obj_file: &CStr, debug_info_compression: &CStr, use_emulated_tls: bool, - args_cstr_buff: &[u8], + argv0: &str, + command_line_args: &str, use_wasm_eh: bool, ) -> Result<Self, LlvmError<'static>> { - // The argument list is passed as the concatenation of one or more C strings. - // This implies that there must be a last byte, and it must be 0. - assert_matches!(args_cstr_buff, [.., b'\0'], "the last byte must be a NUL terminator"); - // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { llvm::LLVMRustCreateTargetMachine( @@ -70,8 +66,10 @@ impl OwnedTargetMachine { output_obj_file.as_ptr(), debug_info_compression.as_ptr(), use_emulated_tls, - args_cstr_buff.as_ptr(), - args_cstr_buff.len(), + argv0.as_ptr(), + argv0.len(), + command_line_args.as_ptr(), + command_line_args.len(), use_wasm_eh, ) }; @@ -97,8 +95,6 @@ impl Drop for OwnedTargetMachine { // SAFETY: constructing ensures we have a valid pointer created by // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. - unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_ptr()); - } + unsafe { llvm::LLVMDisposeTargetMachine(self.tm_unique) }; } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 423f0da4878..1950b8288d1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -31,6 +31,7 @@ use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext, sym}; use rustc_target::spec::{CodeModel, FloatAbi, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use tracing::{debug, trace}; +use crate::back::command_line_args::quote_command_line_args; use crate::back::lto::ThinBuffer; use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ @@ -203,6 +204,9 @@ pub(crate) fn target_machine_factory( optlvl: config::OptLevel, target_features: &[String], ) -> TargetMachineFactoryFn<LlvmCodegenBackend> { + // Self-profile timer for creating a _factory_. + let _prof_timer = sess.prof.generic_activity("target_machine_factory"); + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); @@ -249,23 +253,18 @@ pub(crate) fn target_machine_factory( let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); - // copy the exe path, followed by path all into one buffer - // null terminating them so we can use them as null terminated strings - let args_cstr_buff = { - let mut args_cstr_buff: Vec<u8> = Vec::new(); - let exe_path = std::env::current_exe().unwrap_or_default(); - let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default(); - - args_cstr_buff.extend_from_slice(exe_path_str.as_bytes()); - args_cstr_buff.push(0); - - for arg in sess.expanded_args.iter() { - args_cstr_buff.extend_from_slice(arg.as_bytes()); - args_cstr_buff.push(0); - } - - args_cstr_buff - }; + // Command-line information to be included in the target machine. + // This seems to only be used for embedding in PDB debuginfo files. + // FIXME(Zalathar): Maybe skip this for non-PDB targets? + let argv0 = std::env::current_exe() + .unwrap_or_default() + .into_os_string() + .into_string() + .unwrap_or_default(); + let command_line_args = quote_command_line_args(&sess.expanded_args); + // Self-profile counter for the number of bytes produced by command-line quoting. + // Values are summed, so the summary result is cumulative across all TM factories. + sess.prof.artifact_size("quoted_command_line_args", "-", command_line_args.len() as u64); let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); match sess.opts.debuginfo_compression { @@ -288,7 +287,11 @@ pub(crate) fn target_machine_factory( let use_wasm_eh = wants_wasm_eh(sess); + let prof = SelfProfilerRef::clone(&sess.prof); Arc::new(move |config: TargetMachineFactoryConfig| { + // Self-profile timer for invoking a factory to create a target machine. + let _prof_timer = prof.generic_activity("target_machine_factory_inner"); + let path_to_cstring_helper = |path: Option<PathBuf>| -> CString { let path = path.unwrap_or_default(); let path = path_mapping @@ -323,7 +326,8 @@ pub(crate) fn target_machine_factory( &output_obj_file, &debuginfo_compression, use_emulated_tls, - &args_cstr_buff, + &argv0, + &command_line_args, use_wasm_eh, ) }) @@ -574,7 +578,8 @@ pub(crate) unsafe fn llvm_optimize( // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting // differentiated. - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let run_enzyme = autodiff_stage == AutodiffStage::DuringAD; let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore); let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter); @@ -740,7 +745,8 @@ pub(crate) fn optimize( // If we know that we will later run AD, then we disable vectorization and loop unrolling. // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; // The embedded bitcode is used to run LTO/ThinLTO. // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 978134cc32b..6d12b511e9c 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -105,7 +105,7 @@ pub(crate) fn compile_codegen_unit( if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty()); attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0f17cc9063a..5271d0b4bb8 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,6 +2,7 @@ use std::borrow::{Borrow, Cow}; use std::ops::Deref; use std::{iter, ptr}; +use rustc_ast::expand::typetree::FncTree; pub(crate) mod autodiff; pub(crate) mod gpu_offload; @@ -1107,11 +1108,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + tt: Option<FncTree>, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - unsafe { + let memcpy = unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, dst, @@ -1120,7 +1122,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align.bytes() as c_uint, size, is_volatile, - ); + ) + }; + + // TypeTree metadata for memcpy is especially important: when Enzyme encounters + // a memcpy during autodiff, it needs to know the structure of the data being + // copied to properly track derivatives. For example, copying an array of floats + // vs. copying a struct with mixed types requires different derivative handling. + // The TypeTree tells Enzyme exactly what memory layout to expect. + if let Some(tt) = tt { + crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt); } } @@ -1433,7 +1444,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // If there is an inline attribute and a target feature that matches // we will add the attribute to the callsite otherwise we'll omit // this and not add the attribute to prevent soundness issues. - && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance) + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance) && self.cx.tcx.is_target_feature_call_safe( &fn_call_attrs.target_features, &fn_defn_attrs.target_features, diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 6ddf53cdc87..c3485f56391 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,10 +1,12 @@ use std::ptr; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; +use rustc_ast::expand::typetree::FncTree; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; -use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; +use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv}; use rustc_middle::{bug, ty}; +use rustc_target::callconv::PassMode; use tracing::debug; use crate::builder::{Builder, PlaceRef, UNNAMED}; @@ -16,9 +18,12 @@ use crate::value::Value; pub(crate) fn adjust_activity_to_abi<'tcx>( tcx: TyCtxt<'tcx>, - fn_ty: Ty<'tcx>, + instance: Instance<'tcx>, + typing_env: TypingEnv<'tcx>, da: &mut Vec<DiffActivity>, ) { + let fn_ty = instance.ty(tcx, typing_env); + if !matches!(fn_ty.kind(), ty::FnDef(..)) { bug!("expected fn def for autodiff, got {:?}", fn_ty); } @@ -27,8 +32,16 @@ pub(crate) fn adjust_activity_to_abi<'tcx>( // All we do is decide how to handle the arguments. let sig = fn_ty.fn_sig(tcx).skip_binder(); + // FIXME(Sa4dUs): pass proper varargs once we have support for differentiating variadic functions + let Ok(fn_abi) = + tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) + else { + bug!("failed to get fn_abi of instance with empty varargs"); + }; + let mut new_activities = vec![]; let mut new_positions = vec![]; + let mut del_activities = 0; for (i, ty) in sig.inputs().iter().enumerate() { if let Some(inner_ty) = ty.builtin_deref(true) { if inner_ty.is_slice() { @@ -80,6 +93,34 @@ pub(crate) fn adjust_activity_to_abi<'tcx>( continue; } } + + let pci = PseudoCanonicalInput { typing_env: TypingEnv::fully_monomorphized(), value: *ty }; + + let layout = match tcx.layout_of(pci) { + Ok(layout) => layout.layout, + Err(_) => { + bug!("failed to compute layout for type {:?}", ty); + } + }; + + let pass_mode = &fn_abi.args[i].mode; + + // For ZST, just ignore and don't add its activity, as this arg won't be present + // in the LLVM passed to Enzyme. + // Some targets pass ZST indirectly in the C ABI, in that case, handle it as a normal arg + // FIXME(Sa4dUs): Enforce ZST corresponding diff activity be `Const` + if *pass_mode == PassMode::Ignore { + del_activities += 1; + da.remove(i); + } + + // If the argument is lowered as a `ScalarPair`, we need to duplicate its activity. + // Otherwise, the number of activities won't match the number of LLVM arguments and + // this will lead to errors when verifying the Enzyme call. + if let rustc_abi::BackendRepr::ScalarPair(_, _) = layout.backend_repr() { + new_activities.push(da[i].clone()); + new_positions.push(i + 1 - del_activities); + } } // now add the extra activities coming from slices // Reverse order to not invalidate the indices @@ -254,6 +295,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args: &[&'ll Value], attrs: AutoDiffAttrs, dest: PlaceRef<'tcx, &'ll Value>, + fnc_tree: FncTree, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -330,6 +372,10 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( fn_args, ); + if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() { + crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree); + } + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); builder.store_to_place(call, dest.val); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4a8ea11a3a8..aa5c17269fb 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -370,7 +370,8 @@ pub(crate) unsafe fn create_module<'ll>( ); } - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + if let Some(BranchProtection { bti, pac_ret, gcs }) = sess.opts.unstable_opts.branch_protection + { if sess.target.arch == "aarch64" { llvm::add_module_flag_u32( llmod, @@ -403,6 +404,12 @@ pub(crate) unsafe fn create_module<'ll>( "sign-return-address-with-bkey", u32::from(pac_opts.key == PAuthKey::B), ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "guarded-control-stack", + gcs.into(), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -869,7 +876,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); + let target_cpu = attributes::target_cpu_attr(self, self.sess()); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } @@ -884,15 +891,15 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { + if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); + attrs.push(attributes::target_cpu_attr(self, self.sess())); + attrs.extend(attributes::tune_cpu_attr(self, self.sess())); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } @@ -911,7 +918,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn( llfn, llvm::AttributePlace::Function, - attributes::target_features_attr(self, vec![]).as_slice(), + attributes::target_features_attr(self, self.tcx, vec![]).as_slice(), ); Some(llfn) } else { @@ -1037,7 +1044,10 @@ impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::ReferencesError(_) + | LayoutError::InvalidSimd { .. } = err + { self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) @@ -1054,7 +1064,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { match err { - FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + FnAbiError::Layout( + LayoutError::SizeOverflow(_) + | LayoutError::Cycle(_) + | LayoutError::InvalidSimd { .. }, + ) => { self.tcx.dcx().emit_fatal(Spanned { span, node: err }); } _ => match fn_abi_request { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 7a6dc008c7b..3081badb821 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -2,9 +2,9 @@ use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::DebuggerVisualizerType; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; use crate::builder::Builder; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 0903ddab285..4b74c04ed7a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -117,7 +117,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( .try_to_target_usize(cx.tcx) .expect("expected monomorphic const in codegen") as c_longlong; - let subrange = unsafe { llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; + let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; let subscripts = &[subrange]; let di_node = unsafe { @@ -477,8 +477,8 @@ pub(crate) fn spanned_type_di_node<'ll, 'tcx>( ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), - AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id, span), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id, span), AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), @@ -821,14 +821,15 @@ fn build_basic_type_di_node<'ll, 'tcx>( }; let typedef_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateTypedef( + llvm::LLVMDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_c_char_ptr(), + typedef_name.as_ptr(), typedef_name.len(), unknown_file_metadata(cx), - 0, - None, + 0, // (no line number) + None, // (no scope) + 0u32, // (no alignment specified) ) }; @@ -1034,10 +1035,10 @@ fn create_member_type<'ll, 'tcx>( type_di_node: &'ll DIType, ) -> &'ll DIType { unsafe { - llvm::LLVMRustDIBuilderCreateMemberType( + llvm::LLVMDIBuilderCreateMemberType( DIB(cx), owner, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, @@ -1075,6 +1076,7 @@ fn visibility_di_flags<'ll, 'tcx>( fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let struct_type = unique_type_id.expect_ty(); let ty::Adt(adt_def, _) = struct_type.kind() else { @@ -1082,7 +1084,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( }; assert!(adt_def.is_struct()); let containing_scope = get_namespace_for_item(cx, adt_def.did()); - let struct_type_and_layout = cx.layout_of(struct_type); + let struct_type_and_layout = cx.spanned_layout_of(struct_type, span); let variant_def = adt_def.non_enum_variant(); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) @@ -1275,6 +1277,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( fn build_union_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let union_type = unique_type_id.expect_ty(); let (union_def_id, variant_def) = match union_type.kind() { @@ -1282,7 +1285,7 @@ fn build_union_type_di_node<'ll, 'tcx>( _ => bug!("build_union_type_di_node on a non-ADT"), }; let containing_scope = get_namespace_for_item(cx, union_def_id); - let union_ty_and_layout = cx.layout_of(union_type); + let union_ty_and_layout = cx.spanned_layout_of(union_type, span); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(union_def_id))) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index a5c80895741..4ecc3086e1b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; use smallvec::smallvec; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::dwarf_const::DW_TAG_const_type; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; @@ -378,20 +378,17 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_struct_type_wrapper_di_node, None, ), - unsafe { - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), - enum_type_di_node, - TAG_FIELD_NAME.as_c_char_ptr(), - TAG_FIELD_NAME.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - variant_names_type_di_node, - visibility_flags, - Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), - tag_base_type_align.bits() as u32, - ) - } + create_static_member_type( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + visibility_flags, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align, + ), ] } @@ -570,27 +567,28 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( let build_assoc_const = |name: &str, type_di_node_: &'ll DIType, value: u64, - align: Align| unsafe { + align: Align| + -> &'ll llvm::Metadata { // FIXME: Currently we force all DISCR_* values to be u64's as LLDB seems to have // problems inspecting other value types. Since DISCR_* is typically only going to be // directly inspected via the debugger visualizer - which compares it to the `tag` value // (whose type is not modified at all) it shouldn't cause any real problems. let (t_di, align) = if name == ASSOC_CONST_DISCR_NAME { - (type_di_node_, align.bits() as u32) + (type_di_node_, align) } else { let ty_u64 = Ty::new_uint(cx.tcx, ty::UintTy::U64); - (type_di_node(cx, ty_u64), Align::EIGHT.bits() as u32) + (type_di_node(cx, ty_u64), Align::EIGHT) }; // must wrap type in a `const` modifier for LLDB to be able to inspect the value of the member - let field_type = - llvm::LLVMRustDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di); + let field_type = unsafe { + llvm::LLVMDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di) + }; - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), + create_static_member_type( + cx, wrapper_struct_type_di_node, - name.as_c_char_ptr(), - name.len(), + name, unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, field_type, @@ -975,3 +973,30 @@ fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, s .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) } + +fn create_static_member_type<'ll>( + cx: &CodegenCx<'ll, '_>, + scope: &'ll llvm::Metadata, + name: &str, + file: &'ll llvm::Metadata, + line_number: c_uint, + ty: &'ll llvm::Metadata, + flags: DIFlags, + value: Option<&'ll llvm::Value>, + align: Align, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreateStaticMemberType( + DIB(cx), + scope, + name.as_ptr(), + name.len(), + file, + line_number, + ty, + flags, + value, + align.bits() as c_uint, + ) + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 126082aa3aa..af64e4ebed0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -52,15 +52,6 @@ mod utils; use self::create_scope_map::compute_mir_scopes; pub(crate) use self::metadata::build_global_var_di_node; -// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were -// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp` -// to decide which C++ API to call. Instead, we should just have two separate -// FFI functions and choose the correct one on the Rust side. -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, @@ -174,35 +165,38 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { if direct_offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(direct_offset.bytes()); } for &offset in indirect_offsets { addr_ops.push(DW_OP_deref); if offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); - addr_ops.push(offset.bytes() as u64); + addr_ops.push(offset.bytes()); } } if let Some(fragment) = fragment { // `DW_OP_LLVM_fragment` takes as arguments the fragment's // offset and size, both of them in bits. addr_ops.push(DW_OP_LLVM_fragment); - addr_ops.push(fragment.start.bits() as u64); - addr_ops.push((fragment.end - fragment.start).bits() as u64); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); } + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), + llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( + di_builder, variable_alloca, dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, + addr_expr, dbg_loc, self.llbb(), - ); - } + ) + }; } fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { @@ -630,28 +624,39 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let type_metadata = spanned_type_di_node(self, variable_type, span); - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; let align = self.align_of(variable_type); let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_c_char_ptr(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bits() as u32, - ) + + match variable_kind { + ArgumentVariable(arg_index) => unsafe { + llvm::LLVMDIBuilderCreateParameterVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + arg_index as c_uint, + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + ) + }, + LocalVariable => unsafe { + llvm::LLVMDIBuilderCreateAutoVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + align.bits() as u32, + ) + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index cc1d504b430..7e1e49310f6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } + unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 960a895a203..36cdb498839 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -76,7 +76,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>( attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attrs.extend(attributes::non_lazy_bind_attr(cx)); + attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess)); attributes::apply_to_llfn(llfn, Function, &attrs); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 85f71f331a4..013108d1286 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -18,7 +18,6 @@ use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; use rustc_target::callconv::PassMode; -use rustc_target::spec::PanicStrategy; use tracing::debug; use crate::abi::FnAbiLlvmExt; @@ -674,7 +673,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( catch_func: &'ll Value, dest: PlaceRef<'tcx, &'ll Value>, ) { - if bx.sess().panic_strategy() == PanicStrategy::Abort { + if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.call(try_func_ty, None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; @@ -1208,10 +1207,14 @@ fn codegen_autodiff<'ll, 'tcx>( adjust_activity_to_abi( tcx, - fn_source.ty(tcx, TypingEnv::fully_monomorphized()), + fn_source, + TypingEnv::fully_monomorphized(), &mut diff_attrs.input_activity, ); + let fnc_tree = + rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized())); + // Build body generate_enzyme_call( bx, @@ -1222,6 +1225,7 @@ fn codegen_autodiff<'ll, 'tcx>( &val_arr, diff_attrs.clone(), result, + fnc_tree, ); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 6fb23d09843..b9584a9117f 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -45,6 +45,7 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_span::Symbol; +use rustc_target::spec::{RelocModel, TlsModel}; mod abi; mod allocator; @@ -67,6 +68,7 @@ mod llvm_util; mod mono_item; mod type_; mod type_of; +mod typetree; mod va_arg; mod value; @@ -244,16 +246,7 @@ impl CodegenBackend for LlvmCodegenBackend { match req.kind { PrintKind::RelocationModels => { writeln!(out, "Available relocation models:").unwrap(); - for name in &[ - "static", - "pic", - "pie", - "dynamic-no-pic", - "ropi", - "rwpi", - "ropi-rwpi", - "default", - ] { + for name in RelocModel::ALL.iter().map(RelocModel::desc).chain(["default"]) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -267,9 +260,7 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:").unwrap(); - for name in - &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] - { + for name in TlsModel::ALL.iter().map(TlsModel::desc) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 56d756e52cc..e63043b2122 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -3,9 +3,36 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; -use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; +use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value}; use crate::llvm::{Bool, Builder}; +// TypeTree types +pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct EnzymeTypeTree { + _unused: [u8; 0], +} + +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub(crate) enum CConcreteType { + DT_Anything = 0, + DT_Integer = 1, + DT_Pointer = 2, + DT_Half = 3, + DT_Float = 4, + DT_Double = 5, + DT_Unknown = 6, + DT_FP128 = 9, +} + +pub(crate) struct TypeTree { + pub(crate) inner: CTypeTreeRef, +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { // Enzyme @@ -59,19 +86,49 @@ pub(crate) enum LLVMRustVerifierFailureAction { LLVMReturnStatusAction = 2, } -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) use self::Enzyme_AD::*; -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) mod Enzyme_AD { use std::ffi::{CString, c_char}; use libc::c_void; + use super::{CConcreteType, CTypeTreeRef, Context}; + unsafe extern "C" { pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char); } + + // TypeTree functions + unsafe extern "C" { + pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef; + pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef); + pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool; + pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64); + pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef); + pub(crate) fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ); + pub(crate) fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ); + pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char; + pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char); + } + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; @@ -134,13 +191,74 @@ pub(crate) mod Enzyme_AD { } } -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) use self::Fallback_AD::*; -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) mod Fallback_AD { #![allow(unused_variables)] + use libc::c_char; + + use super::{CConcreteType, CTypeTreeRef, Context}; + + // TypeTree function fallbacks + pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) { + unimplemented!() + } + pub(crate) fn set_inline(val: bool) { unimplemented!() } @@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD { unimplemented!() } } + +impl TypeTree { + pub(crate) fn new() -> TypeTree { + let inner = unsafe { EnzymeNewTypeTree() }; + TypeTree { inner } + } + + pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree { + let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) }; + TypeTree { inner } + } + + pub(crate) fn merge(self, other: Self) -> Self { + unsafe { + EnzymeMergeTypeTree(self.inner, other.inner); + } + drop(other); + self + } + + #[must_use] + pub(crate) fn shift( + self, + layout: &str, + offset: isize, + max_size: isize, + add_offset: usize, + ) -> Self { + let layout = std::ffi::CString::new(layout).unwrap(); + + unsafe { + EnzymeTypeTreeShiftIndiciesEq( + self.inner, + layout.as_ptr(), + offset as i64, + max_size as i64, + add_offset as u64, + ); + } + + self + } + + pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) { + unsafe { + EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx); + } + } +} + +impl Clone for TypeTree { + fn clone(&self) -> Self { + let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) }; + TypeTree { inner } + } +} + +impl std::fmt::Display for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ptr = unsafe { EnzymeTypeTreeToString(self.inner) }; + let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) }; + match cstr.to_str() { + Ok(x) => write!(f, "{}", x)?, + Err(err) => write!(f, "could not parse: {}", err)?, + } + + // delete C string pointer + unsafe { + EnzymeTypeTreeToStringFree(ptr); + } + + Ok(()) + } +} + +impl std::fmt::Debug for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + <Self as std::fmt::Display>::fmt(self, f) + } +} + +impl Drop for TypeTree { + fn drop(&mut self) { + unsafe { EnzymeFreeTypeTree(self.inner) } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 71d8b7d25fe..afd2991a09c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -24,9 +24,9 @@ use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ - DIArray, DIBasicType, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, - DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, DISubrange, - DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, + DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, + DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, + DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind, }; use crate::llvm; @@ -710,6 +710,7 @@ pub(crate) enum MemoryEffects { None, ReadOnly, InaccessibleMemOnly, + ReadOnlyNotPure, } /// LLVMOpcode @@ -806,6 +807,8 @@ unsafe extern "C" { pub(crate) type Metadata; pub(crate) type BasicBlock; pub(crate) type Comdat; + /// `&'ll DbgRecord` represents `LLVMDbgRecordRef`. + pub(crate) type DbgRecord; } #[repr(C)] pub(crate) struct Builder<'a>(InvariantOpaque<'a>); @@ -890,7 +893,6 @@ pub(crate) mod debuginfo { pub(crate) type DIVariable = DIDescriptor; pub(crate) type DIGlobalVariableExpression = DIDescriptor; pub(crate) type DIArray = DIDescriptor; - pub(crate) type DISubrange = DIDescriptor; pub(crate) type DIEnumerator = DIDescriptor; pub(crate) type DITemplateTypeParameter = DIDescriptor; @@ -1052,6 +1054,8 @@ unsafe extern "C" { SLen: c_uint, ) -> MetadataKindId; + pub(crate) fn LLVMDisposeTargetMachine(T: ptr::NonNull<TargetMachine>); + // Create modules. pub(crate) fn LLVMModuleCreateWithNameInContext( ModuleID: *const c_char, @@ -1943,6 +1947,105 @@ unsafe extern "C" { UniqueId: *const c_uchar, // See "PTR_LEN_STR". UniqueIdLen: size_t, ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStaticMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + Type: &'ll Metadata, + Flags: DIFlags, + ConstantVal: Option<&'ll Value>, + AlignInBits: u32, + ) -> &'ll Metadata; + + /// Creates a "qualified type" in the C/C++ sense, by adding modifiers + /// like `const` or `volatile`. + pub(crate) fn LLVMDIBuilderCreateQualifiedType<'ll>( + Builder: &DIBuilder<'ll>, + Tag: c_uint, // (DWARF tag, e.g. `DW_TAG_const_type`) + Type: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateTypedef<'ll>( + Builder: &DIBuilder<'ll>, + Type: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Scope: Option<&'ll Metadata>, + AlignInBits: u32, // (optional; default is 0) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>( + Builder: &DIBuilder<'ll>, + LowerBound: i64, + Count: i64, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>( + Builder: &DIBuilder<'ll>, + Data: *const Option<&'ll Metadata>, + NumElements: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateExpression<'ll>( + Builder: &DIBuilder<'ll>, + Addr: *const u64, + Length: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Storage: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ArgNo: c_uint, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + ) -> &'ll Metadata; } #[link(name = "llvm-wrapper", kind = "static")] @@ -2278,30 +2381,6 @@ unsafe extern "C" { TParam: &'a DIArray, ) -> &'a DISubprogram; - pub(crate) fn LLVMRustDIBuilderCreateTypedef<'a>( - Builder: &DIBuilder<'a>, - Type: &'a DIBasicType, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Scope: Option<&'a DIScope>, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - SizeInBits: u64, - AlignInBits: u32, - OffsetInBits: u64, - Flags: DIFlags, - Ty: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2317,25 +2396,6 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateStaticMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - Flags: DIFlags, - val: Option<&'a Value>, - AlignInBits: u32, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateQualifiedType<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Type: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( Builder: &DIBuilder<'a>, Context: Option<&'a DIScope>, @@ -2352,43 +2412,6 @@ unsafe extern "C" { AlignInBits: u32, ) -> &'a DIGlobalVariableExpression; - pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ); - pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2488,12 +2511,13 @@ unsafe extern "C" { OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, UseEmulatedTls: bool, - ArgsCstrBuff: *const c_uchar, // See "PTR_LEN_STR". - ArgsCstrBuffLen: usize, + Argv0: *const c_uchar, // See "PTR_LEN_STR". + Argv0Len: size_t, + CommandLineArgs: *const c_uchar, // See "PTR_LEN_STR". + CommandLineArgsLen: size_t, UseWasmEH: bool, ) -> *mut TargetMachine; - pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 8461c8b03d5..3b920168e06 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -106,7 +106,7 @@ unsafe fn configure_llvm(sess: &Session) { if sess.target.os == "emscripten" && !sess.opts.unstable_opts.emscripten_wasm_eh - && sess.panic_strategy() == PanicStrategy::Unwind + && sess.panic_strategy().unwinds() { add("-enable-emscripten-cxx-exceptions", false); } @@ -217,27 +217,16 @@ impl<'a> IntoIterator for LLVMFeature<'a> { /// Rust can also be build with an external precompiled version of LLVM which might lead to failures /// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics. pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> { - let arch = if sess.target.arch == "x86_64" { - "x86" - } else if sess.target.arch == "arm64ec" { - "aarch64" - } else if sess.target.arch == "sparc64" { - "sparc" - } else if sess.target.arch == "powerpc64" { - "powerpc" - } else { - &*sess.target.arch + let raw_arch = &*sess.target.arch; + let arch = match raw_arch { + "x86_64" => "x86", + "arm64ec" => "aarch64", + "sparc64" => "sparc", + "powerpc64" => "powerpc", + _ => raw_arch, }; + let (major, _, _) = get_version(); match (arch, s) { - ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( - "sse4.2", - smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], - )), - ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), - ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), - ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), - ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), - ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")), ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")), ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")), @@ -260,14 +249,23 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "fpmr") => None, // only existed in 18 ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), // Filter out features that are not supported by the current LLVM version - ("loongarch32" | "loongarch64", "32s") if get_version().0 < 21 => None, + ("loongarch32" | "loongarch64", "32s") if major < 21 => None, + ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), + ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), + ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( + "sse4.2", + smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], + )), + ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), + ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), + ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), + ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), + ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies( s, smallvec![TargetFeatureFoldStrength::EnableOnly("evex512")], )), - ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), - ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), ("x86", "avx10.1") => Some(LLVMFeature::new("avx10.1-512")), ("x86", "avx10.2") => Some(LLVMFeature::new("avx10.2-512")), ("x86", "apxf") => Some(LLVMFeature::with_dependencies( diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs new file mode 100644 index 00000000000..7e263503700 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -0,0 +1,122 @@ +use rustc_ast::expand::typetree::FncTree; +#[cfg(feature = "llvm_enzyme")] +use { + crate::attributes, + rustc_ast::expand::typetree::TypeTree as RustTypeTree, + std::ffi::{CString, c_char, c_uint}, +}; + +use crate::llvm::{self, Value}; + +#[cfg(feature = "llvm_enzyme")] +fn to_enzyme_typetree( + rust_typetree: RustTypeTree, + _data_layout: &str, + llcx: &llvm::Context, +) -> llvm::TypeTree { + let mut enzyme_tt = llvm::TypeTree::new(); + process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); + enzyme_tt +} +#[cfg(feature = "llvm_enzyme")] +fn process_typetree_recursive( + enzyme_tt: &mut llvm::TypeTree, + rust_typetree: &RustTypeTree, + parent_indices: &[i64], + llcx: &llvm::Context, +) { + for rust_type in &rust_typetree.0 { + let concrete_type = match rust_type.kind { + rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything, + rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer, + rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer, + rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half, + rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float, + rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double, + rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128, + rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown, + }; + + let mut indices = parent_indices.to_vec(); + if !parent_indices.is_empty() { + indices.push(rust_type.offset as i64); + } else if rust_type.offset == -1 { + indices.push(-1); + } else { + indices.push(rust_type.offset as i64); + } + + enzyme_tt.insert(&indices, concrete_type, llcx); + + if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer + && !rust_type.child.0.is_empty() + { + process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx); + } + } +} + +#[cfg(feature = "llvm_enzyme")] +pub(crate) fn add_tt<'ll>( + llmod: &'ll llvm::Module, + llcx: &'ll llvm::Context, + fn_def: &'ll Value, + tt: FncTree, +) { + let inputs = tt.args; + let ret_tt: RustTypeTree = tt.ret; + + let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) }; + let llvm_data_layout = + std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + let attr_name = "enzyme_type"; + let c_attr_name = CString::new(attr_name).unwrap(); + + for (i, input) in inputs.iter().enumerate() { + unsafe { + let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } + } + + unsafe { + let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let ret_attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } +} + +#[cfg(not(feature = "llvm_enzyme"))] +pub(crate) fn add_tt<'ll>( + _llmod: &'ll llvm::Module, + _llcx: &'ll llvm::Context, + _fn_def: &'ll Value, + _tt: FncTree, +) { + unimplemented!() +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ab08125217f..d48c7cf874a 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -738,6 +738,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>( src_align, bx.const_u32(layout.layout.size().bytes() as u32), MemFlags::empty(), + None, ); tmp } else { |