diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
9 files changed, 42 insertions, 26 deletions
diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 6b17b5f6989..9e8e4e1c567 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -62,8 +62,8 @@ fn generate_enzyme_call<'ll>( // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::ffi::CStr::from_bytes_with_nul(name).unwrap().to_str().unwrap(); - ad_name.push_str(outer_fn_name.to_string().as_str()); + let outer_fn_name = std::str::from_utf8(name).unwrap(); + ad_name.push_str(outer_fn_name); // Let us assume the user wrote the following function square: // @@ -255,14 +255,14 @@ fn generate_enzyme_call<'ll>( // have no debug info to copy, which would then be ok. trace!("no dbg info"); } + // Now that we copied the metadata, get rid of dummy code. - llvm::LLVMRustEraseInstBefore(entry, last_inst); - llvm::LLVMRustEraseInstFromParent(last_inst); + llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst); - if cx.val_ty(outer_fn) != cx.type_void() { - builder.ret(call); - } else { + if cx.val_ty(call) == cx.type_void() { builder.ret_void(); + } else { + builder.ret(call); } // Let's crash in case that we messed something up above and generated invalid IR. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index f8b30cb507a..8c94a46ebf3 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -314,7 +314,12 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { GlobalAlloc::VTable(ty, dyn_ty) => { let alloc = self .tcx - .global_alloc(self.tcx.vtable_allocation((ty, dyn_ty.principal()))) + .global_alloc(self.tcx.vtable_allocation(( + ty, + dyn_ty.principal().map(|principal| { + self.tcx.instantiate_bound_regions_with_erased(principal) + }), + ))) .unwrap_memory(); let init = const_alloc_to_llvm(self, alloc, /*static*/ false); let value = self.static_addr_of_impl(init, alloc.inner().align, None); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 771ebf2057f..c6855dd42e5 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -420,8 +420,14 @@ impl<'ll> CodegenCx<'ll, '_> { let g = if val_llty == llty { g } else { - // If we created the global with the wrong type, - // correct the type. + // codegen_static_initializer creates the global value just from the + // `Allocation` data by generating one big struct value that is just + // all the bytes and pointers after each other. This will almost never + // match the type that the static was declared with. Unfortunately + // we can't just LLVMConstBitCast our way out of it because that has very + // specific rules on what can be cast. So instead of adding a new way to + // generate static initializers that match the static's type, we picked + // the easier option and retroactively change the type of the static item itself. let name = llvm::get_value_name(g).to_vec(); llvm::set_value_name(g, b""); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 79381f35a3c..ba4fd75fb94 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -77,8 +77,7 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { /// Cache instances of monomorphic and polymorphic items pub instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>, /// Cache generated vtables - pub vtables: - RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>, + pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>>, /// Cache of constant strings, pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>, @@ -663,15 +662,14 @@ impl<'ll> SimpleCx<'ll> { impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn vtables( &self, - ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>> - { + ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>> { &self.vtables } fn apply_vcall_visibility_metadata( &self, ty: Ty<'tcx>, - poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { apply_vcall_visibility_metadata(self, ty, poly_trait_ref, vtable); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index fd22421c7fc..9a2473d6cf2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -298,7 +298,7 @@ struct UsageSets<'tcx> { /// Prepare sets of definitions that are relevant to deciding whether something /// is an "unused function" for coverage purposes. fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> { - let MonoItemPartitions { all_mono_items, codegen_units } = + let MonoItemPartitions { all_mono_items, codegen_units, .. } = tcx.collect_and_partition_mono_items(()); // Obtain a MIR body for each function participating in codegen, via an diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 6eae16d8be6..3a0c7f007bd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -13,7 +13,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, Instance, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, + self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, }; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Symbol, hygiene}; @@ -1399,7 +1399,7 @@ pub(crate) fn build_global_var_di_node<'ll>( fn build_vtable_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, - poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, ) -> &'ll DIType { let tcx = cx.tcx; @@ -1510,7 +1510,7 @@ fn find_vtable_behind_cast<'ll>(vtable: &'ll Value) -> &'ll Value { pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, - trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in @@ -1531,7 +1531,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( let vtable = find_vtable_behind_cast(vtable); let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); - let trait_def_id = trait_ref_self.def_id(); + let trait_def_id = trait_ref_self.def_id; let trait_vis = cx.tcx.visibility(trait_def_id); let cgus = cx.sess().codegen_units().as_usize(); @@ -1590,7 +1590,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( pub(crate) fn create_vtable_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, - poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { if cx.dbg_cx.is_none() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index a37e719d43f..af1d503ad6a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -6,7 +6,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_macros::HashStable; use rustc_middle::bug; -use rustc_middle::ty::{self, PolyExistentialTraitRef, Ty, TyCtxt}; +use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt}; use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; use crate::common::{AsCCharPtr, CodegenCx}; @@ -44,7 +44,7 @@ pub(super) enum UniqueTypeId<'tcx> { /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode. VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst), /// The ID of the artificial type we create for VTables. - VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst), + VTableTy(Ty<'tcx>, Option<ExistentialTraitRef<'tcx>>, private::HiddenZst), } impl<'tcx> UniqueTypeId<'tcx> { @@ -88,7 +88,7 @@ impl<'tcx> UniqueTypeId<'tcx> { pub(crate) fn for_vtable_ty( tcx: TyCtxt<'tcx>, self_type: Ty<'tcx>, - implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, + implemented_trait: Option<ExistentialTraitRef<'tcx>>, ) -> Self { assert_eq!( self_type, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 5089560784a..b1ce52667bd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -588,7 +588,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn create_vtable_debuginfo( &self, ty: Ty<'tcx>, - trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, vtable: Self::Value, ) { metadata::create_vtable_di_node(self, ty, trait_ref, vtable) diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 729d6f62e24..92b0ce8ffe1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -4,16 +4,23 @@ use libc::{c_char, c_uint}; use super::ffi::{BasicBlock, Metadata, Module, Type, Value}; use crate::llvm::Bool; + +#[link(name = "llvm-wrapper", kind = "static")] extern "C" { // Enzyme pub fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool; - pub fn LLVMRustEraseInstBefore(BB: &BasicBlock, I: &Value); + pub fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value); pub fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>; pub fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>; pub fn LLVMRustEraseInstFromParent(V: &Value); pub fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value; pub fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; +} +extern "C" { + // Enzyme + pub fn LLVMDumpModule(M: &Module); + pub fn LLVMDumpValue(V: &Value); pub fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; pub fn LLVMGetReturnType(T: &Type) -> &Type; pub fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); |