diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder/autodiff.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 71 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 38 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 51 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 77 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 56 |
13 files changed, 234 insertions, 124 deletions
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index d2de62b17f0..2d007416263 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1325,7 +1325,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { fn get_static(&mut self, def_id: DefId) -> &'ll Value { // Forward to the `get_static` method of `CodegenCx` - self.cx().get_static(def_id) + let s = self.cx().get_static(def_id); + // Cast to default address space if globals are in a different addrspace + self.cx().const_pointercast(s, self.type_ptr()) } } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 6b17b5f6989..9e8e4e1c567 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -62,8 +62,8 @@ fn generate_enzyme_call<'ll>( // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::ffi::CStr::from_bytes_with_nul(name).unwrap().to_str().unwrap(); - ad_name.push_str(outer_fn_name.to_string().as_str()); + let outer_fn_name = std::str::from_utf8(name).unwrap(); + ad_name.push_str(outer_fn_name); // Let us assume the user wrote the following function square: // @@ -255,14 +255,14 @@ fn generate_enzyme_call<'ll>( // have no debug info to copy, which would then be ok. trace!("no dbg info"); } + // Now that we copied the metadata, get rid of dummy code. - llvm::LLVMRustEraseInstBefore(entry, last_inst); - llvm::LLVMRustEraseInstFromParent(last_inst); + llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst); - if cx.val_ty(outer_fn) != cx.type_void() { - builder.ret(call); - } else { + if cx.val_ty(call) == cx.type_void() { builder.ret_void(); + } else { + builder.ret(call); } // Let's crash in case that we messed something up above and generated invalid IR. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index b4e9b9f44f4..8c94a46ebf3 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -225,6 +225,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); } llvm::set_linkage(g, llvm::Linkage::InternalLinkage); + // Cast to default address space if globals are in a different addrspace + let g = self.const_pointercast(g, self.type_ptr()); (s.to_owned(), g) }) .1; @@ -289,7 +291,7 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let alloc = alloc.inner(); let value = match alloc.mutability { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), - _ => self.static_addr_of(init, alloc.align, None), + _ => self.static_addr_of_impl(init, alloc.align, None), }; if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() { @@ -312,10 +314,15 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { GlobalAlloc::VTable(ty, dyn_ty) => { let alloc = self .tcx - .global_alloc(self.tcx.vtable_allocation((ty, dyn_ty.principal()))) + .global_alloc(self.tcx.vtable_allocation(( + ty, + dyn_ty.principal().map(|principal| { + self.tcx.instantiate_bound_regions_with_erased(principal) + }), + ))) .unwrap_memory(); let init = const_alloc_to_llvm(self, alloc, /*static*/ false); - let value = self.static_addr_of(init, alloc.inner().align, None); + let value = self.static_addr_of_impl(init, alloc.inner().align, None); (value, AddressSpace::DATA) } GlobalAlloc::Static(def_id) => { @@ -327,7 +334,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let llval = unsafe { llvm::LLVMConstInBoundsGEP2( self.type_i8(), - self.const_bitcast(base_addr, self.type_ptr_ext(base_addr_space)), + // Cast to the required address space if necessary + self.const_pointercast(base_addr, self.type_ptr_ext(base_addr_space)), &self.const_usize(offset.bytes()), 1, ) diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index c7114480d8b..c6855dd42e5 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -210,6 +210,14 @@ impl<'ll> CodegenCx<'ll, '_> { unsafe { llvm::LLVMConstBitCast(val, ty) } } + pub(crate) fn const_pointercast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstPointerCast(val, ty) } + } + + /// Create a global variable. + /// + /// The returned global variable is a pointer in the default address space for globals. + /// Fails if a symbol with the given name already exists. pub(crate) fn static_addr_of_mut( &self, cv: &'ll Value, @@ -233,6 +241,34 @@ impl<'ll> CodegenCx<'ll, '_> { gv } + /// Create a global constant. + /// + /// The returned global variable is a pointer in the default address space for globals. + pub(crate) fn static_addr_of_impl( + &self, + cv: &'ll Value, + align: Align, + kind: Option<&str>, + ) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); @@ -384,8 +420,14 @@ impl<'ll> CodegenCx<'ll, '_> { let g = if val_llty == llty { g } else { - // If we created the global with the wrong type, - // correct the type. + // codegen_static_initializer creates the global value just from the + // `Allocation` data by generating one big struct value that is just + // all the bytes and pointers after each other. This will almost never + // match the type that the static was declared with. Unfortunately + // we can't just LLVMConstBitCast our way out of it because that has very + // specific rules on what can be cast. So instead of adding a new way to + // generate static initializers that match the static's type, we picked + // the easier option and retroactively change the type of the static item itself. let name = llvm::get_value_name(g).to_vec(); llvm::set_value_name(g, b""); @@ -505,24 +547,15 @@ impl<'ll> CodegenCx<'ll, '_> { } impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> { + /// Get a pointer to a global variable. + /// + /// The pointer will always be in the default address space. If global variables default to a + /// different address space, an addrspacecast is inserted. fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { - if let Some(&gv) = self.const_globals.borrow().get(&cv) { - unsafe { - // Upgrade the alignment in cases where the same constant is used with different - // alignment requirements - let llalign = align.bytes() as u32; - if llalign > llvm::LLVMGetAlignment(gv) { - llvm::LLVMSetAlignment(gv, llalign); - } - } - return gv; - } - let gv = self.static_addr_of_mut(cv, align, kind); - unsafe { - llvm::LLVMSetGlobalConstant(gv, True); - } - self.const_globals.borrow_mut().insert(cv, gv); - gv + let gv = self.static_addr_of_impl(cv, align, kind); + // static_addr_of_impl returns the bare global variable, which might not be in the default + // address space. Cast to the default address space if necessary. + self.const_pointercast(gv, self.type_ptr()) } fn codegen_static(&self, def_id: DefId) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 79381f35a3c..ba4fd75fb94 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -77,8 +77,7 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { /// Cache instances of monomorphic and polymorphic items pub instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>, /// Cache generated vtables - pub vtables: - RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>, + pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>>, /// Cache of constant strings, pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>, @@ -663,15 +662,14 @@ impl<'ll> SimpleCx<'ll> { impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn vtables( &self, - ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>> - { + ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>> { &self.vtables } fn apply_vcall_visibility_metadata( &self, ty: Ty<'tcx>, - poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { apply_vcall_visibility_metadata(self, ty, poly_trait_ref, vtable); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index b3ad2a0e409..9a2473d6cf2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -9,6 +9,7 @@ use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; use rustc_middle::mir; +use rustc_middle::mir::mono::MonoItemPartitions; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::RemapFileNameExt; use rustc_session::config::RemapPathScopeComponents; @@ -297,12 +298,13 @@ struct UsageSets<'tcx> { /// Prepare sets of definitions that are relevant to deciding whether something /// is an "unused function" for coverage purposes. fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> { - let (all_mono_items, cgus) = tcx.collect_and_partition_mono_items(()); + let MonoItemPartitions { all_mono_items, codegen_units, .. } = + tcx.collect_and_partition_mono_items(()); // Obtain a MIR body for each function participating in codegen, via an // arbitrary instance. let mut def_ids_seen = FxHashSet::default(); - let def_and_mir_for_all_mono_fns = cgus + let def_and_mir_for_all_mono_fns = codegen_units .iter() .flat_map(|cgu| cgu.items().keys()) .filter_map(|item| match item { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 8d782a618fc..3a0c7f007bd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -13,7 +13,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, Instance, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, + self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, }; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Symbol, hygiene}; @@ -919,8 +919,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( .unwrap_or_default(); let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); - let dwarf_version = - tcx.sess.opts.unstable_opts.dwarf_version.unwrap_or(tcx.sess.target.default_dwarf_version); + let dwarf_version = tcx.sess.dwarf_version(); let is_dwarf_kind = matches!(tcx.sess.target.debuginfo_kind, DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym); // Don't emit `.debug_pubnames` and `.debug_pubtypes` on DWARFv4 or lower. @@ -1400,7 +1399,7 @@ pub(crate) fn build_global_var_di_node<'ll>( fn build_vtable_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, - poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, ) -> &'ll DIType { let tcx = cx.tcx; @@ -1488,10 +1487,30 @@ fn build_vtable_type_di_node<'ll, 'tcx>( .di_node } +/// Get the global variable for the vtable. +/// +/// When using global variables, we may have created an addrspacecast to get a pointer to the +/// default address space if global variables are created in a different address space. +/// For modifying the vtable, we need the real global variable. This function accepts either a +/// global variable (which is simply returned), or an addrspacecast constant expression. +/// If the given value is an addrspacecast, the cast is removed and the global variable behind +/// the cast is returned. +fn find_vtable_behind_cast<'ll>(vtable: &'ll Value) -> &'ll Value { + // The vtable is a global variable, which may be behind an addrspacecast. + unsafe { + if let Some(c) = llvm::LLVMIsAConstantExpr(vtable) { + if llvm::LLVMGetConstOpcode(c) == llvm::Opcode::AddrSpaceCast { + return llvm::LLVMGetOperand(c, 0).unwrap(); + } + } + } + vtable +} + pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, - trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in @@ -1508,9 +1527,11 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( let Some(trait_ref) = trait_ref else { return }; + // Unwrap potential addrspacecast + let vtable = find_vtable_behind_cast(vtable); let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); - let trait_def_id = trait_ref_self.def_id(); + let trait_def_id = trait_ref_self.def_id; let trait_vis = cx.tcx.visibility(trait_def_id); let cgus = cx.sess().codegen_units().as_usize(); @@ -1569,7 +1590,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( pub(crate) fn create_vtable_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, - poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { if cx.dbg_cx.is_none() { @@ -1581,6 +1602,9 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>( return; } + // Unwrap potential addrspacecast + let vtable = find_vtable_behind_cast(vtable); + // When full debuginfo is enabled, we want to try and prevent vtables from being // merged. Otherwise debuggers will have a hard time mapping from dyn pointer // to concrete type. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index a37e719d43f..af1d503ad6a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -6,7 +6,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_macros::HashStable; use rustc_middle::bug; -use rustc_middle::ty::{self, PolyExistentialTraitRef, Ty, TyCtxt}; +use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt}; use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; use crate::common::{AsCCharPtr, CodegenCx}; @@ -44,7 +44,7 @@ pub(super) enum UniqueTypeId<'tcx> { /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode. VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst), /// The ID of the artificial type we create for VTables. - VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst), + VTableTy(Ty<'tcx>, Option<ExistentialTraitRef<'tcx>>, private::HiddenZst), } impl<'tcx> UniqueTypeId<'tcx> { @@ -88,7 +88,7 @@ impl<'tcx> UniqueTypeId<'tcx> { pub(crate) fn for_vtable_ty( tcx: TyCtxt<'tcx>, self_type: Ty<'tcx>, - implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, + implemented_trait: Option<ExistentialTraitRef<'tcx>>, ) -> Self { assert_eq!( self_type, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index e6778411365..b1ce52667bd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -22,6 +22,7 @@ use rustc_session::config::{self, DebugInfo}; use rustc_span::{ BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, Symbol, }; +use rustc_target::spec::DebuginfoKind; use smallvec::SmallVec; use tracing::debug; @@ -93,29 +94,31 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { pub(crate) fn finalize(&self, sess: &Session) { unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder) }; - if !sess.target.is_like_msvc { - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - let dwarf_version = - sess.opts.unstable_opts.dwarf_version.unwrap_or(sess.target.default_dwarf_version); - llvm::add_module_flag_u32( - self.llmod, - llvm::ModuleFlagMergeBehavior::Warning, - "Dwarf Version", - dwarf_version, - ); - } else { - // Indicate that we want CodeView debug information on MSVC - llvm::add_module_flag_u32( - self.llmod, - llvm::ModuleFlagMergeBehavior::Warning, - "CodeView", - 1, - ); + + match sess.target.debuginfo_kind { + DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym => { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "Dwarf Version", + sess.dwarf_version(), + ); + } + DebuginfoKind::Pdb => { + // Indicate that we want CodeView debug information + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "CodeView", + 1, + ); + } } // Prevent bitcode readers from deleting the debug info. @@ -585,7 +588,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn create_vtable_debuginfo( &self, ty: Ty<'tcx>, - trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, vtable: Self::Value, ) { metadata::create_vtable_di_node(self, ty, trait_ref, vtable) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index eab4a9f30c9..43d6ccfcb4a 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1329,7 +1329,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle_generic { - let idx = fn_args[2].expect_const().try_to_valtree().unwrap().0.unwrap_branch(); + let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch(); let n = idx.len() as u64; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 729d6f62e24..92b0ce8ffe1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -4,16 +4,23 @@ use libc::{c_char, c_uint}; use super::ffi::{BasicBlock, Metadata, Module, Type, Value}; use crate::llvm::Bool; + +#[link(name = "llvm-wrapper", kind = "static")] extern "C" { // Enzyme pub fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool; - pub fn LLVMRustEraseInstBefore(BB: &BasicBlock, I: &Value); + pub fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value); pub fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>; pub fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>; pub fn LLVMRustEraseInstFromParent(V: &Value); pub fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value; pub fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool; +} +extern "C" { + // Enzyme + pub fn LLVMDumpModule(M: &Module); + pub fn LLVMDumpValue(V: &Value); pub fn LLVMGetFunctionCallConv(F: &Value) -> c_uint; pub fn LLVMGetReturnType(T: &Type) -> &Type; pub fn LLVMGetParams(Fnc: &Value, parms: *mut &Value); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 009d15a932f..cc7c5231aca 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -661,6 +661,79 @@ pub enum MemoryEffects { InaccessibleMemOnly, } +/// LLVMOpcode +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(C)] +pub enum Opcode { + Ret = 1, + Br = 2, + Switch = 3, + IndirectBr = 4, + Invoke = 5, + Unreachable = 7, + CallBr = 67, + FNeg = 66, + Add = 8, + FAdd = 9, + Sub = 10, + FSub = 11, + Mul = 12, + FMul = 13, + UDiv = 14, + SDiv = 15, + FDiv = 16, + URem = 17, + SRem = 18, + FRem = 19, + Shl = 20, + LShr = 21, + AShr = 22, + And = 23, + Or = 24, + Xor = 25, + Alloca = 26, + Load = 27, + Store = 28, + GetElementPtr = 29, + Trunc = 30, + ZExt = 31, + SExt = 32, + FPToUI = 33, + FPToSI = 34, + UIToFP = 35, + SIToFP = 36, + FPTrunc = 37, + FPExt = 38, + PtrToInt = 39, + IntToPtr = 40, + BitCast = 41, + AddrSpaceCast = 60, + ICmp = 42, + FCmp = 43, + PHI = 44, + Call = 45, + Select = 46, + UserOp1 = 47, + UserOp2 = 48, + VAArg = 49, + ExtractElement = 50, + InsertElement = 51, + ShuffleVector = 52, + ExtractValue = 53, + InsertValue = 54, + Freeze = 68, + Fence = 55, + AtomicCmpXchg = 56, + AtomicRMW = 57, + Resume = 58, + LandingPad = 59, + CleanupRet = 61, + CatchRet = 62, + CatchPad = 63, + CleanupPad = 64, + CatchSwitch = 65, +} + unsafe extern "C" { type Opaque; } @@ -991,7 +1064,10 @@ unsafe extern "C" { pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; + pub fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; + pub fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1048,6 +1124,7 @@ unsafe extern "C" { // Operations on instructions pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; + pub fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; // Operations on call sites pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index c3d7c217861..53611c746a7 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -319,7 +319,6 @@ pub fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<Symbol> sess.target .rust_target_features() .iter() - .filter(|(_, gate, _)| gate.in_cfg()) .filter(|(feature, _, _)| { // skip checking special features, as LLVM may not understand them if RUSTC_SPECIAL_FEATURES.contains(feature) { @@ -388,9 +387,13 @@ pub fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<Symbol> sess.target .rust_target_features() .iter() - .filter(|(_, gate, _)| gate.in_cfg()) .filter_map(|(feature, gate, _)| { - if sess.is_nightly_build() || allow_unstable || gate.requires_nightly().is_none() { + // The `allow_unstable` set is used by rustc internally to determined which target + // features are truly available, so we want to return even perma-unstable "forbidden" + // features. + if allow_unstable + || (gate.in_cfg() && (sess.is_nightly_build() || gate.requires_nightly().is_none())) + { Some(*feature) } else { None @@ -670,12 +673,6 @@ pub(crate) fn global_llvm_features( // Will only be filled when `diagnostics` is set! let mut featsmap = FxHashMap::default(); - // Ensure that all ABI-required features are enabled, and the ABI-forbidden ones - // are disabled. - let abi_feature_constraints = sess.target.abi_required_features(); - let abi_incompatible_set = - FxHashSet::from_iter(abi_feature_constraints.incompatible.iter().copied()); - // Compute implied features let mut all_rust_features = vec![]; for feature in sess.opts.cg.target_feature.split(',') { @@ -746,52 +743,11 @@ pub(crate) fn global_llvm_features( } } - // Ensure that the features we enable/disable are compatible with the ABI. - if enable { - if abi_incompatible_set.contains(feature) { - sess.dcx().emit_warn(ForbiddenCTargetFeature { - feature, - enabled: "enabled", - reason: "this feature is incompatible with the target ABI", - }); - } - } else { - // FIXME: we have to request implied features here since - // negative features do not handle implied features above. - for &required in abi_feature_constraints.required.iter() { - let implied = - sess.target.implied_target_features(std::iter::once(required)); - if implied.contains(feature) { - sess.dcx().emit_warn(ForbiddenCTargetFeature { - feature, - enabled: "disabled", - reason: "this feature is required by the target ABI", - }); - } - } - } - // FIXME(nagisa): figure out how to not allocate a full hashset here. featsmap.insert(feature, enable); } } - // To be sure the ABI-relevant features are all in the right state, we explicitly - // (un)set them here. This means if the target spec sets those features wrong, - // we will silently correct them rather than silently producing wrong code. - // (The target sanity check tries to catch this, but we can't know which features are - // enabled in LLVM by default so we can't be fully sure about that check.) - // We add these at the beginning of the list so that `-Ctarget-features` can - // still override it... that's unsound, but more compatible with past behavior. - all_rust_features.splice( - 0..0, - abi_feature_constraints - .required - .iter() - .map(|&f| (true, f)) - .chain(abi_feature_constraints.incompatible.iter().map(|&f| (false, f))), - ); - // Translate this into LLVM features. let feats = all_rust_features .iter() |