diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 104 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 110 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 31 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 13 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 36 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/mono_item.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 30 |
15 files changed, 221 insertions, 207 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 147939d3a52..e5f5146fac8 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -203,57 +203,63 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { val: &'ll Value, dst: PlaceRef<'tcx, &'ll Value>, ) { - if self.is_ignore() { - return; - } - if self.is_sized_indirect() { - OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) - } else if self.is_unsized_indirect() { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast { cast, pad_i32: _ } = &self.mode { - // FIXME(eddyb): Figure out when the simpler Store is safe, clang - // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. - let can_store_through_cast_ptr = false; - if can_store_through_cast_ptr { - bx.store(val, dst.llval, self.layout.align.abi); - } else { - // The actual return type is a struct, but the ABI - // adaptation code has cast it into some scalar type. The - // code that follows is the only reliable way I have - // found to do a transform like i64 -> {i32,i32}. - // Basically we dump the data onto the stack then memcpy it. - // - // Other approaches I tried: - // - Casting rust ret pointer to the foreign type and using Store - // is (a) unsafe if size of foreign type > size of rust type and - // (b) runs afoul of strict aliasing rules, yielding invalid - // assembly under -O (specifically, the store gets removed). - // - Truncating foreign type to correct integral type and then - // bitcasting to the struct type yields invalid cast errors. - - // We instead thus allocate some scratch space... - let scratch_size = cast.size(bx); - let scratch_align = cast.align(bx); - let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); - bx.lifetime_start(llscratch, scratch_size); - - // ... where we first store the value... - bx.store(val, llscratch, scratch_align); - - // ... and then memcpy it to the intended destination. - bx.memcpy( - dst.llval, - self.layout.align.abi, - llscratch, - scratch_align, - bx.const_usize(self.layout.size.bytes()), - MemFlags::empty(), - ); + match &self.mode { + PassMode::Ignore => {} + // Sized indirect arguments + PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { + let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); + OperandValue::Ref(val, None, align).store(bx, dst); + } + // Unsized indirect qrguments + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + } + PassMode::Cast { cast, pad_i32: _ } => { + // FIXME(eddyb): Figure out when the simpler Store is safe, clang + // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. + let can_store_through_cast_ptr = false; + if can_store_through_cast_ptr { + bx.store(val, dst.llval, self.layout.align.abi); + } else { + // The actual return type is a struct, but the ABI + // adaptation code has cast it into some scalar type. The + // code that follows is the only reliable way I have + // found to do a transform like i64 -> {i32,i32}. + // Basically we dump the data onto the stack then memcpy it. + // + // Other approaches I tried: + // - Casting rust ret pointer to the foreign type and using Store + // is (a) unsafe if size of foreign type > size of rust type and + // (b) runs afoul of strict aliasing rules, yielding invalid + // assembly under -O (specifically, the store gets removed). + // - Truncating foreign type to correct integral type and then + // bitcasting to the struct type yields invalid cast errors. + + // We instead thus allocate some scratch space... + let scratch_size = cast.size(bx); + let scratch_align = cast.align(bx); + let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); + bx.lifetime_start(llscratch, scratch_size); + + // ... where we first store the value... + bx.store(val, llscratch, scratch_align); + + // ... and then memcpy it to the intended destination. + bx.memcpy( + dst.llval, + self.layout.align.abi, + llscratch, + scratch_align, + bx.const_usize(self.layout.size.bytes()), + MemFlags::empty(), + ); - bx.lifetime_end(llscratch, scratch_size); + bx.lifetime_end(llscratch, scratch_size); + } + } + _ => { + OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst); } - } else { - OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst); } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index ca2e2b57580..63e59ea13fc 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1132,9 +1132,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, dst: &'ll Value, - src: &'ll Value, + mut src: &'ll Value, order: rustc_codegen_ssa::common::AtomicOrdering, ) -> &'ll Value { + // The only RMW operation that LLVM supports on pointers is compare-exchange. + if self.val_ty(src) == self.type_ptr() + && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg + { + src = self.ptrtoint(src, self.type_isize()); + } unsafe { llvm::LLVMBuildAtomicRMW( self.llbuilder, diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8173e41aff4..25cbd90460f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -95,11 +95,13 @@ impl<'ll> BackendTypes for CodegenCx<'ll, '_> { impl<'ll> CodegenCx<'ll, '_> { pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } + let len = u64::try_from(elts.len()).expect("LLVMConstArray2 elements len overflow"); + unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstVector(elts.as_ptr(), elts.len() as c_uint) } + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } } pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { @@ -108,8 +110,8 @@ impl<'ll> CodegenCx<'ll, '_> { pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { - assert_eq!(idx as c_uint as u64, idx); - let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); + let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); + let r = llvm::LLVMGetAggregateElement(v, idx).unwrap(); debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); @@ -329,7 +331,7 @@ pub fn val_ty(v: &Value) -> &Type { pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) } } @@ -338,9 +340,8 @@ pub fn struct_in_context<'ll>( elts: &[&'ll Value], packed: bool, ) -> &'ll Value { - unsafe { - llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), elts.len() as c_uint, packed as Bool) - } + let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); + unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed as Bool) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index ec2fb2c6e54..4afa230e598 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -9,6 +9,7 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_codegen_ssa::traits::*; +use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ @@ -17,7 +18,7 @@ use rustc_middle::mir::interpret::{ }; use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use rustc_session::config::Lto; use rustc_target::abi::{ @@ -114,7 +115,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< cx.const_struct(&llvals, true) } -pub fn codegen_static_initializer<'ll, 'tcx>( +fn codegen_static_initializer<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { @@ -147,11 +148,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: fn check_and_apply_linkage<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, attrs: &CodegenFnAttrs, - ty: Ty<'tcx>, + llty: &'ll Type, sym: &str, def_id: DefId, ) -> &'ll Value { - let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); @@ -226,9 +226,28 @@ impl<'ll> CodegenCx<'ll, '_> { } } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); - if let Some(&g) = self.instances.borrow().get(&instance) { + trace!(?instance); + + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out + // the llvm type from the actual evaluated initializer. + let llty = if nested { + self.type_i8() + } else { + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + trace!(?ty); + self.layout_of(ty).llvm_type(self) + }; + self.get_static_inner(def_id, llty) + } + + #[instrument(level = "debug", skip(self, llty))] + pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value { + if let Some(&g) = self.instances.borrow().get(&Instance::mono(self.tcx, def_id)) { + trace!("used cached value"); return g; } @@ -240,14 +259,12 @@ impl<'ll> CodegenCx<'ll, '_> { statics defined in the same CGU, but did not for `{def_id:?}`" ); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let sym = self.tcx.symbol_name(instance).name; + let sym = self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name; let fn_attrs = self.tcx.codegen_fn_attrs(def_id); - debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); + debug!(?sym, ?fn_attrs); let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { - let llty = self.layout_of(ty).llvm_type(self); if let Some(g) = self.get_declared_value(sym) { if self.val_ty(g) != self.type_ptr() { span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); @@ -264,7 +281,7 @@ impl<'ll> CodegenCx<'ll, '_> { g } else { - check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) + check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) }; // Thread-local statics in some other crate need to *always* be linked @@ -332,34 +349,18 @@ impl<'ll> CodegenCx<'ll, '_> { } } - self.instances.borrow_mut().insert(instance, g); + self.instances.borrow_mut().insert(Instance::mono(self.tcx, def_id), g); g } -} - -impl<'ll> StaticMethods for CodegenCx<'ll, '_> { - fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { - if let Some(&gv) = self.const_globals.borrow().get(&cv) { - unsafe { - // Upgrade the alignment in cases where the same constant is used with different - // alignment requirements - let llalign = align.bytes() as u32; - if llalign > llvm::LLVMGetAlignment(gv) { - llvm::LLVMSetAlignment(gv, llalign); - } - } - return gv; - } - let gv = self.static_addr_of_mut(cv, align, kind); - unsafe { - llvm::LLVMSetGlobalConstant(gv, True); - } - self.const_globals.borrow_mut().insert(cv, gv); - gv - } - fn codegen_static(&self, def_id: DefId, is_mutable: bool) { + fn codegen_static_item(&self, def_id: DefId) { unsafe { + assert!( + llvm::LLVMGetInitializer( + self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap() + ) + .is_none() + ); let attrs = self.tcx.codegen_fn_attrs(def_id); let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { @@ -368,13 +369,11 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { }; let alloc = alloc.inner(); - let g = self.get_static(def_id); - let val_llty = self.val_ty(v); - let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let llty = self.layout_of(ty).llvm_type(self); + let g = self.get_static_inner(def_id, val_llty); + let llty = self.val_ty(g); + let g = if val_llty == llty { g } else { @@ -409,16 +408,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; - set_global_alignment(self, g, self.align_of(ty)); + set_global_alignment(self, g, alloc.align); llvm::LLVMSetInitializer(g, v); if self.should_assume_dso_local(g, true) { llvm::LLVMRustSetDSOLocal(g, true); } - // As an optimization, all shared statics which do not have interior - // mutability are placed into read-only memory. - if !is_mutable && self.type_is_freeze(ty) { + // Forward the allocation's mutability (picked by the const interner) to LLVM. + if alloc.mutability.is_not() { llvm::LLVMSetGlobalConstant(g, llvm::True); } @@ -541,6 +539,32 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { } } } +} + +impl<'ll> StaticMethods for CodegenCx<'ll, '_> { + fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + + fn codegen_static(&self, def_id: DefId) { + self.codegen_static_item(def_id) + } /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr. fn add_used_global(&self, global: &'ll Value) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 37c690a0128..df9f066e58a 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -27,9 +27,7 @@ use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Spanned; use rustc_span::Span; -use rustc_target::abi::{ - call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, -}; +use rustc_target::abi::{call::FnAbi, HasDataLayout, TargetDataLayout, VariantIdx}; use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; @@ -77,13 +75,12 @@ pub struct CodegenCx<'ll, 'tcx> { /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details pub compiler_used_statics: RefCell<Vec<&'ll Value>>, - /// Mapping of non-scalar types to llvm types and field remapping if needed. - pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), TypeLowering<'ll>>>, + /// Mapping of non-scalar types to llvm types. + pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>, /// Mapping of scalar types to llvm types. pub scalar_lltypes: RefCell<FxHashMap<Ty<'tcx>, &'ll Type>>, - pub pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>, pub isize_ty: &'ll Type, pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, @@ -105,15 +102,6 @@ pub struct CodegenCx<'ll, 'tcx> { pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, } -pub struct TypeLowering<'ll> { - /// Associated LLVM type - pub lltype: &'ll Type, - - /// If padding is used the slice maps fields from source order - /// to llvm order. - pub field_remapping: Option<SmallVec<[u32; 4]>>, -} - fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { match tls_model { TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic, @@ -135,17 +123,6 @@ pub unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (17, 0, 0) { - if sess.target.arch.starts_with("powerpc") { - // LLVM 17 specifies function pointer alignment for ppc: - // https://reviews.llvm.org/D147016 - target_data_layout = target_data_layout - .replace("-Fn32", "") - .replace("-Fi32", "") - .replace("-Fn64", "") - .replace("-Fi64", ""); - } - } if llvm_version < (18, 0, 0) { if sess.target.arch == "x86" || sess.target.arch == "x86_64" { // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants. @@ -335,6 +312,7 @@ pub unsafe fn create_module<'ll>( // // On the wasm targets it will get hooked up to the "producer" sections // `processed-by` information. + #[allow(clippy::option_env_unwrap)] let rustc_producer = format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION")); let name_metadata = llvm::LLVMMDStringInContext( @@ -469,7 +447,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { compiler_used_statics: RefCell::new(Vec::new()), type_lowering: Default::default(), scalar_lltypes: Default::default(), - pointee_infos: Default::default(), isize_ty, coverage_cx, dbg_cx, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 017843c7e7d..2af28146a51 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -164,6 +164,15 @@ impl CounterMappingRegion { end_line, end_col, ), + MappingKind::Branch { true_term, false_term } => Self::branch_region( + Counter::from_term(true_term), + Counter::from_term(false_term), + local_file_id, + start_line, + start_col, + end_line, + end_col, + ), } } @@ -188,9 +197,6 @@ impl CounterMappingRegion { } } - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] pub(crate) fn branch_region( counter: Counter, false_counter: Counter, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index c45787f35aa..ee7ea342301 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -355,21 +355,20 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; - let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let eligible_def_ids = tcx.mir_keys(()).iter().filter_map(|local_def_id| { let def_id = local_def_id.to_def_id(); let kind = tcx.def_kind(def_id); // `mir_keys` will give us `DefId`s for all kinds of things, not // just "functions", like consts, statics, etc. Filter those out. - // If `ignore_unused_generics` was specified, filter out any - // generic functions from consideration as well. if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) { return None; } - if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { - return None; - } + + // FIXME(79651): Consider trying to filter out dummy instantiations of + // unused generic functions from library crates, because they can produce + // "unused instantiation" in coverage reports even when they are actually + // used by some downstream crate in the same binary. + Some(local_def_id.to_def_id()) }); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 733a77d24c2..54f4bc06340 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -85,14 +85,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let bx = self; - match coverage.kind { - // Marker statements have no effect during codegen, - // so return early and don't create `func_coverage`. - CoverageKind::SpanMarker => return, - // Match exhaustively to ensure that newly-added kinds are classified correctly. - CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. } => {} - } - let Some(function_coverage_info) = bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() else { @@ -108,8 +100,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let Coverage { kind } = coverage; match *kind { - CoverageKind::SpanMarker => unreachable!( - "unexpected marker statement {kind:?} should have caused an early return" + CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( + "marker statement {kind:?} should have been removed by CleanupPostBorrowck" ), CoverageKind::CounterIncrement { id } => { func_coverage.mark_counter_id_seen(id); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 660f1647367..3c76df11e3f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -26,6 +26,7 @@ use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -451,7 +452,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), - ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { + ty::RawPtr(pointee_type, _) | ty::Ref(_, pointee_type, _) => { build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) } // Some `Box` are newtyped pointers, make debuginfo aware of that. @@ -1309,6 +1310,11 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo }; let is_local_to_unit = is_node_local_to_unit(cx, def_id); + + let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else { bug!() }; + if nested { + return; + } let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); let type_di_node = type_di_node(cx, variable_type); let var_name = tcx.item_name(def_id); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 467e02d55e3..2409b2e78d7 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1129,7 +1129,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_shuffle_generic { let idx = fn_args[2] .expect_const() - .eval(tcx, ty::ParamEnv::reveal_all(), Some(span)) + .eval(tcx, ty::ParamEnv::reveal_all(), span) .unwrap() .unwrap_branch(); let n = idx.len() as u64; @@ -1483,7 +1483,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() ), ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()), - ty::RawPtr(_) => format!("v{}p0", vec_len), + ty::RawPtr(_, _) => format!("v{}p0", vec_len), _ => unreachable!(), } } @@ -1493,7 +1493,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ty::Int(v) => cx.type_int_from_ty(v), ty::Uint(v) => cx.type_uint_from_ty(v), ty::Float(v) => cx.type_float_from_ty(v), - ty::RawPtr(_) => cx.type_ptr(), + ty::RawPtr(_, _) => cx.type_ptr(), _ => unreachable!(), }; cx.type_vector(elem_ty, vec_len) @@ -1548,8 +1548,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( require!( matches!( - element_ty1.kind(), - ty::RawPtr(p) if p.ty == in_elem && p.ty.kind() == element_ty0.kind() + *element_ty1.kind(), + ty::RawPtr(p_ty, _) if p_ty == in_elem && p_ty.kind() == element_ty0.kind() ), InvalidMonomorphization::ExpectedElementType { span, @@ -1654,8 +1654,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( require!( matches!( - pointer_ty.kind(), - ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() + *pointer_ty.kind(), + ty::RawPtr(p_ty, _) if p_ty == values_elem && p_ty.kind() == values_elem.kind() ), InvalidMonomorphization::ExpectedElementType { span, @@ -1746,8 +1746,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // The second argument must be a mutable pointer type matching the element type require!( matches!( - pointer_ty.kind(), - ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut() + *pointer_ty.kind(), + ty::RawPtr(p_ty, p_mutbl) if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut() ), InvalidMonomorphization::ExpectedElementType { span, @@ -1843,9 +1843,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( require!( matches!( - element_ty1.kind(), - ty::RawPtr(p) - if p.ty == in_elem && p.mutbl.is_mut() && p.ty.kind() == element_ty0.kind() + *element_ty1.kind(), + ty::RawPtr(p_ty, p_mutbl) + if p_ty == in_elem && p_mutbl.is_mut() && p_ty.kind() == element_ty0.kind() ), InvalidMonomorphization::ExpectedElementType { span, @@ -2074,8 +2074,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); match in_elem.kind() { - ty::RawPtr(p) => { - let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + ty::RawPtr(p_ty, _) => { + let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); require!( @@ -2088,8 +2088,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } } match out_elem.kind() { - ty::RawPtr(p) => { - let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + ty::RawPtr(p_ty, _) => { + let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); require!( @@ -2120,7 +2120,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); match in_elem.kind() { - ty::RawPtr(_) => {} + ty::RawPtr(_, _) => {} _ => { return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) } @@ -2152,7 +2152,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }), } match out_elem.kind() { - ty::RawPtr(_) => {} + ty::RawPtr(_, _) => {} _ => { return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 58e98037067..284bc74d5c4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -936,10 +936,16 @@ extern "C" { pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; // Operations on composite constants - pub fn LLVMConstStringInContext( + pub fn LLVMConstArray2<'a>( + ElementTy: &'a Type, + ConstantVals: *const &'a Value, + Length: u64, + ) -> &'a Value; + pub fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; + pub fn LLVMConstStringInContext2( C: &Context, Str: *const c_char, - Length: c_uint, + Length: size_t, DontNullTerminate: Bool, ) -> &Value; pub fn LLVMConstStructInContext<'a>( @@ -948,14 +954,6 @@ extern "C" { Count: c_uint, Packed: Bool, ) -> &'a Value; - - // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 - // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc - pub fn LLVMConstArray<'a>( - ElementTy: &'a Type, - ConstantVals: *const &'a Value, - Length: c_uint, - ) -> &'a Value; pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions @@ -1521,7 +1519,7 @@ extern "C" { #[link(name = "llvm-wrapper", kind = "static")] extern "C" { - pub fn LLVMRustInstallFatalErrorHandler(); + pub fn LLVMRustInstallErrorHandlers(); pub fn LLVMRustDisableSystemDialogsOnCrash(); // Create and destroy contexts. @@ -1530,9 +1528,6 @@ extern "C" { /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on array, pointer, and vector types (sequence types) - pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; - // Operations on all values pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 78d47f36f91..c9e62e504ae 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -5,6 +5,7 @@ use crate::errors::{ }; use crate::llvm; use libc::c_int; +use rustc_codegen_ssa::base::wants_wasm_eh; use rustc_codegen_ssa::traits::PrintBackendInfo; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; @@ -48,7 +49,7 @@ unsafe fn configure_llvm(sess: &Session) { let mut llvm_c_strs = Vec::with_capacity(n_args + 1); let mut llvm_args = Vec::with_capacity(n_args + 1); - llvm::LLVMRustInstallFatalErrorHandler(); + llvm::LLVMRustInstallErrorHandlers(); // On Windows, an LLVM assertion will open an Abort/Retry/Ignore dialog // box for the purpose of launching a debugger. However, on CI this will // cause it to hang until it times out, which can take several hours. @@ -98,6 +99,10 @@ unsafe fn configure_llvm(sess: &Session) { } } + if wants_wasm_eh(sess) { + add("-wasm-enable-eh", false); + } + if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { add("-enable-emscripten-cxx-exceptions", false); } @@ -523,6 +528,10 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .map(String::from), ); + if wants_wasm_eh(sess) && sess.panic_strategy() == PanicStrategy::Unwind { + features.push("+exception-handling".into()); + } + // -Ctarget-features let supported_features = sess.target.supported_target_features(); let mut featsmap = FxHashMap::default(); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f7630719368..29100a64171 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -5,7 +5,9 @@ use crate::errors::SymbolAlreadyDefined; use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_middle::bug; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; @@ -21,7 +23,14 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { symbol_name: &str, ) { let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out + // the llvm type from the actual evaluated initializer. + let ty = if nested { + self.tcx.types.unit + } else { + instance.ty(self.tcx, ty::ParamEnv::reveal_all()) + }; let llty = self.layout_of(ty).llvm_type(self); let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 07a4861ed73..af1bbda4d08 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -233,7 +233,7 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { - unsafe { llvm::LLVMRustArrayType(ty, len) } + unsafe { llvm::LLVMArrayType2(ty, len) } } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 587adefe1d2..d10a083765b 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,5 +1,4 @@ use crate::common::*; -use crate::context::TypeLowering; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; @@ -10,7 +9,6 @@ use rustc_target::abi::HasDataLayout; use rustc_target::abi::{Abi, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F128, F16, F32, F64}; use rustc_target::abi::{Scalar, Size, Variants}; -use smallvec::{smallvec, SmallVec}; use std::fmt::Write; @@ -18,7 +16,6 @@ fn uncached_llvm_type<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, layout: TyAndLayout<'tcx>, defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, - field_remapping: &mut Option<SmallVec<[u32; 4]>>, ) -> &'a Type { match layout.abi { Abi::Scalar(_) => bug!("handled elsewhere"), @@ -71,8 +68,7 @@ fn uncached_llvm_type<'a, 'tcx>( FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), FieldsShape::Arbitrary { .. } => match name { None => { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); - *field_remapping = new_field_remapping; + let (llfields, packed) = struct_llfields(cx, layout); cx.type_struct(&llfields, packed) } Some(ref name) => { @@ -87,7 +83,7 @@ fn uncached_llvm_type<'a, 'tcx>( fn struct_llfields<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, layout: TyAndLayout<'tcx>, -) -> (Vec<&'a Type>, bool, Option<SmallVec<[u32; 4]>>) { +) -> (Vec<&'a Type>, bool) { debug!("struct_llfields: {:#?}", layout); let field_count = layout.fields.count(); @@ -95,7 +91,6 @@ fn struct_llfields<'a, 'tcx>( let mut offset = Size::ZERO; let mut prev_effective_align = layout.align.abi; let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); - let mut field_remapping = smallvec![0; field_count]; for i in layout.fields.index_by_increasing_offset() { let target_offset = layout.fields.offset(i as usize); let field = layout.field(cx, i); @@ -120,12 +115,10 @@ fn struct_llfields<'a, 'tcx>( result.push(cx.type_padding_filler(padding, padding_align)); debug!(" padding before: {:?}", padding); } - field_remapping[i] = result.len() as u32; result.push(field.llvm_type(cx)); offset = target_offset + field.size; prev_effective_align = effective_field_align; } - let padding_used = result.len() > field_count; if layout.is_sized() && field_count > 0 { if offset > layout.size { bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); @@ -143,8 +136,7 @@ fn struct_llfields<'a, 'tcx>( } else { debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); } - let field_remapping = padding_used.then_some(field_remapping); - (result, packed, field_remapping) + (result, packed) } impl<'a, 'tcx> CodegenCx<'a, 'tcx> { @@ -224,7 +216,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { _ => None, }; if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - return llty.lltype; + return llty; } debug!("llvm_type({:#?})", self); @@ -236,7 +228,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { let normal_ty = cx.tcx.erase_regions(self.ty); let mut defer = None; - let mut field_remapping = None; let llty = if self.ty != normal_ty { let mut layout = cx.layout_of(normal_ty); if let Some(v) = variant_index { @@ -244,22 +235,15 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } layout.llvm_type(cx) } else { - uncached_llvm_type(cx, *self, &mut defer, &mut field_remapping) + uncached_llvm_type(cx, *self, &mut defer) }; debug!("--> mapped {:#?} to llty={:?}", self, llty); - cx.type_lowering - .borrow_mut() - .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); + cx.type_lowering.borrow_mut().insert((self.ty, variant_index), llty); if let Some((llty, layout)) = defer { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + let (llfields, packed) = struct_llfields(cx, layout); cx.set_struct_body(llty, &llfields, packed); - cx.type_lowering - .borrow_mut() - .get_mut(&(self.ty, variant_index)) - .unwrap() - .field_remapping = new_field_remapping; } llty } |