diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 20 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 60 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 110 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 38 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 13 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 35 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/mono_item.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 30 |
15 files changed, 255 insertions, 146 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b5b4f894e4d..147939d3a52 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -424,7 +424,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); - let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); + let sret = llvm::CreateStructRetAttr( + cx.llcx, + cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()), + ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } PassMode::Cast { cast, pad_i32: _ } => { @@ -437,7 +440,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => {} PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { let i = apply(attrs); - let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); + let byval = llvm::CreateByValAttr( + cx.llcx, + cx.type_array(cx.type_i8(), arg.layout.size.bytes()), + ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } PassMode::Direct(attrs) @@ -486,7 +492,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); - let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); + let sret = llvm::CreateStructRetAttr( + bx.cx.llcx, + bx.cx.type_array(bx.cx.type_i8(), self.ret.layout.size.bytes()), + ); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } PassMode::Cast { cast, pad_i32: _ } => { @@ -513,7 +522,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => {} PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); - let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); + let byval = llvm::CreateByValAttr( + bx.cx.llcx, + bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()), + ); attributes::apply_to_callsite( callsite, llvm::AttributePlace::Argument(i), diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a413466093b..74539d4d495 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option<Self::BasicBlock>, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Build the template string + let mut labels = vec![]; let mut template_str = String::new(); for piece in template { match *piece { @@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Only emit the raw symbol name template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); } + InlineAsmOperandRef::Label { label } => { + template_str.push_str(&format!("${{{}:l}}", constraints.len())); + constraints.push("!i".to_owned()); + labels.push(label); + } } } } @@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &constraints.join(","), &inputs, output_type, + &labels, volatile, alignstack, dialect, line_spans, options.contains(InlineAsmOptions::MAY_UNWIND), - dest_catch_funclet, + dest, + catch_funclet, ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); @@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` - if let Some((dest, _, _)) = dest_catch_funclet { + if let Some(dest) = dest { self.switch_to_block(dest); } @@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>( cons: &str, inputs: &[&'ll Value], output: &'ll llvm::Type, + labels: &[&'ll llvm::BasicBlock], volatile: bool, alignstack: bool, dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, - dest_catch_funclet: Option<( - &'ll llvm::BasicBlock, - &'ll llvm::BasicBlock, - Option<&Funclet<'ll>>, - )>, + dest: Option<&'ll llvm::BasicBlock>, + catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { let volatile = if volatile { llvm::True } else { llvm::False }; let alignstack = if alignstack { llvm::True } else { llvm::False }; @@ -457,8 +464,11 @@ pub(crate) fn inline_asm_call<'ll>( can_throw, ); - let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) + let call = if !labels.is_empty() { + assert!(catch_funclet.is_none()); + bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None) + } else if let Some((catch, funclet)) = catch_funclet { + bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet) } else { bx.call(fty, None, None, v, inputs, None) }; diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index eaedaec635f..63e59ea13fc 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1132,9 +1132,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, dst: &'ll Value, - src: &'ll Value, + mut src: &'ll Value, order: rustc_codegen_ssa::common::AtomicOrdering, ) -> &'ll Value { + // The only RMW operation that LLVM supports on pointers is compare-exchange. + if self.val_ty(src) == self.type_ptr() + && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg + { + src = self.ptrtoint(src, self.type_isize()); + } unsafe { llvm::LLVMBuildAtomicRMW( self.llbuilder, @@ -1538,6 +1544,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } + pub(crate) fn callbr( + &mut self, + llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + args: &[&'ll Value], + default_dest: &'ll BasicBlock, + indirect_dest: &[&'ll BasicBlock], + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("invoke {:?} with args ({:?})", llfn, args); + + let args = self.check_call("callbr", llty, llfn, args); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } + + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } + + let callbr = unsafe { + llvm::LLVMRustBuildCallBr( + self.llbuilder, + llty, + llfn, + default_dest, + indirect_dest.as_ptr(), + indirect_dest.len() as c_uint, + args.as_ptr(), + args.len() as c_uint, + bundles.as_ptr(), + bundles.len() as c_uint, + UNNAMED, + ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, callbr); + } + callbr + } + // Emits CFI pointer type membership tests. fn cfi_type_test( &mut self, diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8173e41aff4..25cbd90460f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -95,11 +95,13 @@ impl<'ll> BackendTypes for CodegenCx<'ll, '_> { impl<'ll> CodegenCx<'ll, '_> { pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } + let len = u64::try_from(elts.len()).expect("LLVMConstArray2 elements len overflow"); + unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstVector(elts.as_ptr(), elts.len() as c_uint) } + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } } pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { @@ -108,8 +110,8 @@ impl<'ll> CodegenCx<'ll, '_> { pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { - assert_eq!(idx as c_uint as u64, idx); - let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); + let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); + let r = llvm::LLVMGetAggregateElement(v, idx).unwrap(); debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); @@ -329,7 +331,7 @@ pub fn val_ty(v: &Value) -> &Type { pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) } } @@ -338,9 +340,8 @@ pub fn struct_in_context<'ll>( elts: &[&'ll Value], packed: bool, ) -> &'ll Value { - unsafe { - llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), elts.len() as c_uint, packed as Bool) - } + let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); + unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed as Bool) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index ec2fb2c6e54..4afa230e598 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -9,6 +9,7 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_codegen_ssa::traits::*; +use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ @@ -17,7 +18,7 @@ use rustc_middle::mir::interpret::{ }; use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use rustc_session::config::Lto; use rustc_target::abi::{ @@ -114,7 +115,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< cx.const_struct(&llvals, true) } -pub fn codegen_static_initializer<'ll, 'tcx>( +fn codegen_static_initializer<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { @@ -147,11 +148,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: fn check_and_apply_linkage<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, attrs: &CodegenFnAttrs, - ty: Ty<'tcx>, + llty: &'ll Type, sym: &str, def_id: DefId, ) -> &'ll Value { - let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); @@ -226,9 +226,28 @@ impl<'ll> CodegenCx<'ll, '_> { } } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); - if let Some(&g) = self.instances.borrow().get(&instance) { + trace!(?instance); + + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out + // the llvm type from the actual evaluated initializer. + let llty = if nested { + self.type_i8() + } else { + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + trace!(?ty); + self.layout_of(ty).llvm_type(self) + }; + self.get_static_inner(def_id, llty) + } + + #[instrument(level = "debug", skip(self, llty))] + pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value { + if let Some(&g) = self.instances.borrow().get(&Instance::mono(self.tcx, def_id)) { + trace!("used cached value"); return g; } @@ -240,14 +259,12 @@ impl<'ll> CodegenCx<'ll, '_> { statics defined in the same CGU, but did not for `{def_id:?}`" ); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let sym = self.tcx.symbol_name(instance).name; + let sym = self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name; let fn_attrs = self.tcx.codegen_fn_attrs(def_id); - debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); + debug!(?sym, ?fn_attrs); let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { - let llty = self.layout_of(ty).llvm_type(self); if let Some(g) = self.get_declared_value(sym) { if self.val_ty(g) != self.type_ptr() { span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); @@ -264,7 +281,7 @@ impl<'ll> CodegenCx<'ll, '_> { g } else { - check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) + check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) }; // Thread-local statics in some other crate need to *always* be linked @@ -332,34 +349,18 @@ impl<'ll> CodegenCx<'ll, '_> { } } - self.instances.borrow_mut().insert(instance, g); + self.instances.borrow_mut().insert(Instance::mono(self.tcx, def_id), g); g } -} - -impl<'ll> StaticMethods for CodegenCx<'ll, '_> { - fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { - if let Some(&gv) = self.const_globals.borrow().get(&cv) { - unsafe { - // Upgrade the alignment in cases where the same constant is used with different - // alignment requirements - let llalign = align.bytes() as u32; - if llalign > llvm::LLVMGetAlignment(gv) { - llvm::LLVMSetAlignment(gv, llalign); - } - } - return gv; - } - let gv = self.static_addr_of_mut(cv, align, kind); - unsafe { - llvm::LLVMSetGlobalConstant(gv, True); - } - self.const_globals.borrow_mut().insert(cv, gv); - gv - } - fn codegen_static(&self, def_id: DefId, is_mutable: bool) { + fn codegen_static_item(&self, def_id: DefId) { unsafe { + assert!( + llvm::LLVMGetInitializer( + self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap() + ) + .is_none() + ); let attrs = self.tcx.codegen_fn_attrs(def_id); let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { @@ -368,13 +369,11 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { }; let alloc = alloc.inner(); - let g = self.get_static(def_id); - let val_llty = self.val_ty(v); - let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let llty = self.layout_of(ty).llvm_type(self); + let g = self.get_static_inner(def_id, val_llty); + let llty = self.val_ty(g); + let g = if val_llty == llty { g } else { @@ -409,16 +408,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; - set_global_alignment(self, g, self.align_of(ty)); + set_global_alignment(self, g, alloc.align); llvm::LLVMSetInitializer(g, v); if self.should_assume_dso_local(g, true) { llvm::LLVMRustSetDSOLocal(g, true); } - // As an optimization, all shared statics which do not have interior - // mutability are placed into read-only memory. - if !is_mutable && self.type_is_freeze(ty) { + // Forward the allocation's mutability (picked by the const interner) to LLVM. + if alloc.mutability.is_not() { llvm::LLVMSetGlobalConstant(g, llvm::True); } @@ -541,6 +539,32 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { } } } +} + +impl<'ll> StaticMethods for CodegenCx<'ll, '_> { + fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + + fn codegen_static(&self, def_id: DefId) { + self.codegen_static_item(def_id) + } /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr. fn add_used_global(&self, global: &'ll Value) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 16122e5557e..f89c8c9f836 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -77,8 +77,8 @@ pub struct CodegenCx<'ll, 'tcx> { /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details pub compiler_used_statics: RefCell<Vec<&'ll Value>>, - /// Mapping of non-scalar types to llvm types and field remapping if needed. - pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), TypeLowering<'ll>>>, + /// Mapping of non-scalar types to llvm types. + pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>, /// Mapping of scalar types to llvm types. pub scalar_lltypes: RefCell<FxHashMap<Ty<'tcx>, &'ll Type>>, @@ -105,15 +105,6 @@ pub struct CodegenCx<'ll, 'tcx> { pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, } -pub struct TypeLowering<'ll> { - /// Associated LLVM type - pub lltype: &'ll Type, - - /// If padding is used the slice maps fields from source order - /// to llvm order. - pub field_remapping: Option<SmallVec<[u32; 4]>>, -} - fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { match tls_model { TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic, @@ -260,35 +251,29 @@ pub unsafe fn create_module<'ll>( } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { - let behavior = if llvm_version >= (15, 0, 0) { - llvm::LLVMModFlagBehavior::Min - } else { - llvm::LLVMModFlagBehavior::Error - }; - if sess.target.arch == "aarch64" { llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"branch-target-enforcement".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"sign-return-address".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"sign-return-address-all".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"sign-return-address-with-bkey".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); @@ -564,11 +549,12 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let llfn = match tcx.lang_items().eh_personality() { - Some(def_id) if name.is_none() => self.get_fn_addr( - ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty()) - .unwrap() - .unwrap(), - ), + Some(def_id) if name.is_none() => self.get_fn_addr(ty::Instance::expect_resolve( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + ty::List::empty(), + )), _ => { let name = name.unwrap_or("rust_eh_personality"); if let Some(llfn) = self.get_declared_value(name) { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index c45787f35aa..ee7ea342301 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -355,21 +355,20 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; - let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let eligible_def_ids = tcx.mir_keys(()).iter().filter_map(|local_def_id| { let def_id = local_def_id.to_def_id(); let kind = tcx.def_kind(def_id); // `mir_keys` will give us `DefId`s for all kinds of things, not // just "functions", like consts, statics, etc. Filter those out. - // If `ignore_unused_generics` was specified, filter out any - // generic functions from consideration as well. if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) { return None; } - if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { - return None; - } + + // FIXME(79651): Consider trying to filter out dummy instantiations of + // unused generic functions from library crates, because they can produce + // "unused instantiation" in coverage reports even when they are actually + // used by some downstream crate in the same binary. + Some(local_def_id.to_def_id()) }); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 660f1647367..5782b156335 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -26,6 +26,7 @@ use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -1309,6 +1310,11 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo }; let is_local_to_unit = is_node_local_to_unit(cx, def_id); + + let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else { bug!() }; + if nested { + return; + } let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); let type_di_node = type_di_node(cx, variable_type); let var_name = tcx.item_name(def_id); diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 5bef240340b..e15eda7c66c 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -4,7 +4,7 @@ use std::path::Path; use crate::fluent_generated as fluent; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{Diag, DiagCtxt, EmissionGuarantee, IntoDiagnostic, Level}; +use rustc_errors::{Diag, DiagCtxt, Diagnostic, EmissionGuarantee, Level}; use rustc_macros::{Diagnostic, Subdiagnostic}; use rustc_span::Span; @@ -99,9 +99,9 @@ pub(crate) struct DynamicLinkingWithLTO; pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); -impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { - let diag: Diag<'_, G> = self.0.into_diagnostic(dcx, level); +impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { + fn into_diag(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { + let diag: Diag<'_, G> = self.0.into_diag(dcx, level); let (message, _) = diag.messages.first().expect("`LlvmError` with no message"); let message = dcx.eagerly_translate_to_string(message.clone(), diag.args.iter()); Diag::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) @@ -119,8 +119,8 @@ pub(crate) struct TargetFeatureDisableOrEnable<'a> { #[help(codegen_llvm_missing_features)] pub(crate) struct MissingFeatures; -impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { +impl<G: EmissionGuarantee> Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { + fn into_diag(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { let mut diag = Diag::new(dcx, level, fluent::codegen_llvm_target_feature_disable_or_enable); if let Some(span) = self.span { diag.span(span); @@ -179,8 +179,8 @@ pub enum LlvmError<'a> { pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String); -impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { +impl<G: EmissionGuarantee> Diagnostic<'_, G> for WithLlvmError<'_> { + fn into_diag(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { use LlvmError::*; let msg_with_llvm_err = match &self.0 { WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err, @@ -198,7 +198,7 @@ impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, }; self.0 - .into_diagnostic(dcx, level) + .into_diag(dcx, level) .with_primary_message(msg_with_llvm_err) .with_arg("llvm_err", self.1) } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index f33a672aff0..467e02d55e3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -448,12 +448,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { constraint, inputs, self.type_void(), + &[], true, false, llvm::AsmDialect::Att, &[span], false, None, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 3a34ef0874e..d34c289887e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -936,10 +936,16 @@ extern "C" { pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; // Operations on composite constants - pub fn LLVMConstStringInContext( + pub fn LLVMConstArray2<'a>( + ElementTy: &'a Type, + ConstantVals: *const &'a Value, + Length: u64, + ) -> &'a Value; + pub fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; + pub fn LLVMConstStringInContext2( C: &Context, Str: *const c_char, - Length: c_uint, + Length: size_t, DontNullTerminate: Bool, ) -> &Value; pub fn LLVMConstStructInContext<'a>( @@ -948,14 +954,6 @@ extern "C" { Count: c_uint, Packed: Bool, ) -> &'a Value; - - // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 - // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc - pub fn LLVMConstArray<'a>( - ElementTy: &'a Type, - ConstantVals: *const &'a Value, - Length: c_uint, - ) -> &'a Value; pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions @@ -1530,9 +1528,6 @@ extern "C" { /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on array, pointer, and vector types (sequence types) - pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; - // Operations on all values pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; @@ -1616,6 +1611,20 @@ extern "C" { Name: *const c_char, ) -> &'a Value; + pub fn LLVMRustBuildCallBr<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + DefaultDest: &'a BasicBlock, + IndirectDests: *const &'a BasicBlock, + NumIndirectDests: c_uint, + Args: *const &'a Value, + NumArgs: c_uint, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMRustSetFastMath(Instr: &Value); pub fn LLVMRustSetAlgebraicMath(Instr: &Value); pub fn LLVMRustSetAllowReassoc(Instr: &Value); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 78d47f36f91..e32c38644aa 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -5,6 +5,7 @@ use crate::errors::{ }; use crate::llvm; use libc::c_int; +use rustc_codegen_ssa::base::wants_wasm_eh; use rustc_codegen_ssa::traits::PrintBackendInfo; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; @@ -98,6 +99,10 @@ unsafe fn configure_llvm(sess: &Session) { } } + if wants_wasm_eh(sess) { + add("-wasm-enable-eh", false); + } + if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { add("-enable-emscripten-cxx-exceptions", false); } @@ -523,6 +528,10 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .map(String::from), ); + if wants_wasm_eh(sess) && sess.panic_strategy() == PanicStrategy::Unwind { + features.push("+exception-handling".into()); + } + // -Ctarget-features let supported_features = sess.target.supported_target_features(); let mut featsmap = FxHashMap::default(); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f7630719368..29100a64171 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -5,7 +5,9 @@ use crate::errors::SymbolAlreadyDefined; use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_middle::bug; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; @@ -21,7 +23,14 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { symbol_name: &str, ) { let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out + // the llvm type from the actual evaluated initializer. + let ty = if nested { + self.tcx.types.unit + } else { + instance.ty(self.tcx, ty::ParamEnv::reveal_all()) + }; let llty = self.layout_of(ty).llvm_type(self); let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 07a4861ed73..af1bbda4d08 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -233,7 +233,7 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { - unsafe { llvm::LLVMRustArrayType(ty, len) } + unsafe { llvm::LLVMArrayType2(ty, len) } } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 587adefe1d2..d10a083765b 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,5 +1,4 @@ use crate::common::*; -use crate::context::TypeLowering; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; @@ -10,7 +9,6 @@ use rustc_target::abi::HasDataLayout; use rustc_target::abi::{Abi, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F128, F16, F32, F64}; use rustc_target::abi::{Scalar, Size, Variants}; -use smallvec::{smallvec, SmallVec}; use std::fmt::Write; @@ -18,7 +16,6 @@ fn uncached_llvm_type<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, layout: TyAndLayout<'tcx>, defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, - field_remapping: &mut Option<SmallVec<[u32; 4]>>, ) -> &'a Type { match layout.abi { Abi::Scalar(_) => bug!("handled elsewhere"), @@ -71,8 +68,7 @@ fn uncached_llvm_type<'a, 'tcx>( FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), FieldsShape::Arbitrary { .. } => match name { None => { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); - *field_remapping = new_field_remapping; + let (llfields, packed) = struct_llfields(cx, layout); cx.type_struct(&llfields, packed) } Some(ref name) => { @@ -87,7 +83,7 @@ fn uncached_llvm_type<'a, 'tcx>( fn struct_llfields<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, layout: TyAndLayout<'tcx>, -) -> (Vec<&'a Type>, bool, Option<SmallVec<[u32; 4]>>) { +) -> (Vec<&'a Type>, bool) { debug!("struct_llfields: {:#?}", layout); let field_count = layout.fields.count(); @@ -95,7 +91,6 @@ fn struct_llfields<'a, 'tcx>( let mut offset = Size::ZERO; let mut prev_effective_align = layout.align.abi; let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); - let mut field_remapping = smallvec![0; field_count]; for i in layout.fields.index_by_increasing_offset() { let target_offset = layout.fields.offset(i as usize); let field = layout.field(cx, i); @@ -120,12 +115,10 @@ fn struct_llfields<'a, 'tcx>( result.push(cx.type_padding_filler(padding, padding_align)); debug!(" padding before: {:?}", padding); } - field_remapping[i] = result.len() as u32; result.push(field.llvm_type(cx)); offset = target_offset + field.size; prev_effective_align = effective_field_align; } - let padding_used = result.len() > field_count; if layout.is_sized() && field_count > 0 { if offset > layout.size { bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); @@ -143,8 +136,7 @@ fn struct_llfields<'a, 'tcx>( } else { debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); } - let field_remapping = padding_used.then_some(field_remapping); - (result, packed, field_remapping) + (result, packed) } impl<'a, 'tcx> CodegenCx<'a, 'tcx> { @@ -224,7 +216,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { _ => None, }; if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - return llty.lltype; + return llty; } debug!("llvm_type({:#?})", self); @@ -236,7 +228,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { let normal_ty = cx.tcx.erase_regions(self.ty); let mut defer = None; - let mut field_remapping = None; let llty = if self.ty != normal_ty { let mut layout = cx.layout_of(normal_ty); if let Some(v) = variant_index { @@ -244,22 +235,15 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } layout.llvm_type(cx) } else { - uncached_llvm_type(cx, *self, &mut defer, &mut field_remapping) + uncached_llvm_type(cx, *self, &mut defer) }; debug!("--> mapped {:#?} to llty={:?}", self, llty); - cx.type_lowering - .borrow_mut() - .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); + cx.type_lowering.borrow_mut().insert((self.ty, variant_index), llty); if let Some((llty, layout)) = defer { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + let (llfields, packed) = struct_llfields(cx, layout); cx.set_struct_body(llty, &llfields, packed); - cx.type_lowering - .borrow_mut() - .get_mut(&(self.ty, variant_index)) - .unwrap() - .field_remapping = new_field_remapping; } llty } |