diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
23 files changed, 1946 insertions, 1673 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index e9b66b54c58..f8f6956c47e 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,6 +1,7 @@ +use crate::attributes; use crate::builder::Builder; use crate::context::CodegenCx; -use crate::llvm::{self, AttributePlace}; +use crate::llvm::{self, Attribute, AttributePlace}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -13,33 +14,14 @@ use rustc_middle::bug; use rustc_middle::ty::layout::LayoutOf; pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; use rustc_middle::ty::Ty; +use rustc_session::config; use rustc_target::abi::call::ArgAbi; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int}; pub use rustc_target::spec::abi::Abi; use libc::c_uint; - -macro_rules! for_each_kind { - ($flags: ident, $f: ident, $($kind: ident),+) => ({ - $(if $flags.contains(ArgAttribute::$kind) { $f(llvm::Attribute::$kind) })+ - }) -} - -trait ArgAttributeExt { - fn for_each_kind<F>(&self, f: F) - where - F: FnMut(llvm::Attribute); -} - -impl ArgAttributeExt for ArgAttribute { - fn for_each_kind<F>(&self, mut f: F) - where - F: FnMut(llvm::Attribute), - { - for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, InReg) - } -} +use smallvec::SmallVec; pub trait ArgAttributesExt { fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); @@ -58,36 +40,65 @@ fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) } -impl ArgAttributesExt for ArgAttributes { - fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { - let mut regular = self.regular; - unsafe { - let deref = self.pointee_size.bytes(); - if deref != 0 { - if regular.contains(ArgAttribute::NonNull) { - llvm::LLVMRustAddDereferenceableAttr(llfn, idx.as_uint(), deref); - } else { - llvm::LLVMRustAddDereferenceableOrNullAttr(llfn, idx.as_uint(), deref); - } - regular -= ArgAttribute::NonNull; - } - if let Some(align) = self.pointee_align { - llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32); - } - regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn)); - if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { - llvm::Attribute::NoAlias.apply_llfn(idx, llfn); +const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = + [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; + +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ + (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), + (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), + (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), + (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), + (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), +]; + +fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { + let mut regular = this.regular; + + let mut attrs = SmallVec::new(); + + // ABI-affecting attributes must always be applied + for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + } + } + if let Some(align) = this.pointee_align { + attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes())); + } + match this.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)), + ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)), + } + + // Only apply remaining attributes when optimizing + if cx.sess().opts.optimize != config::OptLevel::No { + let deref = this.pointee_size.bytes(); + if deref != 0 { + if regular.contains(ArgAttribute::NonNull) { + attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref)); + } else { + attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref)); } - match self.arg_ext { - ArgExtension::None => {} - ArgExtension::Zext => { - llvm::Attribute::ZExt.apply_llfn(idx, llfn); - } - ArgExtension::Sext => { - llvm::Attribute::SExt.apply_llfn(idx, llfn); - } + regular -= ArgAttribute::NonNull; + } + for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); } } + if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { + attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); + } + } + + attrs +} + +impl ArgAttributesExt for ArgAttributes { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { + let attrs = get_attrs(self, cx); + attributes::apply_to_llfn(llfn, idx, &attrs); } fn apply_attrs_to_callsite( @@ -96,42 +107,8 @@ impl ArgAttributesExt for ArgAttributes { cx: &CodegenCx<'_, '_>, callsite: &Value, ) { - let mut regular = self.regular; - unsafe { - let deref = self.pointee_size.bytes(); - if deref != 0 { - if regular.contains(ArgAttribute::NonNull) { - llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite, idx.as_uint(), deref); - } else { - llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr( - callsite, - idx.as_uint(), - deref, - ); - } - regular -= ArgAttribute::NonNull; - } - if let Some(align) = self.pointee_align { - llvm::LLVMRustAddAlignmentCallSiteAttr( - callsite, - idx.as_uint(), - align.bytes() as u32, - ); - } - regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite)); - if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { - llvm::Attribute::NoAlias.apply_callsite(idx, callsite); - } - match self.arg_ext { - ArgExtension::None => {} - ArgExtension::Zext => { - llvm::Attribute::ZExt.apply_callsite(idx, callsite); - } - ArgExtension::Sext => { - llvm::Attribute::SExt.apply_callsite(idx, callsite); - } - } - } + let attrs = get_attrs(self, cx); + attributes::apply_to_callsite(callsite, idx, &attrs); } } @@ -433,15 +410,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { - // FIXME(eddyb) can this also be applied to callsites? + let mut func_attrs = SmallVec::<[_; 2]>::new(); if self.ret.layout.abi.is_uninhabited() { - llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); + func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); } - - // FIXME(eddyb, wesleywiser): apply this to callsites as well? if !self.can_unwind { - llvm::Attribute::NoUnwind.apply_llfn(llvm::AttributePlace::Function, llfn); + func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); } + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); let mut i = 0; let mut apply = |attrs: &ArgAttributes| { @@ -456,13 +432,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); - unsafe { - llvm::LLVMRustAddStructRetAttr( - llfn, - llvm::AttributePlace::Argument(i).as_uint(), - self.ret.layout.llvm_type(cx), - ); - } + let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } PassMode::Cast(cast) => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); @@ -477,13 +448,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => {} PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { let i = apply(attrs); - unsafe { - llvm::LLVMRustAddByValAttr( - llfn, - llvm::AttributePlace::Argument(i).as_uint(), - arg.layout.llvm_type(cx), - ); - } + let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } PassMode::Direct(ref attrs) | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { @@ -506,12 +472,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { + let mut func_attrs = SmallVec::<[_; 2]>::new(); if self.ret.layout.abi.is_uninhabited() { - llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite); + func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); } if !self.can_unwind { - llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite); + func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx)); } + attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs }); let mut i = 0; let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { @@ -526,13 +494,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); - unsafe { - llvm::LLVMRustAddStructRetCallSiteAttr( - callsite, - llvm::AttributePlace::Argument(i).as_uint(), - self.ret.layout.llvm_type(bx), - ); - } + let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); + attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } PassMode::Cast(cast) => { cast.attrs.apply_attrs_to_callsite( @@ -561,13 +524,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => {} PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); - unsafe { - llvm::LLVMRustAddByValCallSiteAttr( - callsite, - llvm::AttributePlace::Argument(i).as_uint(), - arg.layout.llvm_type(bx), - ); - } + let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(i), + &[byval], + ); } PassMode::Direct(ref attrs) | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { @@ -599,13 +561,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { if self.conv == Conv::CCmseNonSecureCall { // This will probably get ignored on all targets but those supporting the TrustZone-M // extension (thumbv8m targets). - unsafe { - llvm::AddCallSiteAttrString( - callsite, - llvm::AttributePlace::Function, - cstr::cstr!("cmse_nonsecure_call"), - ); - } + let cmse_nonsecure_call = + llvm::CreateAttrString(bx.cx.llcx, cstr::cstr!("cmse_nonsecure_call")); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Function, + &[cmse_nonsecure_call], + ); } } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 30d91b41a8e..eb19e427217 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -64,7 +64,8 @@ pub(crate) unsafe fn codegen( llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); } if tcx.sess.must_emit_unwind_tables() { - attributes::emit_uwtable(llfn, true); + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } let callee = kind.fn_name(method.name); @@ -105,20 +106,22 @@ pub(crate) unsafe fn codegen( let name = "__rust_alloc_error_handler"; let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); // -> ! DIFlagNoReturn - llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); + let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); if tcx.sess.target.default_hidden_visibility { llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); } if tcx.sess.must_emit_unwind_tables() { - attributes::emit_uwtable(llfn, true); + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default }; let callee = kind.fn_name(sym::oom); let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); // -> ! DIFlagNoReturn - llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee); + attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index caf16c1939d..96c7d884b7b 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -1,3 +1,4 @@ +use crate::attributes; use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; @@ -7,117 +8,21 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use rustc_ast::LlvmAsmDialect; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::mir::operand::OperandValue; -use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; -use rustc_hir as hir; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::{bug, span_bug, ty::Instance}; -use rustc_span::{Pos, Span, Symbol}; +use rustc_span::{Pos, Span}; use rustc_target::abi::*; use rustc_target::asm::*; use libc::{c_char, c_uint}; +use smallvec::SmallVec; use tracing::debug; impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { - fn codegen_llvm_inline_asm( - &mut self, - ia: &hir::LlvmInlineAsmInner, - outputs: Vec<PlaceRef<'tcx, &'ll Value>>, - mut inputs: Vec<&'ll Value>, - span: Span, - ) -> bool { - let mut ext_constraints = vec![]; - let mut output_types = vec![]; - - // Prepare the output operands - let mut indirect_outputs = vec![]; - for (i, (out, &place)) in ia.outputs.iter().zip(&outputs).enumerate() { - if out.is_rw { - let operand = self.load_operand(place); - if let OperandValue::Immediate(_) = operand.val { - inputs.push(operand.immediate()); - } - ext_constraints.push(i.to_string()); - } - if out.is_indirect { - let operand = self.load_operand(place); - if let OperandValue::Immediate(_) = operand.val { - indirect_outputs.push(operand.immediate()); - } - } else { - output_types.push(place.layout.llvm_type(self.cx)); - } - } - if !indirect_outputs.is_empty() { - indirect_outputs.extend_from_slice(&inputs); - inputs = indirect_outputs; - } - - let clobbers = ia.clobbers.iter().map(|s| format!("~{{{}}}", &s)); - - // Default per-arch clobbers - // Basically what clang does - let arch_clobbers = match &self.sess().target.arch[..] { - "x86" | "x86_64" => &["~{dirflag}", "~{fpsr}", "~{flags}"][..], - "mips" | "mips64" => &["~{$1}"], - _ => &[], - }; - - let all_constraints = ia - .outputs - .iter() - .map(|out| out.constraint.to_string()) - .chain(ia.inputs.iter().map(|s| s.to_string())) - .chain(ext_constraints) - .chain(clobbers) - .chain(arch_clobbers.iter().map(|s| (*s).to_string())) - .collect::<Vec<String>>() - .join(","); - - debug!("Asm Constraints: {}", &all_constraints); - - // Depending on how many outputs we have, the return type is different - let num_outputs = output_types.len(); - let output_type = match num_outputs { - 0 => self.type_void(), - 1 => output_types[0], - _ => self.type_struct(&output_types, false), - }; - - let asm = ia.asm.as_str(); - let r = inline_asm_call( - self, - &asm, - &all_constraints, - &inputs, - output_type, - ia.volatile, - ia.alignstack, - ia.dialect, - &[span], - false, - None, - ); - if r.is_none() { - return false; - } - let r = r.unwrap(); - - // Again, based on how many outputs we have - let outputs = ia.outputs.iter().zip(&outputs).filter(|&(o, _)| !o.is_indirect); - for (i, (_, &place)) in outputs.enumerate() { - let v = if num_outputs == 1 { r } else { self.extract_value(r, i as u64) }; - OperandValue::Immediate(v).store(self, place); - } - - true - } - fn codegen_inline_asm( &mut self, template: &[InlineAsmTemplatePiece], @@ -142,9 +47,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { for &(_, feature) in reg_class.supported_types(asm_arch) { if let Some(feature) = feature { let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); - let feature_name = Symbol::intern(feature); - if self.tcx.sess.target_features.contains(&feature_name) - || codegen_fn_attrs.target_features.contains(&feature_name) + if self.tcx.sess.target_features.contains(&feature) + || codegen_fn_attrs.target_features.contains(&feature) { return true; } @@ -330,6 +234,9 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} + InlineAsmArch::Msp430 => { + constraints.push("~{sr}".to_string()); + } } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -349,9 +256,9 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::X86 | InlineAsmArch::X86_64 if !options.contains(InlineAsmOptions::ATT_SYNTAX) => { - LlvmAsmDialect::Intel + llvm::AsmDialect::Intel } - _ => LlvmAsmDialect::Att, + _ => llvm::AsmDialect::Att, }; let result = inline_asm_call( self, @@ -368,19 +275,20 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); + let mut attrs = SmallVec::<[_; 2]>::new(); if options.contains(InlineAsmOptions::PURE) { if options.contains(InlineAsmOptions::NOMEM) { - llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result); + attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::READONLY) { - llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result); + attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx)); } - llvm::Attribute::WillReturn.apply_callsite(llvm::AttributePlace::Function, result); + attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::NOMEM) { - llvm::Attribute::InaccessibleMemOnly - .apply_callsite(llvm::AttributePlace::Function, result); + attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx)); } else { // LLVM doesn't have an attribute to represent ReadOnly + SideEffect } + attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); // Write results to outputs for (idx, op) in operands.iter().enumerate() { @@ -455,7 +363,7 @@ pub(crate) fn inline_asm_call<'ll>( output: &'ll llvm::Type, volatile: bool, alignstack: bool, - dia: LlvmAsmDialect, + dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, dest_catch_funclet: Option<( @@ -498,7 +406,7 @@ pub(crate) fn inline_asm_call<'ll>( cons.len(), volatile, alignstack, - llvm::AsmDialect::from_generic(dia), + dia, can_throw, ); @@ -522,7 +430,7 @@ pub(crate) fn inline_asm_call<'ll>( // we just encode the start position of each line. // FIXME: Figure out a way to pass the entire line spans. let mut srcloc = vec![]; - if dia == LlvmAsmDialect::Intel && line_spans.len() > 1 { + if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { // LLVM inserts an extra line to add the ".intel_syntax", so add // a dummy srcloc entry for it. // @@ -678,6 +586,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -764,6 +673,7 @@ fn modifier_to_llvm( }, InlineAsmRegClass::Avr(_) => None, InlineAsmRegClass::S390x(_) => None, + InlineAsmRegClass::Msp430(_) => None, InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -832,6 +742,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), + InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 7f82ce307d5..e382af8880b 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -7,69 +7,94 @@ use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::OptLevel; -use rustc_session::Session; use rustc_target::spec::abi::Abi; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; +use smallvec::SmallVec; use crate::attributes; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, Attribute}; +use crate::llvm::{self, Attribute, AttributeKind, AttributePlace}; use crate::llvm_util; pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use crate::context::CodegenCx; use crate::value::Value; -/// Mark LLVM function to use provided inline heuristic. +pub fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { + if !attrs.is_empty() { + llvm::AddFunctionAttributes(llfn, idx, attrs); + } +} + +pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { + if !attrs.is_empty() { + llvm::AddCallSiteAttributes(callsite, idx, attrs); + } +} + +/// Get LLVM attribute for the provided inline heuristic. #[inline] -fn inline<'ll>(cx: &CodegenCx<'ll, '_>, val: &'ll Value, inline: InlineAttr) { - use self::InlineAttr::*; +fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { match inline { - Hint => Attribute::InlineHint.apply_llfn(Function, val), - Always => Attribute::AlwaysInline.apply_llfn(Function, val), - Never => { - if cx.tcx().sess.target.arch != "amdgpu" { - Attribute::NoInline.apply_llfn(Function, val); + InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), + InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), + InlineAttr::Never => { + if cx.sess().target.arch != "amdgpu" { + Some(AttributeKind::NoInline.create_attr(cx.llcx)) + } else { + None } } - None => {} - }; + InlineAttr::None => None, + } } -/// Apply LLVM sanitize attributes. +/// Get LLVM sanitize attributes. #[inline] -pub fn sanitize<'ll>(cx: &CodegenCx<'ll, '_>, no_sanitize: SanitizerSet, llfn: &'ll Value) { +pub fn sanitize_attrs<'ll>( + cx: &CodegenCx<'ll, '_>, + no_sanitize: SanitizerSet, +) -> SmallVec<[&'ll Attribute; 4]> { + let mut attrs = SmallVec::new(); let enabled = cx.tcx.sess.opts.debugging_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) { - llvm::Attribute::SanitizeAddress.apply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } if enabled.contains(SanitizerSet::MEMORY) { - llvm::Attribute::SanitizeMemory.apply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::SanitizeMemory.create_attr(cx.llcx)); } if enabled.contains(SanitizerSet::THREAD) { - llvm::Attribute::SanitizeThread.apply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::SanitizeThread.create_attr(cx.llcx)); } if enabled.contains(SanitizerSet::HWADDRESS) { - llvm::Attribute::SanitizeHWAddress.apply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); } -} + if enabled.contains(SanitizerSet::MEMTAG) { + // Check to make sure the mte target feature is actually enabled. + let features = cx.tcx.global_backend_features(()); + let mte_feature = + features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); + if let None | Some("-mte") = mte_feature { + cx.tcx.sess.err("`-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`"); + } -/// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. -#[inline] -pub fn emit_uwtable(val: &Value, emit: bool) { - Attribute::UWTable.toggle_llfn(Function, val, emit); + attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); + } + attrs } -/// Tell LLVM if this function should be 'naked', i.e., skip the epilogue and prologue. +/// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. #[inline] -fn naked(val: &Value, is_naked: bool) { - Attribute::Naked.toggle_llfn(Function, val, is_naked); +pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { + // NOTE: We should determine if we even need async unwind tables, as they + // take have more overhead and if we can use sync unwind tables we + // probably should. + llvm::CreateUWTableAttr(llcx, true) } -pub fn set_frame_pointer_type<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { +pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { let mut fp = cx.sess().target.frame_pointer; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. @@ -80,19 +105,14 @@ pub fn set_frame_pointer_type<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { let attr_value = match fp { FramePointer::Always => cstr!("all"), FramePointer::NonLeaf => cstr!("non-leaf"), - FramePointer::MayOmit => return, + FramePointer::MayOmit => return None, }; - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("frame-pointer"), - attr_value, - ); + Some(llvm::CreateAttrStringValue(cx.llcx, cstr!("frame-pointer"), attr_value)) } /// Tell LLVM what instrument function to insert. #[inline] -fn set_instrument_function<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { +fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { if cx.sess().instrument_mcount() { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. @@ -101,16 +121,17 @@ fn set_instrument_function<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { // See test/CodeGen/mcount.c in clang. let mcount_name = CString::new(cx.sess().target.mcount.as_str().as_bytes()).unwrap(); - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, + Some(llvm::CreateAttrStringValue( + cx.llcx, cstr!("instrument-function-entry-inlined"), &mcount_name, - ); + )) + } else { + None } } -fn set_probestack<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { +fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. @@ -121,107 +142,90 @@ fn set_probestack<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { .sanitizer .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) { - return; + return None; } // probestack doesn't play nice either with `-C profile-generate`. if cx.sess().opts.cg.profile_generate.enabled() { - return; + return None; } // probestack doesn't play nice either with gcov profiling. if cx.sess().opts.debugging_opts.profile { - return; + return None; } let attr_value = match cx.sess().target.stack_probes { - StackProbeType::None => None, + StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). - StackProbeType::Inline => Some(cstr!("inline-asm")), + StackProbeType::Inline => cstr!("inline-asm"), // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => Some(cstr!("__rust_probestack")), + StackProbeType::Call => cstr!("__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - Some(cstr!("__rust_probestack")) + cstr!("__rust_probestack") } else { - Some(cstr!("inline-asm")) + cstr!("inline-asm") } } }; - if let Some(attr_value) = attr_value { - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("probe-stack"), - attr_value, - ); - } + Some(llvm::CreateAttrStringValue(cx.llcx, cstr!("probe-stack"), attr_value)) } -fn set_stackprotector<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { +fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { let sspattr = match cx.sess().stack_protector() { - StackProtector::None => return, - StackProtector::All => Attribute::StackProtectReq, - StackProtector::Strong => Attribute::StackProtectStrong, - StackProtector::Basic => Attribute::StackProtect, + StackProtector::None => return None, + StackProtector::All => AttributeKind::StackProtectReq, + StackProtector::Strong => AttributeKind::StackProtectStrong, + StackProtector::Basic => AttributeKind::StackProtect, }; - sspattr.apply_llfn(Function, llfn) + Some(sspattr.create_attr(cx.llcx)) } -pub fn apply_target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { +pub fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { let target_cpu = SmallCStr::new(llvm_util::target_cpu(cx.tcx.sess)); - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("target-cpu"), - target_cpu.as_c_str(), - ); + llvm::CreateAttrStringValue(cx.llcx, cstr!("target-cpu"), target_cpu.as_c_str()) } -pub fn apply_tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { - if let Some(tune) = llvm_util::tune_cpu(cx.tcx.sess) { +pub fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(cx.tcx.sess).map(|tune| { let tune_cpu = SmallCStr::new(tune); - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("tune-cpu"), - tune_cpu.as_c_str(), - ); - } + llvm::CreateAttrStringValue(cx.llcx, cstr!("tune-cpu"), tune_cpu.as_c_str()) + }) } -/// Sets the `NonLazyBind` LLVM attribute on a given function, -/// assuming the codegen options allow skipping the PLT. -pub fn non_lazy_bind<'ll>(sess: &Session, llfn: &'ll Value) { +/// Get the `NonLazyBind` LLVM attribute, +/// if the codegen options allow skipping the PLT. +pub fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !sess.needs_plt() { - Attribute::NonLazyBind.apply_llfn(Function, llfn); + if !cx.sess().needs_plt() { + Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) + } else { + None } } -pub(crate) fn default_optimisation_attrs<'ll>(sess: &Session, llfn: &'ll Value) { - match sess.opts.optimize { +/// Get the default optimizations attrs for a function. +#[inline] +pub(crate) fn default_optimisation_attrs<'ll>( + cx: &CodegenCx<'ll, '_>, +) -> SmallVec<[&'ll Attribute; 2]> { + let mut attrs = SmallVec::new(); + match cx.sess().opts.optimize { OptLevel::Size => { - llvm::Attribute::MinSize.unapply_llfn(Function, llfn); - llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn); - llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } OptLevel::SizeMin => { - llvm::Attribute::MinSize.apply_llfn(Function, llfn); - llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn); - llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); - } - OptLevel::No => { - llvm::Attribute::MinSize.unapply_llfn(Function, llfn); - llvm::Attribute::OptimizeForSize.unapply_llfn(Function, llfn); - llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); + attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } _ => {} } + attrs } /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) @@ -233,30 +237,27 @@ pub fn from_fn_attrs<'ll, 'tcx>( ) { let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let mut to_add = SmallVec::<[_; 16]>::new(); + match codegen_fn_attrs.optimize { OptimizeAttr::None => { - default_optimisation_attrs(cx.tcx.sess, llfn); - } - OptimizeAttr::Speed => { - llvm::Attribute::MinSize.unapply_llfn(Function, llfn); - llvm::Attribute::OptimizeForSize.unapply_llfn(Function, llfn); - llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + to_add.extend(default_optimisation_attrs(cx)); } OptimizeAttr::Size => { - llvm::Attribute::MinSize.apply_llfn(Function, llfn); - llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn); - llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + to_add.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); + to_add.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } + OptimizeAttr::Speed => {} } - let inline_attr = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { InlineAttr::Never } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { InlineAttr::Hint } else { codegen_fn_attrs.inline }; - inline(cx, llfn, inline_attr); + to_add.extend(inline_attr(cx, inline)); // The `uwtable` attribute according to LLVM is: // @@ -275,62 +276,82 @@ pub fn from_fn_attrs<'ll, 'tcx>( // You can also find more info on why Windows always requires uwtables here: // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 if cx.sess().must_emit_unwind_tables() { - attributes::emit_uwtable(llfn, true); + to_add.push(uwtable_attr(cx.llcx)); } if cx.sess().opts.debugging_opts.profile_sample_use.is_some() { - llvm::AddFunctionAttrString(llfn, Function, cstr!("use-sample-profile")); + to_add.push(llvm::CreateAttrString(cx.llcx, cstr!("use-sample-profile"))); } // FIXME: none of these three functions interact with source level attributes. - set_frame_pointer_type(cx, llfn); - set_instrument_function(cx, llfn); - set_probestack(cx, llfn); - set_stackprotector(cx, llfn); + to_add.extend(frame_pointer_type_attr(cx)); + to_add.extend(instrument_function_attr(cx)); + to_add.extend(probestack_attr(cx)); + to_add.extend(stackprotector_attr(cx)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { - Attribute::Cold.apply_llfn(Function, llfn); + to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { - Attribute::ReturnsTwice.apply_llfn(Function, llfn); + to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { - Attribute::ReadOnly.apply_llfn(Function, llfn); + to_add.push(AttributeKind::ReadOnly.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) { - Attribute::ReadNone.apply_llfn(Function, llfn); + to_add.push(AttributeKind::ReadNone.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - naked(llfn, true); + to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { - Attribute::NoAlias.apply_llfn(llvm::AttributePlace::ReturnValue, llfn); + // apply to return place instead of function (unlike all other attributes applied in this function) + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { - llvm::AddFunctionAttrString(llfn, Function, cstr!("cmse_nonsecure_entry")); + to_add.push(llvm::CreateAttrString(cx.llcx, cstr!("cmse_nonsecure_entry"))); } if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align as usize); } - sanitize(cx, codegen_fn_attrs.no_sanitize, llfn); + to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - apply_target_cpu_attr(cx, llfn); + to_add.push(target_cpu_attr(cx)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - apply_tune_cpu_attr(cx, llfn); + to_add.extend(tune_cpu_attr(cx)); + + let function_features = + codegen_fn_attrs.target_features.iter().map(|f| f.as_str()).collect::<Vec<&str>>(); + + if let Some(f) = llvm_util::check_tied_features( + cx.tcx.sess, + &function_features.iter().map(|f| (*f, true)).collect(), + ) { + let span = cx + .tcx + .get_attrs(instance.def_id()) + .iter() + .find(|a| a.has_name(rustc_span::sym::target_feature)) + .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); + let msg = format!( + "the target features {} must all be either enabled or disabled together", + f.join(", ") + ); + let mut err = cx.tcx.sess.struct_span_err(span, &msg); + err.help("add the missing features in a `target_feature` attribute"); + err.emit(); + return; + } - let mut function_features = codegen_fn_attrs - .target_features + let mut function_features = function_features .iter() - .flat_map(|f| { - let feature = f.as_str(); - llvm_util::to_llvm_feature(cx.tcx.sess, feature) - .into_iter() - .map(|f| format!("+{}", f)) - .collect::<Vec<String>>() + .flat_map(|feat| { + llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{}", f)) }) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), @@ -342,22 +363,12 @@ pub fn from_fn_attrs<'ll, 'tcx>( // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("wasm-import-module"), - &module, - ); + to_add.push(llvm::CreateAttrStringValue(cx.llcx, cstr!("wasm-import-module"), &module)); let name = codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); let name = CString::new(name.as_str()).unwrap(); - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("wasm-import-name"), - &name, - ); + to_add.push(llvm::CreateAttrStringValue(cx.llcx, cstr!("wasm-import-name"), &name)); } // The `"wasm"` abi on wasm targets automatically enables the @@ -373,17 +384,15 @@ pub fn from_fn_attrs<'ll, 'tcx>( } if !function_features.is_empty() { - let mut global_features = llvm_util::llvm_global_features(cx.tcx.sess); - global_features.extend(function_features.into_iter()); - let features = global_features.join(","); - let val = CString::new(features).unwrap(); - llvm::AddFunctionAttrStringValue( - llfn, - llvm::AttributePlace::Function, - cstr!("target-features"), - &val, - ); + let global_features = cx.tcx.global_backend_features(()).iter().map(|s| &s[..]); + let val = global_features + .chain(function_features.iter().map(|s| &s[..])) + .intersperse(",") + .collect::<SmallCStr>(); + to_add.push(llvm::CreateAttrStringValue(cx.llcx, cstr!("target-features"), &val)); } + + attributes::apply_to_llfn(llfn, Function, &to_add); } fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option<CString> { diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 2fb5a0f9faf..f814cc93043 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,6 +1,7 @@ //! A helper class for dealing with static archives -use std::ffi::{CStr, CString}; +use std::env; +use std::ffi::{CStr, CString, OsString}; use std::io; use std::mem; use std::path::{Path, PathBuf}; @@ -26,7 +27,6 @@ pub struct LlvmArchiveBuilder<'a> { config: ArchiveConfig<'a>, removals: Vec<String>, additions: Vec<Addition>, - should_update_symbols: bool, src_archive: Option<Option<ArchiveRO>>, } @@ -74,7 +74,6 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { config, removals: Vec::new(), additions: Vec::new(), - should_update_symbols: false, src_archive: None, } } @@ -128,12 +127,6 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); } - /// Indicate that the next call to `build` should update all symbols in - /// the archive (equivalent to running 'ar s' over it). - fn update_symbols(&mut self) { - self.should_update_symbols = true; - } - /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. fn build(mut self) { @@ -158,54 +151,129 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path.with_extension("lib") }; - // we've checked for \0 characters in the library name already - let dll_name_z = CString::new(lib_name).unwrap(); - // All import names are Rust identifiers and therefore cannot contain \0 characters. - // FIXME: when support for #[link_name] implemented, ensure that import.name values don't - // have any \0 characters - let import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = dll_imports + let target = &self.config.sess.target; + let mingw_gnu_toolchain = + target.vendor == "pc" && target.os == "windows" && target.env == "gnu"; + + let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { if self.config.sess.target.arch == "x86" { - (LlvmArchiveBuilder::i686_decorated_name(import), import.ordinal) + ( + LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), + import.ordinal, + ) } else { - (CString::new(import.name.to_string()).unwrap(), import.ordinal) + (import.name.to_string(), import.ordinal) } }) .collect(); - let output_path_z = rustc_fs_util::path_to_c_string(&output_path); + if mingw_gnu_toolchain { + // The binutils linker used on -windows-gnu targets cannot read the import + // libraries generated by LLVM: in our attempts, the linker produced an .EXE + // that loaded but crashed with an AV upon calling one of the imported + // functions. Therefore, use binutils to create the import library instead, + // by writing a .DEF file to the temp dir and calling binutils's dlltool. + let def_file_path = + tmpdir.as_ref().join(format!("{}_imports", lib_name)).with_extension("def"); + + let def_file_content = format!( + "EXPORTS\n{}", + import_name_and_ordinal_vector + .into_iter() + .map(|(name, ordinal)| { + match ordinal { + Some(n) => format!("{} @{} NONAME", name, n), + None => name, + } + }) + .collect::<Vec<String>>() + .join("\n") + ); - tracing::trace!("invoking LLVMRustWriteImportLibrary"); - tracing::trace!(" dll_name {:#?}", dll_name_z); - tracing::trace!(" output_path {}", output_path.display()); - tracing::trace!( - " import names: {}", - dll_imports.iter().map(|import| import.name.to_string()).collect::<Vec<_>>().join(", "), - ); + match std::fs::write(&def_file_path, def_file_content) { + Ok(_) => {} + Err(e) => { + self.config.sess.fatal(&format!("Error writing .DEF file: {}", e)); + } + }; - let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_and_ordinal_vector - .iter() - .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) - .collect(); - let result = unsafe { - crate::llvm::LLVMRustWriteImportLibrary( - dll_name_z.as_ptr(), - output_path_z.as_ptr(), - ffi_exports.as_ptr(), - ffi_exports.len(), - llvm_machine_type(&self.config.sess.target.arch) as u16, - !self.config.sess.target.is_like_msvc, - ) - }; + let dlltool = find_binutils_dlltool(self.config.sess); + let result = std::process::Command::new(dlltool) + .args([ + "-d", + def_file_path.to_str().unwrap(), + "-D", + lib_name, + "-l", + output_path.to_str().unwrap(), + ]) + .output(); + + match result { + Err(e) => { + self.config.sess.fatal(&format!("Error calling dlltool: {}", e)); + } + Ok(output) if !output.status.success() => self.config.sess.fatal(&format!( + "Dlltool could not create import library: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )), + _ => {} + } + } else { + // we've checked for \0 characters in the library name already + let dll_name_z = CString::new(lib_name).unwrap(); + + let output_path_z = rustc_fs_util::path_to_c_string(&output_path); + + tracing::trace!("invoking LLVMRustWriteImportLibrary"); + tracing::trace!(" dll_name {:#?}", dll_name_z); + tracing::trace!(" output_path {}", output_path.display()); + tracing::trace!( + " import names: {}", + dll_imports + .iter() + .map(|import| import.name.to_string()) + .collect::<Vec<_>>() + .join(", "), + ); - if result == crate::llvm::LLVMRustResult::Failure { - self.config.sess.fatal(&format!( - "Error creating import library for {}: {}", - lib_name, - llvm::last_error().unwrap_or("unknown LLVM error".to_string()) - )); - } + // All import names are Rust identifiers and therefore cannot contain \0 characters. + // FIXME: when support for #[link_name] is implemented, ensure that the import names + // still don't contain any \0 characters. Also need to check that the names don't + // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved + // in definition files. + let cstring_import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = + import_name_and_ordinal_vector + .into_iter() + .map(|(name, ordinal)| (CString::new(name).unwrap(), ordinal)) + .collect(); + + let ffi_exports: Vec<LLVMRustCOFFShortExport> = cstring_import_name_and_ordinal_vector + .iter() + .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) + .collect(); + let result = unsafe { + crate::llvm::LLVMRustWriteImportLibrary( + dll_name_z.as_ptr(), + output_path_z.as_ptr(), + ffi_exports.as_ptr(), + ffi_exports.len(), + llvm_machine_type(&self.config.sess.target.arch) as u16, + !self.config.sess.target.is_like_msvc, + ) + }; + + if result == crate::llvm::LLVMRustResult::Failure { + self.config.sess.fatal(&format!( + "Error creating import library for {}: {}", + lib_name, + llvm::last_error().unwrap_or("unknown LLVM error".to_string()) + )); + } + }; self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { self.config.sess.fatal(&format!( @@ -239,16 +307,12 @@ impl<'a> LlvmArchiveBuilder<'a> { let mut members = Vec::new(); let dst = CString::new(self.config.dst.to_str().unwrap())?; - let should_update_symbols = self.should_update_symbols; unsafe { if let Some(archive) = self.src_archive() { for child in archive.iter() { let child = child.map_err(string_to_io_error)?; - let child_name = match child.name() { - Some(s) => s, - None => continue, - }; + let Some(child_name) = child.name() else { continue }; if removals.iter().any(|r| r == child_name) { continue; } @@ -311,7 +375,7 @@ impl<'a> LlvmArchiveBuilder<'a> { dst.as_ptr(), members.len() as libc::size_t, members.as_ptr() as *const &_, - should_update_symbols, + true, kind, ); let ret = if r.into_result().is_err() { @@ -332,22 +396,61 @@ impl<'a> LlvmArchiveBuilder<'a> { } } - fn i686_decorated_name(import: &DllImport) -> CString { + fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { let name = import.name; - // We verified during construction that `name` does not contain any NULL characters, so the - // conversion to CString is guaranteed to succeed. - CString::new(match import.calling_convention { - DllCallingConvention::C => format!("_{}", name), - DllCallingConvention::Stdcall(arg_list_size) => format!("_{}@{}", name, arg_list_size), + let prefix = if mingw { "" } else { "_" }; + + match import.calling_convention { + DllCallingConvention::C => format!("{}{}", prefix, name), + DllCallingConvention::Stdcall(arg_list_size) => { + format!("{}{}@{}", prefix, name, arg_list_size) + } DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), DllCallingConvention::Vectorcall(arg_list_size) => { format!("{}@@{}", name, arg_list_size) } - }) - .unwrap() + } } } fn string_to_io_error(s: String) -> io::Error { io::Error::new(io::ErrorKind::Other, format!("bad archive: {}", s)) } + +fn find_binutils_dlltool(sess: &Session) -> OsString { + assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); + if let Some(dlltool_path) = &sess.opts.debugging_opts.dlltool { + return dlltool_path.clone().into_os_string(); + } + + let mut tool_name: OsString = if sess.host.arch != sess.target.arch { + // We are cross-compiling, so we need the tool with the prefix matching our target + if sess.target.arch == "x86" { + "i686-w64-mingw32-dlltool" + } else { + "x86_64-w64-mingw32-dlltool" + } + } else { + // We are not cross-compiling, so we just want `dlltool` + "dlltool" + } + .into(); + + if sess.host.options.is_like_windows { + // If we're compiling on Windows, add the .exe suffix + tool_name.push(".exe"); + } + + // NOTE: it's not clear how useful it is to explicitly search PATH. + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let full_path = dir.join(&tool_name); + if full_path.is_file() { + return full_path.into_os_string(); + } + } + + // The user didn't specify the location of the dlltool binary, and we weren't able + // to find the appropriate one on the PATH. Just return the name of the tool + // and let the invocation fail with a hopefully useful error message. + tool_name +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 4bb1fed2d51..6afa649b6de 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -3,7 +3,7 @@ use crate::back::write::{ }; use crate::llvm::archive_ro::ArchiveRO; use crate::llvm::{self, build_string, False, True}; -use crate::{LlvmCodegenBackend, ModuleLlvm}; +use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{ @@ -349,13 +349,6 @@ fn fat_lto( ); save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } - - if cgcx.no_landing_pads { - unsafe { - llvm::LLVMRustMarkAllFunctionsNounwind(llmod); - } - save_temp_bitcode(cgcx, &module, "lto.after-nounwind"); - } } Ok(LtoModuleCodegen::Fat { module: Some(module), _serialized_bitcode: serialized_bitcode }) @@ -596,7 +589,10 @@ pub(crate) fn run_pass_manager( // tools/lto/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { - if write::should_use_new_llvm_pass_manager(cgcx, config) { + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); write::optimize_with_new_llvm_pass_manager( @@ -767,16 +763,6 @@ pub unsafe fn optimize_thin_module( return Err(write::llvm_err(&diag_handler, msg)); } - // Like with "fat" LTO, get some better optimizations if landing pads - // are disabled by removing all landing pads. - if cgcx.no_landing_pads { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_remove_landing_pads", thin_module.name()); - llvm::LLVMRustMarkAllFunctionsNounwind(llmod); - save_temp_bitcode(cgcx, &module, "thin-lto-after-nounwind"); - } - // Up next comes the per-module local analyses that we do for Thin LTO. // Each of these functions is basically copied from the LLVM // implementation and then tailored to suit this implementation. Ideally diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index fb194a98a0d..c18719d4ad7 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -23,7 +23,7 @@ use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath}; +use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; @@ -100,19 +100,30 @@ pub fn write_output_file<'ll>( pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { let config = TargetMachineFactoryConfig { split_dwarf_file: None }; - target_machine_factory(sess, config::OptLevel::No)(config) + // Can't use query system here quite yet because this function is invoked before the query + // system/tcx is set up. + let features = llvm_util::global_llvm_features(sess, false); + target_machine_factory(sess, config::OptLevel::No, &features)(config) .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) } pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { - tcx.output_filenames(()).split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name)) + tcx.output_filenames(()).split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.debugging_opts.split_dwarf_kind, + Some(mod_name), + ) } else { None }; let config = TargetMachineFactoryConfig { split_dwarf_file }; - target_machine_factory(tcx.sess, tcx.backend_optimization_level(()))(config) - .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) + target_machine_factory( + &tcx.sess, + tcx.backend_optimization_level(()), + tcx.global_backend_features(()), + )(config) + .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) } pub fn to_llvm_opt_settings( @@ -167,6 +178,7 @@ pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeMod pub fn target_machine_factory( sess: &Session, optlvl: config::OptLevel, + target_features: &[String], ) -> TargetMachineFactoryFn<LlvmCodegenBackend> { let reloc_model = to_llvm_relocation_model(sess.relocation_model()); @@ -191,8 +203,7 @@ pub fn target_machine_factory( let triple = SmallCStr::new(&sess.target.llvm_target); let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); - let features = llvm_util::llvm_global_features(sess).join(","); - let features = CString::new(features).unwrap(); + let features = CString::new(target_features.join(",")).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); let trap_unreachable = sess.opts.debugging_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); @@ -416,21 +427,6 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } -pub(crate) fn should_use_new_llvm_pass_manager( - cgcx: &CodegenContext<LlvmCodegenBackend>, - config: &ModuleConfig, -) -> bool { - // The new pass manager is enabled by default for LLVM >= 13. - // This matches Clang, which also enables it since Clang 13. - - // FIXME: There are some perf issues with the new pass manager - // when targeting s390x, so it is temporarily disabled for that - // arch, see https://github.com/rust-lang/rust/issues/89609 - config - .new_llvm_pass_manager - .unwrap_or_else(|| cgcx.target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0)) -} - pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, @@ -473,6 +469,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let extra_passes = config.passes.join(","); + let llvm_plugins = config.llvm_plugins.join(","); + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. @@ -502,6 +500,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( selfprofile_after_pass_callback, extra_passes.as_ptr().cast(), extra_passes.len(), + llvm_plugins.as_ptr().cast(), + llvm_plugins.len(), ); result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) } @@ -530,7 +530,10 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { - if should_use_new_llvm_pass_manager(cgcx, config) { + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { let opt_stage = match cgcx.lto { Lto::Fat => llvm::OptStage::PreLinkFatLTO, Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, @@ -830,20 +833,14 @@ pub(crate) unsafe fn codegen( let input = unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) }; - let input = match str::from_utf8(input) { - Ok(s) => s, - Err(_) => return 0, - }; + let Ok(input) = str::from_utf8(input) else { return 0 }; let output = unsafe { slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize) }; let mut cursor = io::Cursor::new(output); - let demangled = match rustc_demangle::try_demangle(input) { - Ok(d) => d, - Err(_) => return 0, - }; + let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 }; if write!(cursor, "{:#}", demangled).is_err() { // Possible only if provided buffer is not big enough @@ -900,17 +897,18 @@ pub(crate) unsafe fn codegen( .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name); let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name); - let dwo_out = match cgcx.split_debuginfo { - // Don't change how DWARF is emitted in single mode (or when disabled). - SplitDebuginfo::Off | SplitDebuginfo::Packed => None, - // Emit (a subset of the) DWARF into a separate file in split mode. - SplitDebuginfo::Unpacked => { - if cgcx.target_can_use_split_dwarf { - Some(dwo_out.as_path()) - } else { - None - } - } + let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) { + // Don't change how DWARF is emitted when disabled. + (SplitDebuginfo::Off, _) => None, + // Don't provide a DWARF object path if split debuginfo is enabled but this is + // a platform that doesn't support Split DWARF. + _ if !cgcx.target_can_use_split_dwarf => None, + // Don't provide a DWARF object path in single mode, sections will be written + // into the object as normal but ignored by linker. + (_, SplitDwarfKind::Single) => None, + // Emit (a subset of the) DWARF into a separate dwarf object file in split + // mode. + (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; with_codegen(tm, llmod, config.no_builtins, |cpm| { @@ -947,7 +945,9 @@ pub(crate) unsafe fn codegen( Ok(module.into_compiled_module( config.emit_obj != EmitObj::None, - cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo == SplitDebuginfo::Unpacked, + cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split, config.emit_bc, &cgcx.output_filenames, )) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 7b6ce5ea89b..dd3ada44389 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -95,7 +95,8 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen // If this codegen unit contains the main function, also create the // wrapper here if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx) { - attributes::sanitize(&cx, SanitizerSet::empty(), entry); + let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } // Run replace-all-uses-with for statics that need it @@ -138,10 +139,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen } pub fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { - let sect = match attrs.link_section { - Some(name) => name, - None => return, - }; + let Some(sect) = attrs.link_section else { return }; unsafe { let buf = SmallCStr::new(sect.as_str()); llvm::LLVMSetSection(llval, buf.as_ptr()); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 8c3054b23ff..6c1d4ce8f3c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1,3 +1,4 @@ +use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, BasicBlock, False}; @@ -166,9 +167,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { Self::append_block(self.cx, self.llfn(), name) } - fn build_sibling_block(&mut self, name: &str) -> Self { - let llbb = self.append_sibling_block(name); - Self::build(self.cx, llbb) + fn switch_to_block(&mut self, llbb: Self::BasicBlock) { + *self = Self::build(self.cx, llbb) } fn ret_void(&mut self) { @@ -478,6 +478,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { load: &'ll Value, scalar: abi::Scalar, ) { + if !scalar.is_always_valid(bx) { + bx.noundef_metadata(load); + } + match scalar.value { abi::Int(..) => { if !scalar.is_always_valid(bx) { @@ -544,16 +548,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let start = dest.project_index(&mut self, zero).llval; let end = dest.project_index(&mut self, count).llval; - let mut header_bx = self.build_sibling_block("repeat_loop_header"); - let mut body_bx = self.build_sibling_block("repeat_loop_body"); - let next_bx = self.build_sibling_block("repeat_loop_next"); + let header_bb = self.append_sibling_block("repeat_loop_header"); + let body_bb = self.append_sibling_block("repeat_loop_body"); + let next_bb = self.append_sibling_block("repeat_loop_next"); + + self.br(header_bb); - self.br(header_bx.llbb()); + let mut header_bx = Self::build(self.cx, header_bb); let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); - header_bx.cond_br(keep_going, body_bx.llbb(), next_bx.llbb()); + header_bx.cond_br(keep_going, body_bb, next_bb); + let mut body_bx = Self::build(self.cx, body_bb); let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); cg_elem .val @@ -564,10 +571,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { current, &[self.const_usize(1)], ); - body_bx.br(header_bx.llbb()); - header_bx.add_incoming_to_phi(current, next, body_bx.llbb()); + body_bx.br(header_bb); + header_bx.add_incoming_to_phi(current, next, body_bb); - next_bx + Self::build(self.cx, next_bb) } fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { @@ -731,27 +738,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - if !self.fptoint_sat_broken_in_llvm() { - let src_ty = self.cx.val_ty(val); - let float_width = self.cx.float_width(src_ty); - let int_width = self.cx.int_width(dest_ty); - let name = format!("llvm.fptoui.sat.i{}.f{}", int_width, float_width); - return Some(self.call_intrinsic(&name, &[val])); - } - - None + self.fptoint_sat(false, val, dest_ty) } fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - if !self.fptoint_sat_broken_in_llvm() { - let src_ty = self.cx.val_ty(val); - let float_width = self.cx.float_width(src_ty); - let int_width = self.cx.int_width(dest_ty); - let name = format!("llvm.fptosi.sat.i{}.f{}", int_width, float_width); - return Some(self.call_intrinsic(&name, &[val])); - } - - None + self.fptoint_sat(true, val, dest_ty) } fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -972,29 +963,24 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) } } - fn landing_pad( - &mut self, - ty: &'ll Type, - pers_fn: &'ll Value, - num_clauses: usize, - ) -> &'ll Value { - // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while, - // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The - // personality lives on the parent function anyway. - self.set_personality_fn(pers_fn); + fn set_personality_fn(&mut self, personality: &'ll Value) { unsafe { - llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) + llvm::LLVMSetPersonalityFn(self.llfn(), personality); } } - fn set_cleanup(&mut self, landing_pad: &'ll Value) { + fn cleanup_landing_pad(&mut self, ty: &'ll Type, pers_fn: &'ll Value) -> &'ll Value { + let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); unsafe { llvm::LLVMSetCleanup(landing_pad, llvm::True); } + landing_pad } - fn resume(&mut self, exn: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMBuildResume(self.llbuilder, exn) } + fn resume(&mut self, exn: &'ll Value) { + unsafe { + llvm::LLVMBuildResume(self.llbuilder, exn); + } } fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { @@ -1011,14 +997,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) } - fn cleanup_ret( - &mut self, - funclet: &Funclet<'ll>, - unwind: Option<&'ll BasicBlock>, - ) -> &'ll Value { - let ret = - unsafe { llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) }; - ret.expect("LLVM does not have support for cleanupret") + fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { + unsafe { + llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + .expect("LLVM does not have support for cleanupret"); + } } fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { @@ -1039,7 +1022,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { &mut self, parent: Option<&'ll Value>, unwind: Option<&'ll BasicBlock>, - num_handlers: usize, + handlers: &[&'ll BasicBlock], ) -> &'ll Value { let name = cstr!("catchswitch"); let ret = unsafe { @@ -1047,23 +1030,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.llbuilder, parent, unwind, - num_handlers as c_uint, + handlers.len() as c_uint, name.as_ptr(), ) }; - ret.expect("LLVM does not have support for catchswitch") - } - - fn add_handler(&mut self, catch_switch: &'ll Value, handler: &'ll BasicBlock) { - unsafe { - llvm::LLVMRustAddHandler(catch_switch, handler); - } - } - - fn set_personality_fn(&mut self, personality: &'ll Value) { - unsafe { - llvm::LLVMSetPersonalityFn(self.llfn(), personality); + let ret = ret.expect("LLVM does not have support for catchswitch"); + for handler in handlers { + unsafe { + llvm::LLVMRustAddHandler(ret, handler); + } } + ret } // Atomic Operations @@ -1202,7 +1179,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn do_not_inline(&mut self, llret: &'ll Value) { - llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret); + let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); } } @@ -1230,6 +1208,16 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } + fn noundef_metadata(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_noundef as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } } @@ -1448,4 +1436,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { _ => false, } } + + fn fptoint_sat( + &mut self, + signed: bool, + val: &'ll Value, + dest_ty: &'ll Type, + ) -> Option<&'ll Value> { + if !self.fptoint_sat_broken_in_llvm() { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector + { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + ( + self.cx.element_type(src_ty), + self.cx.element_type(dest_ty), + Some(self.cx.vector_length(src_ty)), + ) + } else { + (src_ty, dest_ty, None) + }; + let float_width = self.cx.float_width(float_ty); + let int_width = self.cx.int_width(int_ty); + + let instr = if signed { "fptosi" } else { "fptoui" }; + let name = if let Some(vector_length) = vector_length { + format!( + "llvm.{}.sat.v{}i{}.v{}f{}", + instr, vector_length, int_width, vector_length, float_width + ) + } else { + format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + }; + let f = + self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); + Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + } else { + None + } + } + + pub(crate) fn landing_pad( + &mut self, + ty: &'ll Type, + pers_fn: &'ll Value, + num_clauses: usize, + ) -> &'ll Value { + // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while, + // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The + // personality lives on the parent function anyway. + self.set_personality_fn(pers_fn); + unsafe { + llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) + } + } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index d43c7c60651..c98720944c9 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -2,6 +2,7 @@ use crate::base; use crate::common::CodegenCx; use crate::debuginfo; use crate::llvm::{self, True}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -37,7 +38,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> alloc: &'a Allocation, range: Range<usize>, ) { - let mut chunks = alloc + let chunks = alloc .init_mask() .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end)); @@ -53,30 +54,26 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> } }; - // Generating partially-uninit consts inhibits optimizations, so it is disabled by default. - // See https://github.com/rust-lang/rust/issues/84565. - let allow_partially_uninit = - match cx.sess().opts.debugging_opts.partially_uninit_const_threshold { - Some(max) => range.len() <= max, - None => false, - }; + // Generating partially-uninit consts is limited to small numbers of chunks, + // to avoid the cost of generating large complex const expressions. + // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, + // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. + let max = if llvm_util::get_version() < (14, 0, 0) { + // Generating partially-uninit consts inhibits optimizations in LLVM < 14. + // See https://github.com/rust-lang/rust/issues/84565. + 1 + } else { + cx.sess().opts.debugging_opts.uninit_const_chunk_threshold + }; + let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; - if allow_partially_uninit { + if allow_uninit_chunks { llvals.extend(chunks.map(chunk_to_llval)); } else { - let llval = match (chunks.next(), chunks.next()) { - (Some(chunk), None) => { - // exactly one chunk, either fully init or fully uninit - chunk_to_llval(chunk) - } - _ => { - // partially uninit, codegen as if it was initialized - // (using some arbitrary value for uninit bytes) - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); - cx.const_bytes(bytes) - } - }; - llvals.push(llval); + // If this allocation contains any uninit bytes, codegen as if it was initialized + // (using some arbitrary value for uninit bytes). + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + llvals.push(cx.const_bytes(bytes)); } } @@ -369,10 +366,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { unsafe { let attrs = self.tcx.codegen_fn_attrs(def_id); - let (v, alloc) = match codegen_static_initializer(self, def_id) { - Ok(v) => v, + let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { // Error has already been reported - Err(_) => return, + return; }; let g = self.get_static(def_id); @@ -522,6 +518,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { } if attrs.flags.contains(CodegenFnAttrFlags::USED) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); + // The semantics of #[used] in Rust only require the symbol to make it into the // object file. It is explicitly allowed for the linker to strip the symbol if it // is dead. As such, use llvm.compiler.used instead of llvm.used. @@ -530,6 +529,12 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // in some versions of the gold linker. self.add_compiler_used_global(g); } + if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED)); + + self.add_used_global(g); + } } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 9f24a95482c..f102becf2bd 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -21,7 +21,8 @@ use rustc_middle::ty::layout::{ }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; -use rustc_session::config::{CFGuard, CrateType, DebugInfo}; +use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; +use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; use rustc_span::symbol::Symbol; @@ -134,7 +135,8 @@ pub unsafe fn create_module<'ll>( let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); let mut target_data_layout = sess.target.data_layout.clone(); - if llvm_util::get_version() < (13, 0, 0) { + let llvm_version = llvm_util::get_version(); + if llvm_version < (13, 0, 0) { if sess.target.arch == "powerpc64" { target_data_layout = target_data_layout.replace("-S128", ""); } @@ -145,6 +147,18 @@ pub unsafe fn create_module<'ll>( target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); } } + if llvm_version < (14, 0, 0) { + if sess.target.llvm_target == "i686-pc-windows-msvc" + || sess.target.llvm_target == "i586-pc-windows-msvc" + { + target_data_layout = + "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" + .to_string(); + } + if sess.target.arch == "wasm32" { + target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. if sess.target.is_builtin { @@ -215,16 +229,19 @@ pub unsafe fn create_module<'ll>( // to ensure intrinsic calls don't use it. if !sess.needs_plt() { let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); - llvm::LLVMRustAddModuleFlag(llmod, avoid_plt, 1); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } if sess.is_sanitizer_cfi_enabled() { // FIXME(rcvalle): Add support for non canonical jump tables. let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag--LLVMRustAddModuleFlag adds it with - // Warning behavior flag. Add support for specifying the behavior flag to - // LLVMRustAddModuleFlag. - llvm::LLVMRustAddModuleFlag(llmod, canonical_jump_tables, 1); + // FIXME(rcvalle): Add it with Override behavior flag. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + canonical_jump_tables, + 1, + ); } // Control Flow Guard is currently only supported by the MSVC linker on Windows. @@ -233,15 +250,75 @@ pub unsafe fn create_module<'ll>( CFGuard::Disabled => {} CFGuard::NoChecks => { // Set `cfguard=1` module flag to emit metadata only. - llvm::LLVMRustAddModuleFlag(llmod, "cfguard\0".as_ptr() as *const _, 1) + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + "cfguard\0".as_ptr() as *const _, + 1, + ) } CFGuard::Checks => { // Set `cfguard=2` module flag to emit metadata and checks. - llvm::LLVMRustAddModuleFlag(llmod, "cfguard\0".as_ptr() as *const _, 2) + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + "cfguard\0".as_ptr() as *const _, + 2, + ) } } } + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.debugging_opts.branch_protection { + if sess.target.arch != "aarch64" { + sess.err("-Zbranch-protection is only supported on aarch64"); + } else { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "branch-target-enforcement\0".as_ptr().cast(), + bti.into(), + ); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address\0".as_ptr().cast(), + pac_ret.is_some().into(), + ); + let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address-all\0".as_ptr().cast(), + pac_opts.leaf.into(), + ); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address-with-bkey\0".as_ptr().cast(), + u32::from(pac_opts.key == PAuthKey::B), + ); + } + } + + // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). + if let CFProtection::Branch | CFProtection::Full = sess.opts.debugging_opts.cf_protection { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + "cf-protection-branch\0".as_ptr().cast(), + 1, + ) + } + if let CFProtection::Return | CFProtection::Full = sess.opts.debugging_opts.cf_protection { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + "cf-protection-return\0".as_ptr().cast(), + 1, + ) + } + llmod } @@ -443,7 +520,8 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - attributes::apply_target_cpu_attr(self, llfn); + let target_cpu = attributes::target_cpu_attr(self); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } } @@ -473,12 +551,16 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - attributes::set_frame_pointer_type(self, llfn) + if let Some(attr) = attributes::frame_pointer_type_attr(self) { + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); + } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { - attributes::apply_target_cpu_attr(self, llfn); - attributes::apply_tune_cpu_attr(self, llfn); + let mut attrs = SmallVec::<[_; 2]>::new(); + attrs.push(attributes::target_cpu_attr(self)); + attrs.extend(attributes::tune_cpu_attr(self)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } fn create_used_variable(&self) { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index e0af5653753..58f391692c4 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -5,12 +5,14 @@ use crate::llvm; use llvm::coverageinfo::CounterMappingRegion; use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; -use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; -use rustc_hir::def_id::{DefId, DefIdSet}; +use rustc_data_structures::fx::FxIndexSet; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::DefIdSet; use rustc_llvm::RustString; +use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::CodeRegion; use rustc_middle::ty::TyCtxt; -use rustc_span::Symbol; use std::ffi::CString; @@ -37,7 +39,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { // LLVM 12. let version = coverageinfo::mapping_version(); if version < 4 { - tcx.sess.fatal("rustc option `-Z instrument-coverage` requires LLVM 12 or higher."); + tcx.sess.fatal("rustc option `-C instrument-coverage` requires LLVM 12 or higher."); } debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); @@ -46,7 +48,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { // functions exist. Generate synthetic functions with a (required) single counter, and add the // MIR `Coverage` code regions to the `function_coverage_map`, before calling // `ctx.take_function_coverage_map()`. - if !tcx.sess.instrument_coverage_except_unused_functions() { + if cx.codegen_unit.is_code_coverage_dead_code_cgu() { add_unused_functions(cx); } @@ -75,10 +77,18 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| { mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); }); - debug_assert!( - !coverage_mapping_buffer.is_empty(), - "Every `FunctionCoverage` should have at least one counter" - ); + + if coverage_mapping_buffer.is_empty() { + if function_coverage.is_used() { + bug!( + "A used function should have had coverage mapping data but did not: {}", + mangled_function_name + ); + } else { + debug!("unused function had no coverage mapping data: {}", mangled_function_name); + continue; + } + } function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); } @@ -264,33 +274,42 @@ fn save_function_record( /// (functions referenced by other "used" or public items). Any other functions considered unused, /// or "Unreachable", were still parsed and processed through the MIR stage, but were not /// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but -/// that flag is known to cause other errors, when combined with `-Z instrument-coverage`; and +/// that flag is known to cause other errors, when combined with `-C instrument-coverage`; and /// `-Clink-dead-code` will not generate code for unused generic functions.) /// /// We can find the unused functions (including generic functions) by the set difference of all MIR /// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query /// `codegened_and_inlined_items`). /// -/// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and -/// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`) -/// allocated to only one of those CGUs. We must NOT inject any unused functions's `CodeRegion`s -/// more than once, so we have to pick a CGUs `function_coverage_map` into which the unused -/// function will be inserted. +/// These unused functions are then codegen'd in one of the CGUs which is marked as the +/// "code coverage dead code cgu" during the partitioning process. This prevents us from generating +/// code regions for the same function more than once which can lead to linker errors regarding +/// duplicate symbols. fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { - let tcx = cx.tcx; + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); - // FIXME(#79622): Can this solution be simplified and/or improved? Are there other sources - // of compiler state data that might help (or better sources that could be exposed, but - // aren't yet)? + let tcx = cx.tcx; let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let all_def_ids: DefIdSet = tcx + let eligible_def_ids: DefIdSet = tcx .mir_keys(()) .iter() .filter_map(|local_def_id| { let def_id = local_def_id.to_def_id(); - if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { + let kind = tcx.def_kind(def_id); + // `mir_keys` will give us `DefId`s for all kinds of things, not + // just "functions", like consts, statics, etc. Filter those out. + // If `ignore_unused_generics` was specified, filter out any + // generic functions from consideration as well. + if !matches!( + kind, + DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator + ) { + return None; + } else if ignore_unused_generics + && tcx.generics_of(def_id).requires_monomorphization(tcx) + { return None; } Some(local_def_id.to_def_id()) @@ -299,79 +318,17 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { let codegenned_def_ids = tcx.codegened_and_inlined_items(()); - let mut unused_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default(); - for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) { - // Make sure the non-codegenned (unused) function has at least one MIR - // `Coverage` statement with a code region, and return its file name. - if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) { - let def_ids = - unused_def_ids_by_file.entry(*non_codegenned_file_name).or_insert_with(Vec::new); - def_ids.push(non_codegenned_def_id); - } - } + for &non_codegenned_def_id in eligible_def_ids.difference(codegenned_def_ids) { + let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); - if unused_def_ids_by_file.is_empty() { - // There are no unused functions with file names to add (in any CGU) - return; - } - - // Each `CodegenUnit` (CGU) has its own function_coverage_map, and generates a specific binary - // with its own coverage map. - // - // Each covered function `Instance` can be included in only one coverage map, produced from a - // specific function_coverage_map, from a specific CGU. - // - // Since unused functions did not generate code, they are not associated with any CGU yet. - // - // To avoid injecting the unused functions in multiple coverage maps (for multiple CGUs) - // determine which function_coverage_map has the responsibility for publishing unreachable - // coverage, based on file name: For each unused function, find the CGU that generates the - // first function (based on sorted `DefId`) from the same file. - // - // Add a new `FunctionCoverage` to the `function_coverage_map`, with unreachable code regions - // for each region in it's MIR. - - // Convert the `HashSet` of `codegenned_def_ids` to a sortable vector, and sort them. - let mut sorted_codegenned_def_ids: Vec<DefId> = codegenned_def_ids.iter().copied().collect(); - sorted_codegenned_def_ids.sort_unstable(); - - let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default(); - for &def_id in sorted_codegenned_def_ids.iter() { - if let Some(covered_file_name) = tcx.covered_file_name(def_id) { - // Only add files known to have unused functions - if unused_def_ids_by_file.contains_key(covered_file_name) { - first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id); - } + // If a function is marked `#[no_coverage]`, then skip generating a + // dead code stub for it. + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { + debug!("skipping unused fn marked #[no_coverage]: {:?}", non_codegenned_def_id); + continue; } - } - - // Get the set of def_ids with coverage regions, known by *this* CoverageContext. - let cgu_covered_def_ids: DefIdSet = match cx.coverage_context() { - Some(ctx) => ctx - .function_coverage_map - .borrow() - .keys() - .map(|&instance| instance.def.def_id()) - .collect(), - None => return, - }; - let cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file - .iter() - .filter_map( - |(&file_name, def_id)| { - if cgu_covered_def_ids.contains(def_id) { Some(file_name) } else { None } - }, - ) - .collect(); - - // For each file for which this CGU is responsible for adding unused function coverage, - // get the `def_id`s for each unused function (if any), define a synthetic function with a - // single LLVM coverage counter, and add the function's coverage `CodeRegion`s. to the - // function_coverage_map. - for covered_file_name in cgu_covered_files { - for def_id in unused_def_ids_by_file.remove(&covered_file_name).into_iter().flatten() { - cx.define_unused_fn(def_id); - } + debug!("generating unused fn: {:?}", non_codegenned_def_id); + cx.define_unused_fn(non_codegenned_def_id); } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 5f9c4189168..89fc8980037 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -10,6 +10,8 @@ use super::CrateDebugContext; use crate::abi; use crate::common::CodegenCx; +use crate::debuginfo::utils::fat_pointer_kind; +use crate::debuginfo::utils::FatPtrKind; use crate::llvm; use crate::llvm::debuginfo::{ DIArray, DICompositeType, DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, @@ -18,32 +20,33 @@ use crate::llvm::debuginfo::{ use crate::value::Value; use cstr::cstr; +use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; +use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; +use rustc_middle::bug; use rustc_middle::mir::{self, GeneratorLayout}; use rustc_middle::ty::layout::{self, IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ self, AdtKind, GeneratorSubsts, Instance, ParamEnv, Ty, TyCtxt, COMMON_VTABLE_ENTRIES, }; -use rustc_middle::{bug, span_bug}; -use rustc_query_system::ich::NodeIdHashingMode; use rustc_session::config::{self, DebugInfo}; use rustc_span::symbol::Symbol; use rustc_span::FileNameDisplayPreference; -use rustc_span::{self, SourceFile, SourceFileHash, Span}; +use rustc_span::{self, SourceFile, SourceFileHash}; use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, TagEncoding}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{Primitive, Size, VariantIdx, Variants}; +use smallvec::SmallVec; use tracing::debug; use libc::{c_longlong, c_uint}; +use std::cell::RefCell; use std::collections::hash_map::Entry; use std::fmt::{self, Write}; use std::hash::{Hash, Hasher}; @@ -83,7 +86,7 @@ const DW_ATE_signed: c_uint = 0x05; #[allow(non_upper_case_globals)] const DW_ATE_unsigned: c_uint = 0x07; #[allow(non_upper_case_globals)] -const DW_ATE_unsigned_char: c_uint = 0x08; +const DW_ATE_UTF: c_uint = 0x10; pub const UNKNOWN_LINE_NUMBER: c_uint = 0; pub const UNKNOWN_COLUMN_NUMBER: c_uint = 0; @@ -91,187 +94,127 @@ pub const UNKNOWN_COLUMN_NUMBER: c_uint = 0; pub const NO_SCOPE_METADATA: Option<&DIScope> = None; mod unique_type_id { - use super::*; - use rustc_arena::DroplessArena; - - #[derive(Copy, Hash, Eq, PartialEq, Clone)] - pub(super) struct UniqueTypeId(u32); + use rustc_data_structures::{ + fingerprint::Fingerprint, + stable_hasher::{HashStable, NodeIdHashingMode, StableHasher}, + }; + use rustc_middle::ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}; + use rustc_target::abi::VariantIdx; + + // This type cannot be constructed outside of this module because + // it has a private field. We make use of this in order to prevent + // `UniqueTypeId` from being constructed directly, without asserting + // the preconditions. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] + pub struct HiddenZst { + _inaccessible: (), + } - // The `&'static str`s in this type actually point into the arena. - // - // The `FxHashMap`+`Vec` pair could be replaced by `FxIndexSet`, but #75278 - // found that to regress performance up to 2% in some cases. This might be - // revisited after further improvements to `indexmap`. - #[derive(Default)] - pub(super) struct TypeIdInterner { - arena: DroplessArena, - names: FxHashMap<&'static str, UniqueTypeId>, - strings: Vec<&'static str>, + /// A unique identifier for anything that we create a debuginfo node for. + /// The types it contains are expected to already be normalized (which + /// is debug_asserted in the constructors). + /// + /// Note that there are some things that only show up in debuginfo, like + /// the separate type descriptions for each enum variant. These get an ID + /// too because they have their own debuginfo node in LLVM IR. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] + pub(super) enum UniqueTypeId<'tcx> { + /// The ID of a regular type as it shows up at the language level. + Ty(Ty<'tcx>, HiddenZst), + /// The ID for the artificial struct type describing a single enum variant. + Variant(Ty<'tcx>, VariantIdx, HiddenZst), + /// The ID for the single DW_TAG_variant_part nested inside the top-level + /// DW_TAG_structure_type that describes enums and generators. + VariantPart(Ty<'tcx>, HiddenZst), + /// The ID of the artificial type we create for VTables. + VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, HiddenZst), } - impl TypeIdInterner { - #[inline] - pub(super) fn intern(&mut self, string: &str) -> UniqueTypeId { - if let Some(&name) = self.names.get(string) { - return name; - } + impl<'tcx> UniqueTypeId<'tcx> { + pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { + debug_assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); + UniqueTypeId::Ty(t, HiddenZst { _inaccessible: () }) + } + + pub fn for_enum_variant( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty) + ); + UniqueTypeId::Variant(enum_ty, variant_idx, HiddenZst { _inaccessible: () }) + } - let name = UniqueTypeId(self.strings.len() as u32); - - // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be - // UTF-8. - let string: &str = - unsafe { std::str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) }; - // It is safe to extend the arena allocation to `'static` because we only access - // these while the arena is still alive. - let string: &'static str = unsafe { &*(string as *const str) }; - self.strings.push(string); - self.names.insert(string, name); - name + pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { + debug_assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty) + ); + UniqueTypeId::VariantPart(enum_ty, HiddenZst { _inaccessible: () }) } - // Get the symbol as a string. `Symbol::as_str()` should be used in - // preference to this function. - pub(super) fn get(&self, symbol: UniqueTypeId) -> &str { - self.strings[symbol.0 as usize] + pub fn for_vtable_ty( + tcx: TyCtxt<'tcx>, + self_type: Ty<'tcx>, + implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, + ) -> Self { + debug_assert_eq!( + self_type, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type) + ); + debug_assert_eq!( + implemented_trait, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) + ); + UniqueTypeId::VTableTy(self_type, implemented_trait, HiddenZst { _inaccessible: () }) + } + + /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` + /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. + /// + /// Right now this takes the form of a hex-encoded opaque hash value. + pub fn generate_unique_id_string(&self, tcx: TyCtxt<'tcx>) -> String { + let mut hasher = StableHasher::new(); + let mut hcx = tcx.create_stable_hashing_context(); + hcx.while_hashing_spans(false, |hcx| { + hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| { + self.hash_stable(hcx, &mut hasher); + }); + }); + hasher.finish::<Fingerprint>().to_hex() } } } use unique_type_id::*; -/// The `TypeMap` is where the `CrateDebugContext` holds the type metadata nodes -/// created so far. The metadata nodes are indexed by `UniqueTypeId`, and, for -/// faster lookup, also by `Ty`. The `TypeMap` is responsible for creating -/// `UniqueTypeId`s. +/// The `TypeMap` is where the debug context holds the type metadata nodes +/// created so far. The metadata nodes are indexed by `UniqueTypeId`. #[derive(Default)] pub struct TypeMap<'ll, 'tcx> { - /// The `UniqueTypeId`s created so far. - unique_id_interner: TypeIdInterner, - /// A map from `UniqueTypeId` to debuginfo metadata for that type. This is a 1:1 mapping. - unique_id_to_metadata: FxHashMap<UniqueTypeId, &'ll DIType>, - /// A map from types to debuginfo metadata. This is an N:1 mapping. - type_to_metadata: FxHashMap<Ty<'tcx>, &'ll DIType>, - /// A map from types to `UniqueTypeId`. This is an N:1 mapping. - type_to_unique_id: FxHashMap<Ty<'tcx>, UniqueTypeId>, + unique_id_to_metadata: RefCell<FxHashMap<UniqueTypeId<'tcx>, &'ll DIType>>, } impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { - /// Adds a Ty to metadata mapping to the TypeMap. The method will fail if - /// the mapping already exists. - fn register_type_with_metadata(&mut self, type_: Ty<'tcx>, metadata: &'ll DIType) { - if self.type_to_metadata.insert(type_, metadata).is_some() { - bug!("type metadata for `Ty` '{}' is already in the `TypeMap`!", type_); - } - } - - /// Removes a `Ty`-to-metadata mapping. - /// This is useful when computing the metadata for a potentially - /// recursive type (e.g., a function pointer of the form: - /// - /// fn foo() -> impl Copy { foo } - /// - /// This kind of type cannot be properly represented - /// via LLVM debuginfo. As a workaround, - /// we register a temporary Ty to metadata mapping - /// for the function before we compute its actual metadata. - /// If the metadata computation ends up recursing back to the - /// original function, it will use the temporary mapping - /// for the inner self-reference, preventing us from - /// recursing forever. - /// - /// This function is used to remove the temporary metadata - /// mapping after we've computed the actual metadata. - fn remove_type(&mut self, type_: Ty<'tcx>) { - if self.type_to_metadata.remove(type_).is_none() { - bug!("type metadata `Ty` '{}' is not in the `TypeMap`!", type_); - } - } - /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will /// fail if the mapping already exists. fn register_unique_id_with_metadata( - &mut self, - unique_type_id: UniqueTypeId, + &self, + unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType, ) { - if self.unique_id_to_metadata.insert(unique_type_id, metadata).is_some() { - bug!( - "type metadata for unique ID '{}' is already in the `TypeMap`!", - self.get_unique_type_id_as_string(unique_type_id) - ); - } - } - - fn find_metadata_for_type(&self, type_: Ty<'tcx>) -> Option<&'ll DIType> { - self.type_to_metadata.get(&type_).cloned() - } - - fn find_metadata_for_unique_id(&self, unique_type_id: UniqueTypeId) -> Option<&'ll DIType> { - self.unique_id_to_metadata.get(&unique_type_id).cloned() - } - - /// Gets the string representation of a `UniqueTypeId`. This method will fail if - /// the ID is unknown. - fn get_unique_type_id_as_string(&self, unique_type_id: UniqueTypeId) -> &str { - self.unique_id_interner.get(unique_type_id) - } - - /// Gets the `UniqueTypeId` for the given type. If the `UniqueTypeId` for the given - /// type has been requested before, this is just a table lookup. Otherwise, an - /// ID will be generated and stored for later lookup. - fn get_unique_type_id_of_type<'a>( - &mut self, - cx: &CodegenCx<'a, 'tcx>, - type_: Ty<'tcx>, - ) -> UniqueTypeId { - // Let's see if we already have something in the cache. - if let Some(unique_type_id) = self.type_to_unique_id.get(&type_).cloned() { - return unique_type_id; + if self.unique_id_to_metadata.borrow_mut().insert(unique_type_id, metadata).is_some() { + bug!("type metadata for unique ID '{:?}' is already in the `TypeMap`!", unique_type_id); } - // If not, generate one. - - // The hasher we are using to generate the UniqueTypeId. We want - // something that provides more than the 64 bits of the DefaultHasher. - let mut hasher = StableHasher::new(); - let mut hcx = cx.tcx.create_stable_hashing_context(); - let type_ = cx.tcx.erase_regions(type_); - hcx.while_hashing_spans(false, |hcx| { - hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| { - type_.hash_stable(hcx, &mut hasher); - }); - }); - let unique_type_id = hasher.finish::<Fingerprint>().to_hex(); - - let key = self.unique_id_interner.intern(&unique_type_id); - self.type_to_unique_id.insert(type_, key); - - key } - /// Gets the `UniqueTypeId` for an enum variant. Enum variants are not really - /// types of their own, so they need special handling. We still need a - /// `UniqueTypeId` for them, since to debuginfo they *are* real types. - fn get_unique_type_id_of_enum_variant<'a>( - &mut self, - cx: &CodegenCx<'a, 'tcx>, - enum_type: Ty<'tcx>, - variant_name: &str, - ) -> UniqueTypeId { - let enum_type_id = self.get_unique_type_id_of_type(cx, enum_type); - let enum_variant_type_id = - format!("{}::{}", self.get_unique_type_id_as_string(enum_type_id), variant_name); - let interner_key = self.unique_id_interner.intern(&enum_variant_type_id); - interner_key - } - - /// Gets the unique type ID string for an enum variant part. - /// Variant parts are not types and shouldn't really have their own ID, - /// but it makes `set_members_of_composite_type()` simpler. - fn get_unique_type_id_str_of_enum_variant_part( - &mut self, - enum_type_id: UniqueTypeId, - ) -> String { - format!("{}_variant_part", self.get_unique_type_id_as_string(enum_type_id)) + fn find_metadata_for_unique_id( + &self, + unique_type_id: UniqueTypeId<'tcx>, + ) -> Option<&'ll DIType> { + self.unique_id_to_metadata.borrow().get(&unique_type_id).cloned() } } @@ -283,7 +226,7 @@ impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { enum RecursiveTypeDescription<'ll, 'tcx> { UnfinishedMetadata { unfinished_type: Ty<'tcx>, - unique_type_id: UniqueTypeId, + unique_type_id: UniqueTypeId<'tcx>, metadata_stub: &'ll DICompositeType, member_holding_stub: &'ll DICompositeType, member_description_factory: MemberDescriptionFactory<'ll, 'tcx>, @@ -294,15 +237,13 @@ enum RecursiveTypeDescription<'ll, 'tcx> { fn create_and_register_recursive_type_forward_declaration<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unfinished_type: Ty<'tcx>, - unique_type_id: UniqueTypeId, + unique_type_id: UniqueTypeId<'tcx>, metadata_stub: &'ll DICompositeType, member_holding_stub: &'ll DICompositeType, member_description_factory: MemberDescriptionFactory<'ll, 'tcx>, ) -> RecursiveTypeDescription<'ll, 'tcx> { // Insert the stub into the `TypeMap` in order to allow for recursive references. - let mut type_map = debug_context(cx).type_map.borrow_mut(); - type_map.register_unique_id_with_metadata(unique_type_id, metadata_stub); - type_map.register_type_with_metadata(unfinished_type, metadata_stub); + debug_context(cx).type_map.register_unique_id_with_metadata(unique_type_id, metadata_stub); UnfinishedMetadata { unfinished_type, @@ -334,9 +275,10 @@ impl<'ll, 'tcx> RecursiveTypeDescription<'ll, 'tcx> { // `create_and_register_recursive_type_forward_declaration()` // function. { - let type_map = debug_context(cx).type_map.borrow(); - if type_map.find_metadata_for_unique_id(unique_type_id).is_none() - || type_map.find_metadata_for_type(unfinished_type).is_none() + if debug_context(cx) + .type_map + .find_metadata_for_unique_id(unique_type_id) + .is_none() { bug!( "Forward declaration of potentially recursive type \ @@ -348,14 +290,15 @@ impl<'ll, 'tcx> RecursiveTypeDescription<'ll, 'tcx> { // ... then create the member descriptions ... let member_descriptions = member_description_factory.create_member_descriptions(cx); + let type_params = compute_type_parameters(cx, unfinished_type); // ... and attach them to the stub to complete it. set_members_of_composite_type( cx, - unfinished_type, member_holding_stub, member_descriptions, None, + type_params, ); MetadataCreationResult::new(metadata_stub, true) } @@ -368,30 +311,31 @@ impl<'ll, 'tcx> RecursiveTypeDescription<'ll, 'tcx> { macro_rules! return_if_metadata_created_in_meantime { ($cx: expr, $unique_type_id: expr) => { if let Some(metadata) = - debug_context($cx).type_map.borrow().find_metadata_for_unique_id($unique_type_id) + debug_context($cx).type_map.find_metadata_for_unique_id($unique_type_id) { return MetadataCreationResult::new(metadata, true); } }; } -fn fixed_vec_metadata<'ll, 'tcx>( +/// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`). +/// For slices (that is, "arrays" of unknown size) use [slice_type_metadata]. +fn fixed_size_array_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId, - array_or_slice_type: Ty<'tcx>, - element_type: Ty<'tcx>, - span: Span, + unique_type_id: UniqueTypeId<'tcx>, + array_type: Ty<'tcx>, ) -> MetadataCreationResult<'ll> { - let element_type_metadata = type_metadata(cx, element_type, span); + let ty::Array(element_type, len) = array_type.kind() else { + bug!("fixed_size_array_metadata() called with non-ty::Array type `{:?}`", array_type) + }; + + let element_type_metadata = type_metadata(cx, *element_type); return_if_metadata_created_in_meantime!(cx, unique_type_id); - let (size, align) = cx.size_and_align_of(array_or_slice_type); + let (size, align) = cx.size_and_align_of(array_type); - let upper_bound = match array_or_slice_type.kind() { - ty::Array(_, len) => len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong, - _ => -1, - }; + let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; let subrange = unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; @@ -410,243 +354,250 @@ fn fixed_vec_metadata<'ll, 'tcx>( MetadataCreationResult::new(metadata, false) } -fn vec_slice_metadata<'ll, 'tcx>( +/// Creates debuginfo for built-in pointer-like things: +/// +/// - ty::Ref +/// - ty::RawPtr +/// - ty::Adt in the case it's Box +/// +/// At some point we might want to remove the special handling of Box +/// and treat it the same as other smart pointers (like Rc, Arc, ...). +fn pointer_or_reference_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - slice_ptr_type: Ty<'tcx>, - element_type: Ty<'tcx>, - unique_type_id: UniqueTypeId, - span: Span, + ptr_type: Ty<'tcx>, + pointee_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, ) -> MetadataCreationResult<'ll> { - let data_ptr_type = cx.tcx.mk_imm_ptr(element_type); - - let data_ptr_metadata = type_metadata(cx, data_ptr_type, span); + let pointee_type_metadata = type_metadata(cx, pointee_type); return_if_metadata_created_in_meantime!(cx, unique_type_id); - let slice_type_name = compute_debuginfo_type_name(cx.tcx, slice_ptr_type, true); - - let (pointer_size, pointer_align) = cx.size_and_align_of(data_ptr_type); - let (usize_size, usize_align) = cx.size_and_align_of(cx.tcx.types.usize); - - let member_descriptions = vec![ - MemberDescription { - name: "data_ptr".to_owned(), - type_metadata: data_ptr_metadata, - offset: Size::ZERO, - size: pointer_size, - align: pointer_align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - }, - MemberDescription { - name: "length".to_owned(), - type_metadata: type_metadata(cx, cx.tcx.types.usize, span), - offset: pointer_size, - size: usize_size, - align: usize_align, - flags: DIFlags::FlagZero, - discriminant: None, - source_info: None, - }, - ]; + let (thin_pointer_size, thin_pointer_align) = + cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); + let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); + + let pointer_type_metadata = match fat_pointer_kind(cx, pointee_type) { + None => { + // This is a thin pointer. Create a regular pointer type and give it the correct name. + debug_assert_eq!( + (thin_pointer_size, thin_pointer_align), + cx.size_and_align_of(ptr_type), + "ptr_type={}, pointee_type={}", + ptr_type, + pointee_type, + ); - let file_metadata = unknown_file_metadata(cx); + unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_metadata, + thin_pointer_size.bits(), + thin_pointer_align.bits() as u32, + 0, // Ignore DWARF address space. + ptr_type_debuginfo_name.as_ptr().cast(), + ptr_type_debuginfo_name.len(), + ) + } + } + Some(fat_pointer_kind) => { + let layout = cx.layout_of(ptr_type); - let metadata = composite_type_metadata( - cx, - slice_ptr_type, - &slice_type_name, - unique_type_id, - member_descriptions, - NO_SCOPE_METADATA, - file_metadata, - span, - ); - MetadataCreationResult::new(metadata, false) + let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); + let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); + + let (addr_field_name, extra_field_name) = match fat_pointer_kind { + FatPtrKind::Dyn => ("pointer", "vtable"), + FatPtrKind::Slice => ("data_ptr", "length"), + }; + + debug_assert_eq!(abi::FAT_PTR_ADDR, 0); + debug_assert_eq!(abi::FAT_PTR_EXTRA, 1); + + // The data pointer type is a regular, thin pointer, regardless of whether this is a slice + // or a trait object. + let data_ptr_type_metadata = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_metadata, + addr_field.size.bits(), + addr_field.align.abi.bits() as u32, + 0, // Ignore DWARF address space. + std::ptr::null(), + 0, + ) + }; + + let member_descriptions = vec![ + MemberDescription { + name: addr_field_name.into(), + type_metadata: data_ptr_type_metadata, + offset: layout.fields.offset(abi::FAT_PTR_ADDR), + size: addr_field.size, + align: addr_field.align.abi, + flags: DIFlags::FlagZero, + discriminant: None, + source_info: None, + }, + MemberDescription { + name: extra_field_name.into(), + type_metadata: type_metadata(cx, extra_field.ty), + offset: layout.fields.offset(abi::FAT_PTR_EXTRA), + size: extra_field.size, + align: extra_field.align.abi, + flags: DIFlags::FlagZero, + discriminant: None, + source_info: None, + }, + ]; + + composite_type_metadata( + cx, + ptr_type, + &ptr_type_debuginfo_name, + unique_type_id, + member_descriptions, + NO_SCOPE_METADATA, + ) + } + }; + + MetadataCreationResult { metadata: pointer_type_metadata, already_stored_in_typemap: false } } fn subroutine_type_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - unique_type_id: UniqueTypeId, - signature: ty::PolyFnSig<'tcx>, - span: Span, + unique_type_id: UniqueTypeId<'tcx>, ) -> MetadataCreationResult<'ll> { - let signature = - cx.tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), signature); + // It's possible to create a self-referential + // type in Rust by using 'impl trait': + // + // fn foo() -> impl Copy { foo } + // + // Unfortunately LLVM's API does not allow us to create recursive subroutine types. + // In order to work around that restriction we place a marker type in the type map, + // before creating the actual type. If the actual type is recursive, it will hit the + // marker type. So we end up with a type that looks like + // + // fn foo() -> <recursive_type> + // + // Once that is created, we replace the marker in the typemap with the actual type. + debug_context(cx) + .type_map + .unique_id_to_metadata + .borrow_mut() + .insert(unique_type_id, recursion_marker_type(cx)); + + let UniqueTypeId::Ty(fn_ty, _) = unique_type_id else { + bug!("subroutine_type_metadata() called with unexpected input type: {:?}", unique_type_id) + }; + + let signature = cx + .tcx + .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), fn_ty.fn_sig(cx.tcx)); - let signature_metadata: Vec<_> = iter::once( + let signature_metadata: SmallVec<[_; 32]> = iter::once( // return type match signature.output().kind() { - ty::Tuple(tys) if tys.is_empty() => None, - _ => Some(type_metadata(cx, signature.output(), span)), + ty::Tuple(tys) if tys.is_empty() => { + // this is a "void" function + None + } + _ => Some(type_metadata(cx, signature.output())), }, ) .chain( // regular arguments - signature.inputs().iter().map(|argument_type| Some(type_metadata(cx, argument_type, span))), + signature.inputs().iter().map(|&argument_type| Some(type_metadata(cx, argument_type))), ) .collect(); - return_if_metadata_created_in_meantime!(cx, unique_type_id); - - MetadataCreationResult::new( - unsafe { - llvm::LLVMRustDIBuilderCreateSubroutineType( - DIB(cx), - create_DIArray(DIB(cx), &signature_metadata[..]), - ) - }, - false, - ) -} - -// FIXME(1563): This is all a bit of a hack because 'trait pointer' is an ill- -// defined concept. For the case of an actual trait pointer (i.e., `Box<Trait>`, -// `&Trait`), `trait_object_type` should be the whole thing (e.g, `Box<Trait>`) and -// `trait_type` should be the actual trait (e.g., `Trait`). Where the trait is part -// of a DST struct, there is no `trait_object_type` and the results of this -// function will be a little bit weird. -fn trait_pointer_metadata<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - trait_type: Ty<'tcx>, - trait_object_type: Option<Ty<'tcx>>, - unique_type_id: UniqueTypeId, -) -> &'ll DIType { - // The implementation provided here is a stub. It makes sure that the trait - // type is assigned the correct name, size, namespace, and source location. - // However, it does not describe the trait's methods. - - let (containing_scope, trait_type_name) = match trait_object_type { - Some(trait_object_type) => match trait_object_type.kind() { - ty::Adt(def, _) => ( - Some(get_namespace_for_item(cx, def.did)), - compute_debuginfo_type_name(cx.tcx, trait_object_type, false), - ), - ty::RawPtr(_) | ty::Ref(..) => { - (NO_SCOPE_METADATA, compute_debuginfo_type_name(cx.tcx, trait_object_type, true)) - } - _ => { - bug!( - "debuginfo: unexpected trait-object type in \ - trait_pointer_metadata(): {:?}", - trait_object_type - ); - } - }, + debug_context(cx).type_map.unique_id_to_metadata.borrow_mut().remove(&unique_type_id); - // No object type, use the trait type directly (no scope here since the type - // will be wrapped in the dyn$ synthetic type). - None => (NO_SCOPE_METADATA, compute_debuginfo_type_name(cx.tcx, trait_type, true)), + let fn_metadata = unsafe { + llvm::LLVMRustDIBuilderCreateSubroutineType( + DIB(cx), + create_DIArray(DIB(cx), &signature_metadata[..]), + ) }; - let file_metadata = unknown_file_metadata(cx); - - let layout = cx.layout_of(cx.tcx.mk_mut_ptr(trait_type)); - - assert_eq!(abi::FAT_PTR_ADDR, 0); - assert_eq!(abi::FAT_PTR_EXTRA, 1); - - let data_ptr_field = layout.field(cx, 0); - let vtable_field = layout.field(cx, 1); - let member_descriptions = vec![ - MemberDescription { - name: "pointer".to_owned(), - type_metadata: type_metadata( - cx, - cx.tcx.mk_mut_ptr(cx.tcx.types.u8), - rustc_span::DUMMY_SP, - ), - offset: layout.fields.offset(0), - size: data_ptr_field.size, - align: data_ptr_field.align.abi, - flags: DIFlags::FlagArtificial, - discriminant: None, - source_info: None, - }, - MemberDescription { - name: "vtable".to_owned(), - type_metadata: type_metadata(cx, vtable_field.ty, rustc_span::DUMMY_SP), - offset: layout.fields.offset(1), - size: vtable_field.size, - align: vtable_field.align.abi, - flags: DIFlags::FlagArtificial, - discriminant: None, - source_info: None, - }, - ]; + // This is actually a function pointer, so wrap it in pointer DI. + let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); + let metadata = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + fn_metadata, + cx.tcx.data_layout.pointer_size.bits(), + cx.tcx.data_layout.pointer_align.abi.bits() as u32, + 0, // Ignore DWARF address space. + name.as_ptr().cast(), + name.len(), + ) + }; - composite_type_metadata( - cx, - trait_object_type.unwrap_or(trait_type), - &trait_type_name, - unique_type_id, - member_descriptions, - containing_scope, - file_metadata, - rustc_span::DUMMY_SP, - ) + MetadataCreationResult::new(metadata, false) } -pub fn type_metadata<'ll, 'tcx>( +/// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs +/// we with the correct type name (e.g. "dyn SomeTrait<Foo, Item=u32> + Sync"). +fn dyn_type_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - t: Ty<'tcx>, - usage_site_span: Span, + dyn_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, ) -> &'ll DIType { - // Get the unique type ID of this type. - let unique_type_id = { - let mut type_map = debug_context(cx).type_map.borrow_mut(); - // First, try to find the type in `TypeMap`. If we have seen it before, we - // can exit early here. - match type_map.find_metadata_for_type(t) { - Some(metadata) => { - return metadata; - } - None => { - // The Ty is not in the `TypeMap` but maybe we have already seen - // an equivalent type (e.g., only differing in region arguments). - // In order to find out, generate the unique type ID and look - // that up. - let unique_type_id = type_map.get_unique_type_id_of_type(cx, t); - match type_map.find_metadata_for_unique_id(unique_type_id) { - Some(metadata) => { - // There is already an equivalent type in the TypeMap. - // Register this Ty as an alias in the cache and - // return the cached metadata. - type_map.register_type_with_metadata(t, metadata); - return metadata; - } - None => { - // There really is no type metadata for this type, so - // proceed by creating it. - unique_type_id - } - } - } + if let ty::Dynamic(..) = dyn_type.kind() { + let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true); + composite_type_metadata(cx, dyn_type, &type_name, unique_type_id, vec![], NO_SCOPE_METADATA) + } else { + bug!("Only ty::Dynamic is valid for dyn_type_metadata(). Found {:?} instead.", dyn_type) + } +} + +/// Create debuginfo for `[T]` and `str`. These are unsized. +/// +/// NOTE: We currently emit just emit the debuginfo for the element type here +/// (i.e. `T` for slices and `u8` for `str`), so that we end up with +/// `*const T` for the `data_ptr` field of the corresponding fat-pointer +/// debuginfo of `&[T]`. +/// +/// It would be preferable and more accurate if we emitted a DIArray of T +/// without an upper bound instead. That is, LLVM already supports emitting +/// debuginfo of arrays of unknown size. But GDB currently seems to end up +/// in an infinite loop when confronted with such a type. +/// +/// As a side effect of the current encoding every instance of a type like +/// `struct Foo { unsized_field: [u8] }` will look like +/// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the +/// slice is zero, then accessing `unsized_field` in the debugger would +/// result in an out-of-bounds access. +fn slice_type_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + slice_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> MetadataCreationResult<'ll> { + let element_type = match slice_type.kind() { + ty::Slice(element_type) => *element_type, + ty::Str => cx.tcx.types.u8, + _ => { + bug!( + "Only ty::Slice is valid for slice_type_metadata(). Found {:?} instead.", + slice_type + ) } }; - debug!("type_metadata: {:?}", t); + let element_type_metadata = type_metadata(cx, element_type); + return_if_metadata_created_in_meantime!(cx, unique_type_id); + MetadataCreationResult { metadata: element_type_metadata, already_stored_in_typemap: false } +} - let ptr_metadata = |ty: Ty<'tcx>| match *ty.kind() { - ty::Slice(typ) => Ok(vec_slice_metadata(cx, t, typ, unique_type_id, usage_site_span)), - ty::Str => Ok(vec_slice_metadata(cx, t, cx.tcx.types.u8, unique_type_id, usage_site_span)), - ty::Dynamic(..) => Ok(MetadataCreationResult::new( - trait_pointer_metadata(cx, ty, Some(t), unique_type_id), - false, - )), - _ => { - let pointee_metadata = type_metadata(cx, ty, usage_site_span); +pub fn type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { + let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); - if let Some(metadata) = - debug_context(cx).type_map.borrow().find_metadata_for_unique_id(unique_type_id) - { - return Err(metadata); - } + if let Some(metadata) = debug_context(cx).type_map.find_metadata_for_unique_id(unique_type_id) { + return metadata; + } - Ok(MetadataCreationResult::new(pointer_type_metadata(cx, t, pointee_metadata), false)) - } - }; + debug!("type_metadata: {:?}", t); let MetadataCreationResult { metadata, already_stored_in_typemap } = match *t.kind() { ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { @@ -655,80 +606,26 @@ pub fn type_metadata<'ll, 'tcx>( ty::Tuple(elements) if elements.is_empty() => { MetadataCreationResult::new(basic_type_metadata(cx, t), false) } - ty::Array(typ, _) | ty::Slice(typ) => { - fixed_vec_metadata(cx, unique_type_id, t, typ, usage_site_span) - } - ty::Str => fixed_vec_metadata(cx, unique_type_id, t, cx.tcx.types.i8, usage_site_span), + ty::Array(..) => fixed_size_array_metadata(cx, unique_type_id, t), + ty::Slice(_) | ty::Str => slice_type_metadata(cx, t, unique_type_id), ty::Dynamic(..) => { - MetadataCreationResult::new(trait_pointer_metadata(cx, t, None, unique_type_id), false) + MetadataCreationResult::new(dyn_type_metadata(cx, t, unique_type_id), false) } ty::Foreign(..) => { MetadataCreationResult::new(foreign_type_metadata(cx, t, unique_type_id), false) } - ty::RawPtr(ty::TypeAndMut { ty, .. }) | ty::Ref(_, ty, _) => match ptr_metadata(ty) { - Ok(res) => res, - Err(metadata) => return metadata, - }, - ty::Adt(def, _) if def.is_box() => match ptr_metadata(t.boxed_ty()) { - Ok(res) => res, - Err(metadata) => return metadata, - }, - ty::FnDef(..) | ty::FnPtr(_) => { - if let Some(metadata) = - debug_context(cx).type_map.borrow().find_metadata_for_unique_id(unique_type_id) - { - return metadata; - } - - // It's possible to create a self-referential - // type in Rust by using 'impl trait': - // - // fn foo() -> impl Copy { foo } - // - // See `TypeMap::remove_type` for more detals - // about the workaround. - - let temp_type = { - unsafe { - // The choice of type here is pretty arbitrary - - // anything reading the debuginfo for a recursive - // type is going to see *something* weird - the only - // question is what exactly it will see. - let name = "<recur_type>"; - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_ptr().cast(), - name.len(), - cx.size_of(t).bits(), - DW_ATE_unsigned, - ) - } - }; - - let type_map = &debug_context(cx).type_map; - type_map.borrow_mut().register_type_with_metadata(t, temp_type); - - let fn_metadata = - subroutine_type_metadata(cx, unique_type_id, t.fn_sig(cx.tcx), usage_site_span) - .metadata; - - type_map.borrow_mut().remove_type(t); - - // This is actually a function pointer, so wrap it in pointer DI. - MetadataCreationResult::new(pointer_type_metadata(cx, t, fn_metadata), false) + ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { + pointer_or_reference_metadata(cx, t, pointee_type, unique_type_id) } + ty::Adt(def, _) if def.is_box() => { + pointer_or_reference_metadata(cx, t, t.boxed_ty(), unique_type_id) + } + ty::FnDef(..) | ty::FnPtr(_) => subroutine_type_metadata(cx, unique_type_id), ty::Closure(def_id, substs) => { let upvar_tys: Vec<_> = substs.as_closure().upvar_tys().collect(); let containing_scope = get_namespace_for_item(cx, def_id); - prepare_tuple_metadata( - cx, - t, - &upvar_tys, - unique_type_id, - usage_site_span, - Some(containing_scope), - ) - .finalize(cx) + prepare_tuple_metadata(cx, t, &upvar_tys, unique_type_id, Some(containing_scope)) + .finalize(cx) } ty::Generator(def_id, substs, _) => { let upvar_tys: Vec<_> = substs @@ -736,25 +633,17 @@ pub fn type_metadata<'ll, 'tcx>( .prefix_tys() .map(|t| cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) .collect(); - prepare_enum_metadata(cx, t, def_id, unique_type_id, usage_site_span, upvar_tys) - .finalize(cx) + prepare_enum_metadata(cx, t, def_id, unique_type_id, upvar_tys).finalize(cx) } ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => { - prepare_struct_metadata(cx, t, unique_type_id, usage_site_span).finalize(cx) - } - AdtKind::Union => { - prepare_union_metadata(cx, t, unique_type_id, usage_site_span).finalize(cx) - } + AdtKind::Struct => prepare_struct_metadata(cx, t, unique_type_id).finalize(cx), + AdtKind::Union => prepare_union_metadata(cx, t, unique_type_id).finalize(cx), AdtKind::Enum => { - prepare_enum_metadata(cx, t, def.did, unique_type_id, usage_site_span, vec![]) - .finalize(cx) + prepare_enum_metadata(cx, t, def.did, unique_type_id, vec![]).finalize(cx) } }, - ty::Tuple(elements) => { - let tys: Vec<_> = elements.iter().map(|k| k.expect_ty()).collect(); - prepare_tuple_metadata(cx, t, &tys, unique_type_id, usage_site_span, NO_SCOPE_METADATA) - .finalize(cx) + ty::Tuple(tys) => { + prepare_tuple_metadata(cx, t, tys, unique_type_id, NO_SCOPE_METADATA).finalize(cx) } // Type parameters from polymorphized functions. ty::Param(_) => MetadataCreationResult::new(param_type_metadata(cx, t), false), @@ -762,52 +651,56 @@ pub fn type_metadata<'ll, 'tcx>( }; { - let mut type_map = debug_context(cx).type_map.borrow_mut(); - if already_stored_in_typemap { - // Also make sure that we already have a `TypeMap` entry for the unique type ID. - let metadata_for_uid = match type_map.find_metadata_for_unique_id(unique_type_id) { - Some(metadata) => metadata, - None => { - span_bug!( - usage_site_span, - "expected type metadata for unique \ - type ID '{}' to already be in \ + // Make sure that we really do have a `TypeMap` entry for the unique type ID. + let metadata_for_uid = + match debug_context(cx).type_map.find_metadata_for_unique_id(unique_type_id) { + Some(metadata) => metadata, + None => { + bug!( + "expected type metadata for unique \ + type ID '{:?}' to already be in \ the `debuginfo::TypeMap` but it \ - was not. (Ty = {})", - type_map.get_unique_type_id_as_string(unique_type_id), - t - ); - } - }; - - match type_map.find_metadata_for_type(t) { - Some(metadata) => { - if metadata != metadata_for_uid { - span_bug!( - usage_site_span, - "mismatch between `Ty` and \ - `UniqueTypeId` maps in \ - `debuginfo::TypeMap`. \ - UniqueTypeId={}, Ty={}", - type_map.get_unique_type_id_as_string(unique_type_id), - t + was not.", + unique_type_id, ); } - } - None => { - type_map.register_type_with_metadata(t, metadata); - } - } + }; + + debug_assert_eq!(metadata_for_uid as *const _, metadata as *const _); } else { - type_map.register_type_with_metadata(t, metadata); - type_map.register_unique_id_with_metadata(unique_type_id, metadata); + debug_context(cx).type_map.register_unique_id_with_metadata(unique_type_id, metadata); } } metadata } +fn recursion_marker_type<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType { + *debug_context(cx).recursion_marker_type.get_or_init(move || { + unsafe { + // The choice of type here is pretty arbitrary - + // anything reading the debuginfo for a recursive + // type is going to see *something* weird - the only + // question is what exactly it will see. + // + // FIXME: the name `<recur_type>` does not fit the naming scheme + // of other types. + // + // FIXME: it might make sense to use an actual pointer type here + // so that debuggers can show the address. + let name = "<recur_type>"; + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + cx.tcx.data_layout.pointer_size.bits(), + DW_ATE_unsigned, + ) + } + }) +} + fn hex_encode(data: &[u8]) -> String { let mut hex_string = String::with_capacity(data.len() * 2); for byte in data.iter() { @@ -933,16 +826,16 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l // When targeting MSVC, emit MSVC style type names for compatibility with // .natvis visualizers (and perhaps other existing native debuggers?) - let msvc_like_names = cx.tcx.sess.target.is_like_msvc; + let cpp_like_debuginfo = cpp_like_debuginfo(cx.tcx); let (name, encoding) = match t.kind() { ty::Never => ("!", DW_ATE_unsigned), ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned), ty::Bool => ("bool", DW_ATE_boolean), - ty::Char => ("char", DW_ATE_unsigned_char), - ty::Int(int_ty) if msvc_like_names => (int_ty.msvc_basic_name(), DW_ATE_signed), - ty::Uint(uint_ty) if msvc_like_names => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), - ty::Float(float_ty) if msvc_like_names => (float_ty.msvc_basic_name(), DW_ATE_float), + ty::Char => ("char", DW_ATE_UTF), + ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), + ty::Uint(uint_ty) if cpp_like_debuginfo => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), + ty::Float(float_ty) if cpp_like_debuginfo => (float_ty.msvc_basic_name(), DW_ATE_float), ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), @@ -959,7 +852,7 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l ) }; - if !msvc_like_names { + if !cpp_like_debuginfo { return ty_metadata; } @@ -988,32 +881,22 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l fn foreign_type_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>, - unique_type_id: UniqueTypeId, + unique_type_id: UniqueTypeId<'tcx>, ) -> &'ll DIType { debug!("foreign_type_metadata: {:?}", t); let name = compute_debuginfo_type_name(cx.tcx, t, false); - create_struct_stub(cx, t, &name, unique_type_id, NO_SCOPE_METADATA, DIFlags::FlagZero) -} - -fn pointer_type_metadata<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - pointer_type: Ty<'tcx>, - pointee_type_metadata: &'ll DIType, -) -> &'ll DIType { - let (pointer_size, pointer_align) = cx.size_and_align_of(pointer_type); - let name = compute_debuginfo_type_name(cx.tcx, pointer_type, false); - unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_metadata, - pointer_size.bits(), - pointer_align.bits() as u32, - 0, // Ignore DWARF address space. - name.as_ptr().cast(), - name.len(), - ) - } + let (size, align) = cx.size_and_align_of(t); + create_struct_stub( + cx, + size, + align, + &name, + unique_type_id, + NO_SCOPE_METADATA, + DIFlags::FlagZero, + None, + ) } fn param_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { @@ -1072,7 +955,11 @@ pub fn compile_unit_metadata<'ll, 'tcx>( let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { output_filenames - .split_dwarf_path(tcx.sess.split_debuginfo(), Some(codegen_unit_name)) + .split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.debugging_opts.split_dwarf_kind, + Some(codegen_unit_name), + ) // We get a path relative to the working directory from split_dwarf_path .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) } else { @@ -1278,7 +1165,6 @@ impl<'ll, 'tcx> MemberDescriptionFactory<'ll, 'tcx> { struct StructMemberDescriptionFactory<'tcx> { ty: Ty<'tcx>, variant: &'tcx ty::VariantDef, - span: Span, } impl<'tcx> StructMemberDescriptionFactory<'tcx> { @@ -1295,12 +1181,12 @@ impl<'tcx> StructMemberDescriptionFactory<'tcx> { let name = if self.variant.ctor_kind == CtorKind::Fn { format!("__{}", i) } else { - f.ident.to_string() + f.name.to_string() }; let field = layout.field(cx, i); MemberDescription { name, - type_metadata: type_metadata(cx, field.ty, self.span), + type_metadata: type_metadata(cx, field.ty), offset: layout.fields.offset(i), size: field.size, align: field.align.abi, @@ -1316,8 +1202,7 @@ impl<'tcx> StructMemberDescriptionFactory<'tcx> { fn prepare_struct_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, struct_type: Ty<'tcx>, - unique_type_id: UniqueTypeId, - span: Span, + unique_type_id: UniqueTypeId<'tcx>, ) -> RecursiveTypeDescription<'ll, 'tcx> { let struct_name = compute_debuginfo_type_name(cx.tcx, struct_type, false); @@ -1327,14 +1212,17 @@ fn prepare_struct_metadata<'ll, 'tcx>( }; let containing_scope = get_namespace_for_item(cx, struct_def_id); + let (size, align) = cx.size_and_align_of(struct_type); let struct_metadata_stub = create_struct_stub( cx, - struct_type, + size, + align, &struct_name, unique_type_id, Some(containing_scope), DIFlags::FlagZero, + None, ); create_and_register_recursive_type_forward_declaration( @@ -1343,7 +1231,7 @@ fn prepare_struct_metadata<'ll, 'tcx>( unique_type_id, struct_metadata_stub, struct_metadata_stub, - StructMDF(StructMemberDescriptionFactory { ty: struct_type, variant, span }), + StructMDF(StructMemberDescriptionFactory { ty: struct_type, variant }), ) } @@ -1380,7 +1268,6 @@ fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> struct TupleMemberDescriptionFactory<'tcx> { ty: Ty<'tcx>, component_types: Vec<Ty<'tcx>>, - span: Span, } impl<'tcx> TupleMemberDescriptionFactory<'tcx> { @@ -1407,7 +1294,7 @@ impl<'tcx> TupleMemberDescriptionFactory<'tcx> { }; MemberDescription { name, - type_metadata: type_metadata(cx, component_type, self.span), + type_metadata: type_metadata(cx, component_type), offset: layout.fields.offset(i), size, align, @@ -1424,19 +1311,21 @@ fn prepare_tuple_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, tuple_type: Ty<'tcx>, component_types: &[Ty<'tcx>], - unique_type_id: UniqueTypeId, - span: Span, + unique_type_id: UniqueTypeId<'tcx>, containing_scope: Option<&'ll DIScope>, ) -> RecursiveTypeDescription<'ll, 'tcx> { + let (size, align) = cx.size_and_align_of(tuple_type); let tuple_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); let struct_stub = create_struct_stub( cx, - tuple_type, + size, + align, &tuple_name[..], unique_type_id, containing_scope, DIFlags::FlagZero, + None, ); create_and_register_recursive_type_forward_declaration( @@ -1448,7 +1337,6 @@ fn prepare_tuple_metadata<'ll, 'tcx>( TupleMDF(TupleMemberDescriptionFactory { ty: tuple_type, component_types: component_types.to_vec(), - span, }), ) } @@ -1460,7 +1348,6 @@ fn prepare_tuple_metadata<'ll, 'tcx>( struct UnionMemberDescriptionFactory<'tcx> { layout: TyAndLayout<'tcx>, variant: &'tcx ty::VariantDef, - span: Span, } impl<'tcx> UnionMemberDescriptionFactory<'tcx> { @@ -1475,8 +1362,8 @@ impl<'tcx> UnionMemberDescriptionFactory<'tcx> { .map(|(i, f)| { let field = self.layout.field(cx, i); MemberDescription { - name: f.ident.to_string(), - type_metadata: type_metadata(cx, field.ty, self.span), + name: f.name.to_string(), + type_metadata: type_metadata(cx, field.ty), offset: Size::ZERO, size: field.size, align: field.align.abi, @@ -1492,8 +1379,7 @@ impl<'tcx> UnionMemberDescriptionFactory<'tcx> { fn prepare_union_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, union_type: Ty<'tcx>, - unique_type_id: UniqueTypeId, - span: Span, + unique_type_id: UniqueTypeId<'tcx>, ) -> RecursiveTypeDescription<'ll, 'tcx> { let union_name = compute_debuginfo_type_name(cx.tcx, union_type, false); @@ -1513,7 +1399,7 @@ fn prepare_union_metadata<'ll, 'tcx>( unique_type_id, union_metadata_stub, union_metadata_stub, - UnionMDF(UnionMemberDescriptionFactory { layout: cx.layout_of(union_type), variant, span }), + UnionMDF(UnionMemberDescriptionFactory { layout: cx.layout_of(union_type), variant }), ) } @@ -1521,13 +1407,6 @@ fn prepare_union_metadata<'ll, 'tcx>( // Enums //=----------------------------------------------------------------------------- -/// DWARF variant support is only available starting in LLVM 8, but -/// on MSVC we have to use the fallback mode, because LLVM doesn't -/// lower variant parts to PDB. -fn use_enum_fallback(cx: &CodegenCx<'_, '_>) -> bool { - cx.sess().target.is_like_msvc -} - // FIXME(eddyb) maybe precompute this? Right now it's computed once // per generator monomorphization, but it doesn't depend on substs. fn generator_layout_and_saved_local_names<'tcx>( @@ -1540,7 +1419,7 @@ fn generator_layout_and_saved_local_names<'tcx>( let state_arg = mir::Local::new(1); for var in &body.var_debug_info { - let place = if let mir::VarDebugInfoContents::Place(p) = var.value { p } else { continue }; + let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; if place.local != state_arg { continue; } @@ -1575,7 +1454,6 @@ struct EnumMemberDescriptionFactory<'ll, 'tcx> { layout: TyAndLayout<'tcx>, tag_type_metadata: Option<&'ll DIType>, common_members: Vec<Option<&'ll DIType>>, - span: Span, } impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { @@ -1588,7 +1466,7 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { }; let variant_info_for = |index: VariantIdx| match *self.enum_type.kind() { - ty::Adt(adt, _) => VariantInfo::Adt(&adt.variants[index]), + ty::Adt(adt, _) => VariantInfo::Adt(&adt.variants[index], index), ty::Generator(def_id, _, _) => { let (generator_layout, generator_saved_local_names) = generator_variant_info_data.as_ref().unwrap(); @@ -1602,9 +1480,12 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { _ => bug!(), }; - let fallback = use_enum_fallback(cx); + // While LLVM supports generating debuginfo for variant types (enums), it doesn't support + // lowering that debuginfo to CodeView records for msvc targets. So if we are targeting + // msvc, then we need to use a different, fallback encoding of the debuginfo. + let fallback = cpp_like_debuginfo(cx.tcx); // This will always find the metadata in the type map. - let self_metadata = type_metadata(cx, self.enum_type, self.span); + let self_metadata = type_metadata(cx, self.enum_type); match self.layout.variants { Variants::Single { index } => { @@ -1616,16 +1497,17 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { let variant_info = variant_info_for(index); let (variant_type_metadata, member_description_factory) = - describe_enum_variant(cx, self.layout, variant_info, self_metadata, self.span); + describe_enum_variant(cx, self.layout, variant_info, self_metadata); let member_descriptions = member_description_factory.create_member_descriptions(cx); + let type_params = compute_type_parameters(cx, self.enum_type); set_members_of_composite_type( cx, - self.enum_type, variant_type_metadata, member_descriptions, Some(&self.common_members), + type_params, ); vec![MemberDescription { name: variant_info.variant_name(), @@ -1681,23 +1563,19 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { .map(|(i, _)| { let variant = self.layout.for_variant(cx, i); let variant_info = variant_info_for(i); - let (variant_type_metadata, member_desc_factory) = describe_enum_variant( - cx, - variant, - variant_info, - self_metadata, - self.span, - ); + let (variant_type_metadata, member_desc_factory) = + describe_enum_variant(cx, variant, variant_info, self_metadata); let member_descriptions = member_desc_factory.create_member_descriptions(cx); + let type_params = compute_type_parameters(cx, self.enum_type); set_members_of_composite_type( cx, - self.enum_type, variant_type_metadata, member_descriptions, Some(&self.common_members), + type_params, ); MemberDescription { @@ -1806,7 +1684,7 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { tag.value.size(cx).bits(), tag.value.align(cx).abi.bits() as u32, create_DIArray(DIB(cx), &tags), - type_metadata(cx, discr_enum_ty, self.span), + type_metadata(cx, discr_enum_ty), true, ) }; @@ -1817,17 +1695,17 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { dataful_variant_layout, variant_info, self_metadata, - self.span, ); let member_descriptions = member_desc_factory.create_member_descriptions(cx); + let type_params = compute_type_parameters(cx, self.enum_type); set_members_of_composite_type( cx, - self.enum_type, variant_type_metadata, member_descriptions, Some(&self.common_members), + type_params, ); let (size, align) = @@ -1863,23 +1741,18 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { let variant = self.layout.for_variant(cx, i); let variant_info = variant_info_for(i); let (variant_type_metadata, member_desc_factory) = - describe_enum_variant( - cx, - variant, - variant_info, - self_metadata, - self.span, - ); + describe_enum_variant(cx, variant, variant_info, self_metadata); let member_descriptions = member_desc_factory.create_member_descriptions(cx); + let type_params = compute_type_parameters(cx, self.enum_type); set_members_of_composite_type( cx, - self.enum_type, variant_type_metadata, member_descriptions, Some(&self.common_members), + type_params, ); let niche_value = calculate_niche_value(i); @@ -1907,7 +1780,6 @@ struct VariantMemberDescriptionFactory<'tcx> { /// Cloned from the `layout::Struct` describing the variant. offsets: Vec<Size>, args: Vec<(String, Ty<'tcx>)>, - span: Span, } impl<'tcx> VariantMemberDescriptionFactory<'tcx> { @@ -1922,7 +1794,7 @@ impl<'tcx> VariantMemberDescriptionFactory<'tcx> { let (size, align) = cx.size_and_align_of(ty); MemberDescription { name: name.to_string(), - type_metadata: type_metadata(cx, ty, self.span), + type_metadata: type_metadata(cx, ty), offset: self.offsets[i], size, align, @@ -1937,7 +1809,7 @@ impl<'tcx> VariantMemberDescriptionFactory<'tcx> { #[derive(Copy, Clone)] enum VariantInfo<'a, 'tcx> { - Adt(&'tcx ty::VariantDef), + Adt(&'tcx ty::VariantDef, VariantIdx), Generator { def_id: DefId, generator_layout: &'tcx GeneratorLayout<'tcx>, @@ -1947,9 +1819,17 @@ enum VariantInfo<'a, 'tcx> { } impl<'tcx> VariantInfo<'_, 'tcx> { + fn variant_idx(&self) -> VariantIdx { + match self { + VariantInfo::Adt(_, variant_index) | VariantInfo::Generator { variant_index, .. } => { + *variant_index + } + } + } + fn map_struct_name<R>(&self, f: impl FnOnce(&str) -> R) -> R { match self { - VariantInfo::Adt(variant) => f(variant.ident.as_str()), + VariantInfo::Adt(variant, _) => f(variant.name.as_str()), VariantInfo::Generator { variant_index, .. } => { f(&GeneratorSubsts::variant_name(*variant_index)) } @@ -1958,7 +1838,7 @@ impl<'tcx> VariantInfo<'_, 'tcx> { fn variant_name(&self) -> String { match self { - VariantInfo::Adt(variant) => variant.ident.to_string(), + VariantInfo::Adt(variant, _) => variant.name.to_string(), VariantInfo::Generator { variant_index, .. } => { // Since GDB currently prints out the raw discriminant along // with every variant, make each variant name be just the value @@ -1971,8 +1851,8 @@ impl<'tcx> VariantInfo<'_, 'tcx> { fn field_name(&self, i: usize) -> String { let field_name = match *self { - VariantInfo::Adt(variant) if variant.ctor_kind != CtorKind::Fn => { - Some(variant.fields[i].ident.name) + VariantInfo::Adt(variant, _) if variant.ctor_kind != CtorKind::Fn => { + Some(variant.fields[i].name) } VariantInfo::Generator { generator_layout, @@ -2010,20 +1890,22 @@ fn describe_enum_variant<'ll, 'tcx>( layout: layout::TyAndLayout<'tcx>, variant: VariantInfo<'_, 'tcx>, containing_scope: &'ll DIScope, - span: Span, ) -> (&'ll DICompositeType, MemberDescriptionFactory<'ll, 'tcx>) { let metadata_stub = variant.map_struct_name(|variant_name| { - let unique_type_id = debug_context(cx) - .type_map - .borrow_mut() - .get_unique_type_id_of_enum_variant(cx, layout.ty, variant_name); + let unique_type_id = + UniqueTypeId::for_enum_variant(cx.tcx, layout.ty, variant.variant_idx()); + + let (size, align) = cx.size_and_align_of(layout.ty); + create_struct_stub( cx, - layout.ty, + size, + align, variant_name, unique_type_id, Some(containing_scope), DIFlags::FlagZero, + None, ) }); @@ -2032,8 +1914,7 @@ fn describe_enum_variant<'ll, 'tcx>( .map(|i| (variant.field_name(i), layout.field(cx, i).ty)) .collect(); - let member_description_factory = - VariantMDF(VariantMemberDescriptionFactory { offsets, args, span }); + let member_description_factory = VariantMDF(VariantMemberDescriptionFactory { offsets, args }); (metadata_stub, member_description_factory) } @@ -2042,8 +1923,7 @@ fn prepare_enum_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type: Ty<'tcx>, enum_def_id: DefId, - unique_type_id: UniqueTypeId, - span: Span, + unique_type_id: UniqueTypeId<'tcx>, outer_field_tys: Vec<Ty<'tcx>>, ) -> RecursiveTypeDescription<'ll, 'tcx> { let tcx = cx.tcx; @@ -2062,7 +1942,7 @@ fn prepare_enum_metadata<'ll, 'tcx>( let enumerators_metadata: Vec<_> = match enum_type.kind() { ty::Adt(def, _) => iter::zip(def.discriminants(tcx), &def.variants) .map(|((_, discr), v)| { - let name = v.ident.as_str(); + let name = v.name.as_str(); let is_unsigned = match discr.ty.kind() { ty::Int(_) => false, ty::Uint(_) => true, @@ -2108,8 +1988,7 @@ fn prepare_enum_metadata<'ll, 'tcx>( Some(discriminant_type_metadata) => discriminant_type_metadata, None => { let (discriminant_size, discriminant_align) = (discr.size(cx), discr.align(cx)); - let discriminant_base_type_metadata = - type_metadata(cx, discr.to_ty(tcx), rustc_span::DUMMY_SP); + let discriminant_base_type_metadata = type_metadata(cx, discr.to_ty(tcx)); let item_name; let discriminant_name = match enum_type.kind() { @@ -2155,7 +2034,10 @@ fn prepare_enum_metadata<'ll, 'tcx>( return FinalMetadata(discriminant_type_metadata(tag.value)); } - if use_enum_fallback(cx) { + // While LLVM supports generating debuginfo for variant types (enums), it doesn't support + // lowering that debuginfo to CodeView records for msvc targets. So if we are targeting + // msvc, then we need to use a different encoding of the debuginfo. + if cpp_like_debuginfo(tcx) { let discriminant_type_metadata = match layout.variants { Variants::Single { .. } => None, Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } @@ -2165,8 +2047,7 @@ fn prepare_enum_metadata<'ll, 'tcx>( }; let enum_metadata = { - let type_map = debug_context(cx).type_map.borrow(); - let unique_type_id_str = type_map.get_unique_type_id_as_string(unique_type_id); + let unique_type_id_str = unique_type_id.generate_unique_id_string(tcx); unsafe { llvm::LLVMRustDIBuilderCreateUnionType( @@ -2198,7 +2079,6 @@ fn prepare_enum_metadata<'ll, 'tcx>( layout, tag_type_metadata: discriminant_type_metadata, common_members: vec![], - span, }), ); } @@ -2268,11 +2148,8 @@ fn prepare_enum_metadata<'ll, 'tcx>( let outer_fields = match layout.variants { Variants::Single { .. } => vec![], Variants::Multiple { .. } => { - let tuple_mdf = TupleMemberDescriptionFactory { - ty: enum_type, - component_types: outer_field_tys, - span, - }; + let tuple_mdf = + TupleMemberDescriptionFactory { ty: enum_type, component_types: outer_field_tys }; tuple_mdf .create_member_descriptions(cx) .into_iter() @@ -2281,10 +2158,9 @@ fn prepare_enum_metadata<'ll, 'tcx>( } }; - let variant_part_unique_type_id_str = debug_context(cx) - .type_map - .borrow_mut() - .get_unique_type_id_str_of_enum_variant_part(unique_type_id); + let variant_part_unique_type_id_str = + UniqueTypeId::for_enum_variant_part(tcx, enum_type).generate_unique_id_string(tcx); + let empty_array = create_DIArray(DIB(cx), &[]); let name = ""; let variant_part = unsafe { @@ -2312,9 +2188,7 @@ fn prepare_enum_metadata<'ll, 'tcx>( // an equivalent layout but offers us much better integration with // debuggers. let type_array = create_DIArray(DIB(cx), &[Some(variant_part)]); - - let type_map = debug_context(cx).type_map.borrow(); - let unique_type_id_str = type_map.get_unique_type_id_as_string(unique_type_id); + let unique_type_id_str = unique_type_id.generate_unique_id_string(tcx); unsafe { llvm::LLVMRustDIBuilderCreateStructType( @@ -2348,7 +2222,6 @@ fn prepare_enum_metadata<'ll, 'tcx>( layout, tag_type_metadata: None, common_members: outer_fields, - span, }), ) } @@ -2361,31 +2234,31 @@ fn composite_type_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, composite_type: Ty<'tcx>, composite_type_name: &str, - composite_type_unique_id: UniqueTypeId, + composite_type_unique_id: UniqueTypeId<'tcx>, member_descriptions: Vec<MemberDescription<'ll>>, containing_scope: Option<&'ll DIScope>, - - // Ignore source location information as long as it - // can't be reconstructed for non-local crates. - _file_metadata: &'ll DIFile, - _definition_span: Span, ) -> &'ll DICompositeType { + let (size, align) = cx.size_and_align_of(composite_type); + // Create the (empty) struct metadata node ... let composite_type_metadata = create_struct_stub( cx, - composite_type, + size, + align, composite_type_name, composite_type_unique_id, containing_scope, DIFlags::FlagZero, + None, ); + // ... and immediately create and add the member descriptions. set_members_of_composite_type( cx, - composite_type, composite_type_metadata, member_descriptions, None, + compute_type_parameters(cx, composite_type), ); composite_type_metadata @@ -2393,10 +2266,10 @@ fn composite_type_metadata<'ll, 'tcx>( fn set_members_of_composite_type<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - composite_type: Ty<'tcx>, composite_type_metadata: &'ll DICompositeType, member_descriptions: Vec<MemberDescription<'ll>>, common_members: Option<&Vec<Option<&'ll DIType>>>, + type_params: &'ll DIArray, ) { // In some rare cases LLVM metadata uniquing would lead to an existing type // description being used instead of a new one created in @@ -2423,13 +2296,12 @@ fn set_members_of_composite_type<'ll, 'tcx>( member_metadata.extend(other_members.iter()); } - let type_params = compute_type_parameters(cx, composite_type); unsafe { - let type_array = create_DIArray(DIB(cx), &member_metadata); + let field_array = create_DIArray(DIB(cx), &member_metadata); llvm::LLVMRustDICompositeTypeReplaceArrays( DIB(cx), composite_type_metadata, - Some(type_array), + Some(field_array), Some(type_params), ); } @@ -2446,8 +2318,7 @@ fn compute_type_parameters<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) - if let GenericArgKind::Type(ty) = kind.unpack() { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); - let actual_type_metadata = - type_metadata(cx, actual_type, rustc_span::DUMMY_SP); + let actual_type_metadata = type_metadata(cx, actual_type); let name = name.as_str(); Some(unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( @@ -2483,16 +2354,15 @@ fn compute_type_parameters<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) - /// with `set_members_of_composite_type()`. fn create_struct_stub<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - struct_type: Ty<'tcx>, - struct_type_name: &str, - unique_type_id: UniqueTypeId, + size: Size, + align: Align, + type_name: &str, + unique_type_id: UniqueTypeId<'tcx>, containing_scope: Option<&'ll DIScope>, flags: DIFlags, + vtable_holder: Option<&'ll DIType>, ) -> &'ll DICompositeType { - let (struct_size, struct_align) = cx.size_and_align_of(struct_type); - - let type_map = debug_context(cx).type_map.borrow(); - let unique_type_id = type_map.get_unique_type_id_as_string(unique_type_id); + let unique_type_id = unique_type_id.generate_unique_id_string(cx.tcx); let metadata_stub = unsafe { // `LLVMRustDIBuilderCreateStructType()` wants an empty array. A null @@ -2503,17 +2373,17 @@ fn create_struct_stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStructType( DIB(cx), containing_scope, - struct_type_name.as_ptr().cast(), - struct_type_name.len(), + type_name.as_ptr().cast(), + type_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, - struct_size.bits(), - struct_align.bits() as u32, + size.bits(), + align.bits() as u32, flags, None, empty_array, 0, - None, + vtable_holder, unique_type_id.as_ptr().cast(), unique_type_id.len(), ) @@ -2526,13 +2396,11 @@ fn create_union_stub<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, union_type: Ty<'tcx>, union_type_name: &str, - unique_type_id: UniqueTypeId, + unique_type_id: UniqueTypeId<'tcx>, containing_scope: &'ll DIScope, ) -> &'ll DICompositeType { let (union_size, union_align) = cx.size_and_align_of(union_type); - - let type_map = debug_context(cx).type_map.borrow(); - let unique_type_id = type_map.get_unique_type_id_as_string(unique_type_id); + let unique_type_id = unique_type_id.generate_unique_id_string(cx.tcx); let metadata_stub = unsafe { // `LLVMRustDIBuilderCreateUnionType()` wants an empty array. A null @@ -2589,7 +2457,7 @@ pub fn create_global_var_metadata<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, g let is_local_to_unit = is_node_local_to_unit(cx, def_id); let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); - let type_metadata = type_metadata(cx, variable_type, span); + let type_metadata = type_metadata(cx, variable_type); let var_name = tcx.item_name(def_id); let var_name = var_name.as_str(); let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name; @@ -2619,6 +2487,14 @@ pub fn create_global_var_metadata<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, g } /// Generates LLVM debuginfo for a vtable. +/// +/// The vtable type looks like a struct with a field for each function pointer and super-trait +/// pointer it contains (plus the `size` and `align` fields). +/// +/// Except for `size`, `align`, and `drop_in_place`, the field names don't try to mirror +/// the name of the method they implement. This can be implemented in the future once there +/// is a proper disambiguation scheme for dealing with methods from different traits that have +/// the same name. fn vtable_type_metadata<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, @@ -2635,16 +2511,78 @@ fn vtable_type_metadata<'ll, 'tcx>( COMMON_VTABLE_ENTRIES }; - // FIXME: We describe the vtable as an array of *const () pointers. The length of the array is - // correct - but we could create a more accurate description, e.g. by describing it - // as a struct where each field has a name that corresponds to the name of the method - // it points to. - // However, this is not entirely straightforward because there might be multiple - // methods with the same name if the vtable is for multiple traits. So for now we keep - // things simple instead of adding some ad-hoc disambiguation scheme. - let vtable_type = tcx.mk_array(tcx.mk_imm_ptr(tcx.types.unit), vtable_entries.len() as u64); + // All function pointers are described as opaque pointers. This could be improved in the future + // by describing them as actual function pointers. + let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); + let void_pointer_type_debuginfo = type_metadata(cx, void_pointer_ty); + let usize_debuginfo = type_metadata(cx, tcx.types.usize); + let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); + // If `usize` is not pointer-sized and -aligned then the size and alignment computations + // for the vtable as a whole would be wrong. Let's make sure this holds even on weird + // platforms. + assert_eq!(cx.size_and_align_of(tcx.types.usize), (pointer_size, pointer_align)); + + let vtable_type_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::Type); + let unique_type_id = UniqueTypeId::for_vtable_ty(tcx, ty, poly_trait_ref); + let size = pointer_size * vtable_entries.len() as u64; + + // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate + // the vtable to the type it is for. + let vtable_holder = type_metadata(cx, ty); + + let vtable_type_metadata = create_struct_stub( + cx, + size, + pointer_align, + &vtable_type_name, + unique_type_id, + NO_SCOPE_METADATA, + DIFlags::FlagArtificial, + Some(vtable_holder), + ); + + // Create a field for each entry in the vtable. + let fields: Vec<_> = vtable_entries + .iter() + .enumerate() + .filter_map(|(index, vtable_entry)| { + let (field_name, field_type) = match vtable_entry { + ty::VtblEntry::MetadataDropInPlace => { + ("drop_in_place".to_string(), void_pointer_type_debuginfo) + } + ty::VtblEntry::Method(_) => { + // Note: This code does not try to give a proper name to each method + // because there might be multiple methods with the same name + // (coming from different traits). + (format!("__method{}", index), void_pointer_type_debuginfo) + } + ty::VtblEntry::TraitVPtr(_) => { + // Note: In the future we could try to set the type of this pointer + // to the type that we generate for the corresponding vtable. + (format!("__super_trait_ptr{}", index), void_pointer_type_debuginfo) + } + ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_debuginfo), + ty::VtblEntry::MetadataSize => ("size".to_string(), usize_debuginfo), + ty::VtblEntry::Vacant => return None, + }; + + Some(MemberDescription { + name: field_name, + type_metadata: field_type, + offset: pointer_size * index as u64, + size: pointer_size, + align: pointer_align, + flags: DIFlags::FlagZero, + discriminant: None, + source_info: None, + }) + }) + .collect(); - type_metadata(cx, vtable_type, rustc_span::DUMMY_SP) + let type_params = create_DIArray(DIB(cx), &[]); + set_members_of_composite_type(cx, vtable_type_metadata, fields, None, type_params); + vtable_type_metadata } /// Creates debug information for the given vtable, which is for the @@ -2666,11 +2604,12 @@ pub fn create_vtable_metadata<'ll, 'tcx>( return; } - let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref); + let vtable_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); let vtable_type = vtable_type_metadata(cx, ty, poly_trait_ref); + let linkage_name = ""; unsafe { - let linkage_name = ""; llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), NO_SCOPE_METADATA, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 3e7371179cc..1b4a193dbf1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -25,7 +25,7 @@ use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::vec::IndexVec; use rustc_middle::mir; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable}; use rustc_session::config::{self, DebugInfo}; @@ -38,6 +38,7 @@ use libc::c_uint; use smallvec::SmallVec; use std::cell::RefCell; use std::iter; +use std::lazy::OnceCell; use tracing::debug; mod create_scope_map; @@ -63,9 +64,11 @@ pub struct CrateDebugContext<'a, 'tcx> { created_files: RefCell<FxHashMap<(Option<String>, Option<String>), &'a DIFile>>, created_enum_disr_types: RefCell<FxHashMap<(DefId, Primitive), &'a DIType>>, - type_map: RefCell<TypeMap<'a, 'tcx>>, + type_map: TypeMap<'a, 'tcx>, namespace_map: RefCell<DefIdMap<&'a DIScope>>, + recursion_marker_type: OnceCell<&'a DIType>, + // This collection is used to assert that composite types (structs, enums, // ...) have their members only set once: composite_types_completed: RefCell<FxHashSet<&'a DIType>>, @@ -93,6 +96,7 @@ impl<'a, 'tcx> CrateDebugContext<'a, 'tcx> { created_enum_disr_types: Default::default(), type_map: Default::default(), namespace_map: RefCell::new(Default::default()), + recursion_marker_type: OnceCell::new(), composite_types_completed: Default::default(), } } @@ -108,18 +112,29 @@ impl<'a, 'tcx> CrateDebugContext<'a, 'tcx> { // This can be overridden using --llvm-opts -dwarf-version,N. // Android has the same issue (#22398) if let Some(version) = sess.target.dwarf_version { - llvm::LLVMRustAddModuleFlag(self.llmod, "Dwarf Version\0".as_ptr().cast(), version) + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + "Dwarf Version\0".as_ptr().cast(), + version, + ) } // Indicate that we want CodeView debug information on MSVC if sess.target.is_like_msvc { - llvm::LLVMRustAddModuleFlag(self.llmod, "CodeView\0".as_ptr().cast(), 1) + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + "CodeView\0".as_ptr().cast(), + 1, + ) } // Prevent bitcode readers from deleting the debug info. let ptr = "Debug Info Version\0".as_ptr(); llvm::LLVMRustAddModuleFlag( self.llmod, + llvm::LLVMModFlagBehavior::Warning, ptr.cast(), llvm::LLVMRustDebugMetadataVersion(), ); @@ -160,17 +175,17 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { // the values should match the ones in the DWARF standard anyway. let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; - let mut addr_ops = SmallVec::<[_; 8]>::new(); + let mut addr_ops = SmallVec::<[u64; 8]>::new(); if direct_offset.bytes() > 0 { addr_ops.push(op_plus_uconst()); - addr_ops.push(direct_offset.bytes() as i64); + addr_ops.push(direct_offset.bytes() as u64); } for &offset in indirect_offsets { addr_ops.push(op_deref()); if offset.bytes() > 0 { addr_ops.push(op_plus_uconst()); - addr_ops.push(offset.bytes() as i64); + addr_ops.push(offset.bytes() as u64); } } @@ -307,9 +322,11 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn_abi: &FnAbi<'tcx, Ty<'tcx>>, maybe_definition_llfn: Option<&'ll Value>, ) -> &'ll DIScope { + let tcx = self.tcx; + let def_id = instance.def_id(); let containing_scope = get_containing_scope(self, instance); - let span = self.tcx.def_span(def_id); + let span = tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); @@ -319,16 +336,24 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { }; let mut name = String::new(); - type_names::push_item_name(self.tcx(), def_id, false, &mut name); + type_names::push_item_name(tcx, def_id, false, &mut name); // Find the enclosing function, in case this is a closure. - let enclosing_fn_def_id = self.tcx().typeck_root_def_id(def_id); + let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); + + // We look up the generics of the enclosing function and truncate the substs + // to their length in order to cut off extra stuff that might be in there for + // closures or generators. + let generics = tcx.generics_of(enclosing_fn_def_id); + let substs = instance.substs.truncate_to(tcx, generics); + + type_names::push_generic_params( + tcx, + tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), + &mut name, + ); - // Get_template_parameters() will append a `<...>` clause to the function - // name if necessary. - let generics = self.tcx().generics_of(enclosing_fn_def_id); - let substs = instance.substs.truncate_to(self.tcx(), generics); - let template_parameters = get_template_parameters(self, generics, substs, &mut name); + let template_parameters = get_template_parameters(self, generics, substs); let linkage_name = &mangled_name_of_instance(self, instance).name; // Omit the linkage_name if it is the same as subprogram name. @@ -350,7 +375,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { if self.sess().opts.optimize != config::OptLevel::No { spflags |= DISPFlags::SPFlagOptimized; } - if let Some((id, _)) = self.tcx.entry_fn(()) { + if let Some((id, _)) = tcx.entry_fn(()) { if id == def_id { spflags |= DISPFlags::SPFlagMainSubprogram; } @@ -390,7 +415,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { signature.push(if fn_abi.ret.is_ignore() { None } else { - Some(type_metadata(cx, fn_abi.ret.layout.ty, rustc_span::DUMMY_SP)) + Some(type_metadata(cx, fn_abi.ret.layout.ty)) }); // Arguments types @@ -409,21 +434,17 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let t = arg.layout.ty; let t = match t.kind() { ty::Array(ct, _) - if (*ct == cx.tcx.types.u8) || cx.layout_of(ct).is_zst() => + if (*ct == cx.tcx.types.u8) || cx.layout_of(*ct).is_zst() => { - cx.tcx.mk_imm_ptr(ct) + cx.tcx.mk_imm_ptr(*ct) } _ => t, }; - Some(type_metadata(cx, t, rustc_span::DUMMY_SP)) + Some(type_metadata(cx, t)) })); } else { - signature.extend( - fn_abi - .args - .iter() - .map(|arg| Some(type_metadata(cx, arg.layout.ty, rustc_span::DUMMY_SP))), - ); + signature + .extend(fn_abi.args.iter().map(|arg| Some(type_metadata(cx, arg.layout.ty)))); } create_DIArray(DIB(cx), &signature[..]) @@ -433,14 +454,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { cx: &CodegenCx<'ll, 'tcx>, generics: &ty::Generics, substs: SubstsRef<'tcx>, - name_to_append_suffix_to: &mut String, ) -> &'ll DIArray { - type_names::push_generic_params( - cx.tcx, - cx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), - name_to_append_suffix_to, - ); - if substs.types().next().is_none() { return create_DIArray(DIB(cx), &[]); } @@ -453,8 +467,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { if let GenericArgKind::Type(ty) = kind.unpack() { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); - let actual_type_metadata = - type_metadata(cx, actual_type, rustc_span::DUMMY_SP); + let actual_type_metadata = type_metadata(cx, actual_type); let name = name.as_str(); Some(unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( @@ -507,9 +520,9 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { ty::Adt(def, ..) if !def.is_box() => { // Again, only create type information if full debuginfo is enabled if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.definitely_needs_subst(cx.tcx) + && !impl_self_ty.needs_subst() { - Some(type_metadata(cx, impl_self_ty, rustc_span::DUMMY_SP)) + Some(type_metadata(cx, impl_self_ty)) } else { Some(namespace::item_namespace(cx, def.did)) } @@ -584,7 +597,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); - let type_metadata = type_metadata(self, variable_type, span); + let type_metadata = type_metadata(self, variable_type); let (argument_index, dwarf_tag) = match variable_kind { ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 953b6765a48..fa75463067f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -4,7 +4,9 @@ use super::namespace::item_namespace; use super::CrateDebugContext; use rustc_hir::def_id::DefId; -use rustc_middle::ty::DefIdTree; +use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::ty::{self, DefIdTree, Ty}; +use tracing::trace; use crate::common::CodegenCx; use crate::llvm; @@ -46,3 +48,52 @@ pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { item_namespace(cx, cx.tcx.parent(def_id).expect("get_namespace_for_item: missing parent?")) } + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum FatPtrKind { + Slice, + Dyn, +} + +/// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. +/// if the second field of the fat pointer is a length or a vtable-pointer. +/// If `pointee_ty` does not require a fat pointer (because it is Sized) then +/// the function returns `None`. +pub(crate) fn fat_pointer_kind<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + pointee_ty: Ty<'tcx>, +) -> Option<FatPtrKind> { + let pointee_tail_ty = cx.tcx.struct_tail_erasing_lifetimes(pointee_ty, cx.param_env()); + let layout = cx.layout_of(pointee_tail_ty); + trace!( + "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", + pointee_tail_ty, + layout, + layout.is_unsized() + ); + + if !layout.is_unsized() { + return None; + } + + match *pointee_tail_ty.kind() { + ty::Str | ty::Slice(_) => Some(FatPtrKind::Slice), + ty::Dynamic(..) => Some(FatPtrKind::Dyn), + ty::Foreign(_) => { + // Assert that pointers to foreign types really are thin: + debug_assert_eq!( + cx.size_of(cx.tcx.mk_imm_ptr(pointee_tail_ty)), + cx.size_of(cx.tcx.mk_imm_ptr(cx.tcx.types.u8)) + ); + None + } + _ => { + // For all other pointee types we should already have returned None + // at the beginning of the function. + panic!( + "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {:?}", + pointee_tail_ty + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 90d0d5caba1..5a5c4f7f860 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -18,8 +18,8 @@ use crate::llvm; use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; -use rustc_codegen_ssa::traits::*; use rustc_middle::ty::Ty; +use smallvec::SmallVec; use tracing::debug; /// Declare a function. @@ -41,12 +41,16 @@ fn declare_raw_fn<'ll>( llvm::SetFunctionCallConv(llfn, callconv); llvm::SetUnnamedAddress(llfn, unnamed); + let mut attrs = SmallVec::<[_; 4]>::new(); + if cx.tcx.sess.opts.cg.no_redzone.unwrap_or(cx.tcx.sess.target.disable_redzone) { - llvm::Attribute::NoRedZone.apply_llfn(Function, llfn); + attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attributes::default_optimisation_attrs(cx.tcx.sess, llfn); - attributes::non_lazy_bind(cx.sess(), llfn); + attrs.extend(attributes::non_lazy_bind_attr(cx)); + + attributes::apply_to_llfn(llfn, Function, &attrs); + llfn } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 07d49b6e729..e7c13e793d9 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -7,7 +7,6 @@ use crate::type_of::LayoutLlvmExt; use crate::va_arg::emit_va_arg; use crate::value::Value; -use rustc_ast as ast; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; use rustc_codegen_ssa::common::span_invalid_monomorphization_error; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; @@ -89,9 +88,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); - let (def_id, substs) = match *callee_ty.kind() { - ty::FnDef(def_id, substs) => (def_id, substs), - _ => bug!("expected fn item type, found {}", callee_ty), + let ty::FnDef(def_id, substs) = *callee_ty.kind() else { + bug!("expected fn item type, found {}", callee_ty); }; let sig = callee_ty.fn_sig(tcx); @@ -351,7 +349,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.type_void(), true, false, - ast::LlvmAsmDialect::Att, + llvm::AsmDialect::Att, &[span], false, None, @@ -454,11 +452,11 @@ fn codegen_msvc_try<'ll>( let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { bx.set_personality_fn(bx.eh_personality()); - let mut normal = bx.build_sibling_block("normal"); - let mut catchswitch = bx.build_sibling_block("catchswitch"); - let mut catchpad_rust = bx.build_sibling_block("catchpad_rust"); - let mut catchpad_foreign = bx.build_sibling_block("catchpad_foreign"); - let mut caught = bx.build_sibling_block("caught"); + let normal = bx.append_sibling_block("normal"); + let catchswitch = bx.append_sibling_block("catchswitch"); + let catchpad_rust = bx.append_sibling_block("catchpad_rust"); + let catchpad_foreign = bx.append_sibling_block("catchpad_foreign"); + let caught = bx.append_sibling_block("caught"); let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); @@ -522,13 +520,13 @@ fn codegen_msvc_try<'ll>( let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], normal.llbb(), catchswitch.llbb(), None); + bx.invoke(try_func_ty, try_func, &[data], normal, catchswitch, None); - normal.ret(bx.const_i32(0)); + bx.switch_to_block(normal); + bx.ret(bx.const_i32(0)); - let cs = catchswitch.catch_switch(None, None, 2); - catchswitch.add_handler(cs, catchpad_rust.llbb()); - catchswitch.add_handler(cs, catchpad_foreign.llbb()); + bx.switch_to_block(catchswitch); + let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]); // We can't use the TypeDescriptor defined in libpanic_unwind because it // might be in another DLL and the SEH encoding only supports specifying @@ -561,21 +559,24 @@ fn codegen_msvc_try<'ll>( // since our exception object effectively contains a Box. // // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang + bx.switch_to_block(catchpad_rust); let flags = bx.const_i32(8); - let funclet = catchpad_rust.catch_pad(cs, &[tydesc, flags, slot]); - let ptr = catchpad_rust.load(bx.type_i8p(), slot, ptr_align); + let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); + let ptr = bx.load(bx.type_i8p(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - catchpad_rust.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); - catchpad_rust.catch_ret(&funclet, caught.llbb()); + bx.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); + bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". + bx.switch_to_block(catchpad_foreign); let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); - let funclet = catchpad_foreign.catch_pad(cs, &[null, flags, null]); - catchpad_foreign.call(catch_ty, catch_func, &[data, null], Some(&funclet)); - catchpad_foreign.catch_ret(&funclet, caught.llbb()); + let funclet = bx.catch_pad(cs, &[null, flags, null]); + bx.call(catch_ty, catch_func, &[data, null], Some(&funclet)); + bx.catch_ret(&funclet, caught); - caught.ret(bx.const_i32(1)); + bx.switch_to_block(caught); + bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function @@ -616,15 +617,17 @@ fn codegen_gnu_try<'ll>( // (%ptr, _) = landingpad // call %catch_func(%data, %ptr) // ret 1 - let mut then = bx.build_sibling_block("then"); - let mut catch = bx.build_sibling_block("catch"); + let then = bx.append_sibling_block("then"); + let catch = bx.append_sibling_block("catch"); let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then.llbb(), catch.llbb(), None); - then.ret(bx.const_i32(0)); + bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + + bx.switch_to_block(then); + bx.ret(bx.const_i32(0)); // Type indicator for the exception being thrown. // @@ -632,14 +635,15 @@ fn codegen_gnu_try<'ll>( // being thrown. The second value is a "selector" indicating which of // the landing pad clauses the exception's type had been matched to. // rust_try ignores the selector. + bx.switch_to_block(catch); let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); - let vals = catch.landing_pad(lpad_ty, bx.eh_personality(), 1); + let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1); let tydesc = bx.const_null(bx.type_i8p()); - catch.add_clause(vals, tydesc); - let ptr = catch.extract_value(vals, 0); + bx.add_clause(vals, tydesc); + let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - catch.call(catch_ty, catch_func, &[data, ptr], None); - catch.ret(bx.const_i32(1)); + bx.call(catch_ty, catch_func, &[data, ptr], None); + bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function @@ -677,57 +681,54 @@ fn codegen_emcc_try<'ll>( // %catch_data[1] = %is_rust_panic // call %catch_func(%data, %catch_data) // ret 1 - let mut then = bx.build_sibling_block("then"); - let mut catch = bx.build_sibling_block("catch"); + let then = bx.append_sibling_block("then"); + let catch = bx.append_sibling_block("catch"); let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then.llbb(), catch.llbb(), None); - then.ret(bx.const_i32(0)); + bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + + bx.switch_to_block(then); + bx.ret(bx.const_i32(0)); // Type indicator for the exception being thrown. // // The first value in this tuple is a pointer to the exception object // being thrown. The second value is a "selector" indicating which of // the landing pad clauses the exception's type had been matched to. + bx.switch_to_block(catch); let tydesc = bx.eh_catch_typeinfo(); let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); - let vals = catch.landing_pad(lpad_ty, bx.eh_personality(), 2); - catch.add_clause(vals, tydesc); - catch.add_clause(vals, bx.const_null(bx.type_i8p())); - let ptr = catch.extract_value(vals, 0); - let selector = catch.extract_value(vals, 1); + let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2); + bx.add_clause(vals, tydesc); + bx.add_clause(vals, bx.const_null(bx.type_i8p())); + let ptr = bx.extract_value(vals, 0); + let selector = bx.extract_value(vals, 1); // Check if the typeid we got is the one for a Rust panic. - let rust_typeid = catch.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); - let is_rust_panic = catch.icmp(IntPredicate::IntEQ, selector, rust_typeid); - let is_rust_panic = catch.zext(is_rust_panic, bx.type_bool()); + let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); + let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); + let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); // We need to pass two values to catch_func (ptr and is_rust_panic), so // create an alloca and pass a pointer to that. let ptr_align = bx.tcx().data_layout.pointer_align.abi; let i8_align = bx.tcx().data_layout.i8_align.abi; let catch_data_type = bx.type_struct(&[bx.type_i8p(), bx.type_bool()], false); - let catch_data = catch.alloca(catch_data_type, ptr_align); - let catch_data_0 = catch.inbounds_gep( - catch_data_type, - catch_data, - &[bx.const_usize(0), bx.const_usize(0)], - ); - catch.store(ptr, catch_data_0, ptr_align); - let catch_data_1 = catch.inbounds_gep( - catch_data_type, - catch_data, - &[bx.const_usize(0), bx.const_usize(1)], - ); - catch.store(is_rust_panic, catch_data_1, i8_align); - let catch_data = catch.bitcast(catch_data, bx.type_i8p()); + let catch_data = bx.alloca(catch_data_type, ptr_align); + let catch_data_0 = + bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); + bx.store(ptr, catch_data_0, ptr_align); + let catch_data_1 = + bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); + bx.store(is_rust_panic, catch_data_1, i8_align); + let catch_data = bx.bitcast(catch_data, bx.type_i8p()); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - catch.call(catch_ty, catch_func, &[data, catch_data], None); - catch.ret(bx.const_i32(1)); + bx.call(catch_ty, catch_func, &[data, catch_data], None); + bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function @@ -791,7 +792,7 @@ fn get_rust_try_fn<'ll, 'tcx>( ))); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( - vec![try_fn_ty, i8p, catch_fn_ty].into_iter(), + [try_fn_ty, i8p, catch_fn_ty].into_iter(), tcx.types.i32, false, hir::Unsafety::Unsafe, @@ -1002,9 +1003,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } }) .collect(); - let indices = match indices { - Some(i) => i, - None => return Ok(bx.const_null(llret_ty)), + let Some(indices) = indices else { + return Ok(bx.const_null(llret_ty)); }; return Ok(bx.shuffle_vector( @@ -1134,8 +1134,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( fn simd_simple_float_intrinsic<'ll, 'tcx>( name: Symbol, - in_elem: &::rustc_middle::ty::TyS<'_>, - in_ty: &::rustc_middle::ty::TyS<'_>, + in_elem: Ty<'_>, + in_ty: Ty<'_>, in_len: u64, bx: &mut Builder<'_, 'll, 'tcx>, span: Span, @@ -1689,7 +1689,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bitwise_red!(simd_reduce_all: vector_reduce_and, true); bitwise_red!(simd_reduce_any: vector_reduce_or, true); - if name == sym::simd_cast { + if name == sym::simd_cast || name == sym::simd_as { require_simd!(ret_ty, "return"); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( @@ -1715,14 +1715,26 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let (in_style, in_width) = match in_elem.kind() { // vectors of pointer-sized integers should've been // disallowed before here, so this unwrap is safe. - ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), - ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), ty::Float(f) => (Style::Float, f.bit_width()), _ => (Style::Unsupported, 0), }; let (out_style, out_width) = match out_elem.kind() { - ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), - ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), ty::Float(f) => (Style::Float, f.bit_width()), _ => (Style::Unsupported, 0), }; @@ -1749,10 +1761,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, }); } (Style::Float, Style::Int(out_is_signed)) => { - return Ok(if out_is_signed { - bx.fptosi(args[0].immediate(), llret_ty) - } else { - bx.fptoui(args[0].immediate(), llret_ty) + return Ok(match (out_is_signed, name == sym::simd_as) { + (false, false) => bx.fptoui(args[0].immediate(), llret_ty), + (true, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), }); } (Style::Float, Style::Float) => { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index cea4595fbbf..875b4f033d1 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -7,9 +7,16 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(bool_to_option)] #![feature(crate_visibility_modifier)] +#![feature(let_else)] #![feature(extern_types)] +#![feature(once_cell)] #![feature(nll)] +#![feature(iter_intersperse)] #![recursion_limit = "256"] +#![allow(rustc::potential_query_instability)] + +#[macro_use] +extern crate rustc_macros; use back::write::{create_informational_target_machine, create_target_machine}; @@ -23,9 +30,10 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; use rustc_data_structures::fx::FxHashMap; -use rustc_errors::{ErrorReported, FatalError, Handler}; +use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; use rustc_session::Session; @@ -120,8 +128,9 @@ impl ExtraBackendMethods for LlvmCodegenBackend { &self, sess: &Session, optlvl: OptLevel, + target_features: &[String], ) -> TargetMachineFactoryFn<Self> { - back::write::target_machine_factory(sess, optlvl) + back::write::target_machine_factory(sess, optlvl, target_features) } fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { llvm_util::target_cpu(sess) @@ -245,6 +254,11 @@ impl CodegenBackend for LlvmCodegenBackend { llvm_util::init(sess); // Make sure llvm is inited } + fn provide(&self, providers: &mut Providers) { + providers.global_backend_features = + |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) + } + fn print(&self, req: PrintRequest, sess: &Session) { match req { PrintRequest::RelocationModels => { @@ -338,7 +352,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorReported> { + ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") @@ -359,7 +373,7 @@ impl CodegenBackend for LlvmCodegenBackend { sess: &Session, codegen_results: CodegenResults, outputs: &OutputFilenames, - ) -> Result<(), ErrorReported> { + ) -> Result<(), ErrorGuaranteed> { use crate::back::archive::LlvmArchiveBuilder; use rustc_codegen_ssa::back::link::link_binary; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6c911938ccc..6e9e0332faf 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -61,6 +61,26 @@ pub enum LLVMMachineType { ARM = 0x01c0, } +/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. +/// +/// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are +/// resolved according to the merge behaviors specified here. Flags differing only in merge +/// behavior are still considered to be in conflict. +/// +/// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, +/// 'Error' and 'Warning' cannot be mixed for a given flag. +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum LLVMModFlagBehavior { + Error = 1, + Warning = 2, + Require = 3, + Override = 4, + Append = 5, + AppendUnique = 6, + Max = 7, +} + // Consts for the LLVM CallConv type, pre-cast to usize. /// LLVM CallingConv::ID. Should we wrap this? @@ -135,7 +155,7 @@ pub enum DLLStorageClass { /// though it is not ABI compatible (since it's a C++ enum) #[repr(C)] #[derive(Copy, Clone, Debug)] -pub enum Attribute { +pub enum AttributeKind { AlwaysInline = 0, ByVal = 1, Cold = 2, @@ -169,6 +189,8 @@ pub enum Attribute { StackProtectReq = 30, StackProtectStrong = 31, StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, } /// LLVMIntPredicate @@ -420,25 +442,17 @@ pub enum MetadataType { MD_mem_parallel_loop_access = 10, MD_nonnull = 11, MD_type = 19, + MD_noundef = 29, } /// LLVMRustAsmDialect -#[derive(Copy, Clone)] +#[derive(Copy, Clone, PartialEq)] #[repr(C)] pub enum AsmDialect { Att, Intel, } -impl AsmDialect { - pub fn from_generic(asm: rustc_ast::LlvmAsmDialect) -> Self { - match asm { - rustc_ast::LlvmAsmDialect::Att => AsmDialect::Att, - rustc_ast::LlvmAsmDialect::Intel => AsmDialect::Intel, - } - } -} - /// LLVMRustCodeGenOptLevel #[derive(Copy, Clone, PartialEq)] #[repr(C)] @@ -562,12 +576,12 @@ pub enum PassKind { Module, } -/// LLVMRustThinLTOData +// LLVMRustThinLTOData extern "C" { pub type ThinLTOData; } -/// LLVMRustThinLTOBuffer +// LLVMRustThinLTOBuffer extern "C" { pub type ThinLTOBuffer; } @@ -631,6 +645,9 @@ extern "C" { pub type ConstantInt; } extern "C" { + pub type Attribute; +} +extern "C" { pub type Metadata; } extern "C" { @@ -976,6 +993,7 @@ pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); extern "C" { pub fn LLVMRustInstallFatalErrorHandler(); + pub fn LLVMRustDisableSystemDialogsOnCrash(); // Create and destroy contexts. pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; @@ -1155,6 +1173,21 @@ extern "C" { ) -> Option<&Value>; pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + // Operations on attributes + pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; + pub fn LLVMRustCreateAttrString(C: &Context, Name: *const c_char) -> &Attribute; + pub fn LLVMRustCreateAttrStringValue( + C: &Context, + Name: *const c_char, + Value: *const c_char, + ) -> &Attribute; + pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( M: &'a Module, @@ -1163,19 +1196,12 @@ extern "C" { FunctionTy: &'a Type, ) -> &'a Value; pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); - pub fn LLVMRustAddAlignmentAttr(Fn: &Value, index: c_uint, bytes: u32); - pub fn LLVMRustAddDereferenceableAttr(Fn: &Value, index: c_uint, bytes: u64); - pub fn LLVMRustAddDereferenceableOrNullAttr(Fn: &Value, index: c_uint, bytes: u64); - pub fn LLVMRustAddByValAttr(Fn: &Value, index: c_uint, ty: &Type); - pub fn LLVMRustAddStructRetAttr(Fn: &Value, index: c_uint, ty: &Type); - pub fn LLVMRustAddFunctionAttribute(Fn: &Value, index: c_uint, attr: Attribute); - pub fn LLVMRustAddFunctionAttrStringValue( - Fn: &Value, + pub fn LLVMRustAddFunctionAttributes<'a>( + Fn: &'a Value, index: c_uint, - Name: *const c_char, - Value: *const c_char, + Attrs: *const &'a Attribute, + AttrsLen: size_t, ); - pub fn LLVMRustRemoveFunctionAttributes(Fn: &Value, index: c_uint, attr: Attribute); // Operations on parameters pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; @@ -1196,13 +1222,12 @@ extern "C" { // Operations on call sites pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); - pub fn LLVMRustAddCallSiteAttribute(Instr: &Value, index: c_uint, attr: Attribute); - pub fn LLVMRustAddCallSiteAttrString(Instr: &Value, index: c_uint, Name: *const c_char); - pub fn LLVMRustAddAlignmentCallSiteAttr(Instr: &Value, index: c_uint, bytes: u32); - pub fn LLVMRustAddDereferenceableCallSiteAttr(Instr: &Value, index: c_uint, bytes: u64); - pub fn LLVMRustAddDereferenceableOrNullCallSiteAttr(Instr: &Value, index: c_uint, bytes: u64); - pub fn LLVMRustAddByValCallSiteAttr(Instr: &Value, index: c_uint, ty: &Type); - pub fn LLVMRustAddStructRetCallSiteAttr(Instr: &Value, index: c_uint, ty: &Type); + pub fn LLVMRustAddCallSiteAttributes<'a>( + Instr: &'a Value, + index: c_uint, + Attrs: *const &'a Attribute, + AttrsLen: size_t, + ); // Operations on load/store instructions (only) pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); @@ -1902,7 +1927,16 @@ extern "C" { pub fn LLVMRustVersionMinor() -> u32; pub fn LLVMRustVersionPatch() -> u32; - pub fn LLVMRustAddModuleFlag(M: &Module, name: *const c_char, value: u32); + /// Add LLVM module flags. + /// + /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What + /// "compatible" means depends on the merge behaviors involved. + pub fn LLVMRustAddModuleFlag( + M: &Module, + merge_behavior: LLVMModFlagBehavior, + name: *const c_char, + value: u32, + ); pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; @@ -2106,7 +2140,7 @@ extern "C" { Builder: &DIBuilder<'a>, Val: &'a Value, VarInfo: &'a DIVariable, - AddrOps: *const i64, + AddrOps: *const u64, AddrOpsCount: c_uint, DL: &'a DILocation, InsertAtEnd: &'a BasicBlock, @@ -2197,8 +2231,8 @@ extern "C" { Scope: &'a DIScope, InlinedAt: Option<&'a DILocation>, ) -> &'a DILocation; - pub fn LLVMRustDIBuilderCreateOpDeref() -> i64; - pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> i64; + pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; + pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; #[allow(improper_ctypes)] pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); @@ -2313,6 +2347,8 @@ extern "C" { end_callback: SelfProfileAfterPassCallback, ExtraPasses: *const c_char, ExtraPassesLen: size_t, + LLVMPlugins: *const c_char, + LLVMPluginsLen: size_t, ) -> LLVMRustResult; pub fn LLVMRustPrintModule( M: &Module, @@ -2325,7 +2361,6 @@ extern "C" { pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - pub fn LLVMRustMarkAllFunctionsNounwind(M: &Module); pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; pub fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index a1117a11fc7..4892b8d4a84 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -31,20 +31,52 @@ impl LLVMRustResult { } } -pub fn AddFunctionAttrStringValue(llfn: &Value, idx: AttributePlace, attr: &CStr, value: &CStr) { +pub fn AddFunctionAttributes<'ll>(llfn: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute]) { unsafe { - LLVMRustAddFunctionAttrStringValue(llfn, idx.as_uint(), attr.as_ptr(), value.as_ptr()) + LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len()); } } -pub fn AddFunctionAttrString(llfn: &Value, idx: AttributePlace, attr: &CStr) { +pub fn AddCallSiteAttributes<'ll>( + callsite: &'ll Value, + idx: AttributePlace, + attrs: &[&'ll Attribute], +) { unsafe { - LLVMRustAddFunctionAttrStringValue(llfn, idx.as_uint(), attr.as_ptr(), std::ptr::null()) + LLVMRustAddCallSiteAttributes(callsite, idx.as_uint(), attrs.as_ptr(), attrs.len()); } } -pub fn AddCallSiteAttrString(callsite: &Value, idx: AttributePlace, attr: &CStr) { - unsafe { LLVMRustAddCallSiteAttrString(callsite, idx.as_uint(), attr.as_ptr()) } +pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &CStr, value: &CStr) -> &'ll Attribute { + unsafe { LLVMRustCreateAttrStringValue(llcx, attr.as_ptr(), value.as_ptr()) } +} + +pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &CStr) -> &'ll Attribute { + unsafe { LLVMRustCreateAttrStringValue(llcx, attr.as_ptr(), std::ptr::null()) } +} + +pub fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) } +} + +pub fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) } +} + +pub fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) } +} + +pub fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateByValAttr(llcx, ty) } +} + +pub fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } +} + +pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { + unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } } #[derive(Copy, Clone)] @@ -128,25 +160,10 @@ pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { } } -impl Attribute { - pub fn apply_llfn(&self, idx: AttributePlace, llfn: &Value) { - unsafe { LLVMRustAddFunctionAttribute(llfn, idx.as_uint(), *self) } - } - - pub fn apply_callsite(&self, idx: AttributePlace, callsite: &Value) { - unsafe { LLVMRustAddCallSiteAttribute(callsite, idx.as_uint(), *self) } - } - - pub fn unapply_llfn(&self, idx: AttributePlace, llfn: &Value) { - unsafe { LLVMRustRemoveFunctionAttributes(llfn, idx.as_uint(), *self) } - } - - pub fn toggle_llfn(&self, idx: AttributePlace, llfn: &Value, set: bool) { - if set { - self.apply_llfn(idx, llfn); - } else { - self.unapply_llfn(idx, llfn); - } +impl AttributeKind { + /// Create an LLVM Attribute with no associated value. + pub fn create_attr(self, llcx: &Context) -> &Attribute { + unsafe { LLVMRustCreateAttrNoValue(llcx, self) } } } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index e4935ac431c..3b06587061d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -2,14 +2,18 @@ use crate::back::write::create_informational_target_machine; use crate::{llvm, llvm_util}; use libc::c_int; use libloading::Library; -use rustc_codegen_ssa::target_features::supported_target_features; -use rustc_data_structures::fx::FxHashSet; +use rustc_codegen_ssa::target_features::{ + supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, +}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::small_c_str::SmallCStr; use rustc_fs_util::path_to_c_string; use rustc_middle::bug; use rustc_session::config::PrintRequest; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; +use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; use tracing::debug; @@ -46,6 +50,12 @@ unsafe fn configure_llvm(sess: &Session) { let mut llvm_args = Vec::with_capacity(n_args + 1); llvm::LLVMRustInstallFatalErrorHandler(); + // On Windows, an LLVM assertion will open an Abort/Retry/Ignore dialog + // box for the purpose of launching a debugger. However, on CI this will + // cause it to hang until it times out, which can take several hours. + if std::env::var_os("CI").is_some() { + llvm::LLVMRustDisableSystemDialogsOnCrash(); + } fn llvm_arg_to_arg_name(full_arg: &str) -> &str { full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("") @@ -121,14 +131,20 @@ unsafe fn configure_llvm(sess: &Session) { llvm::LLVMInitializePasses(); - // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.debugging_opts.llvm_plugins { - let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); - debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); - - // Intentionally leak the dynamic library. We can't ever unload it - // since the library can make things that will live arbitrarily long. - mem::forget(lib); + // Use the legacy plugin registration if we don't use the new pass manager + if !should_use_new_llvm_pass_manager( + &sess.opts.debugging_opts.new_llvm_pass_manager, + &sess.target.arch, + ) { + // Register LLVM plugins by loading them into the compiler process. + for plugin in &sess.opts.debugging_opts.llvm_plugins { + let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); + debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); + + // Intentionally leak the dynamic library. We can't ever unload it + // since the library can make things that will live arbitrarily long. + mem::forget(lib); + } } rustc_llvm::initialize_available_targets(); @@ -143,9 +159,10 @@ pub fn time_trace_profiler_finish(file_name: &Path) { } } -// WARNING: the features after applying `to_llvm_feature` must be known +// WARNING: the features after applying `to_llvm_features` must be known // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. +// // To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def // where the * matches the architecture's name // Beware to not use the llvm github project for this, but check the git submodule @@ -153,56 +170,83 @@ pub fn time_trace_profiler_finish(file_name: &Path) { // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> Vec<&'a str> { +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { ("x86", "sse4.2") => { if get_version() >= (14, 0, 0) { - vec!["sse4.2", "crc32"] + smallvec!["sse4.2", "crc32"] } else { - vec!["sse4.2"] + smallvec!["sse4.2"] } } - ("x86", "pclmulqdq") => vec!["pclmul"], - ("x86", "rdrand") => vec!["rdrnd"], - ("x86", "bmi1") => vec!["bmi"], - ("x86", "cmpxchg16b") => vec!["cx16"], - ("x86", "avx512vaes") => vec!["vaes"], - ("x86", "avx512gfni") => vec!["gfni"], - ("x86", "avx512vpclmulqdq") => vec!["vpclmulqdq"], - ("aarch64", "fp") => vec!["fp-armv8"], - ("aarch64", "fp16") => vec!["fullfp16"], - ("aarch64", "fhm") => vec!["fp16fml"], - ("aarch64", "rcpc2") => vec!["rcpc-immo"], - ("aarch64", "dpb") => vec!["ccpp"], - ("aarch64", "dpb2") => vec!["ccdp"], - ("aarch64", "frintts") => vec!["fptoint"], - ("aarch64", "fcma") => vec!["complxnum"], - ("aarch64", "pmuv3") => vec!["perfmon"], - (_, s) => vec![s], + ("x86", "pclmulqdq") => smallvec!["pclmul"], + ("x86", "rdrand") => smallvec!["rdrnd"], + ("x86", "bmi1") => smallvec!["bmi"], + ("x86", "cmpxchg16b") => smallvec!["cx16"], + ("x86", "avx512vaes") => smallvec!["vaes"], + ("x86", "avx512gfni") => smallvec!["gfni"], + ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], + ("aarch64", "fp") => smallvec!["fp-armv8"], + ("aarch64", "fp16") => smallvec!["fullfp16"], + ("aarch64", "fhm") => smallvec!["fp16fml"], + ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], + ("aarch64", "dpb") => smallvec!["ccpp"], + ("aarch64", "dpb2") => smallvec!["ccdp"], + ("aarch64", "frintts") => smallvec!["fptoint"], + ("aarch64", "fcma") => smallvec!["complxnum"], + ("aarch64", "pmuv3") => smallvec!["perfmon"], + ("aarch64", "paca") => smallvec!["pauth"], + ("aarch64", "pacg") => smallvec!["pauth"], + (_, s) => smallvec![s], + } +} + +// Given a map from target_features to whether they are enabled or disabled, +// ensure only valid combinations are allowed. +pub fn check_tied_features( + sess: &Session, + features: &FxHashMap<&str, bool>, +) -> Option<&'static [&'static str]> { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } } + None } pub fn target_features(sess: &Session) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); - supported_target_features(sess) - .iter() - .filter_map( - |&(feature, gate)| { + let mut features: Vec<Symbol> = + supported_target_features(sess) + .iter() + .filter_map(|&(feature, gate)| { if sess.is_nightly_build() || gate.is_none() { Some(feature) } else { None } - }, - ) - .filter(|feature| { - for llvm_feature in to_llvm_feature(sess, feature) { - let cstr = CString::new(llvm_feature).unwrap(); - if unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { - return true; + }) + .filter(|feature| { + for llvm_feature in to_llvm_features(sess, feature) { + let cstr = SmallCStr::new(llvm_feature); + if unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return true; + } } - } - false - }) - .map(|feature| Symbol::intern(feature)) - .collect() + false + }) + .map(|feature| Symbol::intern(feature)) + .collect(); + + // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 + // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use + // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI. + // The target feature can be dropped once we no longer support older LLVM versions. + if sess.is_nightly_build() && get_version() >= (14, 0, 0) { + features.push(Symbol::intern("llvm14-builtins-abi")); + } + features } pub fn print_version() { @@ -252,9 +296,9 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { let mut rustc_target_features = supported_target_features(sess) .iter() .filter_map(|(feature, _gate)| { - for llvm_feature in to_llvm_feature(sess, *feature) { + for llvm_feature in to_llvm_features(sess, *feature) { // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. - match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| (*f)).ok().map( + match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok().map( |index| { let (_f, desc) = target_features.remove(index); (*feature, desc) @@ -324,14 +368,7 @@ pub fn target_cpu(sess: &Session) -> &str { /// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, /// `--target` and similar). -// FIXME(nagisa): Cache the output of this somehow? Maybe make this a query? We're calling this -// for every function that has `#[target_feature]` on it. The global features won't change between -// the functions; only crates, maybe… -pub fn llvm_global_features(sess: &Session) -> Vec<String> { - // FIXME(nagisa): this should definitely be available more centrally and to other codegen backends. - /// These features control behaviour of rustc rather than llvm. - const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; - +pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<String> { // Features that come earlier are overriden by conflicting features later in the string. // Typically we'll want more explicit settings to override the implicit ones, so: // @@ -377,37 +414,121 @@ pub fn llvm_global_features(sess: &Session) -> Vec<String> { Some(_) | None => {} }; - let filter = |s: &str| { - if s.is_empty() { - return vec![]; + // Features implied by an implicit or explicit `--target`. + features.extend( + sess.target + .features + .split(',') + .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + .map(String::from), + ); + + // -Ctarget-features + let supported_features = supported_target_features(sess); + let feats = sess + .opts + .cg + .target_feature + .split(',') + .filter_map(|s| { + let enable_disable = match s.chars().next() { + None => return None, + Some(c @ '+' | c @ '-') => c, + Some(_) => { + if diagnostics { + let mut diag = sess.struct_warn(&format!( + "unknown feature specified for `-Ctarget-feature`: `{}`", + s + )); + diag.note("features must begin with a `+` to enable or `-` to disable it"); + diag.emit(); + } + return None; + } + }; + + let feature = backend_feature_name(s)?; + // Warn against use of LLVM specific feature names on the CLI. + if diagnostics && !supported_features.iter().any(|&(v, _)| v == feature) { + let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { + let llvm_features = to_llvm_features(sess, rust_feature); + if llvm_features.contains(&feature) && !llvm_features.contains(&rust_feature) { + Some(rust_feature) + } else { + None + } + }); + let mut diag = sess.struct_warn(&format!( + "unknown feature specified for `-Ctarget-feature`: `{}`", + feature + )); + diag.note("it is still passed through to the codegen backend"); + if let Some(rust_feature) = rust_feature { + diag.help(&format!("you might have meant: `{}`", rust_feature)); + } else { + diag.note("consider filing a feature request"); + } + diag.emit(); + } + Some((enable_disable, feature)) + }) + .collect::<SmallVec<[(char, &str); 8]>>(); + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + let featmap = feats.iter().map(|&(flag, feat)| (feat, flag == '+')).collect(); + if let Some(f) = check_tied_features(sess, &featmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); } - let feature = if s.starts_with('+') || s.starts_with('-') { - &s[1..] - } else { - return vec![s.to_string()]; - }; - // Rustc-specific feature requests like `+crt-static` or `-crt-static` - // are not passed down to LLVM. + } + + features.extend(feats.into_iter().flat_map(|(enable_disable, feature)| { + // rustc-specific features do not get passed down to LLVM… if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return vec![]; + return SmallVec::<[_; 2]>::new(); } - // ... otherwise though we run through `to_llvm_feature` feature when + // ... otherwise though we run through `to_llvm_feature when // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. - to_llvm_feature(sess, feature).iter().map(|f| format!("{}{}", &s[..1], f)).collect() - }; - - // Features implied by an implicit or explicit `--target`. - features.extend(sess.target.features.split(',').flat_map(&filter)); - - // -Ctarget-features - features.extend(sess.opts.cg.target_feature.split(',').flat_map(&filter)); - + to_llvm_features(sess, feature) + .into_iter() + .map(|f| format!("{}{}", enable_disable, f)) + .collect() + })); features } +/// Returns a feature name for the given `+feature` or `-feature` string. +/// +/// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) +fn backend_feature_name(s: &str) -> Option<&str> { + // features must start with a `+` or `-`. + let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { + bug!("target feature `{}` must begin with a `+` or `-`", s); + }); + // Rustc-specific feature requests like `+crt-static` or `-crt-static` + // are not passed down to LLVM. + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + Some(feature) +} + pub fn tune_cpu(sess: &Session) -> Option<&str> { let name = sess.opts.debugging_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } + +pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_arch: &str) -> bool { + // The new pass manager is enabled by default for LLVM >= 13. + // This matches Clang, which also enables it since Clang 13. + + // FIXME: There are some perf issues with the new pass manager + // when targeting s390x, so it is temporarily disabled for that + // arch, see https://github.com/rust-lang/rust/issues/89609 + user_opt.unwrap_or_else(|| target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0)) +} diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index f8c919ec2aa..da378dc6493 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,5 +1,6 @@ use crate::common::*; use crate::context::TypeLowering; +use crate::llvm_util::get_version; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; @@ -42,14 +43,18 @@ fn uncached_llvm_type<'a, 'tcx>( // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str => { - let mut name = - with_no_visible_paths(|| with_no_trimmed_paths(|| layout.ty.to_string())); + ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str + // For performance reasons we use names only when emitting LLVM IR. Unless we are on + // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213, + // #79564, and #79246. + if get_version() < (14, 0, 0) || !cx.sess().fewer_names() => + { + let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string())); if let (&ty::Adt(def, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { if def.is_enum() && !def.variants.is_empty() { - write!(&mut name, "::{}", def.variants[index].ident).unwrap(); + write!(&mut name, "::{}", def.variants[index].name).unwrap(); } } if let (&ty::Generator(_, _, _), &Variants::Single { index }) = @@ -59,6 +64,9 @@ fn uncached_llvm_type<'a, 'tcx>( } Some(name) } + // Use identified structure types for ADT. Due to pointee types in LLVM IR their definition + // might be recursive. Other cases are non-recursive and we can use literal structure types. + ty::Adt(..) => Some(String::new()), _ => None, }; @@ -330,16 +338,17 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { ty::Ref(..) | ty::RawPtr(_) => { return self.field(cx, index).llvm_type(cx); } - ty::Adt(def, _) if def.is_box() => { + // only wide pointer boxes are handled as pointers + // thin pointer boxes with scalar allocators are handled by the general logic below + ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); } _ => {} } - let (a, b) = match self.abi { - Abi::ScalarPair(a, b) => (a, b), - _ => bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self), + let Abi::ScalarPair(a, b) = self.abi else { + bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); }; let scalar = [a, b][index]; diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index f090ae6ecb4..ceb3d5a84ab 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -102,10 +102,10 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( let va_list_ty = va_list_layout.llvm_type(bx); let layout = bx.cx.layout_of(target_ty); - let mut maybe_reg = bx.build_sibling_block("va_arg.maybe_reg"); - let mut in_reg = bx.build_sibling_block("va_arg.in_reg"); - let mut on_stack = bx.build_sibling_block("va_arg.on_stack"); - let mut end = bx.build_sibling_block("va_arg.end"); + let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let on_stack = bx.append_sibling_block("va_arg.on_stack"); + let end = bx.append_sibling_block("va_arg.end"); let zero = bx.const_i32(0); let offset_align = Align::from_bytes(4).unwrap(); @@ -125,53 +125,53 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( // if the offset >= 0 then the value will be on the stack let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align); let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero); - bx.cond_br(use_stack, on_stack.llbb(), maybe_reg.llbb()); + bx.cond_br(use_stack, on_stack, maybe_reg); // The value at this point might be in a register, but there is a chance that // it could be on the stack so we have to update the offset and then check // the offset again. + bx.switch_to_block(maybe_reg); if gr_type && layout.align.abi.bytes() > 8 { - reg_off_v = maybe_reg.add(reg_off_v, bx.const_i32(15)); - reg_off_v = maybe_reg.and(reg_off_v, bx.const_i32(-16)); + reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); + reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); } - let new_reg_off_v = maybe_reg.add(reg_off_v, bx.const_i32(slot_size as i32)); + let new_reg_off_v = bx.add(reg_off_v, bx.const_i32(slot_size as i32)); - maybe_reg.store(new_reg_off_v, reg_off, offset_align); + bx.store(new_reg_off_v, reg_off, offset_align); // Check to see if we have overflowed the registers as a result of this. // If we have then we need to use the stack for this value - let use_stack = maybe_reg.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); - maybe_reg.cond_br(use_stack, on_stack.llbb(), in_reg.llbb()); + let use_stack = bx.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); + bx.cond_br(use_stack, on_stack, in_reg); + bx.switch_to_block(in_reg); let top_type = bx.type_i8p(); - let top = in_reg.struct_gep(va_list_ty, va_list_addr, reg_top_index); - let top = in_reg.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); + let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); + let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); // reg_value = *(@top + reg_off_v); - let mut reg_addr = in_reg.gep(bx.type_i8(), top, &[reg_off_v]); + let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { // On big-endian systems the value is right-aligned in its slot. let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); - reg_addr = in_reg.gep(bx.type_i8(), reg_addr, &[offset]); + reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); } let reg_type = layout.llvm_type(bx); - let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); - let reg_value = in_reg.load(reg_type, reg_addr, layout.align.abi); - in_reg.br(end.llbb()); + let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); + let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); + bx.br(end); // On Stack block + bx.switch_to_block(on_stack); let stack_value = - emit_ptr_va_arg(&mut on_stack, list, target_ty, false, Align::from_bytes(8).unwrap(), true); - on_stack.br(end.llbb()); + emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true); + bx.br(end); - let val = end.phi( - layout.immediate_llvm_type(bx), - &[reg_value, stack_value], - &[in_reg.llbb(), on_stack.llbb()], - ); + bx.switch_to_block(end); + let val = + bx.phi(layout.immediate_llvm_type(bx), &[reg_value, stack_value], &[in_reg, on_stack]); - *bx = end; val } |