diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
33 files changed, 1521 insertions, 869 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b9baa87bac7..0ce161d7e75 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -37,7 +37,7 @@ fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { // LLVM prior to version 12 had known miscompiles in the presence of // noalias attributes (see #54878), but we don't support earlier // versions at all anymore. We now enable mutable noalias by default. - cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) + cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) } const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = @@ -213,7 +213,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) } else if self.is_unsized_indirect() { bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast(cast) = self.mode { + } else if let PassMode::Cast(cast, _) = &self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. let can_store_through_cast_ptr = false; @@ -283,7 +283,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(_) | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } - | PassMode::Cast(_) => { + | PassMode::Cast(..) => { let next_arg = next(); self.store(bx, next_arg, dst); } @@ -325,20 +325,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { // Ignore "extra" args from the call site for C variadic functions. // Only the "fixed" args are part of the LLVM function signature. - let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + let args = + if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args }; - let args_capacity: usize = args.iter().map(|arg| - if arg.pad.is_some() { 1 } else { 0 } + - if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } - ).sum(); + // This capacity calculation is approximate. let mut llargument_tys = Vec::with_capacity( - if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, + self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }, ); - let llreturn_ty = match self.ret.mode { + let llreturn_ty = match &self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, _) => cast.llvm_type(cx), PassMode::Indirect { .. } => { llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); cx.type_void() @@ -346,12 +344,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { }; for arg in args { - // add padding - if let Some(ty) = arg.pad { - llargument_tys.push(ty.llvm_type(cx)); - } - - let llarg_ty = match arg.mode { + let llarg_ty = match &arg.mode { PassMode::Ignore => continue, PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), PassMode::Pair(..) => { @@ -366,7 +359,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, pad_i32) => { + // add padding + if *pad_i32 { + llargument_tys.push(Reg::i32().llvm_type(cx)); + } + cast.llvm_type(cx) + } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { cx.type_ptr_to(arg.memory_ty(cx)) } @@ -393,6 +392,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_cconv(&self) -> llvm::CallConv { match self.conv { Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, @@ -425,46 +425,46 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } _ => {} } - for arg in &self.args { - if arg.pad.is_some() { - apply(&ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(attrs); let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(attrs); } - PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { assert!(!on_stack); apply(attrs); apply(extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(a); apply(b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(&ArgAttributes::new()); + } apply(&cast.attrs); } } @@ -487,17 +487,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_callsite( llvm::AttributePlace::ReturnValue, &bx.cx, @@ -516,13 +516,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } } - for arg in &self.args { - if arg.pad.is_some() { - apply(bx.cx, &ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); attributes::apply_to_callsite( @@ -531,23 +528,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[byval], ); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(bx.cx, attrs); } - PassMode::Indirect { - ref attrs, - extra_attrs: Some(ref extra_attrs), - on_stack: _, - } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack: _ } => { apply(bx.cx, attrs); apply(bx.cx, extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(bx.cx, a); apply(bx.cx, b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(bx.cx, &ArgAttributes::new()); + } apply(bx.cx, &cast.attrs); } } @@ -568,6 +564,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[cmse_nonsecure_call], ); } + + // Some intrinsics require that an elementtype attribute (with the pointee type of a + // pointer argument) is added to the callsite. + let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; + if element_type_index >= 0 { + let arg_ty = self.args[element_type_index as usize].layout.ty; + let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; + let element_type_attr = unsafe { + llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) + }; + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(element_type_index as u32), + &[element_type_attr], + ); + } } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index f935acb1a7e..72961ae888e 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -145,7 +145,7 @@ pub(crate) unsafe fn codegen( if tcx.sess.target.default_hidden_visibility { llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); } - let val = tcx.sess.opts.debugging_opts.oom.should_panic(); + let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a53946995ee..5202ac697e9 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -3,7 +3,6 @@ use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -20,7 +19,6 @@ use rustc_target::asm::*; use libc::{c_char, c_uint}; use smallvec::SmallVec; -use tracing::debug; impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_inline_asm( @@ -419,13 +417,6 @@ pub(crate) fn inline_asm_call<'ll>( let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { - if unwind && llvm_util::get_version() < (13, 0, 0) { - bx.cx.sess().span_fatal( - line_spans[0], - "unwinding from inline assembly is only supported on llvm >= 13.", - ); - } - let v = llvm::LLVMRustInlineAsm( fty, asm.as_ptr().cast(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 9394d60134f..aabbe8ac276 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -13,7 +13,7 @@ use smallvec::SmallVec; use crate::attributes; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, Attribute, AttributeKind, AttributePlace}; +use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace}; use crate::llvm_util; pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; @@ -56,7 +56,7 @@ pub fn sanitize_attrs<'ll>( no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.debugging_opts.sanitizer - no_sanitize; + let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -69,6 +69,9 @@ pub fn sanitize_attrs<'ll>( if enabled.contains(SanitizerSet::HWADDRESS) { attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); } + if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { + attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); + } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. let features = cx.tcx.global_backend_features(()); @@ -136,7 +139,7 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { if cx .sess() .opts - .debugging_opts + .unstable_opts .sanitizer .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) { @@ -149,7 +152,7 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with gcov profiling. - if cx.sess().opts.debugging_opts.profile { + if cx.sess().opts.unstable_opts.profile { return None; } @@ -224,6 +227,10 @@ pub(crate) fn default_optimisation_attrs<'ll>( attrs } +fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { + llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") +} + /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub fn from_fn_attrs<'ll, 'tcx>( @@ -275,7 +282,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( to_add.push(uwtable_attr(cx.llcx)); } - if cx.sess().opts.debugging_opts.profile_sample_use.is_some() { + if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } @@ -299,12 +306,61 @@ pub fn from_fn_attrs<'ll, 'tcx>( } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); + // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. + // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. + // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 + to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); + // Need this for AArch64. + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) + || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) + { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; + } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + } // apply to return place instead of function (unlike all other attributes applied in this function) let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); + } + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + } + } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 8f6438e85ad..38a366095b4 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -8,26 +8,18 @@ use std::path::{Path, PathBuf}; use std::ptr; use std::str; +use crate::common; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; -use rustc_codegen_ssa::back::archive::ArchiveBuilder; -use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_session::cstore::{DllCallingConvention, DllImport}; +use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; +use rustc_session::cstore::DllImport; use rustc_session::Session; -struct ArchiveConfig<'a> { - pub sess: &'a Session, - pub dst: PathBuf, - pub src: Option<PathBuf>, -} - /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] pub struct LlvmArchiveBuilder<'a> { - config: ArchiveConfig<'a>, - removals: Vec<String>, + sess: &'a Session, additions: Vec<Addition>, - src_archive: Option<Option<ArchiveRO>>, } enum Addition { @@ -50,10 +42,6 @@ fn is_relevant_child(c: &Child<'_>) -> bool { } } -fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) -> ArchiveConfig<'a> { - ArchiveConfig { sess, dst: output.to_path_buf(), src: input.map(|p| p.to_path_buf()) } -} - /// Map machine type strings to values of LLVM's MachineTypes enum. fn llvm_machine_type(cpu: &str) -> LLVMMachineType { match cpu { @@ -66,45 +54,11 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { } impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { - /// Creates a new static archive, ready for modifying the archive specified - /// by `config`. - fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> LlvmArchiveBuilder<'a> { - let config = archive_config(sess, output, input); - LlvmArchiveBuilder { - config, - removals: Vec::new(), - additions: Vec::new(), - src_archive: None, - } - } - - /// Removes a file from this archive - fn remove_file(&mut self, file: &str) { - self.removals.push(file.to_string()); - } - - /// Lists all files in an archive - fn src_files(&mut self) -> Vec<String> { - if self.src_archive().is_none() { - return Vec::new(); - } - - let archive = self.src_archive.as_ref().unwrap().as_ref().unwrap(); - - archive - .iter() - .filter_map(|child| child.ok()) - .filter(is_relevant_child) - .filter_map(|child| child.name()) - .filter(|name| !self.removals.iter().any(|x| x == name)) - .map(|name| name.to_owned()) - .collect() - } - - fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> - where - F: FnMut(&str) -> bool + 'static, - { + fn add_archive( + &mut self, + archive: &Path, + skip: Box<dyn FnMut(&str) -> bool + 'static>, + ) -> io::Result<()> { let archive_ro = match ArchiveRO::open(archive) { Ok(ar) => ar, Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), @@ -129,44 +83,47 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. - fn build(mut self) { - let kind = self.llvm_archive_kind().unwrap_or_else(|kind| { - self.config.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) - }); - - if let Err(e) = self.build_with_llvm(kind) { - self.config.sess.fatal(&format!("failed to build archive: {}", e)); + fn build(mut self: Box<Self>, output: &Path) -> bool { + match self.build_with_llvm(output) { + Ok(any_members) => any_members, + Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), } } +} - fn inject_dll_import_lib( - &mut self, +pub struct LlvmArchiveBuilderBuilder; + +impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { + Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + } + + fn create_dll_import_lib( + &self, + sess: &Session, lib_name: &str, dll_imports: &[DllImport], - tmpdir: &MaybeTempDir, - ) { + tmpdir: &Path, + ) -> PathBuf { let output_path = { - let mut output_path: PathBuf = tmpdir.as_ref().to_path_buf(); + let mut output_path: PathBuf = tmpdir.to_path_buf(); output_path.push(format!("{}_imports", lib_name)); output_path.with_extension("lib") }; - let target = &self.config.sess.target; - let mingw_gnu_toolchain = target.vendor == "pc" - && target.os == "windows" - && target.env == "gnu" - && target.abi.is_empty(); + let target = &sess.target; + let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(target); let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { - if self.config.sess.target.arch == "x86" { + if sess.target.arch == "x86" { ( - LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), - import.ordinal, + common::i686_decorated_name(import, mingw_gnu_toolchain, false), + import.ordinal(), ) } else { - (import.name.to_string(), import.ordinal) + (import.name.to_string(), import.ordinal()) } }) .collect(); @@ -177,8 +134,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { // that loaded but crashed with an AV upon calling one of the imported // functions. Therefore, use binutils to create the import library instead, // by writing a .DEF file to the temp dir and calling binutils's dlltool. - let def_file_path = - tmpdir.as_ref().join(format!("{}_imports", lib_name)).with_extension("def"); + let def_file_path = tmpdir.join(format!("{}_imports", lib_name)).with_extension("def"); let def_file_content = format!( "EXPORTS\n{}", @@ -197,11 +153,14 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - self.config.sess.fatal(&format!("Error writing .DEF file: {}", e)); + sess.fatal(&format!("Error writing .DEF file: {}", e)); } }; - let dlltool = find_binutils_dlltool(self.config.sess); + // --no-leading-underscore: For the `import_name_type` feature to work, we need to be + // able to control the *exact* spelling of each of the symbols that are being imported: + // hence we don't want `dlltool` adding leading underscores automatically. + let dlltool = find_binutils_dlltool(sess); let result = std::process::Command::new(dlltool) .args([ "-d", @@ -210,14 +169,15 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { lib_name, "-l", output_path.to_str().unwrap(), + "--no-leading-underscore", ]) .output(); match result { Err(e) => { - self.config.sess.fatal(&format!("Error calling dlltool: {}", e)); + sess.fatal(&format!("Error calling dlltool: {}", e)); } - Ok(output) if !output.status.success() => self.config.sess.fatal(&format!( + Ok(output) if !output.status.success() => sess.fatal(&format!( "Dlltool could not create import library: {}\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) @@ -230,10 +190,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { let output_path_z = rustc_fs_util::path_to_c_string(&output_path); - tracing::trace!("invoking LLVMRustWriteImportLibrary"); - tracing::trace!(" dll_name {:#?}", dll_name_z); - tracing::trace!(" output_path {}", output_path.display()); - tracing::trace!( + trace!("invoking LLVMRustWriteImportLibrary"); + trace!(" dll_name {:#?}", dll_name_z); + trace!(" output_path {}", output_path.display()); + trace!( " import names: {}", dll_imports .iter() @@ -263,13 +223,13 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path_z.as_ptr(), ffi_exports.as_ptr(), ffi_exports.len(), - llvm_machine_type(&self.config.sess.target.arch) as u16, - !self.config.sess.target.is_like_msvc, + llvm_machine_type(&sess.target.arch) as u16, + !sess.target.is_like_msvc, ) }; if result == crate::llvm::LLVMRustResult::Failure { - self.config.sess.fatal(&format!( + sess.fatal(&format!( "Error creating import library for {}: {}", lib_name, llvm::last_error().unwrap_or("unknown LLVM error".to_string()) @@ -277,57 +237,24 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { } }; - self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { - self.config.sess.fatal(&format!( - "failed to add native library {}: {}", - output_path.display(), - e - )); - }); + output_path } } impl<'a> LlvmArchiveBuilder<'a> { - fn src_archive(&mut self) -> Option<&ArchiveRO> { - if let Some(ref a) = self.src_archive { - return a.as_ref(); - } - let src = self.config.src.as_ref()?; - self.src_archive = Some(ArchiveRO::open(src).ok()); - self.src_archive.as_ref().unwrap().as_ref() - } - - fn llvm_archive_kind(&self) -> Result<ArchiveKind, &str> { - let kind = &*self.config.sess.target.archive_format; - kind.parse().map_err(|_| kind) - } + fn build_with_llvm(&mut self, output: &Path) -> io::Result<bool> { + let kind = &*self.sess.target.archive_format; + let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| { + self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) + }); - fn build_with_llvm(&mut self, kind: ArchiveKind) -> io::Result<()> { - let removals = mem::take(&mut self.removals); let mut additions = mem::take(&mut self.additions); let mut strings = Vec::new(); let mut members = Vec::new(); - let dst = CString::new(self.config.dst.to_str().unwrap())?; + let dst = CString::new(output.to_str().unwrap())?; unsafe { - if let Some(archive) = self.src_archive() { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - let Some(child_name) = child.name() else { continue }; - if removals.iter().any(|r| r == child_name) { - continue; - } - - let name = CString::new(child_name)?; - members.push(llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - )); - strings.push(name); - } - } for addition in &mut additions { match addition { Addition::File { path, name_in_archive } => { @@ -389,7 +316,7 @@ impl<'a> LlvmArchiveBuilder<'a> { }; Err(io::Error::new(io::ErrorKind::Other, msg)) } else { - Ok(()) + Ok(!members.is_empty()) }; for member in members { llvm::LLVMRustArchiveMemberFree(member); @@ -397,22 +324,6 @@ impl<'a> LlvmArchiveBuilder<'a> { ret } } - - fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { - let name = import.name; - let prefix = if mingw { "" } else { "_" }; - - match import.calling_convention { - DllCallingConvention::C => format!("{}{}", prefix, name), - DllCallingConvention::Stdcall(arg_list_size) => { - format!("{}{}@{}", prefix, name, arg_list_size) - } - DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), - DllCallingConvention::Vectorcall(arg_list_size) => { - format!("{}@@{}", name, arg_list_size) - } - } - } } fn string_to_io_error(s: String) -> io::Error { @@ -421,7 +332,7 @@ fn string_to_io_error(s: String) -> io::Error { fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.debugging_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { return dlltool_path.clone().into_os_string(); } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index d0724baf9e0..a89df00e248 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,15 +1,16 @@ use crate::back::write::{ self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, }; -use crate::llvm::archive_ro::ArchiveRO; use crate::llvm::{self, build_string, False, True}; use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; +use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; @@ -17,7 +18,6 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; -use tracing::{debug, info}; use std::ffi::{CStr, CString}; use std::fs::File; @@ -107,14 +107,24 @@ fn prepare_lto( .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); } - let archive = ArchiveRO::open(path).expect("wanted an rlib"); + let archive_data = unsafe { + Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib")) + .expect("couldn't map rlib") + }; + let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib"); let obj_files = archive - .iter() - .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) + .members() + .filter_map(|child| { + child.ok().and_then(|c| { + std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c)) + }) + }) .filter(|&(name, _)| looks_like_rust_object_file(name)); for (name, child) in obj_files { info!("adding bitcode from {}", name); - match get_bitcode_slice_from_object_data(child.data()) { + match get_bitcode_slice_from_object_data( + child.data(&*archive_data).expect("corrupt rlib"), + ) { Ok(data) => { let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); @@ -199,7 +209,7 @@ pub(crate) fn run_thin( pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { let name = module.name.clone(); - let buffer = ThinBuffer::new(module.module_llvm.llmod()); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -325,20 +335,6 @@ fn fat_lto( drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); - // Fat LTO also suffers from the invalid DWARF issue similar to Thin LTO. - // Here we rewrite all `DICompileUnit` pointers if there is only one `DICompileUnit`. - // This only works around the problem when codegen-units = 1. - // Refer to the comments in the `optimize_thin_module` function for more details. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - unsafe { llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2) }; - if !cu2.is_null() { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_fat_lto_patch_debuginfo", &*module.name); - unsafe { llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1) }; - save_temp_bitcode(cgcx, &module, "fat-lto-after-patch"); - } - // Internalize everything below threshold to help strip out more modules and such. unsafe { let ptr = symbols_below_threshold.as_ptr(); @@ -354,14 +350,14 @@ fn fat_lto( Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) } -crate struct Linker<'a>(&'a mut llvm::Linker<'a>); +pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); impl<'a> Linker<'a> { - crate fn new(llmod: &'a llvm::Module) -> Self { + pub(crate) fn new(llmod: &'a llvm::Module) -> Self { unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } } - crate fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { + pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { unsafe { if llvm::LLVMRustLinkerAdd( self.0, @@ -586,9 +582,21 @@ pub(crate) fn run_pass_manager( // LTO-specific optimization passes that LLVM provides. // // This code is based off the code found in llvm's LTO code generator: - // tools/lto/LTOCodeGenerator.cpp + // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { + if !llvm::LLVMRustHasModuleFlag( + module.module_llvm.llmod(), + "LTOPostLink".as_ptr().cast(), + 11, + ) { + llvm::LLVMRustAddModuleFlag( + module.module_llvm.llmod(), + llvm::LLVMModFlagBehavior::Error, + "LTOPostLink\0".as_ptr().cast(), + 1, + ); + } if llvm_util::should_use_new_llvm_pass_manager( &config.new_llvm_pass_manager, &cgcx.target_arch, @@ -697,9 +705,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub fn new(m: &llvm::Module) -> ThinBuffer { + pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); ThinBuffer(buffer) } } @@ -757,7 +765,7 @@ pub unsafe fn optimize_thin_module( // an error. let mut cu1 = ptr::null_mut(); let mut cu2 = ptr::null_mut(); - llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); + llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); if !cu2.is_null() { let msg = "multiple source DICompileUnits found"; return Err(write::llvm_err(&diag_handler, msg)); @@ -846,7 +854,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1); + llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 99e30531c22..a695df8409b 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -5,7 +5,7 @@ use crate::back::profiling::{ use crate::base; use crate::common; use crate::consts; -use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; +use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; @@ -28,7 +28,6 @@ use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; -use tracing::debug; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; @@ -56,28 +55,24 @@ pub fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { + debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { let output_c = path_to_c_string(output); - let result = if let Some(dwo_output) = dwo_output { - let dwo_output_c = path_to_c_string(dwo_output); - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - dwo_output_c.as_ptr(), - file_type, - ) + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() } else { - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - std::ptr::null(), - file_type, - ) + std::ptr::null() }; + let result = llvm::LLVMRustWriteOutputFile( + target, + pm, + m, + output_c.as_ptr(), + dwo_output_ptr, + file_type, + ); // Record artifact sizes for self-profiling if result == llvm::LLVMRustResult::Success { @@ -111,7 +106,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { tcx.output_filenames(()).split_dwarf_path( tcx.sess.split_debuginfo(), - tcx.sess.opts.debugging_opts.split_dwarf_kind, + tcx.sess.opts.unstable_opts.split_dwarf_kind, Some(mod_name), ) } else { @@ -186,9 +181,9 @@ pub fn target_machine_factory( let use_softfp = sess.opts.cg.soft_float; let ffunction_sections = - sess.opts.debugging_opts.function_sections.unwrap_or(sess.target.function_sections); + sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); let fdata_sections = ffunction_sections; - let funique_section_names = !sess.opts.debugging_opts.no_unique_section_names; + let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names; let code_model = to_llvm_code_model(sess.code_model()); @@ -206,15 +201,15 @@ pub fn target_machine_factory( let features = CString::new(target_features.join(",")).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); let trap_unreachable = - sess.opts.debugging_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); - let emit_stack_size_section = sess.opts.debugging_opts.emit_stack_sizes; + sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); + let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; let asm_comments = sess.asm_comments(); let relax_elf_relocations = - sess.opts.debugging_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); + sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); let use_init_array = - !sess.opts.debugging_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); + !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); let path_mapping = sess.source_map().path_mapping().clone(); @@ -308,7 +303,6 @@ impl<'a> DiagnosticHandlers<'a> { remark_passes.as_ptr(), remark_passes.len(), ); - llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); DiagnosticHandlers { data, llcx, old_handler } } } @@ -316,9 +310,7 @@ impl<'a> DiagnosticHandlers<'a> { impl<'a> Drop for DiagnosticHandlers<'a> { fn drop(&mut self) { - use std::ptr::null_mut; unsafe { - llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); drop(Box::from_raw(self.data)); } @@ -340,22 +332,12 @@ fn report_inline_asm( } let level = match level { llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning, + llvm::DiagnosticLevel::Warning => Level::Warning(None), llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); } -unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { - if user.is_null() { - return; - } - let (cgcx, _) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); - - let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); - report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); -} - unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { if user.is_null() { return; @@ -427,6 +409,14 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } +fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { + if config.instrument_coverage { + Some(CString::new("default_%m_%p.profraw").unwrap()) + } else { + None + } +} + pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, @@ -442,6 +432,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + let instr_profile_output_path = get_instr_profile_output_path(config); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { @@ -467,7 +458,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); - let extra_passes = config.passes.join(","); + let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; let llvm_plugins = config.llvm_plugins.join(","); @@ -492,6 +483,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, + instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, @@ -794,7 +786,7 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto); let data = thin.data(); if let Some(bitcode_filename) = bc_out.file_name() { @@ -1040,7 +1032,8 @@ unsafe fn embed_bitcode( // reason (see issue #90326 for historical background). let is_apple = cgcx.opts.target_triple.triple().contains("-ios") || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos"); + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos"); if is_apple || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 88b87951ecd..63b63c6a1fa 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,7 +3,6 @@ use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, BasicBlock, False}; use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -28,7 +27,6 @@ use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; -use tracing::debug; // All Builders must have an llfn associated with them #[must_use] @@ -464,15 +462,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + #[instrument(level = "trace", skip(self))] fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { - debug!("PlaceRef::load: {:?}", place); - assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); if place.layout.is_zst() { return OperandRef::new_zst(self, place.layout); } + #[instrument(level = "trace", skip(bx))] fn scalar_load_metadata<'a, 'll, 'tcx>( bx: &mut Builder<'a, 'll, 'tcx>, load: &'ll Value, @@ -509,15 +507,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Ref(place.llval, Some(llextra), place.align) } else if place.layout.is_llvm_immediate() { let mut const_llval = None; + let llty = place.layout.llvm_type(self); unsafe { if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { if llvm::LLVMIsGlobalConstant(global) == llvm::True { - const_llval = llvm::LLVMGetInitializer(global); + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); + } + } } } } let llval = const_llval.unwrap_or_else(|| { - let load = self.load(place.layout.llvm_type(self), place.llval, place.align); + let load = self.load(llty, place.llval, place.align); if let abi::Abi::Scalar(scalar) = place.layout.abi { scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); } @@ -621,32 +624,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn type_metadata(&mut self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); - let v = [self.const_usize(0), typeid_metadata]; - unsafe { - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( - self.cx.llcx, - v.as_ptr(), - v.len() as c_uint, - )), - ) - } - } - - fn typeid_metadata(&mut self, typeid: String) -> Self::Value { - unsafe { - llvm::LLVMMDStringInContext( - self.cx.llcx, - typeid.as_ptr() as *const c_char, - typeid.as_bytes().len() as c_uint, - ) - } - } - fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { self.store_with_flags(val, ptr, align, MemFlags::empty()) } @@ -747,11 +724,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(false, val, dest_ty) } - fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(true, val, dest_ty) } @@ -1412,7 +1389,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } } - crate fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); self.call(ty, f, args, None) } @@ -1451,51 +1428,32 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - fn fptoint_sat_broken_in_llvm(&self) -> bool { - match self.tcx.sess.target.arch.as_ref() { - // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 - "riscv64" => llvm_util::get_version() < (13, 0, 0), - _ => false, - } - } - - fn fptoint_sat( - &mut self, - signed: bool, - val: &'ll Value, - dest_ty: &'ll Type, - ) -> Option<&'ll Value> { - if !self.fptoint_sat_broken_in_llvm() { - let src_ty = self.cx.val_ty(val); - let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector - { - assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); - ( - self.cx.element_type(src_ty), - self.cx.element_type(dest_ty), - Some(self.cx.vector_length(src_ty)), - ) - } else { - (src_ty, dest_ty, None) - }; - let float_width = self.cx.float_width(float_ty); - let int_width = self.cx.int_width(int_ty); - - let instr = if signed { "fptosi" } else { "fptoui" }; - let name = if let Some(vector_length) = vector_length { - format!( - "llvm.{}.sat.v{}i{}.v{}f{}", - instr, vector_length, int_width, vector_length, float_width - ) - } else { - format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) - }; - let f = - self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + ( + self.cx.element_type(src_ty), + self.cx.element_type(dest_ty), + Some(self.cx.vector_length(src_ty)), + ) } else { - None - } + (src_ty, dest_ty, None) + }; + let float_width = self.cx.float_width(float_ty); + let int_width = self.cx.int_width(int_ty); + + let instr = if signed { "fptosi" } else { "fptoui" }; + let name = if let Some(vector_length) = vector_length { + format!( + "llvm.{}.sat.v{}i{}.v{}f{}", + instr, vector_length, int_width, vector_length, float_width + ) + } else { + format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + }; + let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); + self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None) } pub(crate) fn landing_pad( diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index ac423a22703..b83c1e8f08f 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -6,14 +6,14 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; +use crate::common; use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; use rustc_codegen_ssa::traits::*; -use tracing::debug; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::{self, Instance, TypeFoldable}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -79,13 +79,18 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> llfn } } else { - let llfn = cx.declare_fn(sym, fn_abi); + let instance_def_id = instance.def_id(); + let llfn = if tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) + { + cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi) + } else { + cx.declare_fn(sym, fn_abi) + }; debug!("get_fn: not casting pointer!"); attributes::from_fn_attrs(cx, llfn, instance); - let instance_def_id = instance.def_id(); - // Apply an appropriate linkage/visibility value to our item that we // just declared. // diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index b69d7a000ee..13e437cfbf7 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -10,15 +10,17 @@ use crate::value::Value; use rustc_ast::Mutability; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::ScalarInt; -use rustc_span::symbol::Symbol; +use rustc_middle::ty::TyCtxt; +use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_target::spec::Target; use libc::{c_char, c_uint}; -use tracing::debug; +use std::fmt::Write; /* * A note on nomenclature of linking: "extern", "foreign", and "upcall". @@ -109,8 +111,7 @@ impl<'ll> CodegenCx<'ll, '_> { pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { assert_eq!(idx as c_uint as u64, idx); - let us = &[idx as c_uint]; - let r = llvm::LLVMConstExtractValue(v, us.as_ptr(), us.len() as c_uint); + let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); @@ -181,22 +182,27 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMConstReal(t, val) } } - fn const_str(&self, s: Symbol) -> (&'ll Value, &'ll Value) { - let s_str = s.as_str(); - let str_global = *self.const_str_cache.borrow_mut().entry(s).or_insert_with(|| { - let sc = self.const_bytes(s_str.as_bytes()); - let sym = self.generate_local_symbol_name("str"); - let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", sym); - }); - unsafe { - llvm::LLVMSetInitializer(g, sc); - llvm::LLVMSetGlobalConstant(g, True); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); - } - g - }); - let len = s_str.len(); + fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { + let str_global = *self + .const_str_cache + .borrow_mut() + .raw_entry_mut() + .from_key(s) + .or_insert_with(|| { + let sc = self.const_bytes(s.as_bytes()); + let sym = self.generate_local_symbol_name("str"); + let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + unsafe { + llvm::LLVMSetInitializer(g, sc); + llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); + } + (s.to_owned(), g) + }) + .1; + let len = s.len(); let cs = consts::ptrcast( str_global, self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)), @@ -220,13 +226,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } + fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { + self.const_undef(self.type_ix(0)) + } + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { - Scalar::Int(ScalarInt::ZST) => { - assert_eq!(0, layout.size(self).bytes()); - self.const_undef(self.type_ix(0)) - } Scalar::Int(int) => { let data = int.assert_bits(layout.size(self)); let llval = self.const_uint_big(self.type_ix(bitsize), data); @@ -255,6 +261,15 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.get_fn_addr(fn_instance.polymorphize(self.tcx)), self.data_layout().instruction_address_space, ), + GlobalAlloc::VTable(ty, trait_ref) => { + let alloc = self + .tcx + .global_alloc(self.tcx.vtable_allocation((ty, trait_ref))) + .unwrap_memory(); + let init = const_alloc_to_llvm(self, alloc); + let value = self.static_addr_of(init, alloc.inner().align, None); + (value, AddressSpace::DATA) + } GlobalAlloc::Static(def_id) => { assert!(self.tcx.is_static(def_id)); assert!(!self.tcx.is_thread_local_static(def_id)); @@ -346,3 +361,74 @@ fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { unsafe { llvm::LLVMIsAConstantInt(v) } } + +pub(crate) fn get_dllimport<'tcx>( + tcx: TyCtxt<'tcx>, + id: DefId, + name: &str, +) -> Option<&'tcx DllImport> { + tcx.native_library(id) + .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) + .flatten() +} + +pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { + target.vendor == "pc" && target.os == "windows" && target.env == "gnu" && target.abi.is_empty() +} + +pub(crate) fn i686_decorated_name( + dll_import: &DllImport, + mingw: bool, + disable_name_mangling: bool, +) -> String { + let name = dll_import.name.as_str(); + + let (add_prefix, add_suffix) = match dll_import.import_name_type { + Some(PeImportNameType::NoPrefix) => (false, true), + Some(PeImportNameType::Undecorated) => (false, false), + _ => (true, true), + }; + + // Worst case: +1 for disable name mangling, +1 for prefix, +4 for suffix (@@__). + let mut decorated_name = String::with_capacity(name.len() + 6); + + if disable_name_mangling { + // LLVM uses a binary 1 ('\x01') prefix to a name to indicate that mangling needs to be disabled. + decorated_name.push('\x01'); + } + + let prefix = if add_prefix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C | DllCallingConvention::Vectorcall(_) => None, + DllCallingConvention::Stdcall(_) => (!mingw + || dll_import.import_name_type == Some(PeImportNameType::Decorated)) + .then_some('_'), + DllCallingConvention::Fastcall(_) => Some('@'), + } + } else if !dll_import.is_fn && !mingw { + // For static variables, prefix with '_' on MSVC. + Some('_') + } else { + None + }; + if let Some(prefix) = prefix { + decorated_name.push(prefix); + } + + decorated_name.push_str(name); + + if add_suffix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C => {} + DllCallingConvention::Stdcall(arg_list_size) + | DllCallingConvention::Fastcall(arg_list_size) => { + write!(&mut decorated_name, "@{}", arg_list_size).unwrap(); + } + DllCallingConvention::Vectorcall(arg_list_size) => { + write!(&mut decorated_name, "@@{}", arg_list_size).unwrap(); + } + } + } + + decorated_name +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 4d3f3f318b8..a559f7f3d57 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,5 +1,5 @@ use crate::base; -use crate::common::CodegenCx; +use crate::common::{self, CodegenCx}; use crate::debuginfo; use crate::llvm::{self, True}; use crate::llvm_util; @@ -23,16 +23,15 @@ use rustc_target::abi::{ AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, }; use std::ops::Range; -use tracing::debug; pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1); let dl = cx.data_layout(); let pointer_size = dl.pointer_size.bytes() as usize; - // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, - // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` + // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( llvals: &mut Vec<&'ll Value>, cx: &'a CodegenCx<'ll, 'b>, @@ -64,7 +63,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< // See https://github.com/rust-lang/rust/issues/84565. 1 } else { - cx.sess().opts.debugging_opts.uninit_const_chunk_threshold + cx.sess().opts.unstable_opts.uninit_const_chunk_threshold }; let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; @@ -79,12 +78,12 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } let mut next_offset = 0; - for &(offset, alloc_id) in alloc.relocations().iter() { + for &(offset, alloc_id) in alloc.provenance().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a relocation, it + // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); @@ -93,7 +92,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< dl.endian, // This `inspect` is okay since it is within the bounds of the allocation, it doesn't // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the relocation as a relocation pointer offset. + // and we properly interpret the provenance as a relocation pointer offset. alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), ) .expect("const_alloc_to_llvm: could not read relocation pointer") @@ -101,7 +100,9 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< let address_space = match cx.tcx.global_alloc(alloc_id) { GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, - GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) => AddressSpace::DATA, + GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => { + AddressSpace::DATA + } }; llvals.push(cx.scalar_to_backend( @@ -119,7 +120,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } if alloc.len() >= next_offset { let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all relocations, it is + // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); @@ -158,7 +159,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str, - span_def_id: DefId, + def_id: DefId, ) -> &'ll Value { let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.linkage { @@ -173,7 +174,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( cx.layout_of(mt.ty).llvm_type(cx) } else { cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), + cx.tcx.def_span(def_id), "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", ) }; @@ -192,7 +193,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), + cx.tcx.def_span(def_id), &format!("symbol `{}` is already defined", &sym), ) }); @@ -200,6 +201,10 @@ fn check_and_apply_linkage<'ll, 'tcx>( llvm::LLVMSetInitializer(g2, g1); g2 } + } else if cx.tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) + { + cx.declare_global(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), true), llty) } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global @@ -212,11 +217,11 @@ pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { } impl<'ll> CodegenCx<'ll, '_> { - crate fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMConstBitCast(val, ty) } } - crate fn static_addr_of_mut( + pub(crate) fn static_addr_of_mut( &self, cv: &'ll Value, align: Align, @@ -241,7 +246,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn get_static(&self, def_id: DefId) -> &'ll Value { + pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); if let Some(&g) = self.instances.borrow().get(&instance) { return g; @@ -473,7 +478,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // // We could remove this hack whenever we decide to drop macOS 10.10 support. if self.tcx.sess.target.is_like_osx { - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state // (not as part of the interpreter execution). // @@ -481,7 +486,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // happens to be zero. Instead, we should only check the value of defined bytes // and set all undefined bytes to zero if this allocation is headed for the // BSS. - let all_bytes_are_zero = alloc.relocations().is_empty() + let all_bytes_are_zero = alloc.provenance().is_empty() && alloc .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) .iter() @@ -505,9 +510,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { section.as_str().as_ptr().cast(), section.as_str().len() as c_uint, ); - assert!(alloc.relocations().is_empty()); + assert!(alloc.provenance().is_empty()); - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state (not // as part of the interpreter execution). let bytes = @@ -535,10 +540,20 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // The semantics of #[used] in Rust only require the symbol to make it into the // object file. It is explicitly allowed for the linker to strip the symbol if it - // is dead. As such, use llvm.compiler.used instead of llvm.used. + // is dead, which means we are allowed use `llvm.compiler.used` instead of + // `llvm.used` here. + // // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs - // in some versions of the gold linker. + // in the handling of `.init_array` (the static constructor list) in versions of + // the gold linker (prior to the one released with binutils 2.36). + // + // That said, we only ever emit these when compiling for ELF targets, unless + // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage + // on other targets, in particular MachO targets have *their* static constructor + // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, + // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_typeck`, + // so we don't need to take care of it here. self.add_compiler_used_global(g); } if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d296ee3b42c..67ffc7cb951 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -26,7 +26,6 @@ use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; -use rustc_span::symbol::Symbol; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; @@ -56,7 +55,7 @@ pub struct CodegenCx<'ll, 'tcx> { pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>, /// Cache of constant strings, - pub const_str_cache: RefCell<FxHashMap<Symbol, &'ll Value>>, + pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>, /// Reverse-direction for const ptrs cast from globals. /// @@ -143,17 +142,6 @@ pub unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (13, 0, 0) { - if sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-S128", ""); - } - if sess.target.arch == "wasm32" { - target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); - } - if sess.target.arch == "wasm64" { - target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); - } - } if llvm_version < (14, 0, 0) { if sess.target.llvm_target == "i686-pc-windows-msvc" || sess.target.llvm_target == "i586-pc-windows-msvc" @@ -276,7 +264,7 @@ pub unsafe fn create_module<'ll>( } } - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.debugging_opts.branch_protection { + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch != "aarch64" { sess.err("-Zbranch-protection is only supported on aarch64"); } else { @@ -309,7 +297,7 @@ pub unsafe fn create_module<'ll>( } // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). - if let CFProtection::Branch | CFProtection::Full = sess.opts.debugging_opts.cf_protection { + if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, @@ -317,7 +305,7 @@ pub unsafe fn create_module<'ll>( 1, ) } - if let CFProtection::Return | CFProtection::Full = sess.opts.debugging_opts.cf_protection { + if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, @@ -326,11 +314,20 @@ pub unsafe fn create_module<'ll>( ) } + if sess.opts.unstable_opts.virtual_function_elimination { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "Virtual Function Elim\0".as_ptr().cast(), + 1, + ); + } + llmod } impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { - crate fn new( + pub(crate) fn new( tcx: TyCtxt<'tcx>, codegen_unit: &'tcx CodegenUnit<'tcx>, llvm_module: &'ll crate::ModuleLlvm, @@ -447,7 +444,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } } - crate fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> { + pub(crate) fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> { &self.statics_to_rauw } @@ -599,7 +596,7 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll> CodegenCx<'ll, '_> { - crate fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { + pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { if let Some(v) = self.intrinsics.borrow().get(key).cloned() { return v; } @@ -656,6 +653,7 @@ impl<'ll> CodegenCx<'ll, '_> { let t_isize = self.type_isize(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_metadata = self.type_metadata(); ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); @@ -881,21 +879,25 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); } - ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1); + ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void); - ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void); + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } + + ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); + None } - crate fn eh_catch_typeinfo(&self) -> &'ll Value { + pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() { return eh_catch_typeinfo; } let tcx = self.tcx; - assert!(self.sess().target.is_like_emscripten); + assert!(self.sess().target.os == "emscripten"); let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { Some(def_id) => self.get_static(def_id), _ => { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 58f391692c4..0d1df6fb1ac 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -16,8 +16,6 @@ use rustc_middle::ty::TyCtxt; use std::ffi::CString; -use tracing::debug; - /// Generates and exports the Coverage Map. /// /// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 98ba38356a4..964a632b6ee 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -28,7 +28,6 @@ use std::cell::RefCell; use std::ffi::CString; use std::iter; -use tracing::debug; pub mod mapgen; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 31a09242c5a..80fd9726fc7 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -5,11 +5,14 @@ use crate::llvm; use crate::builder::Builder; use crate::common::CodegenCx; use crate::value::Value; +use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_session::config::DebugInfo; +use rustc_session::config::{CrateType, DebugInfo}; use rustc_span::symbol::sym; +use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -37,9 +40,33 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' section_var.unwrap_or_else(|| { let section_name = b".debug_gdb_scripts\0"; - let section_contents = b"\x01gdb_load_rust_pretty_printers.py\0"; + let mut section_contents = Vec::new(); + + // Add the pretty printers for the standard library first. + section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0"); + + // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute. + let visualizers = collect_debugger_visualizers_transitive( + cx.tcx, + DebuggerVisualizerType::GdbPrettyPrinter, + ); + let crate_name = cx.tcx.crate_name(LOCAL_CRATE); + for (index, visualizer) in visualizers.iter().enumerate() { + // The initial byte `4` instructs GDB that the following pretty printer + // is defined inline as opposed to in a standalone file. + section_contents.extend_from_slice(b"\x04"); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); + section_contents.extend_from_slice(vis_name.as_bytes()); + section_contents.extend_from_slice(&visualizer.src); + + // The final byte `0` tells GDB that the pretty printer has been + // fully defined and can continue searching for additional + // pretty printers. + section_contents.extend_from_slice(b"\0"); + } unsafe { + let section_contents = section_contents.as_slice(); let llvm_type = cx.type_array(cx.type_i8(), section_contents.len() as u64); let section_var = cx @@ -62,7 +89,32 @@ pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create + // ODR violations at link time, this section will not be emitted for rlibs since + // each rlib could produce a different set of visualizers that would be embedded + // in the `.debug_gdb_scripts` section. For that reason, we make sure that the + // section is only emitted for leaf crates. + let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { + CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { + // These are crate types for which we will embed pretty printers since they + // are treated as leaf crates. + true + } + CrateType::ProcMacro => { + // We could embed pretty printers for proc macro crates too but it does not + // seem like a good default, since this is a rare use case and we don't + // want to slow down the common case. + false + } + CrateType::Rlib => { + // As per the above description, embedding pretty printers for rlibs could + // lead to ODR violations so we skip this crate type as well. + false + } + }); + !omit_gdb_pretty_printer_section && cx.sess().opts.debuginfo != DebugInfo::None && cx.sess().target.emit_debug_gdb_scripts + && embed_visualizers } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 97d3acb34ce..163ccd9460c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -30,27 +30,26 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{self, GeneratorLayout}; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; -use rustc_middle::ty::{self, AdtKind, Instance, ParamEnv, Ty, TyCtxt, COMMON_VTABLE_ENTRIES}; -use rustc_session::config::{self, DebugInfo}; +use rustc_middle::ty::{ + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, +}; +use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::FileName; -use rustc_span::FileNameDisplayPreference; -use rustc_span::{self, SourceFile}; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; -use tracing::debug; -use libc::{c_longlong, c_uint}; +use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; use std::fmt::{self, Write}; use std::hash::{Hash, Hasher}; use std::iter; use std::path::{Path, PathBuf}; use std::ptr; -use tracing::instrument; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -113,6 +112,7 @@ macro_rules! return_if_di_node_created_in_meantime { } /// Extract size and alignment from a TyAndLayout. +#[inline] fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { (ty_and_layout.size, ty_and_layout.align.abi) } @@ -823,7 +823,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( output_filenames .split_dwarf_path( tcx.sess.split_debuginfo(), - tcx.sess.opts.debugging_opts.split_dwarf_kind, + tcx.sess.opts.unstable_opts.split_dwarf_kind, Some(codegen_unit_name), ) // We get a path relative to the working directory from split_dwarf_path @@ -880,15 +880,15 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( split_name.len(), kind, 0, - tcx.sess.opts.debugging_opts.split_dwarf_inlining, + tcx.sess.opts.unstable_opts.split_dwarf_inlining, ); - if tcx.sess.opts.debugging_opts.profile { + if tcx.sess.opts.unstable_opts.profile { let cu_desc_metadata = llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = - tcx.sess.opts.debugging_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); + tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), @@ -1365,7 +1365,7 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo is_local_to_unit, global, None, - global_align.bytes() as u32, + global_align.bits() as u32, ); } } @@ -1392,7 +1392,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( tcx.vtable_entries(trait_ref) } else { - COMMON_VTABLE_ENTRIES + TyCtxt::COMMON_VTABLE_ENTRIES }; // All function pointers are described as opaque pointers. This could be improved in the future @@ -1419,7 +1419,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( cx, type_map::stub( cx, - Stub::VtableTy { vtable_holder }, + Stub::VTableTy { vtable_holder }, unique_type_id, &vtable_type_name, (size, pointer_align), @@ -1468,6 +1468,78 @@ fn build_vtable_type_di_node<'ll, 'tcx>( .di_node } +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); + + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public, + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => { + VCallVisibility::LinkageUnit + } + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit, + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } +} + /// Creates debug information for the given vtable, which is for the /// given type. /// @@ -1478,6 +1550,12 @@ pub fn create_vtable_di_node<'ll, 'tcx>( poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.unstable_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + if cx.dbg_cx.is_none() { return; } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index d6e2c8ccdf4..daec9303b2c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -1,19 +1,21 @@ use std::borrow::Cow; use libc::c_uint; -use rustc_codegen_ssa::debuginfo::{ - type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +use rustc_codegen_ssa::{ + debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, + traits::ConstMethods, }; + +use rustc_index::vec::IndexVec; use rustc_middle::{ bug, ty::{ self, layout::{LayoutOf, TyAndLayout}, - util::Discr, - AdtDef, GeneratorSubsts, + AdtDef, GeneratorSubsts, Ty, }, }; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::{ @@ -21,9 +23,9 @@ use crate::{ debuginfo::{ metadata::{ build_field_di_node, closure_saved_names_of_captured_variables, - enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, - type_map::{self, UniqueTypeId}, + enums::{tag_base_type, DiscrResult}, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + type_map::{self, Stub, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER, }, @@ -35,59 +37,161 @@ use crate::{ }, }; -/// In CPP-like mode, we generate a union of structs for each variant and an -/// explicit discriminant field roughly equivalent to the following C/C++ code: +// The names of the associated constants in each variant wrapper struct. +// These have to match up with the names being used in `intrinsic.natvis`. +const ASSOC_CONST_DISCR_NAME: &str = "NAME"; +const ASSOC_CONST_DISCR_EXACT: &str = "DISCR_EXACT"; +const ASSOC_CONST_DISCR_BEGIN: &str = "DISCR_BEGIN"; +const ASSOC_CONST_DISCR_END: &str = "DISCR_END"; + +const ASSOC_CONST_DISCR128_EXACT_LO: &str = "DISCR128_EXACT_LO"; +const ASSOC_CONST_DISCR128_EXACT_HI: &str = "DISCR128_EXACT_HI"; +const ASSOC_CONST_DISCR128_BEGIN_LO: &str = "DISCR128_BEGIN_LO"; +const ASSOC_CONST_DISCR128_BEGIN_HI: &str = "DISCR128_BEGIN_HI"; +const ASSOC_CONST_DISCR128_END_LO: &str = "DISCR128_END_LO"; +const ASSOC_CONST_DISCR128_END_HI: &str = "DISCR128_END_HI"; + +// The name of the tag field in the top-level union +const TAG_FIELD_NAME: &str = "tag"; +const TAG_FIELD_NAME_128_LO: &str = "tag128_lo"; +const TAG_FIELD_NAME_128_HI: &str = "tag128_hi"; + +// We assign a "virtual" discriminant value to the sole variant of +// a single-variant enum. +const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; + +/// In CPP-like mode, we generate a union with a field for each variant and an +/// explicit tag field. The field of each variant has a struct type +/// that encodes the discrimiant of the variant and it's data layout. +/// The union also has a nested enumeration type that is only used for encoding +/// variant names in an efficient way. Its enumerator values do _not_ correspond +/// to the enum's discriminant values. +/// It's roughly equivalent to the following C/C++ code: /// /// ```c -/// union enum$<{fully-qualified-name}> { -/// struct {variant 0 name} { -/// <variant 0 fields> +/// union enum2$<{fully-qualified-name}> { +/// struct Variant0 { +/// struct {name-of-variant-0} { +/// <variant 0 fields> +/// } value; +/// +/// static VariantNames NAME = {name-of-variant-0}; +/// static int_type DISCR_EXACT = {discriminant-of-variant-0}; /// } variant0; +/// /// <other variant structs> -/// {name} discriminant; +/// +/// int_type tag; +/// +/// enum VariantNames { +/// <name-of-variant-0> = 0, // The numeric values are variant index, +/// <name-of-variant-1> = 1, // not discriminant values. +/// <name-of-variant-2> = 2, +/// ... +/// } /// } /// ``` /// -/// As you can see, the type name is wrapped `enum$`. This way we can have a -/// single NatVis rule for handling all enums. +/// As you can see, the type name is wrapped in `enum2$<_>`. This way we can +/// have a single NatVis rule for handling all enums. The `2` in `enum2$<_>` +/// is an encoding version tag, so that debuggers can decide to decode this +/// differently than the previous `enum$<_>` encoding emitted by earlier +/// compiler versions. /// -/// At the LLVM IR level this looks like +/// Niche-tag enums have one special variant, usually called the +/// "dataful variant". This variant has a field that +/// doubles as the tag of the enum. The variant is active when the value of +/// that field is within a pre-defined range. Therefore the variant struct +/// has a `DISCR_BEGIN` and `DISCR_END` field instead of `DISCR_EXACT` in +/// that case. Both `DISCR_BEGIN` and `DISCR_END` are inclusive bounds. +/// Note that these ranges can wrap around, so that `DISCR_END < DISCR_BEGIN`. /// -/// ```txt -/// DW_TAG_union_type (top-level type for enum) -/// DW_TAG_member (member for variant 1) -/// DW_TAG_member (member for variant 2) -/// DW_TAG_member (member for variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// DW_TAG_enumeration_type (type of tag) -/// ``` +/// Single-variant enums don't actually have a tag field. In this case we +/// emit a static tag field (that always has the value 0) so we can use the +/// same representation (and NatVis). /// -/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things -/// differently in order to allow a NatVis visualizer to extract all the information needed: -/// We generate a union of two fields, one for the dataful variant -/// and one that just points to the discriminant (which is some field within the dataful variant). -/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful -/// variants and make the discriminant field that type. We then use NatVis to render the enum type -/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: +/// For niche-layout enums it's possible to have a 128-bit tag. NatVis, VS, and +/// WinDbg (the main targets for CPP-like debuginfo at the moment) don't support +/// 128-bit integers, so all values involved get split into two 64-bit fields. +/// Instead of the `tag` field, we generate two fields `tag128_lo` and `tag128_hi`, +/// Instead of `DISCR_EXACT`, we generate `DISCR128_EXACT_LO` and `DISCR128_EXACT_HI`, +/// and so on. /// -/// ```c -/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { -/// struct <dataful variant name> { -/// <fields in dataful variant> -/// } dataful_variant; -/// enum Discriminant$ { -/// <non-dataful variants> -/// } discriminant; +/// +/// The following pseudocode shows how to decode an enum value in a debugger: +/// +/// ```text +/// +/// fn find_active_variant(enum_value) -> (VariantName, VariantValue) { +/// let is_128_bit = enum_value.has_field("tag128_lo"); +/// +/// if !is_128_bit { +/// // Note: `tag` can be a static field for enums with only one +/// // inhabited variant. +/// let tag = enum_value.field("tag").value; +/// +/// // For each variant, check if it is a match. Only one of them will match, +/// // so if we find it we can return it immediately. +/// for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) { +/// if variant_field.has_field("DISCR_EXACT") { +/// // This variant corresponds to a single tag value +/// if variant_field.field("DISCR_EXACT").value == tag { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } else { +/// // This is a range variant +/// let begin = variant_field.field("DISCR_BEGIN"); +/// let end = variant_field.field("DISCR_END"); +/// +/// if is_in_range(tag, begin, end) { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } +/// } +/// } else { +/// // Basically the same as with smaller tags, we just have to +/// // stitch the values together. +/// let tag: u128 = (enum_value.field("tag128_lo").value as u128) | +/// (enum_value.field("tag128_hi").value as u128 << 64); +/// +/// for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) { +/// if variant_field.has_field("DISCR128_EXACT_LO") { +/// let discr_exact = (variant_field.field("DISCR128_EXACT_LO" as u128) | +/// (variant_field.field("DISCR128_EXACT_HI") as u128 << 64); +/// +/// // This variant corresponds to a single tag value +/// if discr_exact.value == tag { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } else { +/// // This is a range variant +/// let begin = (variant_field.field("DISCR128_BEGIN_LO").value as u128) | +/// (variant_field.field("DISCR128_BEGIN_HI").value as u128 << 64); +/// let end = (variant_field.field("DISCR128_END_LO").value as u128) | +/// (variant_field.field("DISCR128_END_HI").value as u128 << 64); +/// +/// if is_in_range(tag, begin, end) { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } +/// } +/// } +/// +/// // We should have found an active variant at this point. +/// unreachable!(); /// } -/// ``` /// -/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` -/// and evaluates `this.discriminant`. If the value is between the min niche and max -/// niche, then the enum is in the dataful variant and `this.dataful_variant` is -/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that -/// case, we just need to render the name of the `this.discriminant` enum. +/// // Check if a value is within the given range +/// // (where the range might wrap around the value space) +/// fn is_in_range(value, start, end) -> bool { +/// if start < end { +/// value >= start && value <= end +/// } else { +/// value >= start || value <= end +/// } +/// } +/// +/// ``` pub(super) fn build_enum_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, @@ -135,27 +239,28 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ref variants, tag_field, .. - } => build_union_fields_for_direct_tag_enum( + } => build_union_fields_for_enum( cx, enum_adt_def, enum_type_and_layout, enum_type_di_node, - &mut variants.indices(), + variants.indices(), tag_field, + None, ), Variants::Multiple { tag_encoding: TagEncoding::Niche { dataful_variant, .. }, ref variants, tag_field, .. - } => build_union_fields_for_niche_tag_enum( + } => build_union_fields_for_enum( cx, enum_adt_def, enum_type_and_layout, enum_type_di_node, - dataful_variant, - &mut variants.indices(), + variants.indices(), tag_field, + Some(dataful_variant), ), } }, @@ -217,137 +322,344 @@ fn build_single_variant_union_fields<'ll, 'tcx>( let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( cx, - enum_type_and_layout.ty, + enum_type_and_layout, enum_type_di_node, variant_index, enum_adt_def.variant(variant_index), variant_layout, ); - // NOTE: The field name of the union is the same as the variant name, not "variant0". - let variant_name = enum_adt_def.variant(variant_index).name.as_str(); + let tag_base_type = cx.tcx.types.u32; + let tag_base_type_di_node = type_di_node(cx, tag_base_type); + let tag_base_type_align = cx.align_of(tag_base_type); - smallvec![build_field_di_node( + let variant_names_type_di_node = build_variant_names_type_di_node( cx, enum_type_di_node, - variant_name, - // NOTE: We use the size and align of the entire type, not from variant_layout - // since the later is sometimes smaller (if it has fewer fields). - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, + std::iter::once(( + variant_index, + Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + )), + ); + + let variant_struct_type_wrapper_di_node = build_variant_struct_wrapper_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_index, + None, variant_struct_type_di_node, - )] + variant_names_type_di_node, + tag_base_type_di_node, + tag_base_type, + DiscrResult::NoDiscriminant, + ); + + smallvec![ + build_field_di_node( + cx, + enum_type_di_node, + &variant_union_field_name(variant_index), + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_wrapper_di_node, + ), + unsafe { + llvm::LLVMRustDIBuilderCreateStaticMemberType( + DIB(cx), + enum_type_di_node, + TAG_FIELD_NAME.as_ptr().cast(), + TAG_FIELD_NAME.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + DIFlags::FlagZero, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align.bits() as u32, + ) + } + ] } -fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( +fn build_union_fields_for_enum<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_adt_def: AdtDef<'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, - variant_indices: &mut dyn Iterator<Item = VariantIdx>, + variant_indices: impl Iterator<Item = VariantIdx> + Clone, tag_field: usize, + dataful_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + enum_type_di_node, + variant_indices.clone().map(|variant_index| { + let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + (variant_index, variant_name) + }), + ); + let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices .map(|variant_index| { let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let variant_def = enum_adt_def.variant(variant_index); + + let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_index, + variant_def, + variant_layout, + ); + VariantFieldInfo { variant_index, - variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( - cx, - enum_type_and_layout.ty, - enum_type_di_node, - variant_index, - enum_adt_def.variant(variant_index), - variant_layout, - ), + variant_struct_type_di_node, source_info: None, + discr: super::compute_discriminant_value(cx, enum_type_and_layout, variant_index), } }) .collect(); - let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name.as_str(), - tag_base_type, - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) - }), - enum_type_di_node, - ); - build_union_fields_for_direct_tag_enum_or_generator( cx, enum_type_and_layout, enum_type_di_node, &variant_field_infos, - discr_type_di_node, + variant_names_type_di_node, + tag_base_type, tag_field, + dataful_variant_index, ) } -fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( +// The base type of the VariantNames DW_AT_enumeration_type is always the same. +// It has nothing to do with the tag of the enum and just has to be big enough +// to hold all variant names. +fn variant_names_enum_base_type<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> Ty<'tcx> { + cx.tcx.types.u32 +} + +/// This function builds a DW_AT_enumeration_type that contains an entry for +/// each variant. Note that this has nothing to do with the discriminant. The +/// numeric value of each enumerator corresponds to the variant index. The +/// type is only used for efficiently encoding the name of each variant in +/// debuginfo. +fn build_variant_names_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - dataful_variant_index: VariantIdx, - variant_indices: &mut dyn Iterator<Item = VariantIdx>, - tag_field: usize, -) -> SmallVec<&'ll DIType> { - let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + containing_scope: &'ll DIType, + variants: impl Iterator<Item = (VariantIdx, Cow<'tcx, str>)>, +) -> &'ll DIType { + // Create an enumerator for each variant. + super::build_enumeration_type_di_node( cx, - enum_type_and_layout.ty, - enum_type_di_node, - dataful_variant_index, - &enum_adt_def.variant(dataful_variant_index), - enum_type_and_layout.for_variant(cx, dataful_variant_index), - ); + "VariantNames", + variant_names_enum_base_type(cx), + variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32() as u64)), + containing_scope, + ) +} - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - // Create an DW_TAG_enumerator for each variant except the dataful one. - let discr_type_di_node = super::build_enumeration_type_di_node( +fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_generator_type_and_layout: TyAndLayout<'tcx>, + enum_or_generator_type_di_node: &'ll DIType, + variant_index: VariantIdx, + dataful_variant_index: Option<VariantIdx>, + variant_struct_type_di_node: &'ll DIType, + variant_names_type_di_node: &'ll DIType, + tag_base_type_di_node: &'ll DIType, + tag_base_type: Ty<'tcx>, + discr: DiscrResult, +) -> &'ll DIType { + type_map::build_type_with_children( cx, - "Discriminant$", - tag_base_type, - &mut variant_indices.filter_map(|variant_index| { - if let Some(discr_val) = - super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) - { - let discr = Discr { val: discr_val as u128, ty: tag_base_type }; - let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); - Some((discr, variant_name)) - } else { - debug_assert_eq!(variant_index, dataful_variant_index); - None - } - }), - enum_type_di_node, - ); - - smallvec![ - build_field_di_node( - cx, - enum_type_di_node, - "dataful_variant", - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, - dataful_variant_struct_type_di_node, - ), - build_field_di_node( + type_map::stub( cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(tag_base_type), - enum_type_and_layout.fields.offset(tag_field), + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type_wrapper( + cx.tcx, + enum_or_generator_type_and_layout.ty, + variant_index, + ), + &variant_struct_wrapper_type_name(variant_index), + // NOTE: We use size and align of enum_type, not from variant_layout: + size_and_align_of(enum_or_generator_type_and_layout), + Some(enum_or_generator_type_di_node), DIFlags::FlagZero, - discr_type_di_node, ), - ] + |cx, wrapper_struct_type_di_node| { + enum DiscrKind { + Exact(u64), + Exact128(u128), + Range(u64, u64), + Range128(u128, u128), + } + + let (tag_base_type_size, tag_base_type_align) = cx.size_and_align_of(tag_base_type); + let is_128_bits = tag_base_type_size.bits() > 64; + + let discr = match discr { + DiscrResult::NoDiscriminant => DiscrKind::Exact(SINGLE_VARIANT_VIRTUAL_DISR), + DiscrResult::Value(discr_val) => { + if is_128_bits { + DiscrKind::Exact128(discr_val) + } else { + debug_assert_eq!(discr_val, discr_val as u64 as u128); + DiscrKind::Exact(discr_val as u64) + } + } + DiscrResult::Range(min, max) => { + assert_eq!(Some(variant_index), dataful_variant_index); + if is_128_bits { + DiscrKind::Range128(min, max) + } else { + debug_assert_eq!(min, min as u64 as u128); + debug_assert_eq!(max, max as u64 as u128); + DiscrKind::Range(min as u64, max as u64) + } + } + }; + + let mut fields = SmallVec::new(); + + // We always have a field for the value + fields.push(build_field_di_node( + cx, + wrapper_struct_type_di_node, + "value", + size_and_align_of(enum_or_generator_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_di_node, + )); + + let build_assoc_const = + |name: &str, type_di_node: &'ll DIType, value: u64, align: Align| unsafe { + llvm::LLVMRustDIBuilderCreateStaticMemberType( + DIB(cx), + wrapper_struct_type_di_node, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + type_di_node, + DIFlags::FlagZero, + Some(cx.const_u64(value)), + align.bits() as u32, + ) + }; + + // We also always have an associated constant for the discriminant value + // of the variant. + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_NAME, + variant_names_type_di_node, + variant_index.as_u32() as u64, + cx.align_of(variant_names_enum_base_type(cx)), + )); + + // Emit the discriminant value (or range) corresponding to the variant. + match discr { + DiscrKind::Exact(discr_val) => { + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_EXACT, + tag_base_type_di_node, + discr_val, + tag_base_type_align, + )); + } + DiscrKind::Exact128(discr_val) => { + let align = cx.align_of(cx.tcx.types.u64); + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let Split128 { hi, lo } = split_128(discr_val); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_EXACT_LO, + type_di_node, + lo, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_EXACT_HI, + type_di_node, + hi, + align, + )); + } + DiscrKind::Range(begin, end) => { + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_BEGIN, + tag_base_type_di_node, + begin, + tag_base_type_align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_END, + tag_base_type_di_node, + end, + tag_base_type_align, + )); + } + DiscrKind::Range128(begin, end) => { + let align = cx.align_of(cx.tcx.types.u64); + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let Split128 { hi: begin_hi, lo: begin_lo } = split_128(begin); + let Split128 { hi: end_hi, lo: end_lo } = split_128(end); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_BEGIN_HI, + type_di_node, + begin_hi, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_BEGIN_LO, + type_di_node, + begin_lo, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_END_HI, + type_di_node, + end_hi, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_END_LO, + type_di_node, + end_lo, + align, + )); + } + } + + fields + }, + NO_GENERICS, + ) + .di_node +} + +struct Split128 { + hi: u64, + lo: u64, +} + +fn split_128(value: u128) -> Split128 { + Split128 { hi: (value >> 64) as u64, lo: value as u64 } } fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( @@ -369,6 +681,29 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); + + let tag_base_type = tag_base_type(cx, generator_type_and_layout); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + generator_type_di_node, + variant_range + .clone() + .map(|variant_index| (variant_index, GeneratorSubsts::variant_name(variant_index))), + ); + + let discriminants: IndexVec<VariantIdx, DiscrResult> = { + let discriminants_iter = generator_substs.discriminants(generator_def_id, cx.tcx); + let mut discriminants: IndexVec<VariantIdx, DiscrResult> = + IndexVec::with_capacity(variant_count); + for (variant_index, discr) in discriminants_iter { + // Assert that the index in the IndexMap matches up with the given VariantIdx. + assert_eq!(variant_index, discriminants.next_index()); + discriminants.push(DiscrResult::Value(discr.val)); + } + discriminants + }; // Build the type node for each field. let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range @@ -391,29 +726,24 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( None }; - VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } + VariantFieldInfo { + variant_index, + variant_struct_type_di_node, + source_info, + discr: discriminants[variant_index], + } }) .collect(); - let tag_base_type = tag_base_type(cx, generator_type_and_layout); - let discr_type_name = "Discriminant$"; - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name, - tag_base_type, - &mut generator_substs - .discriminants(generator_def_id, cx.tcx) - .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), - generator_type_di_node, - ); - build_union_fields_for_direct_tag_enum_or_generator( cx, generator_type_and_layout, generator_type_di_node, &variant_field_infos[..], - discr_type_di_node, + variant_names_type_di_node, + tag_base_type, tag_field, + None, ) } @@ -425,8 +755,11 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( enum_type_di_node: &'ll DIType, variant_field_infos: &[VariantFieldInfo<'ll>], discr_type_di_node: &'ll DIType, + tag_base_type: Ty<'tcx>, tag_field: usize, + dataful_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { + let tag_base_type_di_node = type_di_node(cx, tag_base_type); let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); // We create a field in the union for each variant ... @@ -438,6 +771,19 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( let field_name = variant_union_field_name(variant_member_info.variant_index); let (size, align) = size_and_align_of(enum_type_and_layout); + let variant_struct_type_wrapper = build_variant_struct_wrapper_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_member_info.variant_index, + dataful_variant_index, + variant_member_info.variant_struct_type_di_node, + discr_type_di_node, + tag_base_type_di_node, + tag_base_type, + variant_member_info.discr, + ); + // We use LLVMRustDIBuilderCreateMemberType() member type directly because // the build_field_di_node() function does not support specifying a source location, // which is something that we don't do anywhere else. @@ -456,7 +802,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( // Union fields are always at offset zero Size::ZERO.bits(), DIFlags::FlagZero, - variant_member_info.variant_struct_type_di_node, + variant_struct_type_wrapper, ) } })); @@ -466,16 +812,53 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) ); - // ... and a field for the discriminant. - unions_fields.push(build_field_di_node( - cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - enum_type_and_layout.fields.offset(tag_field), - DIFlags::FlagZero, - discr_type_di_node, - )); + // ... and a field for the tag. If the tag is 128 bits wide, this will actually + // be two 64-bit fields. + let is_128_bits = cx.size_of(tag_base_type).bits() > 64; + + if is_128_bits { + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let size_and_align = cx.size_and_align_of(cx.tcx.types.u64); + + let (lo_offset, hi_offset) = match cx.tcx.data_layout.endian { + Endian::Little => (0, 8), + Endian::Big => (8, 0), + }; + + let tag_field_offset = enum_type_and_layout.fields.offset(tag_field).bytes(); + let lo_offset = Size::from_bytes(tag_field_offset + lo_offset); + let hi_offset = Size::from_bytes(tag_field_offset + hi_offset); + + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME_128_LO, + size_and_align, + lo_offset, + DIFlags::FlagZero, + type_di_node, + )); + + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME_128_HI, + size_and_align, + hi_offset, + DIFlags::FlagZero, + type_di_node, + )); + } else { + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + tag_base_type_di_node, + )); + } unions_fields } @@ -485,6 +868,7 @@ struct VariantFieldInfo<'ll> { variant_index: VariantIdx, variant_struct_type_di_node: &'ll DIType, source_info: Option<(&'ll DIFile, c_uint)>, + discr: DiscrResult, } fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { @@ -512,3 +896,29 @@ fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) } + +fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, str> { + const PRE_ALLOCATED: [&str; 16] = [ + "Variant0", + "Variant1", + "Variant2", + "Variant3", + "Variant4", + "Variant5", + "Variant6", + "Variant7", + "Variant8", + "Variant9", + "Variant10", + "Variant11", + "Variant12", + "Variant13", + "Variant14", + "Variant15", + ]; + + PRE_ALLOCATED + .get(variant_index.as_usize()) + .map(|&s| Cow::from(s)) + .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 73e01d0453b..9b3d080bfd6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -10,7 +10,6 @@ use rustc_middle::{ ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - util::Discr, AdtDef, GeneratorSubsts, Ty, VariantDef, }, }; @@ -90,8 +89,11 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( cx, &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), tag_base_type(cx, enum_type_and_layout), - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + // Is there anything we can do to support 128-bit C-Style enums? + let value = discr.val as u64; + (name, value) }), containing_scope, ), @@ -152,7 +154,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, type_name: &str, base_type: Ty<'tcx>, - variants: &mut dyn Iterator<Item = (Discr<'tcx>, Cow<'tcx, str>)>, + enumerators: impl Iterator<Item = (Cow<'tcx, str>, u64)>, containing_scope: &'ll DIType, ) -> &'ll DIType { let is_unsigned = match base_type.kind() { @@ -161,18 +163,15 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), }; - let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = variants - .map(|(discr, variant_name)| { - unsafe { - Some(llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - variant_name.as_ptr().cast(), - variant_name.len(), - // FIXME: what if enumeration has i128 discriminant? - discr.val as i64, - is_unsigned, - )) - } + let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = enumerators + .map(|(name, value)| unsafe { + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + name.as_ptr().cast(), + name.len(), + value as i64, + is_unsigned, + )) }) .collect(); @@ -247,23 +246,27 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( /// and a DW_TAG_member for each field (but not the discriminant). fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_type: Ty<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, variant_index: VariantIdx, variant_def: &VariantDef, variant_layout: TyAndLayout<'tcx>, ) -> &'ll DIType { - debug_assert_eq!(variant_layout.ty, enum_type); + debug_assert_eq!(variant_layout.ty, enum_type_and_layout.ty); type_map::build_type_with_children( cx, type_map::stub( cx, Stub::Struct, - UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), + UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + enum_type_and_layout.ty, + variant_index, + ), variant_def.name.as_str(), // NOTE: We use size and align of enum_type, not from variant_layout: - cx.size_and_align_of(enum_type), + size_and_align_of(enum_type_and_layout), Some(enum_type_di_node), DIFlags::FlagZero, ), @@ -290,9 +293,9 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( type_di_node(cx, field_layout.ty), ) }) - .collect() + .collect::<SmallVec<_>>() }, - |cx| build_generic_type_param_di_nodes(cx, enum_type), + |cx| build_generic_type_param_di_nodes(cx, enum_type_and_layout.ty), ) .di_node } @@ -398,6 +401,19 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .di_node } +#[derive(Copy, Clone)] +enum DiscrResult { + NoDiscriminant, + Value(u128), + Range(u128, u128), +} + +impl DiscrResult { + fn opt_single_val(&self) -> Option<u128> { + if let Self::Value(d) = *self { Some(d) } else { None } + } +} + /// Returns the discriminant value corresponding to the variant index. /// /// Will return `None` if there is less than two variants (because then the enum won't have) @@ -407,12 +423,11 @@ fn compute_discriminant_value<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, variant_index: VariantIdx, -) -> Option<u64> { +) -> DiscrResult { match enum_type_and_layout.layout.variants() { - &Variants::Single { .. } => None, - &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( - enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val - as u64, + &Variants::Single { .. } => DiscrResult::NoDiscriminant, + &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => DiscrResult::Value( + enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val, ), &Variants::Multiple { tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, @@ -420,17 +435,26 @@ fn compute_discriminant_value<'ll, 'tcx>( .. } => { if variant_index == dataful_variant { - None + let valid_range = enum_type_and_layout + .for_variant(cx, variant_index) + .largest_niche + .as_ref() + .unwrap() + .valid_range; + + let min = valid_range.start.min(valid_range.end); + let min = tag.size(cx).truncate(min); + + let max = valid_range.start.max(valid_range.end); + let max = tag.size(cx).truncate(max); + + DiscrResult::Range(min, max) } else { let value = (variant_index.as_u32() as u128) .wrapping_sub(niche_variants.start().as_u32() as u128) .wrapping_add(niche_start); let value = tag.size(cx).truncate(value); - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - Some(value as u64) + DiscrResult::Value(value) } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index f1935e0ec31..dae90a43f26 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -88,7 +88,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( cx, - enum_type, + enum_type_and_layout, enum_type_di_node, variant_index, enum_adt_def.variant(variant_index), @@ -413,7 +413,13 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.map(|v| cx.const_u64(v)), + discr_value.opt_single_val().map(|value| { + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + cx.const_u64(value as u64) + }), DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 87fbb737ea8..e30622cbdce 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -47,6 +47,8 @@ pub(super) enum UniqueTypeId<'tcx> { VariantPart(Ty<'tcx>, private::HiddenZst), /// The ID for the artificial struct type describing a single enum variant. VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode. + VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst), /// The ID of the artificial type we create for VTables. VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst), } @@ -71,6 +73,15 @@ impl<'tcx> UniqueTypeId<'tcx> { UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) } + pub fn for_enum_variant_struct_type_wrapper( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantStructTypeCppLikeWrapper(enum_ty, variant_idx, private::HiddenZst) + } + pub fn for_vtable_ty( tcx: TyCtxt<'tcx>, self_type: Ty<'tcx>, @@ -93,8 +104,9 @@ impl<'tcx> UniqueTypeId<'tcx> { /// Right now this takes the form of a hex-encoded opaque hash value. pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { let mut hasher = StableHasher::new(); - let mut hcx = tcx.create_stable_hashing_context(); - hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)); + tcx.with_stable_hashing_context(|mut hcx| { + hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) + }); hasher.finish::<Fingerprint>().to_hex() } @@ -145,7 +157,7 @@ impl<'ll> DINodeCreationResult<'ll> { pub enum Stub<'ll> { Struct, Union, - VtableTy { vtable_holder: &'ll DIType }, + VTableTy { vtable_holder: &'ll DIType }, } pub struct StubInfo<'ll, 'tcx> { @@ -179,9 +191,9 @@ pub(super) fn stub<'ll, 'tcx>( let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); let metadata = match kind { - Stub::Struct | Stub::VtableTy { .. } => { + Stub::Struct | Stub::VTableTy { .. } => { let vtable_holder = match kind { - Stub::VtableTy { vtable_holder } => Some(vtable_holder), + Stub::VTableTy { vtable_holder } => Some(vtable_holder), _ => None, }; unsafe { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 0910e7c94ea..b23fe3fc9d5 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -27,7 +27,7 @@ use rustc_index::vec::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable}; +use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -36,10 +36,9 @@ use rustc_target::abi::Size; use libc::c_uint; use smallvec::SmallVec; +use std::cell::OnceCell; use std::cell::RefCell; use std::iter; -use std::lazy::OnceCell; -use tracing::debug; mod create_scope_map; pub mod gdb; @@ -97,23 +96,26 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder); - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - if let Some(version) = sess.target.dwarf_version { + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = sess + .opts + .unstable_opts + .dwarf_version + .unwrap_or(sess.target.default_dwarf_version); llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, "Dwarf Version\0".as_ptr().cast(), - version, - ) - } - - // Indicate that we want CodeView debug information on MSVC - if sess.target.is_like_msvc { + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 8f243673907..a40cfc8b23f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -6,7 +6,7 @@ use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; use rustc_middle::ty::{self, DefIdTree, Ty}; -use tracing::trace; +use trace; use crate::common::CodegenCx; use crate::llvm; diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 5a5c4f7f860..0f663a26732 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -18,9 +18,10 @@ use crate::llvm; use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; +use rustc_codegen_ssa::traits::TypeMembershipMethods; use rustc_middle::ty::Ty; +use rustc_symbol_mangling::typeid::typeid_for_fnabi; use smallvec::SmallVec; -use tracing::debug; /// Declare a function. /// @@ -97,6 +98,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.llvm_type(self), ); fn_abi.apply_attrs_llfn(self, llfn); + + if self.tcx.sess.is_sanitizer_cfi_enabled() { + let typeid = typeid_for_fnabi(self.tcx, fn_abi); + self.set_type_metadata(llfn, typeid); + } + llfn } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4407297c943..3d789904707 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -71,6 +71,7 @@ fn get_simple_intrinsic<'ll>( sym::nearbyintf64 => "llvm.nearbyint.f64", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::ptr_mask => "llvm.ptrmask", _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -161,7 +162,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::volatile_load | sym::unaligned_volatile_load => { let tp_ty = substs.type_at(0); let ptr = args[0].immediate(); - let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { + let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let llty = ty.llvm_type(self); let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); self.volatile_load(llty, ptr) @@ -374,7 +375,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { }; if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { + if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); let ptr = self.pointercast(result.llval, ptr_llty); self.store(llval, ptr, result.align); @@ -406,6 +407,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) } + fn type_checked_load( + &mut self, + llvtable: &'ll Value, + vtable_byte_offset: u64, + typeid: &'ll Value, + ) -> Self::Value { + let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + } + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { self.call_intrinsic("llvm.va_start", &[va_list]) } @@ -431,7 +442,7 @@ fn try_intrinsic<'ll>( bx.store(bx.const_i32(0), dest, ret_align); } else if wants_msvc_seh(bx.sess()) { codegen_msvc_try(bx, try_func, data, catch_func, dest); - } else if bx.sess().target.is_like_emscripten { + } else if bx.sess().target.os == "emscripten" { codegen_emcc_try(bx, try_func, data, catch_func, dest); } else { codegen_gnu_try(bx, try_func, data, catch_func, dest); @@ -816,7 +827,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, ) -> Result<&'ll Value, ()> { // macros for error handling: - #[cfg_attr(not(bootstrap), allow(unused_macro_rules))] + #[allow(unused_macro_rules)] macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) @@ -1145,7 +1156,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { - #[cfg_attr(not(bootstrap), allow(unused_macro_rules))] + #[allow(unused_macro_rules)] macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 0bead4629a6..334425ae55b 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,18 +5,19 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(crate_visibility_modifier)] +#![feature(hash_raw_entry)] #![feature(let_chains)] #![feature(let_else)] #![feature(extern_types)] #![feature(once_cell)] -#![feature(nll)] #![feature(iter_intersperse)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] #[macro_use] extern crate rustc_macros; +#[macro_use] +extern crate tracing; use back::write::{create_informational_target_machine, create_target_machine}; @@ -305,8 +306,8 @@ impl CodegenBackend for LlvmCodegenBackend { local stack variable in the ABI.) basic - Generate stack canaries in functions with: - - local variables of `[T; N]` type, where `T` is byte-sized and `N` > 8. + Generate stack canaries in functions with local variables of `[T; N]` + type, where `T` is byte-sized and `N` >= 8. none Do not generate stack canaries. @@ -325,8 +326,8 @@ impl CodegenBackend for LlvmCodegenBackend { llvm_util::print_version(); } - fn target_features(&self, sess: &Session) -> Vec<Symbol> { - target_features(sess) + fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec<Symbol> { + target_features(sess, allow_unstable) } fn codegen_crate<'tcx>( @@ -356,7 +357,7 @@ impl CodegenBackend for LlvmCodegenBackend { .join(sess); sess.time("llvm_dump_timing_file", || { - if sess.opts.debugging_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { let file_name = outputs.with_extension("llvm_timings.json"); llvm_util::time_trace_profiler_finish(&file_name); } @@ -371,12 +372,12 @@ impl CodegenBackend for LlvmCodegenBackend { codegen_results: CodegenResults, outputs: &OutputFilenames, ) -> Result<(), ErrorGuaranteed> { - use crate::back::archive::LlvmArchiveBuilder; + use crate::back::archive::LlvmArchiveBuilderBuilder; use rustc_codegen_ssa::back::link::link_binary; // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary::<LlvmArchiveBuilder<'_>>(sess, &codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 64db4f7462d..7d948970223 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -83,17 +83,6 @@ impl<'a> Child<'a> { } } } - - pub fn data(&self) -> &'a [u8] { - unsafe { - let mut data_len = 0; - let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); - if data_ptr.is_null() { - panic!("failed to read data from archive child"); - } - slice::from_raw_parts(data_ptr as *const u8, data_len as usize) - } - } } impl<'a> Drop for Child<'a> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index e2fa5e488ed..45de284d22a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -20,19 +20,6 @@ pub enum OptimizationDiagnosticKind { OptimizationRemarkOther, } -impl OptimizationDiagnosticKind { - pub fn describe(self) -> &'static str { - match self { - OptimizationRemark | OptimizationRemarkOther => "remark", - OptimizationMissed => "missed", - OptimizationAnalysis => "analysis", - OptimizationAnalysisFPCommute => "floating-point", - OptimizationAnalysisAliasing => "aliasing", - OptimizationFailure => "failure", - } - } -} - pub struct OptimizationDiagnostic<'ll> { pub kind: OptimizationDiagnosticKind, pub pass_name: String, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 13baaddccd4..172684414fc 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -191,6 +191,11 @@ pub enum AttributeKind { StackProtect = 32, NoUndef = 33, SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, } /// LLVMIntPredicate @@ -381,9 +386,8 @@ pub enum AtomicOrdering { impl AtomicOrdering { pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { match ao { - rustc_codegen_ssa::common::AtomicOrdering::NotAtomic => AtomicOrdering::NotAtomic, rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered, - rustc_codegen_ssa::common::AtomicOrdering::Monotonic => AtomicOrdering::Monotonic, + rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic, rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire, rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release, rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => { @@ -443,6 +447,7 @@ pub enum MetadataType { MD_nonnull = 11, MD_align = 17, MD_type = 19, + MD_vcall_visibility = 28, MD_noundef = 29, } @@ -567,16 +572,6 @@ pub enum ArchiveKind { K_COFF, } -/// LLVMRustPassKind -#[derive(Copy, Clone, PartialEq, Debug)] -#[repr(C)] -#[allow(dead_code)] // Variants constructed by C++. -pub enum PassKind { - Other, - Function, - Module, -} - // LLVMRustThinLTOData extern "C" { pub type ThinLTOData; @@ -587,10 +582,6 @@ extern "C" { pub type ThinLTOBuffer; } -// LLVMRustModuleNameCallback -pub type ThinLTOModuleNameCallback = - unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); - /// LLVMRustThinLTOModule #[repr(C)] pub struct ThinLTOModule { @@ -656,9 +647,6 @@ extern "C" { } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); -extern "C" { - pub type MemoryBuffer; -} #[repr(C)] pub struct PassManager<'a>(InvariantOpaque<'a>); extern "C" { @@ -775,7 +763,7 @@ pub mod coverageinfo { } impl CounterMappingRegion { - crate fn code_region( + pub(crate) fn code_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -799,7 +787,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn branch_region( + pub(crate) fn branch_region( counter: coverage_map::Counter, false_counter: coverage_map::Counter, file_id: u32, @@ -824,7 +812,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn expansion_region( + pub(crate) fn expansion_region( file_id: u32, expanded_file_id: u32, start_line: u32, @@ -848,7 +836,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn skipped_region( + pub(crate) fn skipped_region( file_id: u32, start_line: u32, start_col: u32, @@ -871,7 +859,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn gap_region( + pub(crate) fn gap_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -984,6 +972,22 @@ pub mod debuginfo { } } +use bitflags::bitflags; +// These values **must** match with LLVMRustAllocKindFlags +bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct AllocKindFlags : u64 { + const Unknown = 0; + const Alloc = 1; + const Realloc = 1 << 1; + const Free = 1 << 2; + const Uninitialized = 1 << 3; + const Zeroed = 1 << 4; + const Aligned = 1 << 5; + } +} + extern "C" { pub type ModuleBuffer; } @@ -1011,7 +1015,6 @@ extern "C" { pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMSetModuleInlineAsm2(M: &Module, Asm: *const c_char, AsmLen: size_t); pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); /// See llvm::LLVMTypeKind::getTypeID. @@ -1068,6 +1071,7 @@ extern "C" { pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1081,6 +1085,11 @@ extern "C" { Vals: *const &'a Value, Count: c_uint, ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); // Operations on scalar constants @@ -1128,11 +1137,7 @@ extern "C" { pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstExtractValue( - AggConstant: &Value, - IdxList: *const c_uint, - NumIdx: c_uint, - ) -> &Value; + pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1163,7 +1168,6 @@ extern "C" { pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; - pub fn LLVMSetThreadLocal(GlobalVar: &Value, IsThreadLocal: Bool); pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); @@ -1188,7 +1192,10 @@ extern "C" { pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( @@ -1617,7 +1624,7 @@ extern "C" { B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, - IsSized: bool, + IsSigned: bool, ) -> &'a Value; // Comparisons @@ -1937,6 +1944,7 @@ extern "C" { name: *const c_char, value: u32, ); + pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; @@ -2071,6 +2079,19 @@ extern "C" { Ty: &'a DIType, ) -> &'a DIType; + pub fn LLVMRustDIBuilderCreateStaticMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + val: Option<&'a Value>, + AlignInBits: u32, + ) -> &'a DIDerivedType; + pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2241,7 +2262,6 @@ extern "C" { pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - pub fn LLVMRustPassKind(Pass: &Pass) -> PassKind; pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; @@ -2340,6 +2360,7 @@ extern "C" { PGOGenPath: *const c_char, PGOUsePath: *const c_char, InstrumentCoverage: bool, + InstrProfileOutput: *const c_char, InstrumentGCOV: bool, PGOSampleUsePath: *const c_char, DebugInfoForProfiling: bool, @@ -2358,7 +2379,6 @@ extern "C" { ) -> LLVMRustResult; pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); pub fn LLVMRustPrintPasses(); - pub fn LLVMRustGetInstructionCount(M: &Module) -> u32; pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); @@ -2369,7 +2389,6 @@ extern "C" { AIR: &ArchiveIterator<'a>, ) -> Option<&'a mut ArchiveChild<'a>>; pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); @@ -2404,12 +2423,6 @@ extern "C" { cookie_out: &mut c_uint, ) -> &'a SMDiagnostic; - pub fn LLVMRustSetInlineAsmDiagnosticHandler( - C: &Context, - H: InlineAsmDiagHandlerTy, - CX: *mut c_void, - ); - #[allow(improper_ctypes)] pub fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, @@ -2456,7 +2469,6 @@ extern "C" { pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMRustUnsetComdat(V: &Value); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); @@ -2466,7 +2478,7 @@ extern "C" { pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub fn LLVMRustModuleCost(M: &Module) -> u64; - pub fn LLVMRustThinLTOBufferCreate(M: &Module) -> &'static mut ThinLTOBuffer; + pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; @@ -2488,11 +2500,6 @@ extern "C" { Module: &Module, Target: &TargetMachine, ) -> bool; - pub fn LLVMRustGetThinLTOModuleImports( - Data: *const ThinLTOData, - ModuleNameCallback: ThinLTOModuleNameCallback, - CallbackPayload: *mut c_void, - ); pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); pub fn LLVMRustParseBitcodeForLTO( Context: &Context, @@ -2505,8 +2512,12 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustLTOGetDICompileUnit(M: &Module, CU1: &mut *mut c_void, CU2: &mut *mut c_void); - pub fn LLVMRustLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + pub fn LLVMRustThinLTOGetDICompileUnit( + M: &Module, + CU1: &mut *mut c_void, + CU2: &mut *mut c_void, + ); + pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( @@ -2538,4 +2549,6 @@ extern "C" { #[allow(improper_ctypes)] pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); + + pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 48fbc1de8ee..6602a4ab863 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -95,6 +95,14 @@ pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } } +pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { + unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } +} + +pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { + unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, @@ -158,12 +166,6 @@ pub fn SetUniqueComdat(llmod: &Module, val: &Value) { } } -pub fn UnsetComdat(val: &Value) { - unsafe { - LLVMRustUnsetComdat(val); - } -} - pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { unsafe { LLVMSetUnnamedAddress(global, unnamed); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 7b407c94e7b..1b049dfe979 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -15,7 +15,6 @@ use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; -use tracing::debug; use std::mem; use std::path::Path; @@ -87,22 +86,12 @@ unsafe fn configure_llvm(sess: &Session) { add("-debug-pass=Structure", false); } if sess.target.generate_arange_section - && !sess.opts.debugging_opts.no_generate_arange_section + && !sess.opts.unstable_opts.no_generate_arange_section { add("-generate-arange-section", false); } - // Disable the machine outliner by default in LLVM versions 11 and LLVM - // version 12, where it leads to miscompilation. - // - // Ref: - // - https://github.com/rust-lang/rust/issues/85351 - // - https://reviews.llvm.org/D103167 - if llvm_util::get_version() < (13, 0, 0) { - add("-enable-machine-outliner=never", false); - } - - match sess.opts.debugging_opts.merge_functions.unwrap_or(sess.target.merge_functions) { + match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { MergeFunctions::Disabled | MergeFunctions::Trampolines => {} MergeFunctions::Aliases => { add("-mergefunc-use-aliases", false); @@ -125,7 +114,7 @@ unsafe fn configure_llvm(sess: &Session) { } } - if sess.opts.debugging_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { llvm::LLVMTimeTraceProfilerInitialize(); } @@ -133,11 +122,11 @@ unsafe fn configure_llvm(sess: &Session) { // Use the legacy plugin registration if we don't use the new pass manager if !should_use_new_llvm_pass_manager( - &sess.opts.debugging_opts.new_llvm_pass_manager, + &sess.opts.unstable_opts.new_llvm_pass_manager, &sess.target.arch, ) { // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.debugging_opts.llvm_plugins { + for plugin in &sess.opts.unstable_opts.llvm_plugins { let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); @@ -218,39 +207,44 @@ pub fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, ) -> Option<&'static [&'static str]> { - for tied in tied_target_features(sess) { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); + if !features.is_empty() { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } } } - None + return None; } // Used to generate cfg variables and apply features // Must express features in the way Rust understands them -pub fn target_features(sess: &Session) -> Vec<Symbol> { +pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); - let mut features: Vec<Symbol> = - supported_target_features(sess) - .iter() - .filter_map(|&(feature, gate)| { - if sess.is_nightly_build() || gate.is_none() { Some(feature) } else { None } - }) - .filter(|feature| { - // check that all features in a given smallvec are enabled - for llvm_feature in to_llvm_features(sess, feature) { - let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { - return false; - } + let mut features: Vec<Symbol> = supported_target_features(sess) + .iter() + .filter_map(|&(feature, gate)| { + if sess.is_nightly_build() || allow_unstable || gate.is_none() { + Some(feature) + } else { + None + } + }) + .filter(|feature| { + // check that all features in a given smallvec are enabled + for llvm_feature in to_llvm_features(sess, feature) { + let cstr = SmallCStr::new(llvm_feature); + if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return false; } - true - }) - .map(|feature| Symbol::intern(feature)) - .collect(); + } + true + }) + .map(|feature| Symbol::intern(feature)) + .collect(); // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use @@ -435,11 +429,14 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + // Drop +atomics-32 feature introduced in LLVM 15. + .filter(|v| *v != "+atomics-32" || get_version() >= (15, 0, 0)) .map(String::from), ); // -Ctarget-features let supported_features = supported_target_features(sess); + let mut featsmap = FxHashMap::default(); let feats = sess .opts .cg @@ -485,35 +482,36 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } diag.emit(); } - Some((enable_disable, feature)) + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } + + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some( + to_llvm_features(sess, feature) + .into_iter() + .map(move |f| format!("{}{}", enable_disable, f)), + ) }) - .collect::<SmallVec<[(char, &str); 8]>>(); - - if diagnostics { - // FIXME(nagisa): figure out how to not allocate a full hashset here. - let featmap = feats.iter().map(|&(flag, feat)| (feat, flag == '+')).collect(); - if let Some(f) = check_tied_features(sess, &featmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); - } + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); } - features.extend(feats.into_iter().flat_map(|(enable_disable, feature)| { - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return SmallVec::<[_; 2]>::new(); - } - // ... otherwise though we run through `to_llvm_features` when - // passing requests down to LLVM. This means that all in-language - // features also work on the command line instead of having two - // different names when the LLVM name and the Rust name differ. - to_llvm_features(sess, feature) - .into_iter() - .map(|f| format!("{}{}", enable_disable, f)) - .collect() - })); features } @@ -534,7 +532,7 @@ fn backend_feature_name(s: &str) -> Option<&str> { } pub fn tune_cpu(sess: &Session) -> Option<&str> { - let name = sess.opts.debugging_opts.tune_cpu.as_ref()?; + let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index a3053742aad..1eceb7f5c87 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -8,10 +8,9 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; -use rustc_middle::ty::{self, Instance, TypeFoldable}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; -use tracing::debug; impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 21b77f7dea6..eeb38d4ecf5 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -19,7 +19,7 @@ use rustc_target::abi::{AddressSpace, Align, Integer, Size}; use std::fmt; use std::ptr; -use libc::c_uint; +use libc::{c_char, c_uint}; impl PartialEq for Type { fn eq(&self, other: &Self) -> bool { @@ -39,33 +39,33 @@ impl fmt::Debug for Type { } impl<'ll> CodegenCx<'ll, '_> { - crate fn type_named_struct(&self, name: &str) -> &'ll Type { + pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { let name = SmallCStr::new(name); unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } } - crate fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { + pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } } - crate fn type_void(&self) -> &'ll Type { + pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } } - crate fn type_metadata(&self) -> &'ll Type { + pub(crate) fn type_metadata(&self) -> &'ll Type { unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 - crate fn type_ix(&self, num_bits: u64) -> &'ll Type { + pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } } - crate fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { unsafe { llvm::LLVMVectorType(ty, len as c_uint) } } - crate fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { unsafe { let n_args = llvm::LLVMCountParamTypes(ty) as usize; let mut args = Vec::with_capacity(n_args); @@ -75,11 +75,11 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_bool(&self) -> &'ll Type { + pub(crate) fn type_bool(&self) -> &'ll Type { self.type_i8() } - crate fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { + pub(crate) fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { match t { ty::IntTy::Isize => self.type_isize(), ty::IntTy::I8 => self.type_i8(), @@ -90,7 +90,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { + pub(crate) fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { match t { ty::UintTy::Usize => self.type_isize(), ty::UintTy::U8 => self.type_i8(), @@ -101,14 +101,14 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { + pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { match t { ty::FloatTy::F32 => self.type_f32(), ty::FloatTy::F64 => self.type_f64(), } } - crate fn type_pointee_for_align(&self, align: Align) -> &'ll Type { + pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { // FIXME(eddyb) We could find a better approximation if ity.align < align. let ity = Integer::approximate_align(self, align); self.type_from_integer(ity) @@ -116,7 +116,7 @@ impl<'ll> CodegenCx<'ll, '_> { /// Return a LLVM type that has at most the required alignment, /// and exactly the required size, as a best-effort padding array. - crate fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { + pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { let unit = Integer::approximate_align(self, align); let size = size.bytes(); let unit_size = unit.size().bytes(); @@ -124,11 +124,11 @@ impl<'ll> CodegenCx<'ll, '_> { self.type_array(self.type_from_integer(unit), size / unit_size) } - crate fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } } - crate fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { unsafe { llvm::LLVMRustArrayType(ty, len) } } } @@ -289,3 +289,31 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { ty.llvm_type(self) } } + +impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn typeid_metadata(&self, typeid: String) -> &'ll Value { + unsafe { + llvm::LLVMMDStringInContext( + self.llcx, + typeid.as_ptr() as *const c_char, + typeid.len() as c_uint, + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 86280523631..dc1165835e7 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -6,12 +6,11 @@ use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Ty, TypeFoldable}; +use rustc_middle::ty::{self, Ty, TypeVisitable}; use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; use smallvec::{smallvec, SmallVec}; -use tracing::debug; use std::fmt::Write; |