diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
32 files changed, 1142 insertions, 598 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index d4437bd449d..26f5225f6b4 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -19,6 +19,7 @@ use rustc_target::abi::call::ArgAbi; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int}; pub use rustc_target::spec::abi::Abi; +use rustc_target::spec::SanitizerSet; use libc::c_uint; use smallvec::SmallVec; @@ -90,6 +91,13 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); } + } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { + // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects + // memory sanitizer's behavior. + + if regular.contains(ArgAttribute::NoUndef) { + attrs.push(llvm::AttributeKind::NoUndef.create_attr(cx.llcx)); + } } attrs @@ -213,7 +221,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) } else if self.is_unsized_indirect() { bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast(cast) = self.mode { + } else if let PassMode::Cast(cast, _) = &self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. let can_store_through_cast_ptr = false; @@ -283,7 +291,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(_) | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } - | PassMode::Cast(_) => { + | PassMode::Cast(..) => { let next_arg = next(); self.store(bx, next_arg, dst); } @@ -325,20 +333,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { // Ignore "extra" args from the call site for C variadic functions. // Only the "fixed" args are part of the LLVM function signature. - let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + let args = + if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args }; - let args_capacity: usize = args.iter().map(|arg| - if arg.pad.is_some() { 1 } else { 0 } + - if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } - ).sum(); + // This capacity calculation is approximate. let mut llargument_tys = Vec::with_capacity( - if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, + self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }, ); - let llreturn_ty = match self.ret.mode { + let llreturn_ty = match &self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, _) => cast.llvm_type(cx), PassMode::Indirect { .. } => { llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); cx.type_void() @@ -346,12 +352,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { }; for arg in args { - // add padding - if let Some(ty) = arg.pad { - llargument_tys.push(ty.llvm_type(cx)); - } - - let llarg_ty = match arg.mode { + let llarg_ty = match &arg.mode { PassMode::Ignore => continue, PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), PassMode::Pair(..) => { @@ -366,7 +367,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, pad_i32) => { + // add padding + if *pad_i32 { + llargument_tys.push(Reg::i32().llvm_type(cx)); + } + cast.llvm_type(cx) + } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { cx.type_ptr_to(arg.memory_ty(cx)) } @@ -426,46 +433,46 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } _ => {} } - for arg in &self.args { - if arg.pad.is_some() { - apply(&ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(attrs); let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(attrs); } - PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { assert!(!on_stack); apply(attrs); apply(extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(a); apply(b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(&ArgAttributes::new()); + } apply(&cast.attrs); } } @@ -488,17 +495,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_callsite( llvm::AttributePlace::ReturnValue, &bx.cx, @@ -517,13 +524,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } } - for arg in &self.args { - if arg.pad.is_some() { - apply(bx.cx, &ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); attributes::apply_to_callsite( @@ -532,23 +536,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[byval], ); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(bx.cx, attrs); } - PassMode::Indirect { - ref attrs, - extra_attrs: Some(ref extra_attrs), - on_stack: _, - } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack: _ } => { apply(bx.cx, attrs); apply(bx.cx, extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(bx.cx, a); apply(bx.cx, b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(bx.cx, &ArgAttributes::new()); + } apply(bx.cx, &cast.attrs); } } @@ -569,6 +572,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[cmse_nonsecure_call], ); } + + // Some intrinsics require that an elementtype attribute (with the pointee type of a + // pointer argument) is added to the callsite. + let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; + if element_type_index >= 0 { + let arg_ty = self.args[element_type_index as usize].layout.ty; + let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; + let element_type_attr = unsafe { + llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) + }; + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(element_type_index as u32), + &[element_type_attr], + ); + } } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a53946995ee..5202ac697e9 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -3,7 +3,6 @@ use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -20,7 +19,6 @@ use rustc_target::asm::*; use libc::{c_char, c_uint}; use smallvec::SmallVec; -use tracing::debug; impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_inline_asm( @@ -419,13 +417,6 @@ pub(crate) fn inline_asm_call<'ll>( let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { - if unwind && llvm_util::get_version() < (13, 0, 0) { - bx.cx.sess().span_fatal( - line_spans[0], - "unwinding from inline assembly is only supported on llvm >= 13.", - ); - } - let v = llvm::LLVMRustInlineAsm( fty, asm.as_ptr().cast(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 32bfa5094c3..eff2436d41c 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -13,7 +13,7 @@ use smallvec::SmallVec; use crate::attributes; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, Attribute, AttributeKind, AttributePlace}; +use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace}; use crate::llvm_util; pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; @@ -35,6 +35,10 @@ pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attrib /// Get LLVM attribute for the provided inline heuristic. #[inline] fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { + if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + // disable LLVM inlining + return Some(AttributeKind::NoInline.create_attr(cx.llcx)); + } match inline { InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), @@ -69,6 +73,9 @@ pub fn sanitize_attrs<'ll>( if enabled.contains(SanitizerSet::HWADDRESS) { attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); } + if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { + attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); + } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. let features = cx.tcx.global_backend_features(()); @@ -224,6 +231,10 @@ pub(crate) fn default_optimisation_attrs<'ll>( attrs } +fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { + llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") +} + /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub fn from_fn_attrs<'ll, 'tcx>( @@ -299,12 +310,61 @@ pub fn from_fn_attrs<'ll, 'tcx>( } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); + // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. + // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. + // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 + to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); + // Need this for AArch64. + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) + || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) + { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; + } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + } // apply to return place instead of function (unlike all other attributes applied in this function) let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); + } + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + } + } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); } @@ -330,7 +390,8 @@ pub fn from_fn_attrs<'ll, 'tcx>( ) { let span = cx .tcx - .get_attr(instance.def_id(), sym::target_feature) + .get_attrs(instance.def_id(), sym::target_feature) + .next() .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); let msg = format!( "the target features {} must all be either enabled or disabled together", diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index bccc2a995a3..38a366095b4 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -8,18 +8,17 @@ use std::path::{Path, PathBuf}; use std::ptr; use std::str; +use crate::common; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; -use rustc_codegen_ssa::back::archive::ArchiveBuilder; -use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_session::cstore::{DllCallingConvention, DllImport}; +use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; +use rustc_session::cstore::DllImport; use rustc_session::Session; /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] pub struct LlvmArchiveBuilder<'a> { sess: &'a Session, - dst: PathBuf, additions: Vec<Addition>, } @@ -55,16 +54,11 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { } impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { - /// Creates a new static archive, ready for modifying the archive specified - /// by `config`. - fn new(sess: &'a Session, output: &Path) -> LlvmArchiveBuilder<'a> { - LlvmArchiveBuilder { sess, dst: output.to_path_buf(), additions: Vec::new() } - } - - fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> - where - F: FnMut(&str) -> bool + 'static, - { + fn add_archive( + &mut self, + archive: &Path, + skip: Box<dyn FnMut(&str) -> bool + 'static>, + ) -> io::Result<()> { let archive_ro = match ArchiveRO::open(archive) { Ok(ar) => ar, Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), @@ -89,41 +83,47 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. - fn build(mut self) -> bool { - match self.build_with_llvm() { + fn build(mut self: Box<Self>, output: &Path) -> bool { + match self.build_with_llvm(output) { Ok(any_members) => any_members, Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), } } +} + +pub struct LlvmArchiveBuilderBuilder; + +impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { + Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + } - fn inject_dll_import_lib( - &mut self, + fn create_dll_import_lib( + &self, + sess: &Session, lib_name: &str, dll_imports: &[DllImport], - tmpdir: &MaybeTempDir, - ) { + tmpdir: &Path, + ) -> PathBuf { let output_path = { - let mut output_path: PathBuf = tmpdir.as_ref().to_path_buf(); + let mut output_path: PathBuf = tmpdir.to_path_buf(); output_path.push(format!("{}_imports", lib_name)); output_path.with_extension("lib") }; - let target = &self.sess.target; - let mingw_gnu_toolchain = target.vendor == "pc" - && target.os == "windows" - && target.env == "gnu" - && target.abi.is_empty(); + let target = &sess.target; + let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(target); let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { - if self.sess.target.arch == "x86" { + if sess.target.arch == "x86" { ( - LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), - import.ordinal, + common::i686_decorated_name(import, mingw_gnu_toolchain, false), + import.ordinal(), ) } else { - (import.name.to_string(), import.ordinal) + (import.name.to_string(), import.ordinal()) } }) .collect(); @@ -134,8 +134,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { // that loaded but crashed with an AV upon calling one of the imported // functions. Therefore, use binutils to create the import library instead, // by writing a .DEF file to the temp dir and calling binutils's dlltool. - let def_file_path = - tmpdir.as_ref().join(format!("{}_imports", lib_name)).with_extension("def"); + let def_file_path = tmpdir.join(format!("{}_imports", lib_name)).with_extension("def"); let def_file_content = format!( "EXPORTS\n{}", @@ -154,11 +153,14 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - self.sess.fatal(&format!("Error writing .DEF file: {}", e)); + sess.fatal(&format!("Error writing .DEF file: {}", e)); } }; - let dlltool = find_binutils_dlltool(self.sess); + // --no-leading-underscore: For the `import_name_type` feature to work, we need to be + // able to control the *exact* spelling of each of the symbols that are being imported: + // hence we don't want `dlltool` adding leading underscores automatically. + let dlltool = find_binutils_dlltool(sess); let result = std::process::Command::new(dlltool) .args([ "-d", @@ -167,14 +169,15 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { lib_name, "-l", output_path.to_str().unwrap(), + "--no-leading-underscore", ]) .output(); match result { Err(e) => { - self.sess.fatal(&format!("Error calling dlltool: {}", e)); + sess.fatal(&format!("Error calling dlltool: {}", e)); } - Ok(output) if !output.status.success() => self.sess.fatal(&format!( + Ok(output) if !output.status.success() => sess.fatal(&format!( "Dlltool could not create import library: {}\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) @@ -187,10 +190,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { let output_path_z = rustc_fs_util::path_to_c_string(&output_path); - tracing::trace!("invoking LLVMRustWriteImportLibrary"); - tracing::trace!(" dll_name {:#?}", dll_name_z); - tracing::trace!(" output_path {}", output_path.display()); - tracing::trace!( + trace!("invoking LLVMRustWriteImportLibrary"); + trace!(" dll_name {:#?}", dll_name_z); + trace!(" output_path {}", output_path.display()); + trace!( " import names: {}", dll_imports .iter() @@ -220,13 +223,13 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path_z.as_ptr(), ffi_exports.as_ptr(), ffi_exports.len(), - llvm_machine_type(&self.sess.target.arch) as u16, - !self.sess.target.is_like_msvc, + llvm_machine_type(&sess.target.arch) as u16, + !sess.target.is_like_msvc, ) }; if result == crate::llvm::LLVMRustResult::Failure { - self.sess.fatal(&format!( + sess.fatal(&format!( "Error creating import library for {}: {}", lib_name, llvm::last_error().unwrap_or("unknown LLVM error".to_string()) @@ -234,18 +237,12 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { } }; - self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { - self.sess.fatal(&format!( - "failed to add native library {}: {}", - output_path.display(), - e - )); - }); + output_path } } impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self) -> io::Result<bool> { + fn build_with_llvm(&mut self, output: &Path) -> io::Result<bool> { let kind = &*self.sess.target.archive_format; let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| { self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) @@ -255,7 +252,7 @@ impl<'a> LlvmArchiveBuilder<'a> { let mut strings = Vec::new(); let mut members = Vec::new(); - let dst = CString::new(self.dst.to_str().unwrap())?; + let dst = CString::new(output.to_str().unwrap())?; unsafe { for addition in &mut additions { @@ -327,22 +324,6 @@ impl<'a> LlvmArchiveBuilder<'a> { ret } } - - fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { - let name = import.name; - let prefix = if mingw { "" } else { "_" }; - - match import.calling_convention { - DllCallingConvention::C => format!("{}{}", prefix, name), - DllCallingConvention::Stdcall(arg_list_size) => { - format!("{}{}@{}", prefix, name, arg_list_size) - } - DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), - DllCallingConvention::Vectorcall(arg_list_size) => { - format!("{}@@{}", name, arg_list_size) - } - } - } } fn string_to_io_error(s: String) -> io::Error { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index be539499b56..a89df00e248 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,15 +1,16 @@ use crate::back::write::{ self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, }; -use crate::llvm::archive_ro::ArchiveRO; use crate::llvm::{self, build_string, False, True}; use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; +use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; @@ -17,7 +18,6 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; -use tracing::{debug, info}; use std::ffi::{CStr, CString}; use std::fs::File; @@ -107,14 +107,24 @@ fn prepare_lto( .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); } - let archive = ArchiveRO::open(path).expect("wanted an rlib"); + let archive_data = unsafe { + Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib")) + .expect("couldn't map rlib") + }; + let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib"); let obj_files = archive - .iter() - .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) + .members() + .filter_map(|child| { + child.ok().and_then(|c| { + std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c)) + }) + }) .filter(|&(name, _)| looks_like_rust_object_file(name)); for (name, child) in obj_files { info!("adding bitcode from {}", name); - match get_bitcode_slice_from_object_data(child.data()) { + match get_bitcode_slice_from_object_data( + child.data(&*archive_data).expect("corrupt rlib"), + ) { Ok(data) => { let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); @@ -199,7 +209,7 @@ pub(crate) fn run_thin( pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { let name = module.name.clone(); - let buffer = ThinBuffer::new(module.module_llvm.llmod()); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -695,9 +705,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub fn new(m: &llvm::Module) -> ThinBuffer { + pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); ThinBuffer(buffer) } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 2b465ce40e7..a695df8409b 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -5,7 +5,7 @@ use crate::back::profiling::{ use crate::base; use crate::common; use crate::consts; -use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; +use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; @@ -28,7 +28,6 @@ use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; -use tracing::debug; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; @@ -304,7 +303,6 @@ impl<'a> DiagnosticHandlers<'a> { remark_passes.as_ptr(), remark_passes.len(), ); - llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); DiagnosticHandlers { data, llcx, old_handler } } } @@ -312,9 +310,7 @@ impl<'a> DiagnosticHandlers<'a> { impl<'a> Drop for DiagnosticHandlers<'a> { fn drop(&mut self) { - use std::ptr::null_mut; unsafe { - llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); drop(Box::from_raw(self.data)); } @@ -342,16 +338,6 @@ fn report_inline_asm( cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); } -unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { - if user.is_null() { - return; - } - let (cgcx, _) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); - - let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); - report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); -} - unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { if user.is_null() { return; @@ -423,6 +409,14 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } +fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { + if config.instrument_coverage { + Some(CString::new("default_%m_%p.profraw").unwrap()) + } else { + None + } +} + pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, @@ -438,6 +432,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + let instr_profile_output_path = get_instr_profile_output_path(config); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { @@ -488,6 +483,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, + instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, @@ -790,7 +786,7 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto); let data = thin.data(); if let Some(bitcode_filename) = bc_out.file_name() { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 4a4cccb490d..63b63c6a1fa 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,7 +3,6 @@ use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, BasicBlock, False}; use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -28,7 +27,6 @@ use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; -use tracing::{debug, instrument}; // All Builders must have an llfn associated with them #[must_use] @@ -626,32 +624,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn type_metadata(&mut self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); - let v = [self.const_usize(0), typeid_metadata]; - unsafe { - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( - self.cx.llcx, - v.as_ptr(), - v.len() as c_uint, - )), - ) - } - } - - fn typeid_metadata(&mut self, typeid: String) -> Self::Value { - unsafe { - llvm::LLVMMDStringInContext( - self.cx.llcx, - typeid.as_ptr() as *const c_char, - typeid.as_bytes().len() as c_uint, - ) - } - } - fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { self.store_with_flags(val, ptr, align, MemFlags::empty()) } @@ -752,11 +724,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(false, val, dest_ty) } - fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(true, val, dest_ty) } @@ -1064,25 +1036,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst: &'ll Value, cmp: &'ll Value, src: &'ll Value, - mut order: rustc_codegen_ssa::common::AtomicOrdering, + order: rustc_codegen_ssa::common::AtomicOrdering, failure_order: rustc_codegen_ssa::common::AtomicOrdering, weak: bool, ) -> &'ll Value { let weak = if weak { llvm::True } else { llvm::False }; - if llvm_util::get_version() < (13, 0, 0) { - use rustc_codegen_ssa::common::AtomicOrdering::*; - // Older llvm has the pre-C++17 restriction on - // success and failure memory ordering, - // requiring the former to be at least as strong as the latter. - // So, for llvm 12, we upgrade the success ordering to a stronger - // one if necessary. - match (order, failure_order) { - (Relaxed, Acquire) => order = Acquire, - (Release, Acquire) => order = AcquireRelease, - (_, SequentiallyConsistent) => order = SequentiallyConsistent, - _ => {} - } - } unsafe { llvm::LLVMRustBuildAtomicCmpXchg( self.llbuilder, @@ -1470,51 +1428,32 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - fn fptoint_sat_broken_in_llvm(&self) -> bool { - match self.tcx.sess.target.arch.as_ref() { - // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 - "riscv64" => llvm_util::get_version() < (13, 0, 0), - _ => false, - } - } - - fn fptoint_sat( - &mut self, - signed: bool, - val: &'ll Value, - dest_ty: &'ll Type, - ) -> Option<&'ll Value> { - if !self.fptoint_sat_broken_in_llvm() { - let src_ty = self.cx.val_ty(val); - let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector - { - assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); - ( - self.cx.element_type(src_ty), - self.cx.element_type(dest_ty), - Some(self.cx.vector_length(src_ty)), - ) - } else { - (src_ty, dest_ty, None) - }; - let float_width = self.cx.float_width(float_ty); - let int_width = self.cx.int_width(int_ty); - - let instr = if signed { "fptosi" } else { "fptoui" }; - let name = if let Some(vector_length) = vector_length { - format!( - "llvm.{}.sat.v{}i{}.v{}f{}", - instr, vector_length, int_width, vector_length, float_width - ) - } else { - format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) - }; - let f = - self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + ( + self.cx.element_type(src_ty), + self.cx.element_type(dest_ty), + Some(self.cx.vector_length(src_ty)), + ) } else { - None - } + (src_ty, dest_ty, None) + }; + let float_width = self.cx.float_width(float_ty); + let int_width = self.cx.int_width(int_ty); + + let instr = if signed { "fptosi" } else { "fptoui" }; + let name = if let Some(vector_length) = vector_length { + format!( + "llvm.{}.sat.v{}i{}.v{}f{}", + instr, vector_length, int_width, vector_length, float_width + ) + } else { + format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + }; + let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); + self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None) } pub(crate) fn landing_pad( diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 72155d874a2..b83c1e8f08f 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -6,11 +6,11 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; +use crate::common; use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; use rustc_codegen_ssa::traits::*; -use tracing::debug; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeVisitable}; @@ -79,13 +79,18 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> llfn } } else { - let llfn = cx.declare_fn(sym, fn_abi); + let instance_def_id = instance.def_id(); + let llfn = if tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) + { + cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi) + } else { + cx.declare_fn(sym, fn_abi) + }; debug!("get_fn: not casting pointer!"); attributes::from_fn_attrs(cx, llfn, instance); - let instance_def_id = instance.def_id(); - // Apply an appropriate linkage/visibility value to our item that we // just declared. // diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 77cbbf4c6ca..acee9134fb9 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -10,13 +10,17 @@ use crate::value::Value; use rustc_ast::Mutability; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::TyCtxt; +use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_target::spec::Target; use libc::{c_char, c_uint}; -use tracing::debug; +use std::fmt::Write; /* * A note on nomenclature of linking: "extern", "foreign", and "upcall". @@ -211,7 +215,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_to_opt_uint(&self, v: &'ll Value) -> Option<u64> { - try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) }) + try_as_const_integral(v).and_then(|v| unsafe { + let mut i = 0u64; + let success = llvm::LLVMRustConstIntGetZExtValue(v, &mut i); + success.then_some(i) + }) } fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option<u128> { @@ -222,10 +230,6 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } - fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { - self.const_undef(self.type_ix(0)) - } - fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { @@ -257,6 +261,15 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.get_fn_addr(fn_instance.polymorphize(self.tcx)), self.data_layout().instruction_address_space, ), + GlobalAlloc::VTable(ty, trait_ref) => { + let alloc = self + .tcx + .global_alloc(self.tcx.vtable_allocation((ty, trait_ref))) + .unwrap_memory(); + let init = const_alloc_to_llvm(self, alloc); + let value = self.static_addr_of(init, alloc.inner().align, None); + (value, AddressSpace::DATA) + } GlobalAlloc::Static(def_id) => { assert!(self.tcx.is_static(def_id)); assert!(!self.tcx.is_thread_local_static(def_id)); @@ -348,3 +361,74 @@ fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { unsafe { llvm::LLVMIsAConstantInt(v) } } + +pub(crate) fn get_dllimport<'tcx>( + tcx: TyCtxt<'tcx>, + id: DefId, + name: &str, +) -> Option<&'tcx DllImport> { + tcx.native_library(id) + .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) + .flatten() +} + +pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { + target.vendor == "pc" && target.os == "windows" && target.env == "gnu" && target.abi.is_empty() +} + +pub(crate) fn i686_decorated_name( + dll_import: &DllImport, + mingw: bool, + disable_name_mangling: bool, +) -> String { + let name = dll_import.name.as_str(); + + let (add_prefix, add_suffix) = match dll_import.import_name_type { + Some(PeImportNameType::NoPrefix) => (false, true), + Some(PeImportNameType::Undecorated) => (false, false), + _ => (true, true), + }; + + // Worst case: +1 for disable name mangling, +1 for prefix, +4 for suffix (@@__). + let mut decorated_name = String::with_capacity(name.len() + 6); + + if disable_name_mangling { + // LLVM uses a binary 1 ('\x01') prefix to a name to indicate that mangling needs to be disabled. + decorated_name.push('\x01'); + } + + let prefix = if add_prefix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C | DllCallingConvention::Vectorcall(_) => None, + DllCallingConvention::Stdcall(_) => (!mingw + || dll_import.import_name_type == Some(PeImportNameType::Decorated)) + .then_some('_'), + DllCallingConvention::Fastcall(_) => Some('@'), + } + } else if !dll_import.is_fn && !mingw { + // For static variables, prefix with '_' on MSVC. + Some('_') + } else { + None + }; + if let Some(prefix) = prefix { + decorated_name.push(prefix); + } + + decorated_name.push_str(name); + + if add_suffix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C => {} + DllCallingConvention::Stdcall(arg_list_size) + | DllCallingConvention::Fastcall(arg_list_size) => { + write!(&mut decorated_name, "@{}", arg_list_size).unwrap(); + } + DllCallingConvention::Vectorcall(arg_list_size) => { + write!(&mut decorated_name, "@@{}", arg_list_size).unwrap(); + } + } + } + + decorated_name +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 2b16ae1a88d..a559f7f3d57 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,5 +1,5 @@ use crate::base; -use crate::common::CodegenCx; +use crate::common::{self, CodegenCx}; use crate::debuginfo; use crate::llvm::{self, True}; use crate::llvm_util; @@ -23,16 +23,15 @@ use rustc_target::abi::{ AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, }; use std::ops::Range; -use tracing::debug; pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1); let dl = cx.data_layout(); let pointer_size = dl.pointer_size.bytes() as usize; - // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, - // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` + // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( llvals: &mut Vec<&'ll Value>, cx: &'a CodegenCx<'ll, 'b>, @@ -79,12 +78,12 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } let mut next_offset = 0; - for &(offset, alloc_id) in alloc.relocations().iter() { + for &(offset, alloc_id) in alloc.provenance().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a relocation, it + // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); @@ -93,7 +92,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< dl.endian, // This `inspect` is okay since it is within the bounds of the allocation, it doesn't // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the relocation as a relocation pointer offset. + // and we properly interpret the provenance as a relocation pointer offset. alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), ) .expect("const_alloc_to_llvm: could not read relocation pointer") @@ -101,7 +100,9 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< let address_space = match cx.tcx.global_alloc(alloc_id) { GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, - GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) => AddressSpace::DATA, + GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => { + AddressSpace::DATA + } }; llvals.push(cx.scalar_to_backend( @@ -119,7 +120,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } if alloc.len() >= next_offset { let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all relocations, it is + // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); @@ -158,7 +159,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str, - span_def_id: DefId, + def_id: DefId, ) -> &'ll Value { let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.linkage { @@ -173,7 +174,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( cx.layout_of(mt.ty).llvm_type(cx) } else { cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), + cx.tcx.def_span(def_id), "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", ) }; @@ -192,7 +193,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), + cx.tcx.def_span(def_id), &format!("symbol `{}` is already defined", &sym), ) }); @@ -200,6 +201,10 @@ fn check_and_apply_linkage<'ll, 'tcx>( llvm::LLVMSetInitializer(g2, g1); g2 } + } else if cx.tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) + { + cx.declare_global(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), true), llty) } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global @@ -473,7 +478,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // // We could remove this hack whenever we decide to drop macOS 10.10 support. if self.tcx.sess.target.is_like_osx { - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state // (not as part of the interpreter execution). // @@ -481,7 +486,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // happens to be zero. Instead, we should only check the value of defined bytes // and set all undefined bytes to zero if this allocation is headed for the // BSS. - let all_bytes_are_zero = alloc.relocations().is_empty() + let all_bytes_are_zero = alloc.provenance().is_empty() && alloc .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) .iter() @@ -505,9 +510,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { section.as_str().as_ptr().cast(), section.as_str().len() as c_uint, ); - assert!(alloc.relocations().is_empty()); + assert!(alloc.provenance().is_empty()); - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state (not // as part of the interpreter execution). let bytes = @@ -535,10 +540,20 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // The semantics of #[used] in Rust only require the symbol to make it into the // object file. It is explicitly allowed for the linker to strip the symbol if it - // is dead. As such, use llvm.compiler.used instead of llvm.used. + // is dead, which means we are allowed use `llvm.compiler.used` instead of + // `llvm.used` here. + // // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs - // in some versions of the gold linker. + // in the handling of `.init_array` (the static constructor list) in versions of + // the gold linker (prior to the one released with binutils 2.36). + // + // That said, we only ever emit these when compiling for ELF targets, unless + // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage + // on other targets, in particular MachO targets have *their* static constructor + // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, + // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_typeck`, + // so we don't need to take care of it here. self.add_compiler_used_global(g); } if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 5857b83f6c9..67ffc7cb951 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -142,17 +142,6 @@ pub unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (13, 0, 0) { - if sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-S128", ""); - } - if sess.target.arch == "wasm32" { - target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); - } - if sess.target.arch == "wasm64" { - target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); - } - } if llvm_version < (14, 0, 0) { if sess.target.llvm_target == "i686-pc-windows-msvc" || sess.target.llvm_target == "i586-pc-windows-msvc" @@ -897,6 +886,9 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } + + ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); + None } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 58f391692c4..0d1df6fb1ac 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -16,8 +16,6 @@ use rustc_middle::ty::TyCtxt; use std::ffi::CString; -use tracing::debug; - /// Generates and exports the Coverage Map. /// /// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 98ba38356a4..964a632b6ee 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -28,7 +28,6 @@ use std::cell::RefCell; use std::ffi::CString; use std::iter; -use tracing::debug; pub mod mapgen; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 5186aee57fb..80fd9726fc7 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -55,7 +55,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' // The initial byte `4` instructs GDB that the following pretty printer // is defined inline as opposed to in a standalone file. section_contents.extend_from_slice(b"\x04"); - let vis_name = format!("pretty-printer-{}-{}\n", crate_name.as_str(), index); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); section_contents.extend_from_slice(vis_name.as_bytes()); section_contents.extend_from_slice(&visualizer.src); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index f8bd2d234f3..163ccd9460c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -42,7 +42,6 @@ use rustc_span::{self, FileNameDisplayPreference, SourceFile}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; -use tracing::debug; use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; @@ -51,7 +50,6 @@ use std::hash::{Hash, Hasher}; use std::iter; use std::path::{Path, PathBuf}; use std::ptr; -use tracing::instrument; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -114,6 +112,7 @@ macro_rules! return_if_di_node_created_in_meantime { } /// Extract size and alignment from a TyAndLayout. +#[inline] fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { (ty_and_layout.size, ty_and_layout.align.abi) } @@ -1420,7 +1419,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( cx, type_map::stub( cx, - Stub::VtableTy { vtable_holder }, + Stub::VTableTy { vtable_holder }, unique_type_id, &vtable_type_name, (size, pointer_align), @@ -1499,24 +1498,18 @@ fn vcall_visibility_metadata<'ll, 'tcx>( // If there is not LTO and the visibility in public, we have to assume that the vtable can // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. (Lto::No | Lto::ThinLocal, Visibility::Public, _) - | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { - VCallVisibility::Public - } + | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public, // With LTO and a quasi-public visibility, the usages of the functions of the vtable are // all known by the `LinkageUnit`. // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. (Lto::Fat | Lto::Thin, Visibility::Public, _) - | ( - Lto::ThinLocal | Lto::Thin | Lto::Fat, - Visibility::Restricted(_) | Visibility::Invisible, - false, - ) => VCallVisibility::LinkageUnit, + | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => { + VCallVisibility::LinkageUnit + } // If there is only one CGU, private vtables can only be seen by that CGU/translation unit // and therefore we know of all usages of functions in the vtable. - (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { - VCallVisibility::TranslationUnit - } + (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit, }; let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index d6e2c8ccdf4..129e336c7e4 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -1,19 +1,21 @@ use std::borrow::Cow; use libc::c_uint; -use rustc_codegen_ssa::debuginfo::{ - type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +use rustc_codegen_ssa::{ + debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, + traits::ConstMethods, }; + +use rustc_index::vec::IndexVec; use rustc_middle::{ bug, ty::{ self, layout::{LayoutOf, TyAndLayout}, - util::Discr, - AdtDef, GeneratorSubsts, + AdtDef, GeneratorSubsts, Ty, }, }; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::{ @@ -21,9 +23,9 @@ use crate::{ debuginfo::{ metadata::{ build_field_di_node, closure_saved_names_of_captured_variables, - enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, - type_map::{self, UniqueTypeId}, + enums::{tag_base_type, DiscrResult}, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + type_map::{self, Stub, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER, }, @@ -35,59 +37,161 @@ use crate::{ }, }; -/// In CPP-like mode, we generate a union of structs for each variant and an -/// explicit discriminant field roughly equivalent to the following C/C++ code: +// The names of the associated constants in each variant wrapper struct. +// These have to match up with the names being used in `intrinsic.natvis`. +const ASSOC_CONST_DISCR_NAME: &str = "NAME"; +const ASSOC_CONST_DISCR_EXACT: &str = "DISCR_EXACT"; +const ASSOC_CONST_DISCR_BEGIN: &str = "DISCR_BEGIN"; +const ASSOC_CONST_DISCR_END: &str = "DISCR_END"; + +const ASSOC_CONST_DISCR128_EXACT_LO: &str = "DISCR128_EXACT_LO"; +const ASSOC_CONST_DISCR128_EXACT_HI: &str = "DISCR128_EXACT_HI"; +const ASSOC_CONST_DISCR128_BEGIN_LO: &str = "DISCR128_BEGIN_LO"; +const ASSOC_CONST_DISCR128_BEGIN_HI: &str = "DISCR128_BEGIN_HI"; +const ASSOC_CONST_DISCR128_END_LO: &str = "DISCR128_END_LO"; +const ASSOC_CONST_DISCR128_END_HI: &str = "DISCR128_END_HI"; + +// The name of the tag field in the top-level union +const TAG_FIELD_NAME: &str = "tag"; +const TAG_FIELD_NAME_128_LO: &str = "tag128_lo"; +const TAG_FIELD_NAME_128_HI: &str = "tag128_hi"; + +// We assign a "virtual" discriminant value to the sole variant of +// a single-variant enum. +const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; + +/// In CPP-like mode, we generate a union with a field for each variant and an +/// explicit tag field. The field of each variant has a struct type +/// that encodes the discrimiant of the variant and it's data layout. +/// The union also has a nested enumeration type that is only used for encoding +/// variant names in an efficient way. Its enumerator values do _not_ correspond +/// to the enum's discriminant values. +/// It's roughly equivalent to the following C/C++ code: /// /// ```c -/// union enum$<{fully-qualified-name}> { -/// struct {variant 0 name} { -/// <variant 0 fields> +/// union enum2$<{fully-qualified-name}> { +/// struct Variant0 { +/// struct {name-of-variant-0} { +/// <variant 0 fields> +/// } value; +/// +/// static VariantNames NAME = {name-of-variant-0}; +/// static int_type DISCR_EXACT = {discriminant-of-variant-0}; /// } variant0; +/// /// <other variant structs> -/// {name} discriminant; +/// +/// int_type tag; +/// +/// enum VariantNames { +/// <name-of-variant-0> = 0, // The numeric values are variant index, +/// <name-of-variant-1> = 1, // not discriminant values. +/// <name-of-variant-2> = 2, +/// ... +/// } /// } /// ``` /// -/// As you can see, the type name is wrapped `enum$`. This way we can have a -/// single NatVis rule for handling all enums. +/// As you can see, the type name is wrapped in `enum2$<_>`. This way we can +/// have a single NatVis rule for handling all enums. The `2` in `enum2$<_>` +/// is an encoding version tag, so that debuggers can decide to decode this +/// differently than the previous `enum$<_>` encoding emitted by earlier +/// compiler versions. /// -/// At the LLVM IR level this looks like +/// Niche-tag enums have one special variant, usually called the +/// "untagged variant". This variant has a field that +/// doubles as the tag of the enum. The variant is active when the value of +/// that field is within a pre-defined range. Therefore the variant struct +/// has a `DISCR_BEGIN` and `DISCR_END` field instead of `DISCR_EXACT` in +/// that case. Both `DISCR_BEGIN` and `DISCR_END` are inclusive bounds. +/// Note that these ranges can wrap around, so that `DISCR_END < DISCR_BEGIN`. /// -/// ```txt -/// DW_TAG_union_type (top-level type for enum) -/// DW_TAG_member (member for variant 1) -/// DW_TAG_member (member for variant 2) -/// DW_TAG_member (member for variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// DW_TAG_enumeration_type (type of tag) -/// ``` +/// Single-variant enums don't actually have a tag field. In this case we +/// emit a static tag field (that always has the value 0) so we can use the +/// same representation (and NatVis). /// -/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things -/// differently in order to allow a NatVis visualizer to extract all the information needed: -/// We generate a union of two fields, one for the dataful variant -/// and one that just points to the discriminant (which is some field within the dataful variant). -/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful -/// variants and make the discriminant field that type. We then use NatVis to render the enum type -/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: +/// For niche-layout enums it's possible to have a 128-bit tag. NatVis, VS, and +/// WinDbg (the main targets for CPP-like debuginfo at the moment) don't support +/// 128-bit integers, so all values involved get split into two 64-bit fields. +/// Instead of the `tag` field, we generate two fields `tag128_lo` and `tag128_hi`, +/// Instead of `DISCR_EXACT`, we generate `DISCR128_EXACT_LO` and `DISCR128_EXACT_HI`, +/// and so on. /// -/// ```c -/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { -/// struct <dataful variant name> { -/// <fields in dataful variant> -/// } dataful_variant; -/// enum Discriminant$ { -/// <non-dataful variants> -/// } discriminant; +/// +/// The following pseudocode shows how to decode an enum value in a debugger: +/// +/// ```text +/// +/// fn find_active_variant(enum_value) -> (VariantName, VariantValue) { +/// let is_128_bit = enum_value.has_field("tag128_lo"); +/// +/// if !is_128_bit { +/// // Note: `tag` can be a static field for enums with only one +/// // inhabited variant. +/// let tag = enum_value.field("tag").value; +/// +/// // For each variant, check if it is a match. Only one of them will match, +/// // so if we find it we can return it immediately. +/// for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) { +/// if variant_field.has_field("DISCR_EXACT") { +/// // This variant corresponds to a single tag value +/// if variant_field.field("DISCR_EXACT").value == tag { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } else { +/// // This is a range variant +/// let begin = variant_field.field("DISCR_BEGIN"); +/// let end = variant_field.field("DISCR_END"); +/// +/// if is_in_range(tag, begin, end) { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } +/// } +/// } else { +/// // Basically the same as with smaller tags, we just have to +/// // stitch the values together. +/// let tag: u128 = (enum_value.field("tag128_lo").value as u128) | +/// (enum_value.field("tag128_hi").value as u128 << 64); +/// +/// for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) { +/// if variant_field.has_field("DISCR128_EXACT_LO") { +/// let discr_exact = (variant_field.field("DISCR128_EXACT_LO" as u128) | +/// (variant_field.field("DISCR128_EXACT_HI") as u128 << 64); +/// +/// // This variant corresponds to a single tag value +/// if discr_exact.value == tag { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } else { +/// // This is a range variant +/// let begin = (variant_field.field("DISCR128_BEGIN_LO").value as u128) | +/// (variant_field.field("DISCR128_BEGIN_HI").value as u128 << 64); +/// let end = (variant_field.field("DISCR128_END_LO").value as u128) | +/// (variant_field.field("DISCR128_END_HI").value as u128 << 64); +/// +/// if is_in_range(tag, begin, end) { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } +/// } +/// } +/// +/// // We should have found an active variant at this point. +/// unreachable!(); /// } -/// ``` /// -/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` -/// and evaluates `this.discriminant`. If the value is between the min niche and max -/// niche, then the enum is in the dataful variant and `this.dataful_variant` is -/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that -/// case, we just need to render the name of the `this.discriminant` enum. +/// // Check if a value is within the given range +/// // (where the range might wrap around the value space) +/// fn is_in_range(value, start, end) -> bool { +/// if start < end { +/// value >= start && value <= end +/// } else { +/// value >= start || value <= end +/// } +/// } +/// +/// ``` pub(super) fn build_enum_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, @@ -135,27 +239,28 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ref variants, tag_field, .. - } => build_union_fields_for_direct_tag_enum( + } => build_union_fields_for_enum( cx, enum_adt_def, enum_type_and_layout, enum_type_di_node, - &mut variants.indices(), + variants.indices(), tag_field, + None, ), Variants::Multiple { - tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + tag_encoding: TagEncoding::Niche { untagged_variant, .. }, ref variants, tag_field, .. - } => build_union_fields_for_niche_tag_enum( + } => build_union_fields_for_enum( cx, enum_adt_def, enum_type_and_layout, enum_type_di_node, - dataful_variant, - &mut variants.indices(), + variants.indices(), tag_field, + Some(untagged_variant), ), } }, @@ -217,137 +322,344 @@ fn build_single_variant_union_fields<'ll, 'tcx>( let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( cx, - enum_type_and_layout.ty, + enum_type_and_layout, enum_type_di_node, variant_index, enum_adt_def.variant(variant_index), variant_layout, ); - // NOTE: The field name of the union is the same as the variant name, not "variant0". - let variant_name = enum_adt_def.variant(variant_index).name.as_str(); + let tag_base_type = cx.tcx.types.u32; + let tag_base_type_di_node = type_di_node(cx, tag_base_type); + let tag_base_type_align = cx.align_of(tag_base_type); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + enum_type_di_node, + std::iter::once(( + variant_index, + Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + )), + ); - smallvec![build_field_di_node( + let variant_struct_type_wrapper_di_node = build_variant_struct_wrapper_type_di_node( cx, + enum_type_and_layout, enum_type_di_node, - variant_name, - // NOTE: We use the size and align of the entire type, not from variant_layout - // since the later is sometimes smaller (if it has fewer fields). - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, + variant_index, + None, variant_struct_type_di_node, - )] + variant_names_type_di_node, + tag_base_type_di_node, + tag_base_type, + DiscrResult::NoDiscriminant, + ); + + smallvec![ + build_field_di_node( + cx, + enum_type_di_node, + &variant_union_field_name(variant_index), + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_wrapper_di_node, + ), + unsafe { + llvm::LLVMRustDIBuilderCreateStaticMemberType( + DIB(cx), + enum_type_di_node, + TAG_FIELD_NAME.as_ptr().cast(), + TAG_FIELD_NAME.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + DIFlags::FlagZero, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align.bits() as u32, + ) + } + ] } -fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( +fn build_union_fields_for_enum<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_adt_def: AdtDef<'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, - variant_indices: &mut dyn Iterator<Item = VariantIdx>, + variant_indices: impl Iterator<Item = VariantIdx> + Clone, tag_field: usize, + untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + enum_type_di_node, + variant_indices.clone().map(|variant_index| { + let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + (variant_index, variant_name) + }), + ); + let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices .map(|variant_index| { let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let variant_def = enum_adt_def.variant(variant_index); + + let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_index, + variant_def, + variant_layout, + ); + VariantFieldInfo { variant_index, - variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( - cx, - enum_type_and_layout.ty, - enum_type_di_node, - variant_index, - enum_adt_def.variant(variant_index), - variant_layout, - ), + variant_struct_type_di_node, source_info: None, + discr: super::compute_discriminant_value(cx, enum_type_and_layout, variant_index), } }) .collect(); - let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name.as_str(), - tag_base_type, - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) - }), - enum_type_di_node, - ); - build_union_fields_for_direct_tag_enum_or_generator( cx, enum_type_and_layout, enum_type_di_node, &variant_field_infos, - discr_type_di_node, + variant_names_type_di_node, + tag_base_type, tag_field, + untagged_variant_index, ) } -fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( +// The base type of the VariantNames DW_AT_enumeration_type is always the same. +// It has nothing to do with the tag of the enum and just has to be big enough +// to hold all variant names. +fn variant_names_enum_base_type<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> Ty<'tcx> { + cx.tcx.types.u32 +} + +/// This function builds a DW_AT_enumeration_type that contains an entry for +/// each variant. Note that this has nothing to do with the discriminant. The +/// numeric value of each enumerator corresponds to the variant index. The +/// type is only used for efficiently encoding the name of each variant in +/// debuginfo. +fn build_variant_names_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - dataful_variant_index: VariantIdx, - variant_indices: &mut dyn Iterator<Item = VariantIdx>, - tag_field: usize, -) -> SmallVec<&'ll DIType> { - let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + containing_scope: &'ll DIType, + variants: impl Iterator<Item = (VariantIdx, Cow<'tcx, str>)>, +) -> &'ll DIType { + // Create an enumerator for each variant. + super::build_enumeration_type_di_node( cx, - enum_type_and_layout.ty, - enum_type_di_node, - dataful_variant_index, - &enum_adt_def.variant(dataful_variant_index), - enum_type_and_layout.for_variant(cx, dataful_variant_index), - ); + "VariantNames", + variant_names_enum_base_type(cx), + variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32() as u64)), + containing_scope, + ) +} - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - // Create an DW_TAG_enumerator for each variant except the dataful one. - let discr_type_di_node = super::build_enumeration_type_di_node( +fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_generator_type_and_layout: TyAndLayout<'tcx>, + enum_or_generator_type_di_node: &'ll DIType, + variant_index: VariantIdx, + untagged_variant_index: Option<VariantIdx>, + variant_struct_type_di_node: &'ll DIType, + variant_names_type_di_node: &'ll DIType, + tag_base_type_di_node: &'ll DIType, + tag_base_type: Ty<'tcx>, + discr: DiscrResult, +) -> &'ll DIType { + type_map::build_type_with_children( cx, - "Discriminant$", - tag_base_type, - &mut variant_indices.filter_map(|variant_index| { - if let Some(discr_val) = - super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) - { - let discr = Discr { val: discr_val as u128, ty: tag_base_type }; - let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); - Some((discr, variant_name)) - } else { - debug_assert_eq!(variant_index, dataful_variant_index); - None - } - }), - enum_type_di_node, - ); - - smallvec![ - build_field_di_node( - cx, - enum_type_di_node, - "dataful_variant", - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, - dataful_variant_struct_type_di_node, - ), - build_field_di_node( + type_map::stub( cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(tag_base_type), - enum_type_and_layout.fields.offset(tag_field), + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type_wrapper( + cx.tcx, + enum_or_generator_type_and_layout.ty, + variant_index, + ), + &variant_struct_wrapper_type_name(variant_index), + // NOTE: We use size and align of enum_type, not from variant_layout: + size_and_align_of(enum_or_generator_type_and_layout), + Some(enum_or_generator_type_di_node), DIFlags::FlagZero, - discr_type_di_node, ), - ] + |cx, wrapper_struct_type_di_node| { + enum DiscrKind { + Exact(u64), + Exact128(u128), + Range(u64, u64), + Range128(u128, u128), + } + + let (tag_base_type_size, tag_base_type_align) = cx.size_and_align_of(tag_base_type); + let is_128_bits = tag_base_type_size.bits() > 64; + + let discr = match discr { + DiscrResult::NoDiscriminant => DiscrKind::Exact(SINGLE_VARIANT_VIRTUAL_DISR), + DiscrResult::Value(discr_val) => { + if is_128_bits { + DiscrKind::Exact128(discr_val) + } else { + debug_assert_eq!(discr_val, discr_val as u64 as u128); + DiscrKind::Exact(discr_val as u64) + } + } + DiscrResult::Range(min, max) => { + assert_eq!(Some(variant_index), untagged_variant_index); + if is_128_bits { + DiscrKind::Range128(min, max) + } else { + debug_assert_eq!(min, min as u64 as u128); + debug_assert_eq!(max, max as u64 as u128); + DiscrKind::Range(min as u64, max as u64) + } + } + }; + + let mut fields = SmallVec::new(); + + // We always have a field for the value + fields.push(build_field_di_node( + cx, + wrapper_struct_type_di_node, + "value", + size_and_align_of(enum_or_generator_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_di_node, + )); + + let build_assoc_const = + |name: &str, type_di_node: &'ll DIType, value: u64, align: Align| unsafe { + llvm::LLVMRustDIBuilderCreateStaticMemberType( + DIB(cx), + wrapper_struct_type_di_node, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + type_di_node, + DIFlags::FlagZero, + Some(cx.const_u64(value)), + align.bits() as u32, + ) + }; + + // We also always have an associated constant for the discriminant value + // of the variant. + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_NAME, + variant_names_type_di_node, + variant_index.as_u32() as u64, + cx.align_of(variant_names_enum_base_type(cx)), + )); + + // Emit the discriminant value (or range) corresponding to the variant. + match discr { + DiscrKind::Exact(discr_val) => { + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_EXACT, + tag_base_type_di_node, + discr_val, + tag_base_type_align, + )); + } + DiscrKind::Exact128(discr_val) => { + let align = cx.align_of(cx.tcx.types.u64); + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let Split128 { hi, lo } = split_128(discr_val); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_EXACT_LO, + type_di_node, + lo, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_EXACT_HI, + type_di_node, + hi, + align, + )); + } + DiscrKind::Range(begin, end) => { + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_BEGIN, + tag_base_type_di_node, + begin, + tag_base_type_align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_END, + tag_base_type_di_node, + end, + tag_base_type_align, + )); + } + DiscrKind::Range128(begin, end) => { + let align = cx.align_of(cx.tcx.types.u64); + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let Split128 { hi: begin_hi, lo: begin_lo } = split_128(begin); + let Split128 { hi: end_hi, lo: end_lo } = split_128(end); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_BEGIN_HI, + type_di_node, + begin_hi, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_BEGIN_LO, + type_di_node, + begin_lo, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_END_HI, + type_di_node, + end_hi, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_END_LO, + type_di_node, + end_lo, + align, + )); + } + } + + fields + }, + NO_GENERICS, + ) + .di_node +} + +struct Split128 { + hi: u64, + lo: u64, +} + +fn split_128(value: u128) -> Split128 { + Split128 { hi: (value >> 64) as u64, lo: value as u64 } } fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( @@ -369,6 +681,29 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); + + let tag_base_type = tag_base_type(cx, generator_type_and_layout); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + generator_type_di_node, + variant_range + .clone() + .map(|variant_index| (variant_index, GeneratorSubsts::variant_name(variant_index))), + ); + + let discriminants: IndexVec<VariantIdx, DiscrResult> = { + let discriminants_iter = generator_substs.discriminants(generator_def_id, cx.tcx); + let mut discriminants: IndexVec<VariantIdx, DiscrResult> = + IndexVec::with_capacity(variant_count); + for (variant_index, discr) in discriminants_iter { + // Assert that the index in the IndexMap matches up with the given VariantIdx. + assert_eq!(variant_index, discriminants.next_index()); + discriminants.push(DiscrResult::Value(discr.val)); + } + discriminants + }; // Build the type node for each field. let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range @@ -391,29 +726,24 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( None }; - VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } + VariantFieldInfo { + variant_index, + variant_struct_type_di_node, + source_info, + discr: discriminants[variant_index], + } }) .collect(); - let tag_base_type = tag_base_type(cx, generator_type_and_layout); - let discr_type_name = "Discriminant$"; - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name, - tag_base_type, - &mut generator_substs - .discriminants(generator_def_id, cx.tcx) - .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), - generator_type_di_node, - ); - build_union_fields_for_direct_tag_enum_or_generator( cx, generator_type_and_layout, generator_type_di_node, &variant_field_infos[..], - discr_type_di_node, + variant_names_type_di_node, + tag_base_type, tag_field, + None, ) } @@ -425,8 +755,11 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( enum_type_di_node: &'ll DIType, variant_field_infos: &[VariantFieldInfo<'ll>], discr_type_di_node: &'ll DIType, + tag_base_type: Ty<'tcx>, tag_field: usize, + untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { + let tag_base_type_di_node = type_di_node(cx, tag_base_type); let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); // We create a field in the union for each variant ... @@ -438,6 +771,19 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( let field_name = variant_union_field_name(variant_member_info.variant_index); let (size, align) = size_and_align_of(enum_type_and_layout); + let variant_struct_type_wrapper = build_variant_struct_wrapper_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_member_info.variant_index, + untagged_variant_index, + variant_member_info.variant_struct_type_di_node, + discr_type_di_node, + tag_base_type_di_node, + tag_base_type, + variant_member_info.discr, + ); + // We use LLVMRustDIBuilderCreateMemberType() member type directly because // the build_field_di_node() function does not support specifying a source location, // which is something that we don't do anywhere else. @@ -456,7 +802,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( // Union fields are always at offset zero Size::ZERO.bits(), DIFlags::FlagZero, - variant_member_info.variant_struct_type_di_node, + variant_struct_type_wrapper, ) } })); @@ -466,16 +812,53 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) ); - // ... and a field for the discriminant. - unions_fields.push(build_field_di_node( - cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - enum_type_and_layout.fields.offset(tag_field), - DIFlags::FlagZero, - discr_type_di_node, - )); + // ... and a field for the tag. If the tag is 128 bits wide, this will actually + // be two 64-bit fields. + let is_128_bits = cx.size_of(tag_base_type).bits() > 64; + + if is_128_bits { + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let size_and_align = cx.size_and_align_of(cx.tcx.types.u64); + + let (lo_offset, hi_offset) = match cx.tcx.data_layout.endian { + Endian::Little => (0, 8), + Endian::Big => (8, 0), + }; + + let tag_field_offset = enum_type_and_layout.fields.offset(tag_field).bytes(); + let lo_offset = Size::from_bytes(tag_field_offset + lo_offset); + let hi_offset = Size::from_bytes(tag_field_offset + hi_offset); + + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME_128_LO, + size_and_align, + lo_offset, + DIFlags::FlagZero, + type_di_node, + )); + + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME_128_HI, + size_and_align, + hi_offset, + DIFlags::FlagZero, + type_di_node, + )); + } else { + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + tag_base_type_di_node, + )); + } unions_fields } @@ -485,6 +868,7 @@ struct VariantFieldInfo<'ll> { variant_index: VariantIdx, variant_struct_type_di_node: &'ll DIType, source_info: Option<(&'ll DIFile, c_uint)>, + discr: DiscrResult, } fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { @@ -512,3 +896,29 @@ fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) } + +fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, str> { + const PRE_ALLOCATED: [&str; 16] = [ + "Variant0", + "Variant1", + "Variant2", + "Variant3", + "Variant4", + "Variant5", + "Variant6", + "Variant7", + "Variant8", + "Variant9", + "Variant10", + "Variant11", + "Variant12", + "Variant13", + "Variant14", + "Variant15", + ]; + + PRE_ALLOCATED + .get(variant_index.as_usize()) + .map(|&s| Cow::from(s)) + .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 73e01d0453b..14044d0f99b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -10,7 +10,6 @@ use rustc_middle::{ ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - util::Discr, AdtDef, GeneratorSubsts, Ty, VariantDef, }, }; @@ -90,8 +89,11 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( cx, &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), tag_base_type(cx, enum_type_and_layout), - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + // Is there anything we can do to support 128-bit C-Style enums? + let value = discr.val as u64; + (name, value) }), containing_scope, ), @@ -152,7 +154,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, type_name: &str, base_type: Ty<'tcx>, - variants: &mut dyn Iterator<Item = (Discr<'tcx>, Cow<'tcx, str>)>, + enumerators: impl Iterator<Item = (Cow<'tcx, str>, u64)>, containing_scope: &'ll DIType, ) -> &'ll DIType { let is_unsigned = match base_type.kind() { @@ -161,18 +163,15 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), }; - let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = variants - .map(|(discr, variant_name)| { - unsafe { - Some(llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - variant_name.as_ptr().cast(), - variant_name.len(), - // FIXME: what if enumeration has i128 discriminant? - discr.val as i64, - is_unsigned, - )) - } + let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = enumerators + .map(|(name, value)| unsafe { + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + name.as_ptr().cast(), + name.len(), + value as i64, + is_unsigned, + )) }) .collect(); @@ -247,23 +246,27 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( /// and a DW_TAG_member for each field (but not the discriminant). fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_type: Ty<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, variant_index: VariantIdx, variant_def: &VariantDef, variant_layout: TyAndLayout<'tcx>, ) -> &'ll DIType { - debug_assert_eq!(variant_layout.ty, enum_type); + debug_assert_eq!(variant_layout.ty, enum_type_and_layout.ty); type_map::build_type_with_children( cx, type_map::stub( cx, Stub::Struct, - UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), + UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + enum_type_and_layout.ty, + variant_index, + ), variant_def.name.as_str(), // NOTE: We use size and align of enum_type, not from variant_layout: - cx.size_and_align_of(enum_type), + size_and_align_of(enum_type_and_layout), Some(enum_type_di_node), DIFlags::FlagZero, ), @@ -290,9 +293,9 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( type_di_node(cx, field_layout.ty), ) }) - .collect() + .collect::<SmallVec<_>>() }, - |cx| build_generic_type_param_di_nodes(cx, enum_type), + |cx| build_generic_type_param_di_nodes(cx, enum_type_and_layout.ty), ) .di_node } @@ -398,39 +401,60 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .di_node } +#[derive(Copy, Clone)] +enum DiscrResult { + NoDiscriminant, + Value(u128), + Range(u128, u128), +} + +impl DiscrResult { + fn opt_single_val(&self) -> Option<u128> { + if let Self::Value(d) = *self { Some(d) } else { None } + } +} + /// Returns the discriminant value corresponding to the variant index. /// /// Will return `None` if there is less than two variants (because then the enum won't have) -/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no +/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no /// single discriminant value). fn compute_discriminant_value<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, variant_index: VariantIdx, -) -> Option<u64> { +) -> DiscrResult { match enum_type_and_layout.layout.variants() { - &Variants::Single { .. } => None, - &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( - enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val - as u64, + &Variants::Single { .. } => DiscrResult::NoDiscriminant, + &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => DiscrResult::Value( + enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val, ), &Variants::Multiple { - tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, + tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant }, tag, .. } => { - if variant_index == dataful_variant { - None + if variant_index == untagged_variant { + let valid_range = enum_type_and_layout + .for_variant(cx, variant_index) + .largest_niche + .as_ref() + .unwrap() + .valid_range; + + let min = valid_range.start.min(valid_range.end); + let min = tag.size(cx).truncate(min); + + let max = valid_range.start.max(valid_range.end); + let max = tag.size(cx).truncate(max); + + DiscrResult::Range(min, max) } else { let value = (variant_index.as_u32() as u128) .wrapping_sub(niche_variants.start().as_u32() as u128) .wrapping_add(niche_start); let value = tag.size(cx).truncate(value); - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - Some(value as u64) + DiscrResult::Value(value) } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index f1935e0ec31..becbccc434d 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -88,7 +88,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( cx, - enum_type, + enum_type_and_layout, enum_type_di_node, variant_index, enum_adt_def.variant(variant_index), @@ -378,7 +378,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( /// /// The DW_AT_discr_value is optional, and is omitted if /// - This is the only variant of a univariant enum (i.e. their is no discriminant) -/// - This is the "dataful" variant of a niche-layout enum +/// - This is the "untagged" variant of a niche-layout enum /// (where only the other variants are identified by a single value) /// /// There is only ever a single member, the type of which is a struct that describes the @@ -413,7 +413,13 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.map(|v| cx.const_u64(v)), + discr_value.opt_single_val().map(|value| { + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + cx.const_u64(value as u64) + }), DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 8fc8118849b..e30622cbdce 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -47,6 +47,8 @@ pub(super) enum UniqueTypeId<'tcx> { VariantPart(Ty<'tcx>, private::HiddenZst), /// The ID for the artificial struct type describing a single enum variant. VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode. + VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst), /// The ID of the artificial type we create for VTables. VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst), } @@ -71,6 +73,15 @@ impl<'tcx> UniqueTypeId<'tcx> { UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) } + pub fn for_enum_variant_struct_type_wrapper( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantStructTypeCppLikeWrapper(enum_ty, variant_idx, private::HiddenZst) + } + pub fn for_vtable_ty( tcx: TyCtxt<'tcx>, self_type: Ty<'tcx>, @@ -146,7 +157,7 @@ impl<'ll> DINodeCreationResult<'ll> { pub enum Stub<'ll> { Struct, Union, - VtableTy { vtable_holder: &'ll DIType }, + VTableTy { vtable_holder: &'ll DIType }, } pub struct StubInfo<'ll, 'tcx> { @@ -180,9 +191,9 @@ pub(super) fn stub<'ll, 'tcx>( let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); let metadata = match kind { - Stub::Struct | Stub::VtableTy { .. } => { + Stub::Struct | Stub::VTableTy { .. } => { let vtable_holder = match kind { - Stub::VtableTy { vtable_holder } => Some(vtable_holder), + Stub::VTableTy { vtable_holder } => Some(vtable_holder), _ => None, }; unsafe { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index cf591295b84..b23fe3fc9d5 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -39,7 +39,6 @@ use smallvec::SmallVec; use std::cell::OnceCell; use std::cell::RefCell; use std::iter; -use tracing::debug; mod create_scope_map; pub mod gdb; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 8f243673907..a40cfc8b23f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -6,7 +6,7 @@ use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; use rustc_middle::ty::{self, DefIdTree, Ty}; -use tracing::trace; +use trace; use crate::common::CodegenCx; use crate::llvm; diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 5a5c4f7f860..0f663a26732 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -18,9 +18,10 @@ use crate::llvm; use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; +use rustc_codegen_ssa::traits::TypeMembershipMethods; use rustc_middle::ty::Ty; +use rustc_symbol_mangling::typeid::typeid_for_fnabi; use smallvec::SmallVec; -use tracing::debug; /// Declare a function. /// @@ -97,6 +98,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.llvm_type(self), ); fn_abi.apply_attrs_llfn(self, llfn); + + if self.tcx.sess.is_sanitizer_cfi_enabled() { + let typeid = typeid_for_fnabi(self.tcx, fn_abi); + self.set_type_metadata(llfn, typeid); + } + llfn } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9f364749287..3d789904707 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -71,6 +71,7 @@ fn get_simple_intrinsic<'ll>( sym::nearbyintf64 => "llvm.nearbyint.f64", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::ptr_mask => "llvm.ptrmask", _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -161,7 +162,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::volatile_load | sym::unaligned_volatile_load => { let tp_ty = substs.type_at(0); let ptr = args[0].immediate(); - let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { + let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let llty = ty.llvm_type(self); let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); self.volatile_load(llty, ptr) @@ -374,7 +375,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { }; if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { + if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); let ptr = self.pointercast(result.llval, ptr_llty); self.store(llval, ptr, result.align); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 25ce1cef944..334425ae55b 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -6,7 +6,7 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(hash_raw_entry)] -#![cfg_attr(bootstrap, feature(let_chains))] +#![feature(let_chains)] #![feature(let_else)] #![feature(extern_types)] #![feature(once_cell)] @@ -16,6 +16,8 @@ #[macro_use] extern crate rustc_macros; +#[macro_use] +extern crate tracing; use back::write::{create_informational_target_machine, create_target_machine}; @@ -370,12 +372,12 @@ impl CodegenBackend for LlvmCodegenBackend { codegen_results: CodegenResults, outputs: &OutputFilenames, ) -> Result<(), ErrorGuaranteed> { - use crate::back::archive::LlvmArchiveBuilder; + use crate::back::archive::LlvmArchiveBuilderBuilder; use rustc_codegen_ssa::back::link::link_binary; // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary::<LlvmArchiveBuilder<'_>>(sess, &codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 64db4f7462d..7d948970223 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -83,17 +83,6 @@ impl<'a> Child<'a> { } } } - - pub fn data(&self) -> &'a [u8] { - unsafe { - let mut data_len = 0; - let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); - if data_ptr.is_null() { - panic!("failed to read data from archive child"); - } - slice::from_raw_parts(data_ptr as *const u8, data_len as usize) - } - } } impl<'a> Drop for Child<'a> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index e2fa5e488ed..45de284d22a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -20,19 +20,6 @@ pub enum OptimizationDiagnosticKind { OptimizationRemarkOther, } -impl OptimizationDiagnosticKind { - pub fn describe(self) -> &'static str { - match self { - OptimizationRemark | OptimizationRemarkOther => "remark", - OptimizationMissed => "missed", - OptimizationAnalysis => "analysis", - OptimizationAnalysisFPCommute => "floating-point", - OptimizationAnalysisAliasing => "aliasing", - OptimizationFailure => "failure", - } - } -} - pub struct OptimizationDiagnostic<'ll> { pub kind: OptimizationDiagnosticKind, pub pass_name: String, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 0ad65e5d99b..ce27dc5a5d1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -191,6 +191,11 @@ pub enum AttributeKind { StackProtect = 32, NoUndef = 33, SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, } /// LLVMIntPredicate @@ -567,16 +572,6 @@ pub enum ArchiveKind { K_COFF, } -/// LLVMRustPassKind -#[derive(Copy, Clone, PartialEq, Debug)] -#[repr(C)] -#[allow(dead_code)] // Variants constructed by C++. -pub enum PassKind { - Other, - Function, - Module, -} - // LLVMRustThinLTOData extern "C" { pub type ThinLTOData; @@ -587,10 +582,6 @@ extern "C" { pub type ThinLTOBuffer; } -// LLVMRustModuleNameCallback -pub type ThinLTOModuleNameCallback = - unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); - /// LLVMRustThinLTOModule #[repr(C)] pub struct ThinLTOModule { @@ -656,9 +647,6 @@ extern "C" { } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); -extern "C" { - pub type MemoryBuffer; -} #[repr(C)] pub struct PassManager<'a>(InvariantOpaque<'a>); extern "C" { @@ -984,6 +972,22 @@ pub mod debuginfo { } } +use bitflags::bitflags; +// These values **must** match with LLVMRustAllocKindFlags +bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct AllocKindFlags : u64 { + const Unknown = 0; + const Alloc = 1; + const Realloc = 1 << 1; + const Free = 1 << 2; + const Uninitialized = 1 << 3; + const Zeroed = 1 << 4; + const Aligned = 1 << 5; + } +} + extern "C" { pub type ModuleBuffer; } @@ -1011,7 +1015,6 @@ extern "C" { pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMSetModuleInlineAsm2(M: &Module, Asm: *const c_char, AsmLen: size_t); pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); /// See llvm::LLVMTypeKind::getTypeID. @@ -1093,7 +1096,7 @@ extern "C" { pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; - pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; + pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; pub fn LLVMRustConstInt128Get( ConstantVal: &ConstantInt, SExt: bool, @@ -1165,7 +1168,6 @@ extern "C" { pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; - pub fn LLVMSetThreadLocal(GlobalVar: &Value, IsThreadLocal: Bool); pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); @@ -1190,7 +1192,10 @@ extern "C" { pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( @@ -2074,6 +2079,19 @@ extern "C" { Ty: &'a DIType, ) -> &'a DIType; + pub fn LLVMRustDIBuilderCreateStaticMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + val: Option<&'a Value>, + AlignInBits: u32, + ) -> &'a DIDerivedType; + pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2244,7 +2262,6 @@ extern "C" { pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - pub fn LLVMRustPassKind(Pass: &Pass) -> PassKind; pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; @@ -2343,6 +2360,7 @@ extern "C" { PGOGenPath: *const c_char, PGOUsePath: *const c_char, InstrumentCoverage: bool, + InstrProfileOutput: *const c_char, InstrumentGCOV: bool, PGOSampleUsePath: *const c_char, DebugInfoForProfiling: bool, @@ -2361,7 +2379,6 @@ extern "C" { ) -> LLVMRustResult; pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); pub fn LLVMRustPrintPasses(); - pub fn LLVMRustGetInstructionCount(M: &Module) -> u32; pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); @@ -2372,7 +2389,6 @@ extern "C" { AIR: &ArchiveIterator<'a>, ) -> Option<&'a mut ArchiveChild<'a>>; pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); @@ -2407,12 +2423,6 @@ extern "C" { cookie_out: &mut c_uint, ) -> &'a SMDiagnostic; - pub fn LLVMRustSetInlineAsmDiagnosticHandler( - C: &Context, - H: InlineAsmDiagHandlerTy, - CX: *mut c_void, - ); - #[allow(improper_ctypes)] pub fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, @@ -2459,7 +2469,6 @@ extern "C" { pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMRustUnsetComdat(V: &Value); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); @@ -2469,7 +2478,7 @@ extern "C" { pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub fn LLVMRustModuleCost(M: &Module) -> u64; - pub fn LLVMRustThinLTOBufferCreate(M: &Module) -> &'static mut ThinLTOBuffer; + pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; @@ -2491,11 +2500,6 @@ extern "C" { Module: &Module, Target: &TargetMachine, ) -> bool; - pub fn LLVMRustGetThinLTOModuleImports( - Data: *const ThinLTOData, - ModuleNameCallback: ThinLTOModuleNameCallback, - CallbackPayload: *mut c_void, - ); pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); pub fn LLVMRustParseBitcodeForLTO( Context: &Context, @@ -2545,4 +2549,6 @@ extern "C" { #[allow(improper_ctypes)] pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); + + pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 48fbc1de8ee..6602a4ab863 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -95,6 +95,14 @@ pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } } +pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { + unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } +} + +pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { + unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, @@ -158,12 +166,6 @@ pub fn SetUniqueComdat(llmod: &Module, val: &Value) { } } -pub fn UnsetComdat(val: &Value) { - unsafe { - LLVMRustUnsetComdat(val); - } -} - pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { unsafe { LLVMSetUnnamedAddress(global, unnamed); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 47590c8578c..60707a1c34e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -15,7 +15,6 @@ use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; -use tracing::debug; use std::mem; use std::path::Path; @@ -92,16 +91,6 @@ unsafe fn configure_llvm(sess: &Session) { add("-generate-arange-section", false); } - // Disable the machine outliner by default in LLVM versions 11 and LLVM - // version 12, where it leads to miscompilation. - // - // Ref: - // - https://github.com/rust-lang/rust/issues/85351 - // - https://reviews.llvm.org/D103167 - if llvm_util::get_version() < (13, 0, 0) { - add("-enable-machine-outliner=never", false); - } - match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { MergeFunctions::Disabled | MergeFunctions::Trampolines => {} MergeFunctions::Aliases => { @@ -444,6 +433,8 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + // Drop +atomics-32 feature introduced in LLVM 15. + .filter(|v| *v != "+atomics-32" || get_version() >= (15, 0, 0)) .map(String::from), ); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 6e94284852f..1eceb7f5c87 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -11,7 +11,6 @@ use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitable}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; -use tracing::debug; impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index cf2d3c423c3..eeb38d4ecf5 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -19,7 +19,7 @@ use rustc_target::abi::{AddressSpace, Align, Integer, Size}; use std::fmt; use std::ptr; -use libc::c_uint; +use libc::{c_char, c_uint}; impl PartialEq for Type { fn eq(&self, other: &Self) -> bool { @@ -289,3 +289,31 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { ty.llvm_type(self) } } + +impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn typeid_metadata(&self, typeid: String) -> &'ll Value { + unsafe { + llvm::LLVMMDStringInContext( + self.llcx, + typeid.as_ptr() as *const c_char, + typeid.len() as c_uint, + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 9f0e6c80b19..dc1165835e7 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -11,7 +11,6 @@ use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; use smallvec::{smallvec, SmallVec}; -use tracing::debug; use std::fmt::Write; |