diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
28 files changed, 546 insertions, 717 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 9eb3574e77b..d478efc863a 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -19,6 +19,7 @@ use rustc_target::abi::call::ArgAbi; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int}; pub use rustc_target::spec::abi::Abi; +use rustc_target::spec::SanitizerSet; use libc::c_uint; use smallvec::SmallVec; @@ -90,6 +91,13 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); } + } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { + // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects + // memory sanitizer's behavior. + + if regular.contains(ArgAttribute::NoUndef) { + attrs.push(llvm::AttributeKind::NoUndef.create_attr(cx.llcx)); + } } attrs @@ -213,7 +221,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) } else if self.is_unsized_indirect() { bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast(cast) = self.mode { + } else if let PassMode::Cast(cast, _) = &self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. let can_store_through_cast_ptr = false; @@ -283,7 +291,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(_) | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } - | PassMode::Cast(_) => { + | PassMode::Cast(..) => { let next_arg = next(); self.store(bx, next_arg, dst); } @@ -325,20 +333,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { // Ignore "extra" args from the call site for C variadic functions. // Only the "fixed" args are part of the LLVM function signature. - let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + let args = + if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args }; - let args_capacity: usize = args.iter().map(|arg| - if arg.pad.is_some() { 1 } else { 0 } + - if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } - ).sum(); + // This capacity calculation is approximate. let mut llargument_tys = Vec::with_capacity( - if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, + self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }, ); - let llreturn_ty = match self.ret.mode { + let llreturn_ty = match &self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, _) => cast.llvm_type(cx), PassMode::Indirect { .. } => { llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); cx.type_void() @@ -346,12 +352,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { }; for arg in args { - // add padding - if let Some(ty) = arg.pad { - llargument_tys.push(ty.llvm_type(cx)); - } - - let llarg_ty = match arg.mode { + let llarg_ty = match &arg.mode { PassMode::Ignore => continue, PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), PassMode::Pair(..) => { @@ -366,7 +367,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, pad_i32) => { + // add padding + if *pad_i32 { + llargument_tys.push(Reg::i32().llvm_type(cx)); + } + cast.llvm_type(cx) + } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { cx.type_ptr_to(arg.memory_ty(cx)) } @@ -426,46 +433,46 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } _ => {} } - for arg in &self.args { - if arg.pad.is_some() { - apply(&ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(attrs); let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(attrs); } - PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { assert!(!on_stack); apply(attrs); apply(extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(a); apply(b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(&ArgAttributes::new()); + } apply(&cast.attrs); } } @@ -488,17 +495,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_callsite( llvm::AttributePlace::ReturnValue, &bx.cx, @@ -517,13 +524,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } } - for arg in &self.args { - if arg.pad.is_some() { - apply(bx.cx, &ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); attributes::apply_to_callsite( @@ -532,23 +536,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[byval], ); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(bx.cx, attrs); } - PassMode::Indirect { - ref attrs, - extra_attrs: Some(ref extra_attrs), - on_stack: _, - } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack: _ } => { apply(bx.cx, attrs); apply(bx.cx, extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(bx.cx, a); apply(bx.cx, b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(bx.cx, &ArgAttributes::new()); + } apply(bx.cx, &cast.attrs); } } @@ -589,10 +592,6 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { - fn_abi.apply_attrs_callsite(self, callsite) - } - fn get_param(&mut self, index: usize) -> Self::Value { llvm::get_param(self.llfn(), index as c_uint) } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 2a6612eb86f..017513721b7 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -19,7 +19,6 @@ use rustc_target::asm::*; use libc::{c_char, c_uint}; use smallvec::SmallVec; -use tracing::debug; impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_inline_asm( @@ -431,9 +430,9 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, v, inputs, dest, catch, funclet) + bx.invoke(fty, None, v, inputs, dest, catch, funclet) } else { - bx.call(fty, v, inputs, None) + bx.call(fty, None, v, inputs, None) }; // Store mark in a metadata node so we can map LLVM errors @@ -552,6 +551,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> format!("{{{}}}", reg.name()) } } + // The constraints can be retrieved from + // https://llvm.org/docs/LangRef.html#supported-constraint-code-list InlineAsmRegOrRegClass::RegClass(reg) => match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", @@ -625,6 +626,8 @@ fn modifier_to_llvm( reg: InlineAsmRegClass, modifier: Option<char>, ) -> Option<char> { + // The modifiers can be retrieved from + // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index aabbe8ac276..eff2436d41c 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -35,6 +35,10 @@ pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attrib /// Get LLVM attribute for the provided inline heuristic. #[inline] fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { + if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + // disable LLVM inlining + return Some(AttributeKind::NoInline.create_attr(cx.llcx)); + } match inline { InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), @@ -386,7 +390,8 @@ pub fn from_fn_attrs<'ll, 'tcx>( ) { let span = cx .tcx - .get_attr(instance.def_id(), sym::target_feature) + .get_attrs(instance.def_id(), sym::target_feature) + .next() .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); let msg = format!( "the target features {} must all be either enabled or disabled together", diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 27039cda253..20a063f80fd 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -2,16 +2,21 @@ use std::env; use std::ffi::{CStr, CString, OsString}; -use std::io; +use std::fs; +use std::io::{self, Write}; use std::mem; use std::path::{Path, PathBuf}; use std::ptr; use std::str; +use object::read::macho::FatArch; + +use crate::common; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; -use rustc_session::cstore::{DllCallingConvention, DllImport}; +use rustc_data_structures::memmap::Mmap; +use rustc_session::cstore::DllImport; use rustc_session::Session; /// Helper for adding many files to an archive. @@ -52,13 +57,70 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { } } +fn try_filter_fat_archs( + archs: object::read::Result<&[impl FatArch]>, + target_arch: object::Architecture, + archive_path: &Path, + archive_map_data: &[u8], +) -> io::Result<Option<PathBuf>> { + let archs = archs.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + let desired = match archs.iter().filter(|a| a.architecture() == target_arch).next() { + Some(a) => a, + None => return Ok(None), + }; + + let (mut new_f, extracted_path) = tempfile::Builder::new() + .suffix(archive_path.file_name().unwrap()) + .tempfile()? + .keep() + .unwrap(); + + new_f.write_all( + desired.data(archive_map_data).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?, + )?; + + Ok(Some(extracted_path)) +} + +fn try_extract_macho_fat_archive( + sess: &Session, + archive_path: &Path, +) -> io::Result<Option<PathBuf>> { + let archive_map = unsafe { Mmap::map(fs::File::open(&archive_path)?)? }; + let target_arch = match sess.target.arch.as_ref() { + "aarch64" => object::Architecture::Aarch64, + "x86_64" => object::Architecture::X86_64, + _ => return Ok(None), + }; + + match object::macho::FatHeader::parse(&*archive_map) { + Ok(h) if h.magic.get(object::endian::BigEndian) == object::macho::FAT_MAGIC => { + let archs = object::macho::FatHeader::parse_arch32(&*archive_map); + try_filter_fat_archs(archs, target_arch, archive_path, &*archive_map) + } + Ok(h) if h.magic.get(object::endian::BigEndian) == object::macho::FAT_MAGIC_64 => { + let archs = object::macho::FatHeader::parse_arch64(&*archive_map); + try_filter_fat_archs(archs, target_arch, archive_path, &*archive_map) + } + // Not a FatHeader at all, just return None. + _ => Ok(None), + } +} + impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { fn add_archive( &mut self, archive: &Path, skip: Box<dyn FnMut(&str) -> bool + 'static>, ) -> io::Result<()> { - let archive_ro = match ArchiveRO::open(archive) { + let mut archive = archive.to_path_buf(); + if self.sess.target.llvm_target.contains("-apple-macosx") { + if let Some(new_archive) = try_extract_macho_fat_archive(&self.sess, &archive)? { + archive = new_archive + } + } + let archive_ro = match ArchiveRO::open(&archive) { Ok(ar) => ar, Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), }; @@ -66,7 +128,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { return Ok(()); } self.additions.push(Addition::Archive { - path: archive.to_path_buf(), + path: archive, archive: archive_ro, skip: Box::new(skip), }); @@ -111,21 +173,18 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { }; let target = &sess.target; - let mingw_gnu_toolchain = target.vendor == "pc" - && target.os == "windows" - && target.env == "gnu" - && target.abi.is_empty(); + let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(target); let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { if sess.target.arch == "x86" { ( - LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), - import.ordinal, + common::i686_decorated_name(import, mingw_gnu_toolchain, false), + import.ordinal(), ) } else { - (import.name.to_string(), import.ordinal) + (import.name.to_string(), import.ordinal()) } }) .collect(); @@ -159,6 +218,9 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { } }; + // --no-leading-underscore: For the `import_name_type` feature to work, we need to be + // able to control the *exact* spelling of each of the symbols that are being imported: + // hence we don't want `dlltool` adding leading underscores automatically. let dlltool = find_binutils_dlltool(sess); let result = std::process::Command::new(dlltool) .args([ @@ -168,6 +230,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { lib_name, "-l", output_path.to_str().unwrap(), + "--no-leading-underscore", ]) .output(); @@ -188,10 +251,10 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { let output_path_z = rustc_fs_util::path_to_c_string(&output_path); - tracing::trace!("invoking LLVMRustWriteImportLibrary"); - tracing::trace!(" dll_name {:#?}", dll_name_z); - tracing::trace!(" output_path {}", output_path.display()); - tracing::trace!( + trace!("invoking LLVMRustWriteImportLibrary"); + trace!(" dll_name {:#?}", dll_name_z); + trace!(" output_path {}", output_path.display()); + trace!( " import names: {}", dll_imports .iter() @@ -322,22 +385,6 @@ impl<'a> LlvmArchiveBuilder<'a> { ret } } - - fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { - let name = import.name; - let prefix = if mingw { "" } else { "_" }; - - match import.calling_convention { - DllCallingConvention::C => format!("{}{}", prefix, name), - DllCallingConvention::Stdcall(arg_list_size) => { - format!("{}{}@{}", prefix, name, arg_list_size) - } - DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), - DllCallingConvention::Vectorcall(arg_list_size) => { - format!("{}@@{}", name, arg_list_size) - } - } - } } fn string_to_io_error(s: String) -> io::Error { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 3731c6bcfe7..a49cc7f8d66 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,15 +1,14 @@ -use crate::back::write::{ - self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, -}; -use crate::llvm::archive_ro::ArchiveRO; -use crate::llvm::{self, build_string, False, True}; -use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; +use crate::back::write::{self, save_temp_bitcode, DiagnosticHandlers}; +use crate::llvm::{self, build_string}; +use crate::{LlvmCodegenBackend, ModuleLlvm}; +use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; @@ -17,7 +16,6 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; -use tracing::{debug, info}; use std::ffi::{CStr, CString}; use std::fs::File; @@ -34,8 +32,8 @@ pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { - CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, - CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, + CrateType::Executable | CrateType::Dylib | CrateType::Staticlib | CrateType::Cdylib => true, + CrateType::Rlib | CrateType::ProcMacro => false, } } @@ -75,17 +73,6 @@ fn prepare_lto( // with either fat or thin LTO let mut upstream_modules = Vec::new(); if cgcx.lto != Lto::ThinLocal { - if cgcx.opts.cg.prefer_dynamic { - diag_handler - .struct_err("cannot prefer dynamic linking when performing LTO") - .note( - "only 'staticlib', 'bin', and 'cdylib' outputs are \ - supported with LTO", - ) - .emit(); - return Err(FatalError); - } - // Make sure we actually can run LTO for crate_type in cgcx.crate_types.iter() { if !crate_type_allows_lto(*crate_type) { @@ -94,9 +81,25 @@ fn prepare_lto( static library outputs", ); return Err(e); + } else if *crate_type == CrateType::Dylib { + if !cgcx.opts.unstable_opts.dylib_lto { + return Err(diag_handler + .fatal("lto cannot be used for `dylib` crate type without `-Zdylib-lto`")); + } } } + if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto { + diag_handler + .struct_err("cannot prefer dynamic linking when performing LTO") + .note( + "only 'staticlib', 'bin', and 'cdylib' outputs are \ + supported with LTO", + ) + .emit(); + return Err(FatalError); + } + for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); @@ -107,14 +110,24 @@ fn prepare_lto( .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); } - let archive = ArchiveRO::open(path).expect("wanted an rlib"); + let archive_data = unsafe { + Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib")) + .expect("couldn't map rlib") + }; + let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib"); let obj_files = archive - .iter() - .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) + .members() + .filter_map(|child| { + child.ok().and_then(|c| { + std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c)) + }) + }) .filter(|&(name, _)| looks_like_rust_object_file(name)); for (name, child) in obj_files { info!("adding bitcode from {}", name); - match get_bitcode_slice_from_object_data(child.data()) { + match get_bitcode_slice_from_object_data( + child.data(&*archive_data).expect("corrupt rlib"), + ) { Ok(data) => { let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); @@ -565,7 +578,7 @@ pub(crate) fn run_pass_manager( module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, ) -> Result<(), FatalError> { - let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); + let _timer = cgcx.prof.verbose_generic_activity_with_arg("LLVM_lto_optimize", &*module.name); let config = cgcx.config(module.kind); // Now we have one massive module inside of llmod. Time to run the @@ -587,61 +600,9 @@ pub(crate) fn run_pass_manager( 1, ); } - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; - let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - )?; - debug!("lto done"); - return Ok(()); - } - - let pm = llvm::LLVMCreatePassManager(); - llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); - - if config.verify_llvm_ir { - let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - let opt_level = config - .opt_level - .map(|x| to_llvm_opt_settings(x).0) - .unwrap_or(llvm::CodeGenOptLevel::None); - with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { - if thin { - llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); - } else { - llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( - b, pm, /* Internalize = */ False, /* RunInliner = */ True, - ); - } - }); - - // We always generate bitcode through ThinLTOBuffers, - // which do not support anonymous globals - if config.bitcode_needed() { - let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - if config.verify_llvm_ir { - let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()); - - llvm::LLVMDisposePassManager(pm); + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + write::llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage)?; } debug!("lto done"); Ok(()) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 740a68d0772..11053a8f6c4 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -21,14 +21,12 @@ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; -use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; -use tracing::debug; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; @@ -418,7 +416,7 @@ fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { } } -pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( +pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, module: &ModuleCodegen<ModuleLlvm>, @@ -466,7 +464,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. - let result = llvm::LLVMRustOptimizeWithNewPassManager( + let result = llvm::LLVMRustOptimize( module.module_llvm.llmod(), &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), @@ -510,18 +508,11 @@ pub(crate) unsafe fn optimize( let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; - let tm = &*module.module_llvm.tm; let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); - if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { - diag_handler.warn( - "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", - ); - } - if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); let out = path_to_c_string(&out); @@ -529,191 +520,24 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = match cgcx.lto { - Lto::Fat => llvm::OptStage::PreLinkFatLTO, - Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, - _ => llvm::OptStage::PreLinkNoLTO, - }; - return optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - ); - } - - if cgcx.prof.llvm_recording_enabled() { - diag_handler - .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); - } - - // Create the two optimizing pass managers. These mirror what clang - // does, and are by populated by LLVM's default PassManagerBuilder. - // Each manager has a different set of passes, but they also share - // some common passes. - let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); - let mpm = llvm::LLVMCreatePassManager(); - - { - let find_pass = |pass_name: &str| { - let pass_name = SmallCStr::new(pass_name); - llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) - }; - - if config.verify_llvm_ir { - // Verification should run as the very first pass. - llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); - } - - let mut extra_passes = Vec::new(); - let mut have_name_anon_globals_pass = false; - - for pass_name in &config.passes { - if pass_name == "lint" { - // Linting should also be performed early, directly on the generated IR. - llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); - continue; - } - - if let Some(pass) = find_pass(pass_name) { - extra_passes.push(pass); - } else { - diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); - } - - if pass_name == "name-anon-globals" { - have_name_anon_globals_pass = true; - } - } - - // Instrumentation must be inserted before optimization, - // otherwise LLVM may optimize some functions away which - // breaks llvm-cov. - // - // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. - if config.instrument_gcov { - llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); - } - if config.instrument_coverage { - llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); - } - if config.debug_info_for_profiling { - llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); - } - - add_sanitizer_passes(config, &mut extra_passes); - - // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need - // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise - // we'll get errors in LLVM. - let using_thin_buffers = config.bitcode_needed(); - if !config.no_prepopulate_passes { - llvm::LLVMAddAnalysisPasses(tm, fpm); - llvm::LLVMAddAnalysisPasses(tm, mpm); - let opt_level = to_llvm_opt_settings(opt_level).0; - let prepare_for_thin_lto = cgcx.lto == Lto::Thin - || cgcx.lto == Lto::ThinLocal - || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); - with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { - llvm::LLVMRustAddLastExtensionPasses( - b, - extra_passes.as_ptr(), - extra_passes.len() as size_t, - ); - llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); - llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); - }); - - have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; - if using_thin_buffers && !prepare_for_thin_lto { - llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); - have_name_anon_globals_pass = true; - } - } else { - // If we don't use the standard pipeline, directly populate the MPM - // with the extra passes. - for pass in extra_passes { - llvm::LLVMRustAddPass(mpm, pass); - } - } - - if using_thin_buffers && !have_name_anon_globals_pass { - // As described above, this will probably cause an error in LLVM - if config.no_prepopulate_passes { - diag_handler.err( - "The current compilation is going to use thin LTO buffers \ - without running LLVM's NameAnonGlobals pass. \ - This will likely cause errors in LLVM. Consider adding \ - -C passes=name-anon-globals to the compiler command line.", - ); - } else { - bug!( - "We are using thin LTO buffers without running the NameAnonGlobals pass. \ - This will likely cause errors in LLVM and should never happen." - ); - } - } - } - - diag_handler.abort_if_errors(); - - // Finally, run the actual optimization passes - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_function_passes", - &*module.name, - ); - llvm::LLVMRustRunFunctionPassManager(fpm, llmod); - } - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_module_passes", - &*module.name, - ); - llvm::LLVMRunPassManager(mpm, llmod); - } - - // Deallocate managers that we're now done with - llvm::LLVMDisposePassManager(fpm); - llvm::LLVMDisposePassManager(mpm); + let opt_stage = match cgcx.lto { + Lto::Fat => llvm::OptStage::PreLinkFatLTO, + Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ => llvm::OptStage::PreLinkNoLTO, + }; + return llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage); } Ok(()) } -unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { - if config.sanitizer.contains(SanitizerSet::ADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); - passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); - passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); - } - if config.sanitizer.contains(SanitizerSet::MEMORY) { - let track_origins = config.sanitizer_memory_track_origins as c_int; - let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); - passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); - } - if config.sanitizer.contains(SanitizerSet::THREAD) { - passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); - } - if config.sanitizer.contains(SanitizerSet::HWADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); - passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); - } -} - pub(crate) fn link( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, mut modules: Vec<ModuleCodegen<ModuleLlvm>>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { use super::lto::{Linker, ModuleBuffer}; - // Sort the modules by name to ensure to ensure deterministic behavior. + // Sort the modules by name to ensure deterministic behavior. modules.sort_by(|a, b| a.name.cmp(&b.name)); let (first, elements) = modules.split_first().expect("Bug! modules must contain at least one module."); @@ -1073,72 +897,6 @@ unsafe fn embed_bitcode( } } -pub unsafe fn with_llvm_pmb( - llmod: &llvm::Module, - config: &ModuleConfig, - opt_level: llvm::CodeGenOptLevel, - prepare_for_thin_lto: bool, - f: &mut dyn FnMut(&llvm::PassManagerBuilder), -) { - use std::ptr; - - // Create the PassManagerBuilder for LLVM. We configure it with - // reasonable defaults and prepare it to actually populate the pass - // manager. - let builder = llvm::LLVMRustPassManagerBuilderCreate(); - let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); - let inline_threshold = config.inline_threshold; - let pgo_gen_path = get_pgo_gen_path(config); - let pgo_use_path = get_pgo_use_path(config); - let pgo_sample_use_path = get_pgo_sample_use_path(config); - - llvm::LLVMRustConfigurePassManagerBuilder( - builder, - opt_level, - config.merge_functions, - config.vectorize_slp, - config.vectorize_loop, - prepare_for_thin_lto, - pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - opt_size as c_int, - ); - - llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); - - // Here we match what clang does (kinda). For O0 we only inline - // always-inline functions (but don't add lifetime intrinsics), at O1 we - // inline with lifetime intrinsics, and O2+ we add an inliner with a - // thresholds copied from clang. - match (opt_level, opt_size, inline_threshold) { - (.., Some(t)) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); - } - (llvm::CodeGenOptLevel::Aggressive, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); - } - (_, llvm::CodeGenOptSizeDefault, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); - } - (_, llvm::CodeGenOptSizeAggressive, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); - } - (llvm::CodeGenOptLevel::None, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Less, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Default, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); - } - } - - f(builder); - llvm::LLVMRustPassManagerBuilderDispose(builder); -} - // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. // This is required to satisfy `dllimport` references to static data in .rlibs // when using MSVC linker. We do this only for data, as linker can fix up diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 86f92dc0239..5b2bbdb4bde 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -19,6 +19,8 @@ use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; +use cstr::cstr; + use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; @@ -107,11 +109,14 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen } // Create the llvm.used and llvm.compiler.used variables. - if !cx.used_statics().borrow().is_empty() { - cx.create_used_variable() + if !cx.used_statics.borrow().is_empty() { + cx.create_used_variable_impl(cstr!("llvm.used"), &*cx.used_statics.borrow()); } - if !cx.compiler_used_statics().borrow().is_empty() { - cx.create_compiler_used_variable() + if !cx.compiler_used_statics.borrow().is_empty() { + cx.create_used_variable_impl( + cstr!("llvm.compiler.used"), + &*cx.compiler_used_statics.borrow(), + ); } // Run replace-all-uses-with for statics that need it. This must diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 073feecb164..fca43a0d86d 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1,14 +1,14 @@ +use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; -use crate::llvm::{self, BasicBlock, False}; -use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; +use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use cstr::cstr; use libc::{c_char, c_uint}; -use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind}; +use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; @@ -27,7 +27,6 @@ use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; -use tracing::{debug, instrument}; // All Builders must have an llfn associated with them #[must_use] @@ -216,6 +215,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, llty: &'ll Type, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], then: &'ll BasicBlock, @@ -228,7 +228,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let bundle = funclet.map(|funclet| funclet.bundle()); let bundle = bundle.as_ref().map(|b| &*b.raw); - unsafe { + let invoke = unsafe { llvm::LLVMRustBuildInvoke( self.llbuilder, llty, @@ -240,7 +240,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { bundle, UNNAMED, ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, invoke); } + invoke } fn unreachable(&mut self) { @@ -407,20 +411,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { let mut bx = Builder::with_cx(self.cx); bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); - bx.dynamic_alloca(ty, align) - } - - fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { unsafe { - let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED); + let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); alloca } } - fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value { + fn byte_array_alloca(&mut self, len: &'ll Value, align: Align) -> &'ll Value { unsafe { - let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED); + let alloca = + llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), len, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); alloca } @@ -725,11 +726,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(false, val, dest_ty) } - fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(true, val, dest_ty) } @@ -1043,15 +1044,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) -> &'ll Value { let weak = if weak { llvm::True } else { llvm::False }; unsafe { - llvm::LLVMRustBuildAtomicCmpXchg( + let value = llvm::LLVMBuildAtomicCmpXchg( self.llbuilder, dst, cmp, src, AtomicOrdering::from_generic(order), AtomicOrdering::from_generic(failure_order), - weak, - ) + llvm::False, // SingleThreaded + ); + llvm::LLVMSetWeak(value, weak); + value } } fn atomic_rmw( @@ -1068,7 +1071,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst, src, AtomicOrdering::from_generic(order), - False, + llvm::False, // SingleThreaded ) } } @@ -1076,13 +1079,18 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn atomic_fence( &mut self, order: rustc_codegen_ssa::common::AtomicOrdering, - scope: rustc_codegen_ssa::common::SynchronizationScope, + scope: SynchronizationScope, ) { + let single_threaded = match scope { + SynchronizationScope::SingleThread => llvm::True, + SynchronizationScope::CrossThread => llvm::False, + }; unsafe { - llvm::LLVMRustBuildAtomicFence( + llvm::LLVMBuildFence( self.llbuilder, AtomicOrdering::from_generic(order), - SynchronizationScope::from_generic(scope), + single_threaded, + UNNAMED, ); } } @@ -1140,6 +1148,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, @@ -1150,7 +1159,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let bundle = funclet.map(|funclet| funclet.bundle()); let bundle = bundle.as_ref().map(|b| &*b.raw); - unsafe { + let call = unsafe { llvm::LLVMRustBuildCall( self.llbuilder, llty, @@ -1159,7 +1168,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { args.len() as c_uint, bundle, ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, call); } + call } fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -1392,7 +1405,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, f, args, None) + self.call(ty, None, f, args, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { @@ -1429,12 +1442,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - fn fptoint_sat( - &mut self, - signed: bool, - val: &'ll Value, - dest_ty: &'ll Type, - ) -> Option<&'ll Value> { + fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { let src_ty = self.cx.val_ty(val); let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector { assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); @@ -1459,7 +1467,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) }; let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None) } pub(crate) fn landing_pad( diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 72155d874a2..6f0d1b7ce84 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -6,11 +6,11 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; +use crate::common; use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; use rustc_codegen_ssa::traits::*; -use tracing::debug; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeVisitable}; @@ -79,13 +79,18 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> llfn } } else { - let llfn = cx.declare_fn(sym, fn_abi); + let instance_def_id = instance.def_id(); + let llfn = if tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) + { + cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi) + } else { + cx.declare_fn(sym, fn_abi) + }; debug!("get_fn: not casting pointer!"); attributes::from_fn_attrs(cx, llfn, instance); - let instance_def_id = instance.def_id(); - // Apply an appropriate linkage/visibility value to our item that we // just declared. // @@ -174,7 +179,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> // MinGW: For backward compatibility we rely on the linker to decide whether it // should use dllimport for functions. if cx.use_dll_storage_attrs - && tcx.is_dllimport_foreign_item(instance_def_id) + && let Some(library) = tcx.native_library(instance_def_id) + && library.kind.is_dllimport() && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc") { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index fb4da9a5f33..acee9134fb9 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -10,13 +10,17 @@ use crate::value::Value; use rustc_ast::Mutability; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::TyCtxt; +use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_target::spec::Target; use libc::{c_char, c_uint}; -use tracing::debug; +use std::fmt::Write; /* * A note on nomenclature of linking: "extern", "foreign", and "upcall". @@ -211,7 +215,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_to_opt_uint(&self, v: &'ll Value) -> Option<u64> { - try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) }) + try_as_const_integral(v).and_then(|v| unsafe { + let mut i = 0u64; + let success = llvm::LLVMRustConstIntGetZExtValue(v, &mut i); + success.then_some(i) + }) } fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option<u128> { @@ -222,10 +230,6 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } - fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { - self.const_undef(self.type_ix(0)) - } - fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { @@ -357,3 +361,74 @@ fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { unsafe { llvm::LLVMIsAConstantInt(v) } } + +pub(crate) fn get_dllimport<'tcx>( + tcx: TyCtxt<'tcx>, + id: DefId, + name: &str, +) -> Option<&'tcx DllImport> { + tcx.native_library(id) + .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) + .flatten() +} + +pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { + target.vendor == "pc" && target.os == "windows" && target.env == "gnu" && target.abi.is_empty() +} + +pub(crate) fn i686_decorated_name( + dll_import: &DllImport, + mingw: bool, + disable_name_mangling: bool, +) -> String { + let name = dll_import.name.as_str(); + + let (add_prefix, add_suffix) = match dll_import.import_name_type { + Some(PeImportNameType::NoPrefix) => (false, true), + Some(PeImportNameType::Undecorated) => (false, false), + _ => (true, true), + }; + + // Worst case: +1 for disable name mangling, +1 for prefix, +4 for suffix (@@__). + let mut decorated_name = String::with_capacity(name.len() + 6); + + if disable_name_mangling { + // LLVM uses a binary 1 ('\x01') prefix to a name to indicate that mangling needs to be disabled. + decorated_name.push('\x01'); + } + + let prefix = if add_prefix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C | DllCallingConvention::Vectorcall(_) => None, + DllCallingConvention::Stdcall(_) => (!mingw + || dll_import.import_name_type == Some(PeImportNameType::Decorated)) + .then_some('_'), + DllCallingConvention::Fastcall(_) => Some('@'), + } + } else if !dll_import.is_fn && !mingw { + // For static variables, prefix with '_' on MSVC. + Some('_') + } else { + None + }; + if let Some(prefix) = prefix { + decorated_name.push(prefix); + } + + decorated_name.push_str(name); + + if add_suffix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C => {} + DllCallingConvention::Stdcall(arg_list_size) + | DllCallingConvention::Fastcall(arg_list_size) => { + write!(&mut decorated_name, "@{}", arg_list_size).unwrap(); + } + DllCallingConvention::Vectorcall(arg_list_size) => { + write!(&mut decorated_name, "@@{}", arg_list_size).unwrap(); + } + } + } + + decorated_name +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 18467e37082..dd3c43ba5ca 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,5 +1,5 @@ use crate::base; -use crate::common::CodegenCx; +use crate::common::{self, CodegenCx}; use crate::debuginfo; use crate::llvm::{self, True}; use crate::llvm_util; @@ -23,16 +23,15 @@ use rustc_target::abi::{ AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, }; use std::ops::Range; -use tracing::debug; pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1); let dl = cx.data_layout(); let pointer_size = dl.pointer_size.bytes() as usize; - // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, - // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` + // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( llvals: &mut Vec<&'ll Value>, cx: &'a CodegenCx<'ll, 'b>, @@ -79,12 +78,12 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } let mut next_offset = 0; - for &(offset, alloc_id) in alloc.relocations().iter() { + for &(offset, alloc_id) in alloc.provenance().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a relocation, it + // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); @@ -93,7 +92,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< dl.endian, // This `inspect` is okay since it is within the bounds of the allocation, it doesn't // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the relocation as a relocation pointer offset. + // and we properly interpret the provenance as a relocation pointer offset. alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), ) .expect("const_alloc_to_llvm: could not read relocation pointer") @@ -121,7 +120,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } if alloc.len() >= next_offset { let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all relocations, it is + // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); @@ -160,7 +159,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str, - span_def_id: DefId, + def_id: DefId, ) -> &'ll Value { let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.linkage { @@ -175,7 +174,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( cx.layout_of(mt.ty).llvm_type(cx) } else { cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), + cx.tcx.def_span(def_id), "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", ) }; @@ -194,7 +193,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), + cx.tcx.def_span(def_id), &format!("symbol `{}` is already defined", &sym), ) }); @@ -202,6 +201,10 @@ fn check_and_apply_linkage<'ll, 'tcx>( llvm::LLVMSetInitializer(g2, g1); g2 } + } else if cx.tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) + { + cx.declare_global(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), true), llty) } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global @@ -329,7 +332,10 @@ impl<'ll> CodegenCx<'ll, '_> { } } - if self.use_dll_storage_attrs && self.tcx.is_dllimport_foreign_item(def_id) { + if self.use_dll_storage_attrs + && let Some(library) = self.tcx.native_library(def_id) + && library.kind.is_dllimport() + { // For foreign (native) libs we know the exact storage type to use. unsafe { llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); @@ -475,7 +481,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // // We could remove this hack whenever we decide to drop macOS 10.10 support. if self.tcx.sess.target.is_like_osx { - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state // (not as part of the interpreter execution). // @@ -483,7 +489,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // happens to be zero. Instead, we should only check the value of defined bytes // and set all undefined bytes to zero if this allocation is headed for the // BSS. - let all_bytes_are_zero = alloc.relocations().is_empty() + let all_bytes_are_zero = alloc.provenance().is_empty() && alloc .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) .iter() @@ -507,9 +513,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { section.as_str().as_ptr().cast(), section.as_str().len() as c_uint, ); - assert!(alloc.relocations().is_empty()); + assert!(alloc.provenance().is_empty()); - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state (not // as part of the interpreter execution). let bytes = @@ -549,7 +555,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage // on other targets, in particular MachO targets have *their* static constructor // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, - // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_typeck`, + // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_hir_analysis`, // so we don't need to take care of it here. self.add_compiler_used_global(g); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d4d84147239..79ddfd884df 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -154,6 +154,11 @@ pub unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); } } + if llvm_version < (16, 0, 0) { + if sess.target.arch == "s390x" { + target_data_layout = target_data_layout.replace("-v128:64", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. if sess.target.is_builtin { @@ -453,7 +458,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { self.coverage_cx.as_ref() } - fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { + pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let section = cstr!("llvm.metadata"); let array = self.const_array(self.type_ptr_to(self.type_i8()), values); @@ -551,14 +556,6 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.codegen_unit } - fn used_statics(&self) -> &RefCell<Vec<&'ll Value>> { - &self.used_statics - } - - fn compiler_used_statics(&self) -> &RefCell<Vec<&'ll Value>> { - &self.compiler_used_statics - } - fn set_frame_pointer_type(&self, llfn: &'ll Value) { if let Some(attr) = attributes::frame_pointer_type_attr(self) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); @@ -572,17 +569,6 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } - fn create_used_variable(&self) { - self.create_used_variable_impl(cstr!("llvm.used"), &*self.used_statics.borrow()); - } - - fn create_compiler_used_variable(&self) { - self.create_used_variable_impl( - cstr!("llvm.compiler.used"), - &*self.compiler_used_statics.borrow(), - ); - } - fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { if self.get_declared_value("main").is_none() { Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) @@ -886,6 +872,9 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } + + ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); + None } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 58f391692c4..433f043209e 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -16,8 +16,6 @@ use rustc_middle::ty::TyCtxt; use std::ffi::CString; -use tracing::debug; - /// Generates and exports the Coverage Map. /// /// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions @@ -131,7 +129,7 @@ impl CoverageMapGenerator { // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) // requires setting the first filename to the compilation directory. // Since rustc generates coverage maps with relative paths, the - // compilation directory can be combined with the the relative paths + // compilation directory can be combined with the relative paths // to get absolute paths, if needed. let working_dir = tcx .sess diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 98ba38356a4..964a632b6ee 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -28,7 +28,6 @@ use std::cell::RefCell; use std::ffi::CString; use std::iter; -use tracing::debug; pub mod mapgen; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 25a989bdf05..163ccd9460c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -42,7 +42,6 @@ use rustc_span::{self, FileNameDisplayPreference, SourceFile}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; -use tracing::debug; use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; @@ -51,7 +50,6 @@ use std::hash::{Hash, Hasher}; use std::iter; use std::path::{Path, PathBuf}; use std::ptr; -use tracing::instrument; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -1500,24 +1498,18 @@ fn vcall_visibility_metadata<'ll, 'tcx>( // If there is not LTO and the visibility in public, we have to assume that the vtable can // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. (Lto::No | Lto::ThinLocal, Visibility::Public, _) - | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { - VCallVisibility::Public - } + | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public, // With LTO and a quasi-public visibility, the usages of the functions of the vtable are // all known by the `LinkageUnit`. // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. (Lto::Fat | Lto::Thin, Visibility::Public, _) - | ( - Lto::ThinLocal | Lto::Thin | Lto::Fat, - Visibility::Restricted(_) | Visibility::Invisible, - false, - ) => VCallVisibility::LinkageUnit, + | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => { + VCallVisibility::LinkageUnit + } // If there is only one CGU, private vtables can only be seen by that CGU/translation unit // and therefore we know of all usages of functions in the vtable. - (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { - VCallVisibility::TranslationUnit - } + (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit, }; let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index daec9303b2c..129e336c7e4 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -99,7 +99,7 @@ const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; /// compiler versions. /// /// Niche-tag enums have one special variant, usually called the -/// "dataful variant". This variant has a field that +/// "untagged variant". This variant has a field that /// doubles as the tag of the enum. The variant is active when the value of /// that field is within a pre-defined range. Therefore the variant struct /// has a `DISCR_BEGIN` and `DISCR_END` field instead of `DISCR_EXACT` in @@ -249,7 +249,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( None, ), Variants::Multiple { - tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + tag_encoding: TagEncoding::Niche { untagged_variant, .. }, ref variants, tag_field, .. @@ -260,7 +260,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( enum_type_di_node, variants.indices(), tag_field, - Some(dataful_variant), + Some(untagged_variant), ), } }, @@ -391,7 +391,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( enum_type_di_node: &'ll DIType, variant_indices: impl Iterator<Item = VariantIdx> + Clone, tag_field: usize, - dataful_variant_index: Option<VariantIdx>, + untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); @@ -436,7 +436,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( variant_names_type_di_node, tag_base_type, tag_field, - dataful_variant_index, + untagged_variant_index, ) } @@ -472,7 +472,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( enum_or_generator_type_and_layout: TyAndLayout<'tcx>, enum_or_generator_type_di_node: &'ll DIType, variant_index: VariantIdx, - dataful_variant_index: Option<VariantIdx>, + untagged_variant_index: Option<VariantIdx>, variant_struct_type_di_node: &'ll DIType, variant_names_type_di_node: &'ll DIType, tag_base_type_di_node: &'ll DIType, @@ -517,7 +517,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( } } DiscrResult::Range(min, max) => { - assert_eq!(Some(variant_index), dataful_variant_index); + assert_eq!(Some(variant_index), untagged_variant_index); if is_128_bits { DiscrKind::Range128(min, max) } else { @@ -757,7 +757,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( discr_type_di_node: &'ll DIType, tag_base_type: Ty<'tcx>, tag_field: usize, - dataful_variant_index: Option<VariantIdx>, + untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { let tag_base_type_di_node = type_di_node(cx, tag_base_type); let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); @@ -776,7 +776,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( enum_type_and_layout, enum_type_di_node, variant_member_info.variant_index, - dataful_variant_index, + untagged_variant_index, variant_member_info.variant_struct_type_di_node, discr_type_di_node, tag_base_type_di_node, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 9b3d080bfd6..14044d0f99b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -417,7 +417,7 @@ impl DiscrResult { /// Returns the discriminant value corresponding to the variant index. /// /// Will return `None` if there is less than two variants (because then the enum won't have) -/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no +/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no /// single discriminant value). fn compute_discriminant_value<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, @@ -430,11 +430,11 @@ fn compute_discriminant_value<'ll, 'tcx>( enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val, ), &Variants::Multiple { - tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, + tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant }, tag, .. } => { - if variant_index == dataful_variant { + if variant_index == untagged_variant { let valid_range = enum_type_and_layout .for_variant(cx, variant_index) .largest_niche diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index dae90a43f26..becbccc434d 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -378,7 +378,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( /// /// The DW_AT_discr_value is optional, and is omitted if /// - This is the only variant of a univariant enum (i.e. their is no discriminant) -/// - This is the "dataful" variant of a niche-layout enum +/// - This is the "untagged" variant of a niche-layout enum /// (where only the other variants are identified by a single value) /// /// There is only ever a single member, the type of which is a struct that describes the diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index cf591295b84..b23fe3fc9d5 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -39,7 +39,6 @@ use smallvec::SmallVec; use std::cell::OnceCell; use std::cell::RefCell; use std::iter; -use tracing::debug; mod create_scope_map; pub mod gdb; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 8f243673907..a40cfc8b23f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -6,7 +6,7 @@ use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; use rustc_middle::ty::{self, DefIdTree, Ty}; -use tracing::trace; +use trace; use crate::common::CodegenCx; use crate::llvm; diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index fa0ecd18fc8..f79ef11720d 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -22,7 +22,6 @@ use rustc_codegen_ssa::traits::TypeMembershipMethods; use rustc_middle::ty::Ty; use rustc_symbol_mangling::typeid::typeid_for_fnabi; use smallvec::SmallVec; -use tracing::debug; /// Declare a function. /// @@ -33,6 +32,7 @@ fn declare_raw_fn<'ll>( name: &str, callconv: llvm::CallConv, unnamed: llvm::UnnamedAddr, + visibility: llvm::Visibility, ty: &'ll Type, ) -> &'ll Value { debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); @@ -42,6 +42,7 @@ fn declare_raw_fn<'ll>( llvm::SetFunctionCallConv(llfn, callconv); llvm::SetUnnamedAddress(llfn, unnamed); + llvm::set_visibility(llfn, visibility); let mut attrs = SmallVec::<[_; 4]>::new(); @@ -79,7 +80,14 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - declare_raw_fn(self, name, llvm::CCallConv, unnamed, fn_type) + // Declare C ABI functions with the visibility used by C by default. + let visibility = if self.tcx.sess.target.default_hidden_visibility { + llvm::Visibility::Hidden + } else { + llvm::Visibility::Default + }; + + declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type) } /// Declare a Rust function. @@ -96,6 +104,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { name, fn_abi.llvm_cconv(), llvm::UnnamedAddr::Global, + llvm::Visibility::Default, fn_abi.llvm_type(self), ); fn_abi.apply_attrs_llfn(self, llfn); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9f364749287..825011941a2 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -71,6 +71,7 @@ fn get_simple_intrinsic<'ll>( sym::nearbyintf64 => "llvm.nearbyint.f64", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::ptr_mask => "llvm.ptrmask", _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -107,6 +108,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let (simple_ty, simple_fn) = simple.unwrap(); self.call( simple_ty, + None, simple_fn, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -161,7 +163,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::volatile_load | sym::unaligned_volatile_load => { let tp_ty = substs.type_at(0); let ptr = args[0].immediate(); - let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { + let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let llty = ty.llvm_type(self); let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); self.volatile_load(llty, ptr) @@ -374,7 +376,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { }; if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { + if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); let ptr = self.pointercast(result.llval, ptr_llty); self.store(llval, ptr, result.align); @@ -434,7 +436,7 @@ fn try_intrinsic<'ll>( ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, try_func, &[data], None); + bx.call(try_func_ty, None, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; @@ -533,7 +535,7 @@ fn codegen_msvc_try<'ll>( let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], normal, catchswitch, None); + bx.invoke(try_func_ty, None, try_func, &[data], normal, catchswitch, None); bx.switch_to_block(normal); bx.ret(bx.const_i32(0)); @@ -577,7 +579,7 @@ fn codegen_msvc_try<'ll>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_i8p(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); + bx.call(catch_ty, None, catch_func, &[data, ptr], Some(&funclet)); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -585,7 +587,7 @@ fn codegen_msvc_try<'ll>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, catch_func, &[data, null], Some(&funclet)); + bx.call(catch_ty, None, catch_func, &[data, null], Some(&funclet)); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -594,7 +596,7 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -637,7 +639,7 @@ fn codegen_gnu_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -655,13 +657,13 @@ fn codegen_gnu_try<'ll>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, ptr], None); + bx.call(catch_ty, None, catch_func, &[data, ptr], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -701,7 +703,7 @@ fn codegen_emcc_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -740,13 +742,13 @@ fn codegen_emcc_try<'ll>( let catch_data = bx.bitcast(catch_data, bx.type_i8p()); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, catch_data], None); + bx.call(catch_ty, None, catch_func, &[data, catch_data], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -1216,8 +1218,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); - let c = - bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); + let c = bx.call( + fn_ty, + None, + f, + &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), + None, + ); Ok(c) } @@ -1416,8 +1423,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( llvm_elem_vec_ty, ); let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = - bx.call(fn_ty, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); + let v = bx.call( + fn_ty, + None, + f, + &[args[1].immediate(), alignment, mask, args[0].immediate()], + None, + ); return Ok(v); } @@ -1542,8 +1554,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = - bx.call(fn_ty, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); + let v = bx.call( + fn_ty, + None, + f, + &[args[0].immediate(), args[1].immediate(), alignment, mask], + None, + ); return Ok(v); } @@ -1704,6 +1721,97 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bitwise_red!(simd_reduce_all: vector_reduce_and, true); bitwise_red!(simd_reduce_any: vector_reduce_or, true); + if name == sym::simd_cast_ptr { + require_simd!(ret_ty, "return"); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + + match in_elem.kind() { + ty::RawPtr(p) => { + let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) + }); + assert!(!check_sized); // we are in codegen, so we shouldn't see these types + require!(metadata.is_unit(), "cannot cast fat pointer `{}`", in_elem) + } + _ => return_error!("expected pointer, got `{}`", in_elem), + } + match out_elem.kind() { + ty::RawPtr(p) => { + let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) + }); + assert!(!check_sized); // we are in codegen, so we shouldn't see these types + require!(metadata.is_unit(), "cannot cast to fat pointer `{}`", out_elem) + } + _ => return_error!("expected pointer, got `{}`", out_elem), + } + + if in_elem == out_elem { + return Ok(args[0].immediate()); + } else { + return Ok(bx.pointercast(args[0].immediate(), llret_ty)); + } + } + + if name == sym::simd_expose_addr { + require_simd!(ret_ty, "return"); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + + match in_elem.kind() { + ty::RawPtr(_) => {} + _ => return_error!("expected pointer, got `{}`", in_elem), + } + match out_elem.kind() { + ty::Uint(ty::UintTy::Usize) => {} + _ => return_error!("expected `usize`, got `{}`", out_elem), + } + + return Ok(bx.ptrtoint(args[0].immediate(), llret_ty)); + } + + if name == sym::simd_from_exposed_addr { + require_simd!(ret_ty, "return"); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + + match in_elem.kind() { + ty::Uint(ty::UintTy::Usize) => {} + _ => return_error!("expected `usize`, got `{}`", in_elem), + } + match out_elem.kind() { + ty::RawPtr(_) => {} + _ => return_error!("expected pointer, got `{}`", out_elem), + } + + return Ok(bx.inttoptr(args[0].immediate(), llret_ty)); + } + if name == sym::simd_cast || name == sym::simd_as { require_simd!(ret_ty, "return"); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); @@ -1900,7 +2008,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, f, &[lhs, rhs], None); + let v = bx.call(fn_ty, None, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index c80ad692000..89c7e51d09e 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -6,7 +6,7 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(hash_raw_entry)] -#![feature(let_else)] +#![feature(let_chains)] #![feature(extern_types)] #![feature(once_cell)] #![feature(iter_intersperse)] @@ -15,6 +15,8 @@ #[macro_use] extern crate rustc_macros; +#[macro_use] +extern crate tracing; use back::write::{create_informational_target_machine, create_target_machine}; @@ -129,12 +131,6 @@ impl ExtraBackendMethods for LlvmCodegenBackend { ) -> TargetMachineFactoryFn<Self> { back::write::target_machine_factory(sess, optlvl, target_features) } - fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { - llvm_util::target_cpu(sess) - } - fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { - llvm_util::tune_cpu(sess) - } fn spawn_thread<F, T>(time_trace: bool, f: F) -> std::thread::JoinHandle<T> where @@ -168,7 +164,6 @@ impl ExtraBackendMethods for LlvmCodegenBackend { impl WriteBackendMethods for LlvmCodegenBackend { type Module = ModuleLlvm; type ModuleBuffer = back::lto::ModuleBuffer; - type Context = llvm::Context; type TargetMachine = &'static mut llvm::TargetMachine; type ThinData = back::lto::ThinData; type ThinBuffer = back::lto::ThinBuffer; diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 64db4f7462d..7d948970223 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -83,17 +83,6 @@ impl<'a> Child<'a> { } } } - - pub fn data(&self) -> &'a [u8] { - unsafe { - let mut data_len = 0; - let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); - if data_ptr.is_null() { - panic!("failed to read data from archive child"); - } - slice::from_raw_parts(data_ptr as *const u8, data_len as usize) - } - } } impl<'a> Drop for Child<'a> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4ca3dd78cae..42cb694c0e7 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -400,27 +400,6 @@ impl AtomicOrdering { } } -/// LLVMRustSynchronizationScope -#[derive(Copy, Clone)] -#[repr(C)] -pub enum SynchronizationScope { - SingleThread, - CrossThread, -} - -impl SynchronizationScope { - pub fn from_generic(sc: rustc_codegen_ssa::common::SynchronizationScope) -> Self { - match sc { - rustc_codegen_ssa::common::SynchronizationScope::SingleThread => { - SynchronizationScope::SingleThread - } - rustc_codegen_ssa::common::SynchronizationScope::CrossThread => { - SynchronizationScope::CrossThread - } - } - } -} - /// LLVMRustFileType #[derive(Copy, Clone)] #[repr(C)] @@ -1096,7 +1075,7 @@ extern "C" { pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; - pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; + pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; pub fn LLVMRustConstInt128Get( ConstantVal: &ConstantInt, SExt: bool, @@ -1782,16 +1761,18 @@ extern "C" { Order: AtomicOrdering, ) -> &'a Value; - pub fn LLVMRustBuildAtomicCmpXchg<'a>( + pub fn LLVMBuildAtomicCmpXchg<'a>( B: &Builder<'a>, LHS: &'a Value, CMP: &'a Value, RHS: &'a Value, Order: AtomicOrdering, FailureOrder: AtomicOrdering, - Weak: Bool, + SingleThreaded: Bool, ) -> &'a Value; + pub fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool); + pub fn LLVMBuildAtomicRMW<'a>( B: &Builder<'a>, Op: AtomicRmwBinOp, @@ -1801,27 +1782,19 @@ extern "C" { SingleThreaded: Bool, ) -> &'a Value; - pub fn LLVMRustBuildAtomicFence( - B: &Builder<'_>, + pub fn LLVMBuildFence<'a>( + B: &Builder<'a>, Order: AtomicOrdering, - Scope: SynchronizationScope, - ); + SingleThreaded: Bool, + Name: *const c_char, + ) -> &'a Value; /// Writes a module to the specified path. Returns 0 on success. pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; - /// Creates a pass manager. + /// Creates a legacy pass manager -- only used for final codegen. pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; - /// Creates a function-by-function pass manager - pub fn LLVMCreateFunctionPassManagerForModule(M: &Module) -> &mut PassManager<'_>; - - /// Disposes a pass manager. - pub fn LLVMDisposePassManager<'a>(PM: &'a mut PassManager<'a>); - - /// Runs a pass manager on a module. - pub fn LLVMRunPassManager<'a>(PM: &PassManager<'a>, M: &'a Module) -> Bool; - pub fn LLVMInitializePasses(); pub fn LLVMTimeTraceProfilerInitialize(); @@ -1832,32 +1805,6 @@ extern "C" { pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; - pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); - pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( - PMB: &PassManagerBuilder, - threshold: c_uint, - ); - pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - - pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - Internalize: Bool, - RunInliner: Bool, - ); - pub fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - pub fn LLVMGetHostCPUFeatures() -> *mut c_char; pub fn LLVMDisposeMessage(message: *mut c_char); @@ -2262,22 +2209,6 @@ extern "C" { pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; - pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustCreateMemorySanitizerPass( - TrackOrigins: c_int, - Recover: bool, - ) -> &'static mut Pass; - pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; - pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); - pub fn LLVMRustAddLastExtensionPasses( - PMB: &PassManagerBuilder, - Passes: *const &'static mut Pass, - NumPasses: size_t, - ); - pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); @@ -2311,29 +2242,11 @@ extern "C" { SplitDwarfFile: *const c_char, ) -> Option<&'static mut TargetMachine>; pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); - pub fn LLVMRustAddBuilderLibraryInfo<'a>( - PMB: &'a PassManagerBuilder, - M: &'a Module, - DisableSimplifyLibCalls: bool, - ); - pub fn LLVMRustConfigurePassManagerBuilder( - PMB: &PassManagerBuilder, - OptLevel: CodeGenOptLevel, - MergeFunctions: bool, - SLPVectorize: bool, - LoopVectorize: bool, - PrepareForThinLTO: bool, - PGOGenPath: *const c_char, - PGOUsePath: *const c_char, - PGOSampleUsePath: *const c_char, - SizeLevel: c_int, - ); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, DisableSimplifyLibCalls: bool, ); - pub fn LLVMRustRunFunctionPassManager<'a>(PM: &PassManager<'a>, M: &'a Module); pub fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, PM: &PassManager<'a>, @@ -2342,7 +2255,7 @@ extern "C" { DwoOutput: *const c_char, FileType: FileType, ) -> LLVMRustResult; - pub fn LLVMRustOptimizeWithNewPassManager<'a>( + pub fn LLVMRustOptimize<'a>( M: &'a Module, TM: &'a TargetMachine, OptLevel: PassBuilderOptLevel, @@ -2380,7 +2293,6 @@ extern "C" { pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); pub fn LLVMRustPrintPasses(); pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); - pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; @@ -2389,7 +2301,6 @@ extern "C" { AIR: &ArchiveIterator<'a>, ) -> Option<&'a mut ArchiveChild<'a>>; pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index f5d676c44e3..2fd58567c48 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,7 +1,6 @@ use crate::back::write::create_informational_target_machine; -use crate::{llvm, llvm_util}; +use crate::llvm; use libc::c_int; -use libloading::Library; use rustc_codegen_ssa::target_features::{ supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, }; @@ -15,9 +14,7 @@ use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; -use tracing::debug; -use std::mem; use std::path::Path; use std::ptr; use std::slice; @@ -121,22 +118,6 @@ unsafe fn configure_llvm(sess: &Session) { llvm::LLVMInitializePasses(); - // Use the legacy plugin registration if we don't use the new pass manager - if !should_use_new_llvm_pass_manager( - &sess.opts.unstable_opts.new_llvm_pass_manager, - &sess.target.arch, - ) { - // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.unstable_opts.llvm_plugins { - let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); - debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); - - // Intentionally leak the dynamic library. We can't ever unload it - // since the library can make things that will live arbitrarily long. - mem::forget(lib); - } - } - rustc_llvm::initialize_available_targets(); llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); @@ -155,6 +136,10 @@ pub fn time_trace_profiler_finish(file_name: &Path) { // // To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def // where the * matches the architecture's name +// +// For targets not present in the above location, see llvm-project/llvm/lib/Target/{ARCH}/*.td +// where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`. +// // Beware to not use the llvm github project for this, but check the git submodule // found in src/llvm-project // Though note that Rust can also be build with an external precompiled version of LLVM @@ -536,19 +521,3 @@ pub fn tune_cpu(sess: &Session) -> Option<&str> { let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } - -pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_arch: &str) -> bool { - // The new pass manager is enabled by default for LLVM >= 13. - // This matches Clang, which also enables it since Clang 13. - - // Since LLVM 15, the legacy pass manager is no longer supported. - if llvm_util::get_version() >= (15, 0, 0) { - return true; - } - - // There are some perf issues with the new pass manager when targeting - // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. - // See https://github.com/rust-lang/rust/issues/89609. - let min_version = if target_arch == "s390x" { 14 } else { 13 }; - user_opt.unwrap_or_else(|| llvm_util::get_version() >= (min_version, 0, 0)) -} diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 6e94284852f..1eceb7f5c87 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -11,7 +11,6 @@ use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitable}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; -use tracing::debug; impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 9f0e6c80b19..dc1165835e7 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -11,7 +11,6 @@ use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; use smallvec::{smallvec, SmallVec}; -use tracing::debug; use std::fmt::Write; |