diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
23 files changed, 567 insertions, 405 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b9baa87bac7..d4437bd449d 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -37,7 +37,7 @@ fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { // LLVM prior to version 12 had known miscompiles in the presence of // noalias attributes (see #54878), but we don't support earlier // versions at all anymore. We now enable mutable noalias by default. - cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) + cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) } const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = @@ -393,6 +393,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_cconv(&self) -> llvm::CallConv { match self.conv { Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index f935acb1a7e..72961ae888e 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -145,7 +145,7 @@ pub(crate) unsafe fn codegen( if tcx.sess.target.default_hidden_visibility { llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); } - let val = tcx.sess.opts.debugging_opts.oom.should_panic(); + let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index e994001f96f..a53946995ee 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -604,7 +604,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::X86( X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg - | X86InlineAsmRegClass::kreg0, + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, ) => unreachable!("clobber-only"), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", @@ -692,7 +693,8 @@ fn modifier_to_llvm( InlineAsmRegClass::X86( X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg - | X86InlineAsmRegClass::kreg0, + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, ) => { unreachable!("clobber-only") } @@ -766,7 +768,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::X86( X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg - | X86InlineAsmRegClass::kreg0, + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, ) => { unreachable!("clobber-only") } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c098ce36f02..62da99ac3fb 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -6,6 +6,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::OptLevel; +use rustc_span::symbol::sym; use rustc_target::spec::abi::Abi; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; @@ -55,7 +56,7 @@ pub fn sanitize_attrs<'ll>( no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.debugging_opts.sanitizer - no_sanitize; + let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -135,7 +136,7 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { if cx .sess() .opts - .debugging_opts + .unstable_opts .sanitizer .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) { @@ -148,7 +149,7 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with gcov profiling. - if cx.sess().opts.debugging_opts.profile { + if cx.sess().opts.unstable_opts.profile { return None; } @@ -274,7 +275,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( to_add.push(uwtable_attr(cx.llcx)); } - if cx.sess().opts.debugging_opts.profile_sample_use.is_some() { + if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } @@ -298,6 +299,12 @@ pub fn from_fn_attrs<'ll, 'tcx>( } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); + // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. + // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. + // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 + to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); + // Need this for AArch64. + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { // apply to return place instead of function (unlike all other attributes applied in this function) @@ -329,9 +336,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( ) { let span = cx .tcx - .get_attrs(instance.def_id()) - .iter() - .find(|a| a.has_name(rustc_span::sym::target_feature)) + .get_attr(instance.def_id(), sym::target_feature) .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); let msg = format!( "the target features {} must all be either enabled or disabled together", diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index f814cc93043..bccc2a995a3 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -15,19 +15,12 @@ use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_session::cstore::{DllCallingConvention, DllImport}; use rustc_session::Session; -struct ArchiveConfig<'a> { - pub sess: &'a Session, - pub dst: PathBuf, - pub src: Option<PathBuf>, -} - /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] pub struct LlvmArchiveBuilder<'a> { - config: ArchiveConfig<'a>, - removals: Vec<String>, + sess: &'a Session, + dst: PathBuf, additions: Vec<Addition>, - src_archive: Option<Option<ArchiveRO>>, } enum Addition { @@ -50,10 +43,6 @@ fn is_relevant_child(c: &Child<'_>) -> bool { } } -fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) -> ArchiveConfig<'a> { - ArchiveConfig { sess, dst: output.to_path_buf(), src: input.map(|p| p.to_path_buf()) } -} - /// Map machine type strings to values of LLVM's MachineTypes enum. fn llvm_machine_type(cpu: &str) -> LLVMMachineType { match cpu { @@ -68,37 +57,8 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Creates a new static archive, ready for modifying the archive specified /// by `config`. - fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> LlvmArchiveBuilder<'a> { - let config = archive_config(sess, output, input); - LlvmArchiveBuilder { - config, - removals: Vec::new(), - additions: Vec::new(), - src_archive: None, - } - } - - /// Removes a file from this archive - fn remove_file(&mut self, file: &str) { - self.removals.push(file.to_string()); - } - - /// Lists all files in an archive - fn src_files(&mut self) -> Vec<String> { - if self.src_archive().is_none() { - return Vec::new(); - } - - let archive = self.src_archive.as_ref().unwrap().as_ref().unwrap(); - - archive - .iter() - .filter_map(|child| child.ok()) - .filter(is_relevant_child) - .filter_map(|child| child.name()) - .filter(|name| !self.removals.iter().any(|x| x == name)) - .map(|name| name.to_owned()) - .collect() + fn new(sess: &'a Session, output: &Path) -> LlvmArchiveBuilder<'a> { + LlvmArchiveBuilder { sess, dst: output.to_path_buf(), additions: Vec::new() } } fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> @@ -129,13 +89,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. - fn build(mut self) { - let kind = self.llvm_archive_kind().unwrap_or_else(|kind| { - self.config.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) - }); - - if let Err(e) = self.build_with_llvm(kind) { - self.config.sess.fatal(&format!("failed to build archive: {}", e)); + fn build(mut self) -> bool { + match self.build_with_llvm() { + Ok(any_members) => any_members, + Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), } } @@ -151,14 +108,16 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path.with_extension("lib") }; - let target = &self.config.sess.target; - let mingw_gnu_toolchain = - target.vendor == "pc" && target.os == "windows" && target.env == "gnu"; + let target = &self.sess.target; + let mingw_gnu_toolchain = target.vendor == "pc" + && target.os == "windows" + && target.env == "gnu" + && target.abi.is_empty(); let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { - if self.config.sess.target.arch == "x86" { + if self.sess.target.arch == "x86" { ( LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), import.ordinal, @@ -195,11 +154,11 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - self.config.sess.fatal(&format!("Error writing .DEF file: {}", e)); + self.sess.fatal(&format!("Error writing .DEF file: {}", e)); } }; - let dlltool = find_binutils_dlltool(self.config.sess); + let dlltool = find_binutils_dlltool(self.sess); let result = std::process::Command::new(dlltool) .args([ "-d", @@ -213,9 +172,9 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match result { Err(e) => { - self.config.sess.fatal(&format!("Error calling dlltool: {}", e)); + self.sess.fatal(&format!("Error calling dlltool: {}", e)); } - Ok(output) if !output.status.success() => self.config.sess.fatal(&format!( + Ok(output) if !output.status.success() => self.sess.fatal(&format!( "Dlltool could not create import library: {}\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) @@ -261,13 +220,13 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path_z.as_ptr(), ffi_exports.as_ptr(), ffi_exports.len(), - llvm_machine_type(&self.config.sess.target.arch) as u16, - !self.config.sess.target.is_like_msvc, + llvm_machine_type(&self.sess.target.arch) as u16, + !self.sess.target.is_like_msvc, ) }; if result == crate::llvm::LLVMRustResult::Failure { - self.config.sess.fatal(&format!( + self.sess.fatal(&format!( "Error creating import library for {}: {}", lib_name, llvm::last_error().unwrap_or("unknown LLVM error".to_string()) @@ -276,7 +235,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { }; self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { - self.config.sess.fatal(&format!( + self.sess.fatal(&format!( "failed to add native library {}: {}", output_path.display(), e @@ -286,46 +245,19 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { } impl<'a> LlvmArchiveBuilder<'a> { - fn src_archive(&mut self) -> Option<&ArchiveRO> { - if let Some(ref a) = self.src_archive { - return a.as_ref(); - } - let src = self.config.src.as_ref()?; - self.src_archive = Some(ArchiveRO::open(src).ok()); - self.src_archive.as_ref().unwrap().as_ref() - } - - fn llvm_archive_kind(&self) -> Result<ArchiveKind, &str> { - let kind = &*self.config.sess.target.archive_format; - kind.parse().map_err(|_| kind) - } + fn build_with_llvm(&mut self) -> io::Result<bool> { + let kind = &*self.sess.target.archive_format; + let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| { + self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) + }); - fn build_with_llvm(&mut self, kind: ArchiveKind) -> io::Result<()> { - let removals = mem::take(&mut self.removals); let mut additions = mem::take(&mut self.additions); let mut strings = Vec::new(); let mut members = Vec::new(); - let dst = CString::new(self.config.dst.to_str().unwrap())?; + let dst = CString::new(self.dst.to_str().unwrap())?; unsafe { - if let Some(archive) = self.src_archive() { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - let Some(child_name) = child.name() else { continue }; - if removals.iter().any(|r| r == child_name) { - continue; - } - - let name = CString::new(child_name)?; - members.push(llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - )); - strings.push(name); - } - } for addition in &mut additions { match addition { Addition::File { path, name_in_archive } => { @@ -387,7 +319,7 @@ impl<'a> LlvmArchiveBuilder<'a> { }; Err(io::Error::new(io::ErrorKind::Other, msg)) } else { - Ok(()) + Ok(!members.is_empty()) }; for member in members { llvm::LLVMRustArchiveMemberFree(member); @@ -419,7 +351,7 @@ fn string_to_io_error(s: String) -> io::Error { fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.debugging_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { return dlltool_path.clone().into_os_string(); } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index d0724baf9e0..be539499b56 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -325,20 +325,6 @@ fn fat_lto( drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); - // Fat LTO also suffers from the invalid DWARF issue similar to Thin LTO. - // Here we rewrite all `DICompileUnit` pointers if there is only one `DICompileUnit`. - // This only works around the problem when codegen-units = 1. - // Refer to the comments in the `optimize_thin_module` function for more details. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - unsafe { llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2) }; - if !cu2.is_null() { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_fat_lto_patch_debuginfo", &*module.name); - unsafe { llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1) }; - save_temp_bitcode(cgcx, &module, "fat-lto-after-patch"); - } - // Internalize everything below threshold to help strip out more modules and such. unsafe { let ptr = symbols_below_threshold.as_ptr(); @@ -354,14 +340,14 @@ fn fat_lto( Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) } -crate struct Linker<'a>(&'a mut llvm::Linker<'a>); +pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); impl<'a> Linker<'a> { - crate fn new(llmod: &'a llvm::Module) -> Self { + pub(crate) fn new(llmod: &'a llvm::Module) -> Self { unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } } - crate fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { + pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { unsafe { if llvm::LLVMRustLinkerAdd( self.0, @@ -586,9 +572,21 @@ pub(crate) fn run_pass_manager( // LTO-specific optimization passes that LLVM provides. // // This code is based off the code found in llvm's LTO code generator: - // tools/lto/LTOCodeGenerator.cpp + // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { + if !llvm::LLVMRustHasModuleFlag( + module.module_llvm.llmod(), + "LTOPostLink".as_ptr().cast(), + 11, + ) { + llvm::LLVMRustAddModuleFlag( + module.module_llvm.llmod(), + llvm::LLVMModFlagBehavior::Error, + "LTOPostLink\0".as_ptr().cast(), + 1, + ); + } if llvm_util::should_use_new_llvm_pass_manager( &config.new_llvm_pass_manager, &cgcx.target_arch, @@ -757,7 +755,7 @@ pub unsafe fn optimize_thin_module( // an error. let mut cu1 = ptr::null_mut(); let mut cu2 = ptr::null_mut(); - llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); + llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); if !cu2.is_null() { let msg = "multiple source DICompileUnits found"; return Err(write::llvm_err(&diag_handler, msg)); @@ -846,7 +844,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1); + llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 99e30531c22..2b465ce40e7 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -56,28 +56,24 @@ pub fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { + debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { let output_c = path_to_c_string(output); - let result = if let Some(dwo_output) = dwo_output { - let dwo_output_c = path_to_c_string(dwo_output); - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - dwo_output_c.as_ptr(), - file_type, - ) + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() } else { - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - std::ptr::null(), - file_type, - ) + std::ptr::null() }; + let result = llvm::LLVMRustWriteOutputFile( + target, + pm, + m, + output_c.as_ptr(), + dwo_output_ptr, + file_type, + ); // Record artifact sizes for self-profiling if result == llvm::LLVMRustResult::Success { @@ -111,7 +107,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { tcx.output_filenames(()).split_dwarf_path( tcx.sess.split_debuginfo(), - tcx.sess.opts.debugging_opts.split_dwarf_kind, + tcx.sess.opts.unstable_opts.split_dwarf_kind, Some(mod_name), ) } else { @@ -186,9 +182,9 @@ pub fn target_machine_factory( let use_softfp = sess.opts.cg.soft_float; let ffunction_sections = - sess.opts.debugging_opts.function_sections.unwrap_or(sess.target.function_sections); + sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); let fdata_sections = ffunction_sections; - let funique_section_names = !sess.opts.debugging_opts.no_unique_section_names; + let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names; let code_model = to_llvm_code_model(sess.code_model()); @@ -206,15 +202,15 @@ pub fn target_machine_factory( let features = CString::new(target_features.join(",")).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); let trap_unreachable = - sess.opts.debugging_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); - let emit_stack_size_section = sess.opts.debugging_opts.emit_stack_sizes; + sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); + let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; let asm_comments = sess.asm_comments(); let relax_elf_relocations = - sess.opts.debugging_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); + sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); let use_init_array = - !sess.opts.debugging_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); + !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); let path_mapping = sess.source_map().path_mapping().clone(); @@ -340,7 +336,7 @@ fn report_inline_asm( } let level = match level { llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning, + llvm::DiagnosticLevel::Warning => Level::Warning(None), llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); @@ -467,7 +463,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); - let extra_passes = config.passes.join(","); + let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; let llvm_plugins = config.llvm_plugins.join(","); @@ -1040,7 +1036,8 @@ unsafe fn embed_bitcode( // reason (see issue #90326 for historical background). let is_apple = cgcx.opts.target_triple.triple().contains("-ios") || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos"); + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos"); if is_apple || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 88b87951ecd..4a4cccb490d 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -28,7 +28,7 @@ use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; -use tracing::debug; +use tracing::{debug, instrument}; // All Builders must have an llfn associated with them #[must_use] @@ -464,15 +464,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + #[instrument(level = "trace", skip(self))] fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { - debug!("PlaceRef::load: {:?}", place); - assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); if place.layout.is_zst() { return OperandRef::new_zst(self, place.layout); } + #[instrument(level = "trace", skip(bx))] fn scalar_load_metadata<'a, 'll, 'tcx>( bx: &mut Builder<'a, 'll, 'tcx>, load: &'ll Value, @@ -509,15 +509,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Ref(place.llval, Some(llextra), place.align) } else if place.layout.is_llvm_immediate() { let mut const_llval = None; + let llty = place.layout.llvm_type(self); unsafe { if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { if llvm::LLVMIsGlobalConstant(global) == llvm::True { - const_llval = llvm::LLVMGetInitializer(global); + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); + } + } } } } let llval = const_llval.unwrap_or_else(|| { - let load = self.load(place.layout.llvm_type(self), place.llval, place.align); + let load = self.load(llty, place.llval, place.align); if let abi::Abi::Scalar(scalar) = place.layout.abi { scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); } @@ -1059,11 +1064,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst: &'ll Value, cmp: &'ll Value, src: &'ll Value, - order: rustc_codegen_ssa::common::AtomicOrdering, + mut order: rustc_codegen_ssa::common::AtomicOrdering, failure_order: rustc_codegen_ssa::common::AtomicOrdering, weak: bool, ) -> &'ll Value { let weak = if weak { llvm::True } else { llvm::False }; + if llvm_util::get_version() < (13, 0, 0) { + use rustc_codegen_ssa::common::AtomicOrdering::*; + // Older llvm has the pre-C++17 restriction on + // success and failure memory ordering, + // requiring the former to be at least as strong as the latter. + // So, for llvm 12, we upgrade the success ordering to a stronger + // one if necessary. + match (order, failure_order) { + (Relaxed, Acquire) => order = Acquire, + (Release, Acquire) => order = AcquireRelease, + (_, SequentiallyConsistent) => order = SequentiallyConsistent, + _ => {} + } + } unsafe { llvm::LLVMRustBuildAtomicCmpXchg( self.llbuilder, @@ -1412,7 +1431,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } } - crate fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); self.call(ty, f, args, None) } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index ac423a22703..72155d874a2 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::*; use tracing::debug; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::{self, Instance, TypeFoldable}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index b69d7a000ee..77cbbf4c6ca 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -13,8 +13,6 @@ use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::ScalarInt; -use rustc_span::symbol::Symbol; use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; use libc::{c_char, c_uint}; @@ -109,8 +107,7 @@ impl<'ll> CodegenCx<'ll, '_> { pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { assert_eq!(idx as c_uint as u64, idx); - let us = &[idx as c_uint]; - let r = llvm::LLVMConstExtractValue(v, us.as_ptr(), us.len() as c_uint); + let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); @@ -181,22 +178,27 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMConstReal(t, val) } } - fn const_str(&self, s: Symbol) -> (&'ll Value, &'ll Value) { - let s_str = s.as_str(); - let str_global = *self.const_str_cache.borrow_mut().entry(s).or_insert_with(|| { - let sc = self.const_bytes(s_str.as_bytes()); - let sym = self.generate_local_symbol_name("str"); - let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", sym); - }); - unsafe { - llvm::LLVMSetInitializer(g, sc); - llvm::LLVMSetGlobalConstant(g, True); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); - } - g - }); - let len = s_str.len(); + fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { + let str_global = *self + .const_str_cache + .borrow_mut() + .raw_entry_mut() + .from_key(s) + .or_insert_with(|| { + let sc = self.const_bytes(s.as_bytes()); + let sym = self.generate_local_symbol_name("str"); + let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + unsafe { + llvm::LLVMSetInitializer(g, sc); + llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); + } + (s.to_owned(), g) + }) + .1; + let len = s.len(); let cs = consts::ptrcast( str_global, self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)), @@ -220,13 +222,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } + fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { + self.const_undef(self.type_ix(0)) + } + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { - Scalar::Int(ScalarInt::ZST) => { - assert_eq!(0, layout.size(self).bytes()); - self.const_undef(self.type_ix(0)) - } Scalar::Int(int) => { let data = int.assert_bits(layout.size(self)); let llval = self.const_uint_big(self.type_ix(bitsize), data); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index ea5df912438..9d40f9c5d86 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -64,7 +64,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< // See https://github.com/rust-lang/rust/issues/84565. 1 } else { - cx.sess().opts.debugging_opts.uninit_const_chunk_threshold + cx.sess().opts.unstable_opts.uninit_const_chunk_threshold }; let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; @@ -212,11 +212,11 @@ pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { } impl<'ll> CodegenCx<'ll, '_> { - crate fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMConstBitCast(val, ty) } } - crate fn static_addr_of_mut( + pub(crate) fn static_addr_of_mut( &self, cv: &'ll Value, align: Align, @@ -241,7 +241,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn get_static(&self, def_id: DefId) -> &'ll Value { + pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); if let Some(&g) = self.instances.borrow().get(&instance) { return g; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d296ee3b42c..5857b83f6c9 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -26,7 +26,6 @@ use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; -use rustc_span::symbol::Symbol; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; @@ -56,7 +55,7 @@ pub struct CodegenCx<'ll, 'tcx> { pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>, /// Cache of constant strings, - pub const_str_cache: RefCell<FxHashMap<Symbol, &'ll Value>>, + pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>, /// Reverse-direction for const ptrs cast from globals. /// @@ -276,7 +275,7 @@ pub unsafe fn create_module<'ll>( } } - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.debugging_opts.branch_protection { + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch != "aarch64" { sess.err("-Zbranch-protection is only supported on aarch64"); } else { @@ -309,7 +308,7 @@ pub unsafe fn create_module<'ll>( } // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). - if let CFProtection::Branch | CFProtection::Full = sess.opts.debugging_opts.cf_protection { + if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, @@ -317,7 +316,7 @@ pub unsafe fn create_module<'ll>( 1, ) } - if let CFProtection::Return | CFProtection::Full = sess.opts.debugging_opts.cf_protection { + if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, @@ -326,11 +325,20 @@ pub unsafe fn create_module<'ll>( ) } + if sess.opts.unstable_opts.virtual_function_elimination { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "Virtual Function Elim\0".as_ptr().cast(), + 1, + ); + } + llmod } impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { - crate fn new( + pub(crate) fn new( tcx: TyCtxt<'tcx>, codegen_unit: &'tcx CodegenUnit<'tcx>, llvm_module: &'ll crate::ModuleLlvm, @@ -447,7 +455,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } } - crate fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> { + pub(crate) fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> { &self.statics_to_rauw } @@ -599,7 +607,7 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll> CodegenCx<'ll, '_> { - crate fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { + pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { if let Some(v) = self.intrinsics.borrow().get(key).cloned() { return v; } @@ -656,6 +664,7 @@ impl<'ll> CodegenCx<'ll, '_> { let t_isize = self.type_isize(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_metadata = self.type_metadata(); ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); @@ -881,21 +890,22 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); } - ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1); + ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void); - ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void); + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } None } - crate fn eh_catch_typeinfo(&self) -> &'ll Value { + pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() { return eh_catch_typeinfo; } let tcx = self.tcx; - assert!(self.sess().target.is_like_emscripten); + assert!(self.sess().target.os == "emscripten"); let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { Some(def_id) => self.get_static(def_id), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 31a09242c5a..80fd9726fc7 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -5,11 +5,14 @@ use crate::llvm; use crate::builder::Builder; use crate::common::CodegenCx; use crate::value::Value; +use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_session::config::DebugInfo; +use rustc_session::config::{CrateType, DebugInfo}; use rustc_span::symbol::sym; +use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -37,9 +40,33 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' section_var.unwrap_or_else(|| { let section_name = b".debug_gdb_scripts\0"; - let section_contents = b"\x01gdb_load_rust_pretty_printers.py\0"; + let mut section_contents = Vec::new(); + + // Add the pretty printers for the standard library first. + section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0"); + + // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute. + let visualizers = collect_debugger_visualizers_transitive( + cx.tcx, + DebuggerVisualizerType::GdbPrettyPrinter, + ); + let crate_name = cx.tcx.crate_name(LOCAL_CRATE); + for (index, visualizer) in visualizers.iter().enumerate() { + // The initial byte `4` instructs GDB that the following pretty printer + // is defined inline as opposed to in a standalone file. + section_contents.extend_from_slice(b"\x04"); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); + section_contents.extend_from_slice(vis_name.as_bytes()); + section_contents.extend_from_slice(&visualizer.src); + + // The final byte `0` tells GDB that the pretty printer has been + // fully defined and can continue searching for additional + // pretty printers. + section_contents.extend_from_slice(b"\0"); + } unsafe { + let section_contents = section_contents.as_slice(); let llvm_type = cx.type_array(cx.type_i8(), section_contents.len() as u64); let section_var = cx @@ -62,7 +89,32 @@ pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create + // ODR violations at link time, this section will not be emitted for rlibs since + // each rlib could produce a different set of visualizers that would be embedded + // in the `.debug_gdb_scripts` section. For that reason, we make sure that the + // section is only emitted for leaf crates. + let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { + CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { + // These are crate types for which we will embed pretty printers since they + // are treated as leaf crates. + true + } + CrateType::ProcMacro => { + // We could embed pretty printers for proc macro crates too but it does not + // seem like a good default, since this is a rare use case and we don't + // want to slow down the common case. + false + } + CrateType::Rlib => { + // As per the above description, embedding pretty printers for rlibs could + // lead to ODR violations so we skip this crate type as well. + false + } + }); + !omit_gdb_pretty_printer_section && cx.sess().opts.debuginfo != DebugInfo::None && cx.sess().target.emit_debug_gdb_scripts + && embed_visualizers } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index f2cf3b1ef5c..f8bd2d234f3 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -30,26 +30,28 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{self, GeneratorLayout}; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; -use rustc_middle::ty::{self, AdtKind, Instance, ParamEnv, Ty, TyCtxt, COMMON_VTABLE_ENTRIES}; -use rustc_session::config::{self, DebugInfo}; +use rustc_middle::ty::{ + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, +}; +use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; -use rustc_span::FileNameDisplayPreference; -use rustc_span::{self, SourceFile, SourceFileHash}; +use rustc_span::FileName; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; use tracing::debug; -use libc::{c_longlong, c_uint}; +use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; -use std::collections::hash_map::Entry; use std::fmt::{self, Write}; use std::hash::{Hash, Hasher}; use std::iter; use std::path::{Path, PathBuf}; use std::ptr; +use tracing::instrument; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -527,78 +529,105 @@ fn hex_encode(data: &[u8]) -> String { } pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { - debug!("file_metadata: file_name: {:?}", source_file.name); - - let hash = Some(&source_file.src_hash); - let file_name = Some(source_file.name.prefer_remapped().to_string()); - let directory = if source_file.is_real_file() && !source_file.is_imported() { - Some( - cx.sess() - .opts - .working_dir - .to_string_lossy(FileNameDisplayPreference::Remapped) - .to_string(), - ) - } else { - // If the path comes from an upstream crate we assume it has been made - // independent of the compiler's working directory one way or another. - None - }; - file_metadata_raw(cx, file_name, directory, hash) -} - -pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { - file_metadata_raw(cx, None, None, None) -} - -fn file_metadata_raw<'ll>( - cx: &CodegenCx<'ll, '_>, - file_name: Option<String>, - directory: Option<String>, - hash: Option<&SourceFileHash>, -) -> &'ll DIFile { - let key = (file_name, directory); - - match debug_context(cx).created_files.borrow_mut().entry(key) { - Entry::Occupied(o) => o.get(), - Entry::Vacant(v) => { - let (file_name, directory) = v.key(); - debug!("file_metadata: file_name: {:?}, directory: {:?}", file_name, directory); - - let file_name = file_name.as_deref().unwrap_or("<unknown>"); - let directory = directory.as_deref().unwrap_or(""); - - let (hash_kind, hash_value) = match hash { - Some(hash) => { - let kind = match hash.kind { - rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, - rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, - rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, - }; - (kind, hex_encode(hash.hash_bytes())) + let cache_key = Some((source_file.name_hash, source_file.src_hash)); + return debug_context(cx) + .created_files + .borrow_mut() + .entry(cache_key) + .or_insert_with(|| alloc_new_file_metadata(cx, source_file)); + + #[instrument(skip(cx, source_file), level = "debug")] + fn alloc_new_file_metadata<'ll>( + cx: &CodegenCx<'ll, '_>, + source_file: &SourceFile, + ) -> &'ll DIFile { + debug!(?source_file.name); + + let (directory, file_name) = match &source_file.name { + FileName::Real(filename) => { + let working_directory = &cx.sess().opts.working_dir; + debug!(?working_directory); + + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. + // Because of path remapping we sometimes see strange things here: `abs_path` + // might actually look like a relative path + // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without + // taking the working directory into account, downstream tooling will + // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`, + // which makes no sense. Usually in such cases the working directory will also + // be remapped to `<crate-name-and-version>` or some other prefix of the path + // we are remapping, so we end up with + // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) } - None => (llvm::ChecksumKind::None, String::new()), - }; + } + other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), + }; - let file_metadata = unsafe { - llvm::LLVMRustDIBuilderCreateFile( - DIB(cx), - file_name.as_ptr().cast(), - file_name.len(), - directory.as_ptr().cast(), - directory.len(), - hash_kind, - hash_value.as_ptr().cast(), - hash_value.len(), - ) - }; + let hash_kind = match source_file.src_hash.kind { + rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, + rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, + rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, + }; + let hash_value = hex_encode(source_file.src_hash.hash_bytes()); - v.insert(file_metadata); - file_metadata + unsafe { + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + hash_kind, + hash_value.as_ptr().cast(), + hash_value.len(), + ) } } } +pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { + debug_context(cx).created_files.borrow_mut().entry(None).or_insert_with(|| unsafe { + let file_name = "<unknown>"; + let directory = ""; + let hash_value = ""; + + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + llvm::ChecksumKind::None, + hash_value.as_ptr().cast(), + hash_value.len(), + ) + }) +} + trait MsvcBasicName { fn msvc_basic_name(self) -> &'static str; } @@ -795,7 +824,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( output_filenames .split_dwarf_path( tcx.sess.split_debuginfo(), - tcx.sess.opts.debugging_opts.split_dwarf_kind, + tcx.sess.opts.unstable_opts.split_dwarf_kind, Some(codegen_unit_name), ) // We get a path relative to the working directory from split_dwarf_path @@ -852,15 +881,15 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( split_name.len(), kind, 0, - tcx.sess.opts.debugging_opts.split_dwarf_inlining, + tcx.sess.opts.unstable_opts.split_dwarf_inlining, ); - if tcx.sess.opts.debugging_opts.profile { + if tcx.sess.opts.unstable_opts.profile { let cu_desc_metadata = llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = - tcx.sess.opts.debugging_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); + tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), @@ -1337,7 +1366,7 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo is_local_to_unit, global, None, - global_align.bytes() as u32, + global_align.bits() as u32, ); } } @@ -1364,7 +1393,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( tcx.vtable_entries(trait_ref) } else { - COMMON_VTABLE_ENTRIES + TyCtxt::COMMON_VTABLE_ENTRIES }; // All function pointers are described as opaque pointers. This could be improved in the future @@ -1440,6 +1469,84 @@ fn build_vtable_type_di_node<'ll, 'tcx>( .di_node } +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); + + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { + VCallVisibility::Public + } + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | ( + Lto::ThinLocal | Lto::Thin | Lto::Fat, + Visibility::Restricted(_) | Visibility::Invisible, + false, + ) => VCallVisibility::LinkageUnit, + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { + VCallVisibility::TranslationUnit + } + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } +} + /// Creates debug information for the given vtable, which is for the /// given type. /// @@ -1450,6 +1557,12 @@ pub fn create_vtable_di_node<'ll, 'tcx>( poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.unstable_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + if cx.dbg_cx.is_none() { return; } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 87fbb737ea8..8fc8118849b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -93,8 +93,9 @@ impl<'tcx> UniqueTypeId<'tcx> { /// Right now this takes the form of a hex-encoded opaque hash value. pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { let mut hasher = StableHasher::new(); - let mut hcx = tcx.create_stable_hashing_context(); - hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)); + tcx.with_stable_hashing_context(|mut hcx| { + hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) + }); hasher.finish::<Fingerprint>().to_hex() } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 6a164557a47..cf591295b84 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -27,18 +27,18 @@ use rustc_index::vec::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable}; +use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, Span}; +use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; use rustc_target::abi::Size; use libc::c_uint; use smallvec::SmallVec; +use std::cell::OnceCell; use std::cell::RefCell; use std::iter; -use std::lazy::OnceCell; use tracing::debug; mod create_scope_map; @@ -61,7 +61,7 @@ pub struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell<FxHashMap<(Option<String>, Option<String>), &'ll DIFile>>, + created_files: RefCell<FxHashMap<Option<(u128, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, @@ -97,23 +97,26 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder); - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - if let Some(version) = sess.target.dwarf_version { + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = sess + .opts + .unstable_opts + .dwarf_version + .unwrap_or(sess.target.default_dwarf_version); llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, "Dwarf Version\0".as_ptr().cast(), - version, - ) - } - - // Indicate that we want CodeView debug information on MSVC - if sess.target.is_like_msvc { + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index cf9cf1b70aa..9f364749287 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -406,6 +406,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) } + fn type_checked_load( + &mut self, + llvtable: &'ll Value, + vtable_byte_offset: u64, + typeid: &'ll Value, + ) -> Self::Value { + let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + } + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { self.call_intrinsic("llvm.va_start", &[va_list]) } @@ -431,7 +441,7 @@ fn try_intrinsic<'ll>( bx.store(bx.const_i32(0), dest, ret_align); } else if wants_msvc_seh(bx.sess()) { codegen_msvc_try(bx, try_func, data, catch_func, dest); - } else if bx.sess().target.is_like_emscripten { + } else if bx.sess().target.os == "emscripten" { codegen_emcc_try(bx, try_func, data, catch_func, dest); } else { codegen_gnu_try(bx, try_func, data, catch_func, dest); @@ -816,6 +826,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, ) -> Result<&'ll Value, ()> { // macros for error handling: + #[allow(unused_macro_rules)] macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) @@ -1144,6 +1155,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { + #[allow(unused_macro_rules)] macro_rules! emit_error { ($msg: tt) => { emit_error!($msg, ) diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 0bead4629a6..25ce1cef944 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,12 +5,11 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(crate_visibility_modifier)] -#![feature(let_chains)] +#![feature(hash_raw_entry)] +#![cfg_attr(bootstrap, feature(let_chains))] #![feature(let_else)] #![feature(extern_types)] #![feature(once_cell)] -#![feature(nll)] #![feature(iter_intersperse)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] @@ -305,8 +304,8 @@ impl CodegenBackend for LlvmCodegenBackend { local stack variable in the ABI.) basic - Generate stack canaries in functions with: - - local variables of `[T; N]` type, where `T` is byte-sized and `N` > 8. + Generate stack canaries in functions with local variables of `[T; N]` + type, where `T` is byte-sized and `N` >= 8. none Do not generate stack canaries. @@ -325,8 +324,8 @@ impl CodegenBackend for LlvmCodegenBackend { llvm_util::print_version(); } - fn target_features(&self, sess: &Session) -> Vec<Symbol> { - target_features(sess) + fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec<Symbol> { + target_features(sess, allow_unstable) } fn codegen_crate<'tcx>( @@ -356,7 +355,7 @@ impl CodegenBackend for LlvmCodegenBackend { .join(sess); sess.time("llvm_dump_timing_file", || { - if sess.opts.debugging_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { let file_name = outputs.with_extension("llvm_timings.json"); llvm_util::time_trace_profiler_finish(&file_name); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 13baaddccd4..73cedb59349 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -191,6 +191,7 @@ pub enum AttributeKind { StackProtect = 32, NoUndef = 33, SanitizeMemTag = 34, + NoCfCheck = 35, } /// LLVMIntPredicate @@ -381,9 +382,8 @@ pub enum AtomicOrdering { impl AtomicOrdering { pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { match ao { - rustc_codegen_ssa::common::AtomicOrdering::NotAtomic => AtomicOrdering::NotAtomic, rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered, - rustc_codegen_ssa::common::AtomicOrdering::Monotonic => AtomicOrdering::Monotonic, + rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic, rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire, rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release, rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => { @@ -443,6 +443,7 @@ pub enum MetadataType { MD_nonnull = 11, MD_align = 17, MD_type = 19, + MD_vcall_visibility = 28, MD_noundef = 29, } @@ -775,7 +776,7 @@ pub mod coverageinfo { } impl CounterMappingRegion { - crate fn code_region( + pub(crate) fn code_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -799,7 +800,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn branch_region( + pub(crate) fn branch_region( counter: coverage_map::Counter, false_counter: coverage_map::Counter, file_id: u32, @@ -824,7 +825,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn expansion_region( + pub(crate) fn expansion_region( file_id: u32, expanded_file_id: u32, start_line: u32, @@ -848,7 +849,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn skipped_region( + pub(crate) fn skipped_region( file_id: u32, start_line: u32, start_col: u32, @@ -871,7 +872,7 @@ pub mod coverageinfo { // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] - crate fn gap_region( + pub(crate) fn gap_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -1068,6 +1069,7 @@ extern "C" { pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1081,6 +1083,11 @@ extern "C" { Vals: *const &'a Value, Count: c_uint, ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); // Operations on scalar constants @@ -1128,11 +1135,7 @@ extern "C" { pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; - pub fn LLVMConstExtractValue( - AggConstant: &Value, - IdxList: *const c_uint, - NumIdx: c_uint, - ) -> &Value; + pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1617,7 +1620,7 @@ extern "C" { B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, - IsSized: bool, + IsSigned: bool, ) -> &'a Value; // Comparisons @@ -1937,6 +1940,7 @@ extern "C" { name: *const c_char, value: u32, ); + pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; @@ -2505,8 +2509,12 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustLTOGetDICompileUnit(M: &Module, CU1: &mut *mut c_void, CU2: &mut *mut c_void); - pub fn LLVMRustLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + pub fn LLVMRustThinLTOGetDICompileUnit( + M: &Module, + CU1: &mut *mut c_void, + CU2: &mut *mut c_void, + ); + pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 7b407c94e7b..a0a640473eb 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -87,7 +87,7 @@ unsafe fn configure_llvm(sess: &Session) { add("-debug-pass=Structure", false); } if sess.target.generate_arange_section - && !sess.opts.debugging_opts.no_generate_arange_section + && !sess.opts.unstable_opts.no_generate_arange_section { add("-generate-arange-section", false); } @@ -102,7 +102,7 @@ unsafe fn configure_llvm(sess: &Session) { add("-enable-machine-outliner=never", false); } - match sess.opts.debugging_opts.merge_functions.unwrap_or(sess.target.merge_functions) { + match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { MergeFunctions::Disabled | MergeFunctions::Trampolines => {} MergeFunctions::Aliases => { add("-mergefunc-use-aliases", false); @@ -125,7 +125,7 @@ unsafe fn configure_llvm(sess: &Session) { } } - if sess.opts.debugging_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { llvm::LLVMTimeTraceProfilerInitialize(); } @@ -133,11 +133,11 @@ unsafe fn configure_llvm(sess: &Session) { // Use the legacy plugin registration if we don't use the new pass manager if !should_use_new_llvm_pass_manager( - &sess.opts.debugging_opts.new_llvm_pass_manager, + &sess.opts.unstable_opts.new_llvm_pass_manager, &sess.target.arch, ) { // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.debugging_opts.llvm_plugins { + for plugin in &sess.opts.unstable_opts.llvm_plugins { let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); @@ -218,39 +218,44 @@ pub fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, ) -> Option<&'static [&'static str]> { - for tied in tied_target_features(sess) { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); + if !features.is_empty() { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } } } - None + return None; } // Used to generate cfg variables and apply features // Must express features in the way Rust understands them -pub fn target_features(sess: &Session) -> Vec<Symbol> { +pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); - let mut features: Vec<Symbol> = - supported_target_features(sess) - .iter() - .filter_map(|&(feature, gate)| { - if sess.is_nightly_build() || gate.is_none() { Some(feature) } else { None } - }) - .filter(|feature| { - // check that all features in a given smallvec are enabled - for llvm_feature in to_llvm_features(sess, feature) { - let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { - return false; - } + let mut features: Vec<Symbol> = supported_target_features(sess) + .iter() + .filter_map(|&(feature, gate)| { + if sess.is_nightly_build() || allow_unstable || gate.is_none() { + Some(feature) + } else { + None + } + }) + .filter(|feature| { + // check that all features in a given smallvec are enabled + for llvm_feature in to_llvm_features(sess, feature) { + let cstr = SmallCStr::new(llvm_feature); + if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return false; } - true - }) - .map(|feature| Symbol::intern(feature)) - .collect(); + } + true + }) + .map(|feature| Symbol::intern(feature)) + .collect(); // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use @@ -440,6 +445,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // -Ctarget-features let supported_features = supported_target_features(sess); + let mut featsmap = FxHashMap::default(); let feats = sess .opts .cg @@ -485,35 +491,36 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } diag.emit(); } - Some((enable_disable, feature)) + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } + + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some( + to_llvm_features(sess, feature) + .into_iter() + .map(move |f| format!("{}{}", enable_disable, f)), + ) }) - .collect::<SmallVec<[(char, &str); 8]>>(); - - if diagnostics { - // FIXME(nagisa): figure out how to not allocate a full hashset here. - let featmap = feats.iter().map(|&(flag, feat)| (feat, flag == '+')).collect(); - if let Some(f) = check_tied_features(sess, &featmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); - } + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); } - features.extend(feats.into_iter().flat_map(|(enable_disable, feature)| { - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return SmallVec::<[_; 2]>::new(); - } - // ... otherwise though we run through `to_llvm_features` when - // passing requests down to LLVM. This means that all in-language - // features also work on the command line instead of having two - // different names when the LLVM name and the Rust name differ. - to_llvm_features(sess, feature) - .into_iter() - .map(|f| format!("{}{}", enable_disable, f)) - .collect() - })); features } @@ -534,7 +541,7 @@ fn backend_feature_name(s: &str) -> Option<&str> { } pub fn tune_cpu(sess: &Session) -> Option<&str> { - let name = sess.opts.debugging_opts.tune_cpu.as_ref()?; + let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index a3053742aad..6e94284852f 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -8,7 +8,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; -use rustc_middle::ty::{self, Instance, TypeFoldable}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; use tracing::debug; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 21b77f7dea6..cf2d3c423c3 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -39,33 +39,33 @@ impl fmt::Debug for Type { } impl<'ll> CodegenCx<'ll, '_> { - crate fn type_named_struct(&self, name: &str) -> &'ll Type { + pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { let name = SmallCStr::new(name); unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } } - crate fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { + pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } } - crate fn type_void(&self) -> &'ll Type { + pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } } - crate fn type_metadata(&self) -> &'ll Type { + pub(crate) fn type_metadata(&self) -> &'ll Type { unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 - crate fn type_ix(&self, num_bits: u64) -> &'ll Type { + pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } } - crate fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { unsafe { llvm::LLVMVectorType(ty, len as c_uint) } } - crate fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { unsafe { let n_args = llvm::LLVMCountParamTypes(ty) as usize; let mut args = Vec::with_capacity(n_args); @@ -75,11 +75,11 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_bool(&self) -> &'ll Type { + pub(crate) fn type_bool(&self) -> &'ll Type { self.type_i8() } - crate fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { + pub(crate) fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { match t { ty::IntTy::Isize => self.type_isize(), ty::IntTy::I8 => self.type_i8(), @@ -90,7 +90,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { + pub(crate) fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { match t { ty::UintTy::Usize => self.type_isize(), ty::UintTy::U8 => self.type_i8(), @@ -101,14 +101,14 @@ impl<'ll> CodegenCx<'ll, '_> { } } - crate fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { + pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { match t { ty::FloatTy::F32 => self.type_f32(), ty::FloatTy::F64 => self.type_f64(), } } - crate fn type_pointee_for_align(&self, align: Align) -> &'ll Type { + pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { // FIXME(eddyb) We could find a better approximation if ity.align < align. let ity = Integer::approximate_align(self, align); self.type_from_integer(ity) @@ -116,7 +116,7 @@ impl<'ll> CodegenCx<'ll, '_> { /// Return a LLVM type that has at most the required alignment, /// and exactly the required size, as a best-effort padding array. - crate fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { + pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { let unit = Integer::approximate_align(self, align); let size = size.bytes(); let unit_size = unit.size().bytes(); @@ -124,11 +124,11 @@ impl<'ll> CodegenCx<'ll, '_> { self.type_array(self.type_from_integer(unit), size / unit_size) } - crate fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } } - crate fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { unsafe { llvm::LLVMRustArrayType(ty, len) } } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 86280523631..9f0e6c80b19 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -6,7 +6,7 @@ use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Ty, TypeFoldable}; +use rustc_middle::ty::{self, Ty, TypeVisitable}; use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; |