diff options
Diffstat (limited to 'compiler/rustc_codegen_ssa')
30 files changed, 1153 insertions, 783 deletions
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 83dd6256110..8bbf25ce030 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -9,16 +9,19 @@ test = false [dependencies] bitflags = "1.2.1" cc = "1.0.69" -itertools = "0.9" +itertools = "0.10" tracing = "0.1" libc = "0.2.50" jobserver = "0.1.22" tempfile = "3.2" +thorin-dwp = "0.2" pathdiff = "0.2.0" +snap = "1" smallvec = { version = "1.6.1", features = ["union", "may_dangle"] } regex = "1.4" rustc_serialize = { path = "../rustc_serialize" } +rustc_arena = { path = "../rustc_arena" } rustc_ast = { path = "../rustc_ast" } rustc_span = { path = "../rustc_span" } rustc_middle = { path = "../rustc_middle" } @@ -38,6 +41,6 @@ rustc_target = { path = "../rustc_target" } rustc_session = { path = "../rustc_session" } [dependencies.object] -version = "0.26.2" +version = "0.28.0" default-features = false features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"] diff --git a/compiler/rustc_codegen_ssa/src/back/command.rs b/compiler/rustc_codegen_ssa/src/back/command.rs index 503c51d24b6..17071ba1b5b 100644 --- a/compiler/rustc_codegen_ssa/src/back/command.rs +++ b/compiler/rustc_codegen_ssa/src/back/command.rs @@ -48,7 +48,7 @@ impl Command { } pub fn sym_arg(&mut self, arg: Symbol) -> &mut Command { - self.arg(&*arg.as_str()); + self.arg(arg.as_str()); self } diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index be50911f4e1..7a13e424f9a 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -1,11 +1,13 @@ +use rustc_arena::TypedArena; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; +use rustc_data_structures::memmap::Mmap; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{ErrorReported, Handler}; use rustc_fs_util::fix_windows_verbatim_for_gcc; use rustc_hir::def_id::CrateNum; use rustc_middle::middle::dependency_format::Linkage; use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; -use rustc_session::config::{OutputFilenames, OutputType, PrintRequest}; +use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SplitDwarfKind}; use rustc_session::cstore::DllImport; use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; use rustc_session::search_paths::PathKind; @@ -14,7 +16,6 @@ use rustc_session::utils::NativeLibKind; /// need out of the shared crate context before we get rid of it. use rustc_session::{filesearch, Session}; use rustc_span::symbol::Symbol; -use rustc_target::abi::Endian; use rustc_target::spec::crt_objects::{CrtObjects, CrtObjectsFallback}; use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo}; use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target}; @@ -22,6 +23,7 @@ use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Ta use super::archive::{find_library, ArchiveBuilder}; use super::command::Command; use super::linker::{self, Linker}; +use super::metadata::create_rmeta_file; use super::rpath::{self, RPathConfig}; use crate::{ looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib, @@ -29,13 +31,13 @@ use crate::{ }; use cc::windows_registry; -use object::elf; -use object::write::Object; -use object::{Architecture, BinaryFormat, Endianness, FileFlags, SectionFlags, SectionKind}; use regex::Regex; use tempfile::Builder as TempFileBuilder; +use std::borrow::Borrow; use std::ffi::OsString; +use std::fs::{File, OpenOptions}; +use std::io::{BufWriter, Write}; use std::lazy::OnceCell; use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Output, Stdio}; @@ -91,7 +93,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>( sess, crate_type, outputs, - &codegen_results.crate_info.local_crate_name.as_str(), + codegen_results.crate_info.local_crate_name.as_str(), ); match crate_type { CrateType::Rlib => { @@ -121,34 +123,63 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>( if sess.opts.json_artifact_notifications { sess.parse_sess.span_diagnostic.emit_artifact_notification(&out_filename, "link"); } + + if sess.prof.enabled() { + if let Some(artifact_name) = out_filename.file_name() { + // Record size for self-profiling + let file_size = std::fs::metadata(&out_filename).map(|m| m.len()).unwrap_or(0); + + sess.prof.artifact_size( + "linked_artifact", + artifact_name.to_string_lossy(), + file_size, + ); + } + } } } - // Remove the temporary object file and metadata if we aren't saving temps + // Remove the temporary object file and metadata if we aren't saving temps. sess.time("link_binary_remove_temps", || { - if !sess.opts.cg.save_temps { - let remove_temps_from_module = |module: &CompiledModule| { - if let Some(ref obj) = module.object { - ensure_removed(sess.diagnostic(), obj); - } - - if let Some(ref obj) = module.dwarf_object { - ensure_removed(sess.diagnostic(), obj); - } - }; + // If the user requests that temporaries are saved, don't delete any. + if sess.opts.cg.save_temps { + return; + } - if sess.opts.output_types.should_link() && !preserve_objects_for_their_debuginfo(sess) { - for module in &codegen_results.modules { - remove_temps_from_module(module); - } + let remove_temps_from_module = |module: &CompiledModule| { + if let Some(ref obj) = module.object { + ensure_removed(sess.diagnostic(), obj); } + }; + + // Otherwise, always remove the metadata and allocator module temporaries. + if let Some(ref metadata_module) = codegen_results.metadata_module { + remove_temps_from_module(metadata_module); + } + + if let Some(ref allocator_module) = codegen_results.allocator_module { + remove_temps_from_module(allocator_module); + } + + // If no requested outputs require linking, then the object temporaries should + // be kept. + if !sess.opts.output_types.should_link() { + return; + } + + // Potentially keep objects for their debuginfo. + let (preserve_objects, preserve_dwarf_objects) = preserve_objects_for_their_debuginfo(sess); + debug!(?preserve_objects, ?preserve_dwarf_objects); - if let Some(ref metadata_module) = codegen_results.metadata_module { - remove_temps_from_module(metadata_module); + for module in &codegen_results.modules { + if !preserve_objects { + remove_temps_from_module(module); } - if let Some(ref allocator_module) = codegen_results.allocator_module { - remove_temps_from_module(allocator_module); + if !preserve_dwarf_objects { + if let Some(ref obj) = module.dwarf_object { + ensure_removed(sess.diagnostic(), obj); + } } } }); @@ -235,8 +266,14 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( let mut ab = <B as ArchiveBuilder>::new(sess, out_filename, None); - for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) { - ab.add_file(obj); + for m in &codegen_results.modules { + if let Some(obj) = m.object.as_ref() { + ab.add_file(obj); + } + + if let Some(dwarf_obj) = m.dwarf_object.as_ref() { + ab.add_file(dwarf_obj); + } } // Note that in this loop we are ignoring the value of `lib.cfg`. That is, @@ -326,7 +363,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( // metadata in rlib files is wrapped in a "dummy" object file for // the target platform so the rlib can be processed entirely by // normal linkers for the platform. - let metadata = create_metadata_file(sess, codegen_results.metadata.raw_data()); + let metadata = create_rmeta_file(sess, codegen_results.metadata.raw_data()); ab.add_file(&emit_metadata(sess, &metadata, tmpdir)); // After adding all files to the archive, we need to update the @@ -345,136 +382,6 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( } } return Ok(ab); - - // For rlibs we "pack" rustc metadata into a dummy object file. When rustc - // creates a dylib crate type it will pass `--whole-archive` (or the - // platform equivalent) to include all object files from an rlib into the - // final dylib itself. This causes linkers to iterate and try to include all - // files located in an archive, so if metadata is stored in an archive then - // it needs to be of a form that the linker will be able to process. - // - // Note, though, that we don't actually want this metadata to show up in any - // final output of the compiler. Instead this is purely for rustc's own - // metadata tracking purposes. - // - // With the above in mind, each "flavor" of object format gets special - // handling here depending on the target: - // - // * MachO - macos-like targets will insert the metadata into a section that - // is sort of fake dwarf debug info. Inspecting the source of the macos - // linker this causes these sections to be skipped automatically because - // it's not in an allowlist of otherwise well known dwarf section names to - // go into the final artifact. - // - // * WebAssembly - we actually don't have any container format for this - // target. WebAssembly doesn't support the `dylib` crate type anyway so - // there's no need for us to support this at this time. Consequently the - // metadata bytes are simply stored as-is into an rlib. - // - // * COFF - Windows-like targets create an object with a section that has - // the `IMAGE_SCN_LNK_REMOVE` flag set which ensures that if the linker - // ever sees the section it doesn't process it and it's removed. - // - // * ELF - All other targets are similar to Windows in that there's a - // `SHF_EXCLUDE` flag we can set on sections in an object file to get - // automatically removed from the final output. - // - // Note that this metdata format is kept in sync with - // `rustc_codegen_ssa/src/back/metadata.rs`. - fn create_metadata_file(sess: &Session, metadata: &[u8]) -> Vec<u8> { - let endianness = match sess.target.options.endian { - Endian::Little => Endianness::Little, - Endian::Big => Endianness::Big, - }; - let architecture = match &sess.target.arch[..] { - "arm" => Architecture::Arm, - "aarch64" => Architecture::Aarch64, - "x86" => Architecture::I386, - "s390x" => Architecture::S390x, - "mips" => Architecture::Mips, - "mips64" => Architecture::Mips64, - "x86_64" => { - if sess.target.pointer_width == 32 { - Architecture::X86_64_X32 - } else { - Architecture::X86_64 - } - } - "powerpc" => Architecture::PowerPc, - "powerpc64" => Architecture::PowerPc64, - "riscv32" => Architecture::Riscv32, - "riscv64" => Architecture::Riscv64, - "sparc64" => Architecture::Sparc64, - - // This is used to handle all "other" targets. This includes targets - // in two categories: - // - // * Some targets don't have support in the `object` crate just yet - // to write an object file. These targets are likely to get filled - // out over time. - // - // * Targets like WebAssembly don't support dylibs, so the purpose - // of putting metadata in object files, to support linking rlibs - // into dylibs, is moot. - // - // In both of these cases it means that linking into dylibs will - // not be supported by rustc. This doesn't matter for targets like - // WebAssembly and for targets not supported by the `object` crate - // yet it means that work will need to be done in the `object` crate - // to add a case above. - _ => return metadata.to_vec(), - }; - - if sess.target.is_like_osx { - let mut file = Object::new(BinaryFormat::MachO, architecture, endianness); - - let section = - file.add_section(b"__DWARF".to_vec(), b".rmeta".to_vec(), SectionKind::Debug); - file.append_section_data(section, metadata, 1); - file.write().unwrap() - } else if sess.target.is_like_windows { - const IMAGE_SCN_LNK_REMOVE: u32 = 0; - let mut file = Object::new(BinaryFormat::Coff, architecture, endianness); - - let section = file.add_section(Vec::new(), b".rmeta".to_vec(), SectionKind::Debug); - file.section_mut(section).flags = - SectionFlags::Coff { characteristics: IMAGE_SCN_LNK_REMOVE }; - file.append_section_data(section, metadata, 1); - file.write().unwrap() - } else { - const SHF_EXCLUDE: u64 = 0x80000000; - let mut file = Object::new(BinaryFormat::Elf, architecture, endianness); - - match &sess.target.arch[..] { - // copied from `mipsel-linux-gnu-gcc foo.c -c` and - // inspecting the resulting `e_flags` field. - "mips" => { - let e_flags = elf::EF_MIPS_ARCH_32R2 | elf::EF_MIPS_CPIC | elf::EF_MIPS_PIC; - file.flags = FileFlags::Elf { e_flags }; - } - // copied from `mips64el-linux-gnuabi64-gcc foo.c -c` - "mips64" => { - let e_flags = elf::EF_MIPS_ARCH_64R2 | elf::EF_MIPS_CPIC | elf::EF_MIPS_PIC; - file.flags = FileFlags::Elf { e_flags }; - } - - // copied from `riscv64-linux-gnu-gcc foo.c -c`, note though - // that the `+d` target feature represents whether the double - // float abi is enabled. - "riscv64" if sess.target.options.features.contains("+d") => { - let e_flags = elf::EF_RISCV_RVC | elf::EF_RISCV_FLOAT_ABI_DOUBLE; - file.flags = FileFlags::Elf { e_flags }; - } - - _ => {} - } - - let section = file.add_section(Vec::new(), b".rmeta".to_vec(), SectionKind::Debug); - file.section_mut(section).flags = SectionFlags::Elf { sh_flags: SHF_EXCLUDE }; - file.append_section_data(section, metadata, 1); - file.write().unwrap() - } - } } /// Extract all symbols defined in raw-dylib libraries, collated by library name. @@ -622,57 +529,108 @@ fn escape_stdout_stderr_string(s: &[u8]) -> String { }) } -const LLVM_DWP_EXECUTABLE: &'static str = "rust-llvm-dwp"; - -/// Invoke `llvm-dwp` (shipped alongside rustc) to link `dwo` files from Split DWARF into a `dwp` -/// file. -fn link_dwarf_object<'a>(sess: &'a Session, executable_out_filename: &Path) { - info!("preparing dwp to {}.dwp", executable_out_filename.to_str().unwrap()); - +/// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a +/// DWARF package. +fn link_dwarf_object<'a>( + sess: &'a Session, + cg_results: &CodegenResults, + executable_out_filename: &Path, +) { let dwp_out_filename = executable_out_filename.with_extension("dwp"); - let mut cmd = Command::new(LLVM_DWP_EXECUTABLE); - cmd.arg("-e"); - cmd.arg(executable_out_filename); - cmd.arg("-o"); - cmd.arg(&dwp_out_filename); + debug!(?dwp_out_filename, ?executable_out_filename); - let mut new_path = sess.get_tools_search_paths(false); - if let Some(path) = env::var_os("PATH") { - new_path.extend(env::split_paths(&path)); + #[derive(Default)] + struct ThorinSession<Relocations> { + arena_data: TypedArena<Vec<u8>>, + arena_mmap: TypedArena<Mmap>, + arena_relocations: TypedArena<Relocations>, } - let new_path = env::join_paths(new_path).unwrap(); - cmd.env("PATH", new_path); - info!("{:?}", &cmd); - match sess.time("run_dwp", || cmd.output()) { - Ok(prog) if !prog.status.success() => { - sess.struct_err(&format!( - "linking dwarf objects with `{}` failed: {}", - LLVM_DWP_EXECUTABLE, prog.status - )) - .note(&format!("{:?}", &cmd)) - .note(&escape_stdout_stderr_string(&prog.stdout)) - .note(&escape_stdout_stderr_string(&prog.stderr)) - .emit(); - info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr)); - info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout)); + impl<Relocations> ThorinSession<Relocations> { + fn alloc_mmap<'arena>(&'arena self, data: Mmap) -> &'arena Mmap { + (*self.arena_mmap.alloc(data)).borrow() } - Ok(_) => {} - Err(e) => { - let dwp_not_found = e.kind() == io::ErrorKind::NotFound; - let mut err = if dwp_not_found { - sess.struct_err(&format!("linker `{}` not found", LLVM_DWP_EXECUTABLE)) - } else { - sess.struct_err(&format!("could not exec the linker `{}`", LLVM_DWP_EXECUTABLE)) - }; + } + + impl<Relocations> thorin::Session<Relocations> for ThorinSession<Relocations> { + fn alloc_data<'arena>(&'arena self, data: Vec<u8>) -> &'arena [u8] { + (*self.arena_data.alloc(data)).borrow() + } + + fn alloc_relocation<'arena>(&'arena self, data: Relocations) -> &'arena Relocations { + (*self.arena_relocations.alloc(data)).borrow() + } + + fn read_input<'arena>(&'arena self, path: &Path) -> std::io::Result<&'arena [u8]> { + let file = File::open(&path)?; + let mmap = (unsafe { Mmap::map(file) })?; + Ok(self.alloc_mmap(mmap)) + } + } - err.note(&e.to_string()); + match sess.time("run_thorin", || -> Result<(), thorin::Error> { + let thorin_sess = ThorinSession::default(); + let mut package = thorin::DwarfPackage::new(&thorin_sess); - if !dwp_not_found { - err.note(&format!("{:?}", &cmd)); + // Input objs contain .o/.dwo files from the current crate. + match sess.opts.debugging_opts.split_dwarf_kind { + SplitDwarfKind::Single => { + for input_obj in cg_results.modules.iter().filter_map(|m| m.object.as_ref()) { + package.add_input_object(input_obj)?; + } + } + SplitDwarfKind::Split => { + for input_obj in cg_results.modules.iter().filter_map(|m| m.dwarf_object.as_ref()) { + package.add_input_object(input_obj)?; + } } + } + + // Input rlibs contain .o/.dwo files from dependencies. + let input_rlibs = cg_results + .crate_info + .used_crate_source + .values() + .filter_map(|csource| csource.rlib.as_ref()) + .map(|(path, _)| path); + for input_rlib in input_rlibs { + debug!(?input_rlib); + package.add_input_object(input_rlib)?; + } + + // Failing to read the referenced objects is expected for dependencies where the path in the + // executable will have been cleaned by Cargo, but the referenced objects will be contained + // within rlibs provided as inputs. + // + // If paths have been remapped, then .o/.dwo files from the current crate also won't be + // found, but are provided explicitly above. + // + // Adding an executable is primarily done to make `thorin` check that all the referenced + // dwarf objects are found in the end. + package.add_executable( + &executable_out_filename, + thorin::MissingReferencedObjectBehaviour::Skip, + )?; + + let output = package.finish()?.write()?; + let mut output_stream = BufWriter::new( + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(dwp_out_filename)?, + ); + output_stream.write_all(&output)?; + output_stream.flush()?; - err.emit(); + Ok(()) + }) { + Ok(()) => {} + Err(e) => { + sess.struct_err("linking dwarf objects with thorin failed") + .note(&format!("{:?}", e)) + .emit(); } } } @@ -709,7 +667,7 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( cmd.env_remove(k); } - if sess.opts.debugging_opts.print_link_args { + if sess.opts.prints.contains(&PrintRequest::LinkArgs) { println!("{:?}", &cmd); } @@ -974,7 +932,7 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( but `link.exe` was not found", ); sess.note_without_error( - "please ensure that VS 2013, VS 2015, VS 2017 or VS 2019 \ + "please ensure that VS 2013, VS 2015, VS 2017, VS 2019 or VS 2022 \ was installed with the Visual C++ option", ); } @@ -1018,11 +976,17 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( SplitDebuginfo::Packed if sess.target.is_like_msvc => {} // ... and otherwise we're processing a `*.dwp` packed dwarf file. - SplitDebuginfo::Packed => link_dwarf_object(sess, &out_filename), + // + // We cannot rely on the .o paths in the exectuable because they may have been + // remapped by --remap-path-prefix and therefore invalid, so we need to provide + // the .o/.dwo paths explicitly. + SplitDebuginfo::Packed => link_dwarf_object(sess, codegen_results, out_filename), } + let strip = strip_value(sess); + if sess.target.is_like_osx { - match sess.opts.debugging_opts.strip { + match strip { Strip::Debuginfo => strip_symbols_in_osx(sess, &out_filename, Some("-S")), Strip::Symbols => strip_symbols_in_osx(sess, &out_filename, None), Strip::None => {} @@ -1030,6 +994,14 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>( } } +// Temporarily support both -Z strip and -C strip +fn strip_value(sess: &Session) -> Strip { + match (sess.opts.debugging_opts.strip, sess.opts.cg.strip) { + (s, Strip::None) => s, + (_, s) => s, + } +} + fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: Option<&str>) { let mut cmd = Command::new("strip"); if let Some(option) = option { @@ -1095,10 +1067,10 @@ fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut d } fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) { - fn find_sanitizer_runtime(sess: &Session, filename: &String) -> PathBuf { + fn find_sanitizer_runtime(sess: &Session, filename: &str) -> PathBuf { let session_tlib = filesearch::make_target_lib_path(&sess.sysroot, sess.opts.target_triple.triple()); - let path = session_tlib.join(&filename); + let path = session_tlib.join(filename); if path.exists() { return session_tlib; } else { @@ -1187,6 +1159,7 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { LinkerFlavor::Lld(_) => "lld", LinkerFlavor::PtxLinker => "rust-ptx-linker", LinkerFlavor::BpfLinker => "bpf-linker", + LinkerFlavor::L4Bender => "l4-bender", }), flavor, )), @@ -1239,26 +1212,36 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { bug!("Not enough information provided to determine how to invoke the linker"); } -/// Returns a boolean indicating whether we should preserve the object files on -/// the filesystem for their debug information. This is often useful with -/// split-dwarf like schemes. -fn preserve_objects_for_their_debuginfo(sess: &Session) -> bool { +/// Returns a pair of boolean indicating whether we should preserve the object and +/// dwarf object files on the filesystem for their debug information. This is often +/// useful with split-dwarf like schemes. +fn preserve_objects_for_their_debuginfo(sess: &Session) -> (bool, bool) { // If the objects don't have debuginfo there's nothing to preserve. if sess.opts.debuginfo == config::DebugInfo::None { - return false; + return (false, false); } // If we're only producing artifacts that are archives, no need to preserve // the objects as they're losslessly contained inside the archives. - let output_linked = - sess.crate_types().iter().any(|&x| x != CrateType::Rlib && x != CrateType::Staticlib); - if !output_linked { - return false; + if sess.crate_types().iter().all(|&x| x.is_archive()) { + return (false, false); + } + + match (sess.split_debuginfo(), sess.opts.debugging_opts.split_dwarf_kind) { + // If there is no split debuginfo then do not preserve objects. + (SplitDebuginfo::Off, _) => (false, false), + // If there is packed split debuginfo, then the debuginfo in the objects + // has been packaged and the objects can be deleted. + (SplitDebuginfo::Packed, _) => (false, false), + // If there is unpacked split debuginfo and the current target can not use + // split dwarf, then keep objects. + (SplitDebuginfo::Unpacked, _) if !sess.target_can_use_split_dwarf() => (true, false), + // If there is unpacked split debuginfo and the target can use split dwarf, then + // keep the object containing that debuginfo (whether that is an object file or + // dwarf object file depends on the split dwarf kind). + (SplitDebuginfo::Unpacked, SplitDwarfKind::Single) => (true, false), + (SplitDebuginfo::Unpacked, SplitDwarfKind::Split) => (false, true), } - - // "unpacked" split debuginfo means that we leave object files as the - // debuginfo is found in the original object files themselves - sess.split_debuginfo() == SplitDebuginfo::Unpacked } fn archive_search_paths(sess: &Session) -> Vec<PathBuf> { @@ -2001,7 +1984,7 @@ fn add_order_independent_options( cmd.optimize(); // Pass debuginfo and strip flags down to the linker. - cmd.debuginfo(sess.opts.debugging_opts.strip); + cmd.debuginfo(strip_value(sess)); // We want to prevent the compiler from accidentally leaking in any system libraries, // so by default we tell linkers not to link to any default libraries. @@ -2336,8 +2319,8 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( continue; } - let canonical = f.replace("-", "_"); - let canonical_name = name.replace("-", "_"); + let canonical = f.replace('-', "_"); + let canonical_name = name.replace('-', "_"); let is_rust_object = canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 429dc45d6a4..3fb56f42b8c 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -126,7 +126,6 @@ pub fn get_linker<'a>( // FIXME: Move `/LIBPATH` addition for uwp targets from the linker construction // to the linker args construction. assert!(cmd.get_args().is_empty() || sess.target.vendor == "uwp"); - match flavor { LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => { Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker> @@ -149,6 +148,8 @@ pub fn get_linker<'a>( LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>, LinkerFlavor::BpfLinker => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>, + + LinkerFlavor::L4Bender => Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>, } } @@ -219,19 +220,36 @@ pub struct GccLinker<'a> { } impl<'a> GccLinker<'a> { - /// Argument that must be passed *directly* to the linker + /// Passes an argument directly to the linker. /// - /// These arguments need to be prepended with `-Wl`, when a GCC-style linker is used. - fn linker_arg<S>(&mut self, arg: S) -> &mut Self - where - S: AsRef<OsStr>, - { - if !self.is_ld { - let mut os = OsString::from("-Wl,"); - os.push(arg.as_ref()); - self.cmd.arg(os); + /// When the linker is not ld-like such as when using a compiler as a linker, the argument is + /// prepended by `-Wl,`. + fn linker_arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self { + self.linker_args(&[arg]); + self + } + + /// Passes a series of arguments directly to the linker. + /// + /// When the linker is ld-like, the arguments are simply appended to the command. When the + /// linker is not ld-like such as when using a compiler as a linker, the arguments are joined by + /// commas to form an argument that is then prepended with `-Wl`. In this situation, only a + /// single argument is appended to the command to ensure that the order of the arguments is + /// preserved by the compiler. + fn linker_args(&mut self, args: &[impl AsRef<OsStr>]) -> &mut Self { + if self.is_ld { + args.into_iter().for_each(|a| { + self.cmd.arg(a); + }); } else { - self.cmd.arg(arg); + if !args.is_empty() { + let mut s = OsString::from("-Wl"); + for a in args { + s.push(","); + s.push(a); + } + self.cmd.arg(s); + } } self } @@ -289,14 +307,19 @@ impl<'a> GccLinker<'a> { if let Some(path) = &self.sess.opts.debugging_opts.profile_sample_use { self.linker_arg(&format!("-plugin-opt=sample-profile={}", path.display())); }; - self.linker_arg(&format!("-plugin-opt={}", opt_level)); - self.linker_arg(&format!("-plugin-opt=mcpu={}", self.target_cpu)); + self.linker_args(&[ + &format!("-plugin-opt={}", opt_level), + &format!("-plugin-opt=mcpu={}", self.target_cpu), + ]); } fn build_dylib(&mut self, out_filename: &Path) { // On mac we need to tell the linker to let this library be rpathed if self.sess.target.is_like_osx { - self.cmd.arg("-dynamiclib"); + if !self.is_ld { + self.cmd.arg("-dynamiclib"); + } + self.linker_arg("-dylib"); // Note that the `osx_rpath_install_name` option here is a hack @@ -304,10 +327,9 @@ impl<'a> GccLinker<'a> { // principled solution at some point to force the compiler to pass // the right `-Wl,-install_name` with an `@rpath` in it. if self.sess.opts.cg.rpath || self.sess.opts.debugging_opts.osx_rpath_install_name { - self.linker_arg("-install_name"); - let mut v = OsString::from("@rpath/"); - v.push(out_filename.file_name().unwrap()); - self.linker_arg(&v); + let mut rpath = OsString::from("@rpath/"); + rpath.push(out_filename.file_name().unwrap()); + self.linker_args(&[OsString::from("-install_name"), rpath]); } } else { self.cmd.arg("-shared"); @@ -381,8 +403,7 @@ impl<'a> Linker for GccLinker<'a> { self.build_dylib(out_filename); } LinkOutputKind::WasiReactorExe => { - self.linker_arg("--entry"); - self.linker_arg("_initialize"); + self.linker_args(&["--entry", "_initialize"]); } } // VxWorks compiler driver introduced `--static-crt` flag specifically for rustc, @@ -454,8 +475,7 @@ impl<'a> Linker for GccLinker<'a> { self.cmd.arg(path); } fn full_relro(&mut self) { - self.linker_arg("-zrelro"); - self.linker_arg("-znow"); + self.linker_args(&["-zrelro", "-znow"]); } fn partial_relro(&mut self) { self.linker_arg("-zrelro"); @@ -639,7 +659,6 @@ impl<'a> Linker for GccLinker<'a> { } let is_windows = self.sess.target.is_like_windows; - let mut arg = OsString::new(); let path = tmpdir.join(if is_windows { "list.def" } else { "list" }); debug!("EXPORTED SYMBOLS:"); @@ -691,27 +710,18 @@ impl<'a> Linker for GccLinker<'a> { } if self.sess.target.is_like_osx { - if !self.is_ld { - arg.push("-Wl,") - } - arg.push("-exported_symbols_list,"); + self.linker_args(&[OsString::from("-exported_symbols_list"), path.into()]); } else if self.sess.target.is_like_solaris { - if !self.is_ld { - arg.push("-Wl,") - } - arg.push("-M,"); + self.linker_args(&[OsString::from("-M"), path.into()]); } else { - if !self.is_ld { - arg.push("-Wl,") - } - // Both LD and LLD accept export list in *.def file form, there are no flags required - if !is_windows { - arg.push("--version-script=") + if is_windows { + self.linker_arg(path); + } else { + let mut arg = OsString::from("--version-script="); + arg.push(path); + self.linker_arg(arg); } } - - arg.push(&path); - self.cmd.arg(arg); } fn subsystem(&mut self, subsystem: &str) { @@ -769,8 +779,7 @@ impl<'a> Linker for GccLinker<'a> { self.linker_arg("--as-needed"); } else if self.sess.target.is_like_solaris { // -z ignore is the Solaris equivalent to the GNU ld --as-needed option - self.linker_arg("-z"); - self.linker_arg("ignore"); + self.linker_args(&["-z", "ignore"]); } } } @@ -1347,6 +1356,157 @@ impl<'a> Linker for WasmLd<'a> { } } +/// Linker shepherd script for L4Re (Fiasco) +pub struct L4Bender<'a> { + cmd: Command, + sess: &'a Session, + hinted_static: bool, +} + +impl<'a> Linker for L4Bender<'a> { + fn link_dylib(&mut self, _lib: Symbol, _verbatim: bool, _as_needed: bool) { + bug!("dylibs are not supported on L4Re"); + } + fn link_staticlib(&mut self, lib: Symbol, _verbatim: bool) { + self.hint_static(); + self.cmd.arg(format!("-PC{}", lib)); + } + fn link_rlib(&mut self, lib: &Path) { + self.hint_static(); + self.cmd.arg(lib); + } + fn include_path(&mut self, path: &Path) { + self.cmd.arg("-L").arg(path); + } + fn framework_path(&mut self, _: &Path) { + bug!("frameworks are not supported on L4Re"); + } + fn output_filename(&mut self, path: &Path) { + self.cmd.arg("-o").arg(path); + } + + fn add_object(&mut self, path: &Path) { + self.cmd.arg(path); + } + + fn full_relro(&mut self) { + self.cmd.arg("-zrelro"); + self.cmd.arg("-znow"); + } + + fn partial_relro(&mut self) { + self.cmd.arg("-zrelro"); + } + + fn no_relro(&mut self) { + self.cmd.arg("-znorelro"); + } + + fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {} + + fn link_rust_dylib(&mut self, _: Symbol, _: &Path) { + panic!("Rust dylibs not supported"); + } + + fn link_framework(&mut self, _framework: Symbol, _as_needed: bool) { + bug!("frameworks not supported on L4Re"); + } + + fn link_whole_staticlib(&mut self, lib: Symbol, _verbatim: bool, _search_path: &[PathBuf]) { + self.hint_static(); + self.cmd.arg("--whole-archive").arg(format!("-l{}", lib)); + self.cmd.arg("--no-whole-archive"); + } + + fn link_whole_rlib(&mut self, lib: &Path) { + self.hint_static(); + self.cmd.arg("--whole-archive").arg(lib).arg("--no-whole-archive"); + } + + fn gc_sections(&mut self, keep_metadata: bool) { + if !keep_metadata { + self.cmd.arg("--gc-sections"); + } + } + + fn no_gc_sections(&mut self) { + self.cmd.arg("--no-gc-sections"); + } + + fn optimize(&mut self) { + // GNU-style linkers support optimization with -O. GNU ld doesn't + // need a numeric argument, but other linkers do. + if self.sess.opts.optimize == config::OptLevel::Default + || self.sess.opts.optimize == config::OptLevel::Aggressive + { + self.cmd.arg("-O1"); + } + } + + fn pgo_gen(&mut self) {} + + fn debuginfo(&mut self, strip: Strip) { + match strip { + Strip::None => {} + Strip::Debuginfo => { + self.cmd().arg("--strip-debug"); + } + Strip::Symbols => { + self.cmd().arg("--strip-all"); + } + } + } + + fn no_default_libraries(&mut self) { + self.cmd.arg("-nostdlib"); + } + + fn export_symbols(&mut self, _: &Path, _: CrateType, _: &[String]) { + // ToDo, not implemented, copy from GCC + self.sess.warn("exporting symbols not implemented yet for L4Bender"); + return; + } + + fn subsystem(&mut self, subsystem: &str) { + self.cmd.arg(&format!("--subsystem {}", subsystem)); + } + + fn reset_per_library_state(&mut self) { + self.hint_static(); // Reset to default before returning the composed command line. + } + + fn group_start(&mut self) { + self.cmd.arg("--start-group"); + } + + fn group_end(&mut self) { + self.cmd.arg("--end-group"); + } + + fn linker_plugin_lto(&mut self) {} + + fn control_flow_guard(&mut self) {} + + fn no_crt_objects(&mut self) {} +} + +impl<'a> L4Bender<'a> { + pub fn new(cmd: Command, sess: &'a Session) -> L4Bender<'a> { + L4Bender { cmd: cmd, sess: sess, hinted_static: false } + } + + fn hint_static(&mut self) { + if !self.hinted_static { + self.cmd.arg("-static"); + self.hinted_static = true; + } + } +} + pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<String> { if let Some(ref exports) = tcx.sess.target.override_export_symbols { return exports.clone(); diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index ffeb926e648..6849533abc0 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -1,14 +1,25 @@ //! Reading of the rustc metadata for rlibs and dylibs use std::fs::File; +use std::io::Write; use std::path::Path; -use object::{Object, ObjectSection}; +use object::write::{self, StandardSegment, Symbol, SymbolSection}; +use object::{ + elf, pe, Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, + SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, +}; + +use snap::write::FrameEncoder; + use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; +use rustc_metadata::EncodedMetadata; use rustc_session::cstore::MetadataLoader; +use rustc_session::Session; +use rustc_target::abi::Endian; use rustc_target::spec::Target; use crate::METADATA_FILENAME; @@ -83,3 +94,205 @@ fn search_for_metadata<'a>( .data() .map_err(|e| format!("failed to read {} section in '{}': {}", section, path.display(), e)) } + +fn create_object_file(sess: &Session) -> Option<write::Object<'static>> { + let endianness = match sess.target.options.endian { + Endian::Little => Endianness::Little, + Endian::Big => Endianness::Big, + }; + let architecture = match &sess.target.arch[..] { + "arm" => Architecture::Arm, + "aarch64" => Architecture::Aarch64, + "x86" => Architecture::I386, + "s390x" => Architecture::S390x, + "mips" => Architecture::Mips, + "mips64" => Architecture::Mips64, + "x86_64" => { + if sess.target.pointer_width == 32 { + Architecture::X86_64_X32 + } else { + Architecture::X86_64 + } + } + "powerpc" => Architecture::PowerPc, + "powerpc64" => Architecture::PowerPc64, + "riscv32" => Architecture::Riscv32, + "riscv64" => Architecture::Riscv64, + "sparc64" => Architecture::Sparc64, + // Unsupported architecture. + _ => return None, + }; + let binary_format = if sess.target.is_like_osx { + BinaryFormat::MachO + } else if sess.target.is_like_windows { + BinaryFormat::Coff + } else { + BinaryFormat::Elf + }; + + let mut file = write::Object::new(binary_format, architecture, endianness); + match architecture { + Architecture::Mips => { + // copied from `mipsel-linux-gnu-gcc foo.c -c` and + // inspecting the resulting `e_flags` field. + let e_flags = elf::EF_MIPS_CPIC + | elf::EF_MIPS_PIC + | if sess.target.options.cpu.contains("r6") { + elf::EF_MIPS_ARCH_32R6 | elf::EF_MIPS_NAN2008 + } else { + elf::EF_MIPS_ARCH_32R2 + }; + file.flags = FileFlags::Elf { e_flags }; + } + Architecture::Mips64 => { + // copied from `mips64el-linux-gnuabi64-gcc foo.c -c` + let e_flags = elf::EF_MIPS_CPIC + | elf::EF_MIPS_PIC + | if sess.target.options.cpu.contains("r6") { + elf::EF_MIPS_ARCH_64R6 | elf::EF_MIPS_NAN2008 + } else { + elf::EF_MIPS_ARCH_64R2 + }; + file.flags = FileFlags::Elf { e_flags }; + } + Architecture::Riscv64 if sess.target.options.features.contains("+d") => { + // copied from `riscv64-linux-gnu-gcc foo.c -c`, note though + // that the `+d` target feature represents whether the double + // float abi is enabled. + let e_flags = elf::EF_RISCV_RVC | elf::EF_RISCV_FLOAT_ABI_DOUBLE; + file.flags = FileFlags::Elf { e_flags }; + } + _ => {} + }; + Some(file) +} + +// For rlibs we "pack" rustc metadata into a dummy object file. When rustc +// creates a dylib crate type it will pass `--whole-archive` (or the +// platform equivalent) to include all object files from an rlib into the +// final dylib itself. This causes linkers to iterate and try to include all +// files located in an archive, so if metadata is stored in an archive then +// it needs to be of a form that the linker will be able to process. +// +// Note, though, that we don't actually want this metadata to show up in any +// final output of the compiler. Instead this is purely for rustc's own +// metadata tracking purposes. +// +// With the above in mind, each "flavor" of object format gets special +// handling here depending on the target: +// +// * MachO - macos-like targets will insert the metadata into a section that +// is sort of fake dwarf debug info. Inspecting the source of the macos +// linker this causes these sections to be skipped automatically because +// it's not in an allowlist of otherwise well known dwarf section names to +// go into the final artifact. +// +// * WebAssembly - we actually don't have any container format for this +// target. WebAssembly doesn't support the `dylib` crate type anyway so +// there's no need for us to support this at this time. Consequently the +// metadata bytes are simply stored as-is into an rlib. +// +// * COFF - Windows-like targets create an object with a section that has +// the `IMAGE_SCN_LNK_REMOVE` flag set which ensures that if the linker +// ever sees the section it doesn't process it and it's removed. +// +// * ELF - All other targets are similar to Windows in that there's a +// `SHF_EXCLUDE` flag we can set on sections in an object file to get +// automatically removed from the final output. +pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> Vec<u8> { + let mut file = if let Some(file) = create_object_file(sess) { + file + } else { + // This is used to handle all "other" targets. This includes targets + // in two categories: + // + // * Some targets don't have support in the `object` crate just yet + // to write an object file. These targets are likely to get filled + // out over time. + // + // * Targets like WebAssembly don't support dylibs, so the purpose + // of putting metadata in object files, to support linking rlibs + // into dylibs, is moot. + // + // In both of these cases it means that linking into dylibs will + // not be supported by rustc. This doesn't matter for targets like + // WebAssembly and for targets not supported by the `object` crate + // yet it means that work will need to be done in the `object` crate + // to add a case above. + return metadata.to_vec(); + }; + let section = file.add_section( + file.segment_name(StandardSegment::Debug).to_vec(), + b".rmeta".to_vec(), + SectionKind::Debug, + ); + match file.format() { + BinaryFormat::Coff => { + file.section_mut(section).flags = + SectionFlags::Coff { characteristics: pe::IMAGE_SCN_LNK_REMOVE }; + } + BinaryFormat::Elf => { + file.section_mut(section).flags = + SectionFlags::Elf { sh_flags: elf::SHF_EXCLUDE as u64 }; + } + _ => {} + }; + file.append_section_data(section, metadata, 1); + file.write().unwrap() +} + +// Historical note: +// +// When using link.exe it was seen that the section name `.note.rustc` +// was getting shortened to `.note.ru`, and according to the PE and COFF +// specification: +// +// > Executable images do not use a string table and do not support +// > section names longer than 8Â characters +// +// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format +// +// As a result, we choose a slightly shorter name! As to why +// `.note.rustc` works on MinGW, see +// https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197 +pub fn create_compressed_metadata_file( + sess: &Session, + metadata: &EncodedMetadata, + symbol_name: &str, +) -> Vec<u8> { + let mut compressed = rustc_metadata::METADATA_HEADER.to_vec(); + FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap(); + let mut file = if let Some(file) = create_object_file(sess) { + file + } else { + return compressed.to_vec(); + }; + let section = file.add_section( + file.segment_name(StandardSegment::Data).to_vec(), + b".rustc".to_vec(), + SectionKind::ReadOnlyData, + ); + match file.format() { + BinaryFormat::Elf => { + // Explicitly set no flags to avoid SHF_ALLOC default for data section. + file.section_mut(section).flags = SectionFlags::Elf { sh_flags: 0 }; + } + _ => {} + }; + let offset = file.append_section_data(section, &compressed, 1); + + // For MachO and probably PE this is necessary to prevent the linker from throwing away the + // .rustc section. For ELF this isn't necessary, but it also doesn't harm. + file.add_symbol(Symbol { + name: symbol_name.as_bytes().to_vec(), + value: offset, + size: compressed.len() as u64, + kind: SymbolKind::Data, + scope: SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(section), + flags: SymbolFlags::None, + }); + + file.write().unwrap() +} diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs index 61c3ef62fb1..0b5656c9ad1 100644 --- a/compiler/rustc_codegen_ssa/src/back/rpath.rs +++ b/compiler/rustc_codegen_ssa/src/back/rpath.rs @@ -23,9 +23,12 @@ pub fn get_rpath_flags(config: &mut RPathConfig<'_>) -> Vec<String> { let rpaths = get_rpaths(config); let mut flags = rpaths_to_flags(&rpaths); - // Use DT_RUNPATH instead of DT_RPATH if available if config.linker_is_gnu { + // Use DT_RUNPATH instead of DT_RPATH if available flags.push("-Wl,--enable-new-dtags".to_owned()); + + // Set DF_ORIGIN for substitute $ORIGIN + flags.push("-Wl,-z,origin".to_owned()); } flags diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index f80f9965f4d..aeddd926896 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -76,7 +76,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap< // // As a result, if this id is an FFI item (foreign item) then we only // let it through if it's included statically. - match tcx.hir().get(tcx.hir().local_def_id_to_hir_id(def_id)) { + match tcx.hir().get_by_def_id(def_id) { Node::ForeignItem(..) => { tcx.is_statically_included_foreign_item(def_id).then_some(def_id) } @@ -154,7 +154,7 @@ fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> b tcx.reachable_non_generics(def_id.krate).contains_key(&def_id) } -fn exported_symbols_provider_local( +fn exported_symbols_provider_local<'tcx>( tcx: TyCtxt<'tcx>, cnum: CrateNum, ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportLevel)] { diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index b2edc6c0183..540979ce02d 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -32,7 +32,7 @@ use rustc_session::Session; use rustc_span::source_map::SourceMap; use rustc_span::symbol::sym; use rustc_span::{BytePos, FileName, InnerSpan, Pos, Span}; -use rustc_target::spec::{MergeFunctions, PanicStrategy, SanitizerSet}; +use rustc_target::spec::{MergeFunctions, SanitizerSet}; use std::any::Any; use std::fs; @@ -113,6 +113,7 @@ pub struct ModuleConfig { pub inline_threshold: Option<u32>, pub new_llvm_pass_manager: Option<bool>, pub emit_lifetime_markers: bool, + pub llvm_plugins: Vec<String>, } impl ModuleConfig { @@ -260,6 +261,7 @@ impl ModuleConfig { inline_threshold: sess.opts.cg.inline_threshold, new_llvm_pass_manager: sess.opts.debugging_opts.new_llvm_pass_manager, emit_lifetime_markers: sess.emit_lifetime_markers(), + llvm_plugins: if_regular!(sess.opts.debugging_opts.llvm_plugins.clone(), vec![]), } } @@ -284,7 +286,11 @@ impl TargetMachineFactoryConfig { module_name: &str, ) -> TargetMachineFactoryConfig { let split_dwarf_file = if cgcx.target_can_use_split_dwarf { - cgcx.output_filenames.split_dwarf_path(cgcx.split_debuginfo, Some(module_name)) + cgcx.output_filenames.split_dwarf_path( + cgcx.split_debuginfo, + cgcx.split_dwarf_kind, + Some(module_name), + ) } else { None }; @@ -307,7 +313,6 @@ pub struct CodegenContext<B: WriteBackendMethods> { pub backend: B, pub prof: SelfProfilerRef, pub lto: Lto, - pub no_landing_pads: bool, pub save_temps: bool, pub fewer_names: bool, pub time_trace: bool, @@ -327,6 +332,7 @@ pub struct CodegenContext<B: WriteBackendMethods> { pub target_arch: String, pub debuginfo: config::DebugInfo, pub split_debuginfo: rustc_target::spec::SplitDebuginfo, + pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, // Number of cgus excluding the allocator/metadata modules pub total_cgus: usize, @@ -397,7 +403,6 @@ fn generate_lto_work<B: ExtraBackendMethods>( pub struct CompiledModules { pub modules: Vec<CompiledModule>, - pub metadata_module: Option<CompiledModule>, pub allocator_module: Option<CompiledModule>, } @@ -425,6 +430,7 @@ pub fn start_async_codegen<B: ExtraBackendMethods>( tcx: TyCtxt<'_>, target_cpu: String, metadata: EncodedMetadata, + metadata_module: Option<CompiledModule>, total_cgus: usize, ) -> OngoingCodegen<B> { let (coordinator_send, coordinator_receive) = channel(); @@ -464,6 +470,7 @@ pub fn start_async_codegen<B: ExtraBackendMethods>( OngoingCodegen { backend, metadata, + metadata_module, crate_info, coordinator_send, @@ -640,12 +647,6 @@ fn produce_final_output_artifacts( } if !user_wants_bitcode { - if let Some(ref metadata_module) = compiled_modules.metadata_module { - if let Some(ref path) = metadata_module.bytecode { - ensure_removed(sess.diagnostic(), &path); - } - } - if let Some(ref allocator_module) = compiled_modules.allocator_module { if let Some(ref path) = allocator_module.bytecode { ensure_removed(sess.diagnostic(), path); @@ -682,11 +683,11 @@ impl<B: WriteBackendMethods> WorkItem<B> { fn start_profiling<'a>(&self, cgcx: &'a CodegenContext<B>) -> TimingGuard<'a> { match *self { WorkItem::Optimize(ref m) => { - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &m.name[..]) + cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name) } WorkItem::CopyPostLtoArtifacts(ref m) => cgcx .prof - .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &m.name[..]), + .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name), WorkItem::LTO(ref m) => { cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()) } @@ -1037,7 +1038,6 @@ fn start_executing_work<B: ExtraBackendMethods>( crate_types: sess.crate_types().to_vec(), each_linked_rlib_for_lto, lto: sess.lto(), - no_landing_pads: sess.panic_strategy() == PanicStrategy::Abort, fewer_names: sess.fewer_names(), save_temps: sess.opts.cg.save_temps, time_trace: sess.opts.debugging_opts.llvm_time_trace, @@ -1063,6 +1063,7 @@ fn start_executing_work<B: ExtraBackendMethods>( target_arch: tcx.sess.target.arch.clone(), debuginfo: tcx.sess.opts.debuginfo, split_debuginfo: tcx.sess.split_debuginfo(), + split_dwarf_kind: tcx.sess.opts.debugging_opts.split_dwarf_kind, }; // This is the "main loop" of parallel work happening for parallel codegen. @@ -1216,7 +1217,6 @@ fn start_executing_work<B: ExtraBackendMethods>( // This is where we collect codegen units that have gone all the way // through codegen and LLVM. let mut compiled_modules = vec![]; - let mut compiled_metadata_module = None; let mut compiled_allocator_module = None; let mut needs_link = Vec::new(); let mut needs_fat_lto = Vec::new(); @@ -1475,14 +1475,11 @@ fn start_executing_work<B: ExtraBackendMethods>( ModuleKind::Regular => { compiled_modules.push(compiled_module); } - ModuleKind::Metadata => { - assert!(compiled_metadata_module.is_none()); - compiled_metadata_module = Some(compiled_module); - } ModuleKind::Allocator => { assert!(compiled_allocator_module.is_none()); compiled_allocator_module = Some(compiled_module); } + ModuleKind::Metadata => bug!("Should be handled separately"), } } Message::NeedsLink { module, worker_id } => { @@ -1539,7 +1536,6 @@ fn start_executing_work<B: ExtraBackendMethods>( Ok(CompiledModules { modules: compiled_modules, - metadata_module: compiled_metadata_module, allocator_module: compiled_allocator_module, }) }); @@ -1757,7 +1753,7 @@ impl SharedEmitterMain { let msg = msg.strip_prefix("error: ").unwrap_or(&msg); let mut err = match level { - Level::Error => sess.struct_err(&msg), + Level::Error { lint: false } => sess.struct_err(&msg), Level::Warning => sess.struct_warn(&msg), Level::Note => sess.struct_note_without_error(&msg), _ => bug!("Invalid inline asm diagnostic level"), @@ -1800,6 +1796,7 @@ impl SharedEmitterMain { pub struct OngoingCodegen<B: ExtraBackendMethods> { pub backend: B, pub metadata: EncodedMetadata, + pub metadata_module: Option<CompiledModule>, pub crate_info: CrateInfo, pub coordinator_send: Sender<Box<dyn Any + Send>>, pub codegen_worker_receive: Receiver<Message<B>>, @@ -1846,7 +1843,7 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> { modules: compiled_modules.modules, allocator_module: compiled_modules.allocator_module, - metadata_module: compiled_modules.metadata_module, + metadata_module: self.metadata_module, }, work_products, ) diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 9bb4982754c..49b785afa69 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -1,3 +1,4 @@ +use crate::back::metadata::create_compressed_metadata_file; use crate::back::write::{ compute_per_cgu_lto_type, start_async_codegen, submit_codegened_module_to_llvm, submit_post_lto_module_to_llvm, submit_pre_lto_module_to_llvm, ComputedLtoType, OngoingCodegen, @@ -8,7 +9,7 @@ use crate::mir; use crate::mir::operand::OperandValue; use crate::mir::place::PlaceRef; use crate::traits::*; -use crate::{CachedModuleCodegen, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}; +use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind}; use rustc_attr as attr; use rustc_data_structures::fx::FxHashMap; @@ -20,13 +21,14 @@ use rustc_hir::lang_items::LangItem; use rustc_index::vec::Idx; use rustc_metadata::EncodedMetadata; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; +use rustc_middle::middle::exported_symbols; use rustc_middle::middle::lang_items; use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_session::cgu_reuse_tracker::CguReuse; -use rustc_session::config::{self, EntryFnType}; +use rustc_session::config::{self, EntryFnType, OutputType}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_target::abi::{Align, VariantIdx}; @@ -484,14 +486,14 @@ fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( pub fn codegen_crate<B: ExtraBackendMethods>( backend: B, - tcx: TyCtxt<'tcx>, + tcx: TyCtxt<'_>, target_cpu: String, metadata: EncodedMetadata, need_metadata_module: bool, ) -> OngoingCodegen<B> { // Skip crate items and just output metadata in -Z no-codegen mode. if tcx.sess.opts.debugging_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, 1); + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, metadata, None, 1); ongoing_codegen.codegen_finished(tcx); @@ -517,8 +519,41 @@ pub fn codegen_crate<B: ExtraBackendMethods>( } } - let ongoing_codegen = - start_async_codegen(backend.clone(), tcx, target_cpu, metadata, codegen_units.len()); + let metadata_module = if need_metadata_module { + // Emit compressed metadata object. + let metadata_cgu_name = + cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string(); + tcx.sess.time("write_compressed_metadata", || { + let file_name = + tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + let data = create_compressed_metadata_file( + tcx.sess, + &metadata, + &exported_symbols::metadata_symbol_name(tcx), + ); + if let Err(err) = std::fs::write(&file_name, data) { + tcx.sess.fatal(&format!("error writing metadata object file: {}", err)); + } + Some(CompiledModule { + name: metadata_cgu_name, + kind: ModuleKind::Metadata, + object: Some(file_name), + dwarf_object: None, + bytecode: None, + }) + }) + } else { + None + }; + + let ongoing_codegen = start_async_codegen( + backend.clone(), + tcx, + target_cpu, + metadata, + metadata_module, + codegen_units.len(), + ); let ongoing_codegen = AbortCodegenOnDrop::<B>(Some(ongoing_codegen)); // Codegen an allocator shim, if necessary. @@ -558,27 +593,6 @@ pub fn codegen_crate<B: ExtraBackendMethods>( ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module); } - if need_metadata_module { - // Codegen the encoded metadata. - let metadata_cgu_name = - cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string(); - let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name); - tcx.sess.time("write_compressed_metadata", || { - backend.write_compressed_metadata( - tcx, - &ongoing_codegen.metadata, - &mut metadata_llvm_module, - ); - }); - - let metadata_module = ModuleCodegen { - name: metadata_cgu_name, - module_llvm: metadata_llvm_module, - kind: ModuleKind::Metadata, - }; - ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, metadata_module); - } - // For better throughput during parallel processing by LLVM, we used to sort // CGUs largest to smallest. This would lead to better thread utilization // by, for example, preventing a large CGU from being processed last and @@ -658,7 +672,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>( } let cgu_reuse = cgu_reuse[i]; - tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); + tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); match cgu_reuse { CguReuse::No => { diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs index 1fa60612d26..2df58ecc9f6 100644 --- a/compiler/rustc_codegen_ssa/src/common.rs +++ b/compiler/rustc_codegen_ssa/src/common.rs @@ -122,8 +122,8 @@ pub fn langcall(tcx: TyCtxt<'_>, span: Option<Span>, msg: &str, li: LangItem) -> tcx.lang_items().require(li).unwrap_or_else(|s| { let msg = format!("{} {}", msg, s); match span { - Some(span) => tcx.sess.span_fatal(span, &msg[..]), - None => tcx.sess.fatal(&msg[..]), + Some(span) => tcx.sess.span_fatal(span, &msg), + None => tcx.sess.fatal(&msg), } }) } diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs index 962c01c2ee7..e288760a02b 100644 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs @@ -1,6 +1,6 @@ use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; -/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L206-L222) +/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L95) #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum CounterKind { @@ -17,7 +17,7 @@ pub enum CounterKind { /// `instrprof.increment()`) /// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of /// counter expressions. -/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L99-L100) +/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L102-L103) /// Important: The Rust struct layout (order and types of fields) must match its C++ counterpart. #[derive(Copy, Clone, Debug)] #[repr(C)] @@ -59,7 +59,7 @@ impl Counter { } } -/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L147) +/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L150) #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum ExprKind { @@ -67,7 +67,7 @@ pub enum ExprKind { Add = 1, } -/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L148-L149) +/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L152) /// Important: The Rust struct layout (order and types of fields) must match its C++ /// counterpart. #[derive(Copy, Clone, Debug)] diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs index c1dfe1ef856..1a6495cb15c 100644 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/map.rs @@ -150,9 +150,9 @@ impl<'tcx> FunctionCoverage<'tcx> { /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create /// `CounterMappingRegion`s. - pub fn get_expressions_and_counter_regions<'a>( - &'a self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &'a CodeRegion)>) { + pub fn get_expressions_and_counter_regions( + &self, + ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { assert!( self.source_hash != 0 || !self.is_used, "No counters provided the source_hash for used function: {:?}", @@ -168,7 +168,7 @@ impl<'tcx> FunctionCoverage<'tcx> { (counter_expressions, counter_regions) } - fn counter_regions<'a>(&'a self) -> impl Iterator<Item = (Counter, &'a CodeRegion)> { + fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { self.counters.iter_enumerated().filter_map(|(index, entry)| { // Option::map() will return None to filter out missing counters. This may happen // if, for example, a MIR-instrumented counter is removed during an optimization. @@ -177,8 +177,8 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn expressions_with_regions( - &'a self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &'a CodeRegion)>) { + &self, + ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { let mut counter_expressions = Vec::with_capacity(self.expressions.len()); let mut expression_regions = Vec::with_capacity(self.expressions.len()); let mut new_indexes = IndexVec::from_elem_n(None, self.expressions.len()); @@ -336,7 +336,7 @@ impl<'tcx> FunctionCoverage<'tcx> { (counter_expressions, expression_regions.into_iter()) } - fn unreachable_regions<'a>(&'a self) -> impl Iterator<Item = (Counter, &'a CodeRegion)> { + fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) } diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index ab119ae25f5..9687fd09a53 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -53,14 +53,14 @@ fn push_debuginfo_type_name<'tcx>( ) { // When targeting MSVC, emit C++ style type names for compatibility with // .natvis visualizers (and perhaps other existing native debuggers?) - let cpp_like_names = cpp_like_names(tcx); + let cpp_like_debuginfo = cpp_like_debuginfo(tcx); match *t.kind() { ty::Bool => output.push_str("bool"), ty::Char => output.push_str("char"), ty::Str => output.push_str("str"), ty::Never => { - if cpp_like_names { + if cpp_like_debuginfo { output.push_str("never$"); } else { output.push('!'); @@ -71,7 +71,7 @@ fn push_debuginfo_type_name<'tcx>( ty::Float(float_ty) => output.push_str(float_ty.name_str()), ty::Foreign(def_id) => push_item_name(tcx, def_id, qualified, output), ty::Adt(def, substs) => { - if def.is_enum() && cpp_like_names { + if def.is_enum() && cpp_like_debuginfo { msvc_enum_fallback(tcx, t, def, substs, output, visited); } else { push_item_name(tcx, def.did, qualified, output); @@ -79,7 +79,7 @@ fn push_debuginfo_type_name<'tcx>( } } ty::Tuple(component_types) => { - if cpp_like_names { + if cpp_like_debuginfo { output.push_str("tuple$<"); } else { output.push('('); @@ -87,20 +87,20 @@ fn push_debuginfo_type_name<'tcx>( for component_type in component_types { push_debuginfo_type_name(tcx, component_type.expect_ty(), true, output, visited); - push_arg_separator(cpp_like_names, output); + push_arg_separator(cpp_like_debuginfo, output); } if !component_types.is_empty() { pop_arg_separator(output); } - if cpp_like_names { - push_close_angle_bracket(cpp_like_names, output); + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); } else { output.push(')'); } } ty::RawPtr(ty::TypeAndMut { ty: inner_type, mutbl }) => { - if cpp_like_names { + if cpp_like_debuginfo { match mutbl { hir::Mutability::Not => output.push_str("ptr_const$<"), hir::Mutability::Mut => output.push_str("ptr_mut$<"), @@ -115,8 +115,8 @@ fn push_debuginfo_type_name<'tcx>( push_debuginfo_type_name(tcx, inner_type, qualified, output, visited); - if cpp_like_names { - push_close_angle_bracket(cpp_like_names, output); + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); } } ty::Ref(_, inner_type, mutbl) => { @@ -126,7 +126,7 @@ fn push_debuginfo_type_name<'tcx>( // types out to aid debugging in MSVC. let is_slice_or_str = matches!(*inner_type.kind(), ty::Slice(_) | ty::Str); - if !cpp_like_names { + if !cpp_like_debuginfo { output.push('&'); output.push_str(mutbl.prefix_str()); } else if !is_slice_or_str { @@ -138,12 +138,12 @@ fn push_debuginfo_type_name<'tcx>( push_debuginfo_type_name(tcx, inner_type, qualified, output, visited); - if cpp_like_names && !is_slice_or_str { - push_close_angle_bracket(cpp_like_names, output); + if cpp_like_debuginfo && !is_slice_or_str { + push_close_angle_bracket(cpp_like_debuginfo, output); } } ty::Array(inner_type, len) => { - if cpp_like_names { + if cpp_like_debuginfo { output.push_str("array$<"); push_debuginfo_type_name(tcx, inner_type, true, output, visited); match len.val { @@ -162,7 +162,7 @@ fn push_debuginfo_type_name<'tcx>( } } ty::Slice(inner_type) => { - if cpp_like_names { + if cpp_like_debuginfo { output.push_str("slice$<"); } else { output.push('['); @@ -170,8 +170,8 @@ fn push_debuginfo_type_name<'tcx>( push_debuginfo_type_name(tcx, inner_type, true, output, visited); - if cpp_like_names { - push_close_angle_bracket(cpp_like_names, output); + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); } else { output.push(']'); } @@ -179,7 +179,7 @@ fn push_debuginfo_type_name<'tcx>( ty::Dynamic(ref trait_data, ..) => { let auto_traits: SmallVec<[DefId; 4]> = trait_data.auto_traits().collect(); - let has_enclosing_parens = if cpp_like_names { + let has_enclosing_parens = if cpp_like_debuginfo { output.push_str("dyn$<"); false } else { @@ -203,8 +203,9 @@ fn push_debuginfo_type_name<'tcx>( let projection_bounds: SmallVec<[_; 4]> = trait_data .projection_bounds() .map(|bound| { - let ExistentialProjection { item_def_id, ty, .. } = bound.skip_binder(); - (item_def_id, ty) + let ExistentialProjection { item_def_id, term, .. } = bound.skip_binder(); + // FIXME(associated_const_equality): allow for consts here + (item_def_id, term.ty().unwrap()) }) .collect(); @@ -216,14 +217,14 @@ fn push_debuginfo_type_name<'tcx>( } for (item_def_id, ty) in projection_bounds { - push_arg_separator(cpp_like_names, output); + push_arg_separator(cpp_like_debuginfo, output); - if cpp_like_names { + if cpp_like_debuginfo { output.push_str("assoc$<"); push_item_name(tcx, item_def_id, false, output); - push_arg_separator(cpp_like_names, output); + push_arg_separator(cpp_like_debuginfo, output); push_debuginfo_type_name(tcx, ty, true, output, visited); - push_close_angle_bracket(cpp_like_names, output); + push_close_angle_bracket(cpp_like_debuginfo, output); } else { push_item_name(tcx, item_def_id, false, output); output.push('='); @@ -231,11 +232,11 @@ fn push_debuginfo_type_name<'tcx>( } } - push_close_angle_bracket(cpp_like_names, output); + push_close_angle_bracket(cpp_like_debuginfo, output); } if auto_traits.len() != 0 { - push_auto_trait_separator(cpp_like_names, output); + push_auto_trait_separator(cpp_like_debuginfo, output); } } @@ -252,14 +253,14 @@ fn push_debuginfo_type_name<'tcx>( for auto_trait in auto_traits { output.push_str(&auto_trait); - push_auto_trait_separator(cpp_like_names, output); + push_auto_trait_separator(cpp_like_debuginfo, output); } pop_auto_trait_separator(output); } - if cpp_like_names { - push_close_angle_bracket(cpp_like_names, output); + if cpp_like_debuginfo { + push_close_angle_bracket(cpp_like_debuginfo, output); } else if has_enclosing_parens { output.push(')'); } @@ -279,7 +280,7 @@ fn push_debuginfo_type_name<'tcx>( // use a dummy string that should make it clear // that something unusual is going on if !visited.insert(t) { - output.push_str(if cpp_like_names { + output.push_str(if cpp_like_debuginfo { "recursive_type$" } else { "<recursive_type>" @@ -290,7 +291,7 @@ fn push_debuginfo_type_name<'tcx>( let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), t.fn_sig(tcx)); - if cpp_like_names { + if cpp_like_debuginfo { // Format as a C++ function pointer: return_type (*)(params...) if sig.output().is_unit() { output.push_str("void"); @@ -313,7 +314,7 @@ fn push_debuginfo_type_name<'tcx>( if !sig.inputs().is_empty() { for ¶meter_type in sig.inputs() { push_debuginfo_type_name(tcx, parameter_type, true, output, visited); - push_arg_separator(cpp_like_names, output); + push_arg_separator(cpp_like_debuginfo, output); } pop_arg_separator(output); } @@ -328,7 +329,7 @@ fn push_debuginfo_type_name<'tcx>( output.push(')'); - if !cpp_like_names && !sig.output().is_unit() { + if !cpp_like_debuginfo && !sig.output().is_unit() { output.push_str(" -> "); push_debuginfo_type_name(tcx, sig.output(), true, output, visited); } @@ -376,7 +377,7 @@ fn push_debuginfo_type_name<'tcx>( // format (natvis) is able to understand enums and render the active variant correctly in the // debugger. For more information, look in `src/etc/natvis/intrinsic.natvis` and // `EnumMemberDescriptionFactor::create_member_descriptions`. - fn msvc_enum_fallback( + fn msvc_enum_fallback<'tcx>( tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, def: &AdtDef, @@ -409,14 +410,14 @@ fn push_debuginfo_type_name<'tcx>( let max = dataful_discriminant_range.end; let max = tag.value.size(&tcx).truncate(max); - let dataful_variant_name = def.variants[*dataful_variant].ident.as_str(); + let dataful_variant_name = def.variants[*dataful_variant].name.as_str(); output.push_str(&format!(", {}, {}, {}", min, max, dataful_variant_name)); } else if let Variants::Single { index: variant_idx } = &layout.variants { // Uninhabited enums can't be constructed and should never need to be visualized so // skip this step for them. if def.variants.len() != 0 { - let variant = def.variants[*variant_idx].ident.as_str(); + let variant = def.variants[*variant_idx].name.as_str(); output.push_str(&format!(", {}", variant)); } @@ -426,9 +427,9 @@ fn push_debuginfo_type_name<'tcx>( const NON_CPP_AUTO_TRAIT_SEPARATOR: &str = " + "; - fn push_auto_trait_separator(cpp_like_names: bool, output: &mut String) { - if cpp_like_names { - push_arg_separator(cpp_like_names, output); + fn push_auto_trait_separator(cpp_like_debuginfo: bool, output: &mut String) { + if cpp_like_debuginfo { + push_arg_separator(cpp_like_debuginfo, output); } else { output.push_str(NON_CPP_AUTO_TRAIT_SEPARATOR); } @@ -457,11 +458,11 @@ pub fn compute_debuginfo_vtable_name<'tcx>( t: Ty<'tcx>, trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, ) -> String { - let cpp_like_names = cpp_like_names(tcx); + let cpp_like_debuginfo = cpp_like_debuginfo(tcx); let mut vtable_name = String::with_capacity(64); - if cpp_like_names { + if cpp_like_debuginfo { vtable_name.push_str("impl$<"); } else { vtable_name.push('<'); @@ -470,7 +471,7 @@ pub fn compute_debuginfo_vtable_name<'tcx>( let mut visited = FxHashSet::default(); push_debuginfo_type_name(tcx, t, true, &mut vtable_name, &mut visited); - if cpp_like_names { + if cpp_like_debuginfo { vtable_name.push_str(", "); } else { vtable_name.push_str(" as "); @@ -486,9 +487,9 @@ pub fn compute_debuginfo_vtable_name<'tcx>( vtable_name.push_str("_"); } - push_close_angle_bracket(cpp_like_names, &mut vtable_name); + push_close_angle_bracket(cpp_like_debuginfo, &mut vtable_name); - let suffix = if cpp_like_names { "::vtable$" } else { "::{vtable}" }; + let suffix = if cpp_like_debuginfo { "::vtable$" } else { "::{vtable}" }; vtable_name.reserve_exact(suffix.len()); vtable_name.push_str(suffix); @@ -496,7 +497,7 @@ pub fn compute_debuginfo_vtable_name<'tcx>( vtable_name } -pub fn push_item_name(tcx: TyCtxt<'tcx>, def_id: DefId, qualified: bool, output: &mut String) { +pub fn push_item_name(tcx: TyCtxt<'_>, def_id: DefId, qualified: bool, output: &mut String) { let def_key = tcx.def_key(def_id); if qualified { if let Some(parent) = def_key.parent { @@ -509,30 +510,36 @@ pub fn push_item_name(tcx: TyCtxt<'tcx>, def_id: DefId, qualified: bool, output: } fn push_unqualified_item_name( - tcx: TyCtxt<'tcx>, + tcx: TyCtxt<'_>, def_id: DefId, disambiguated_data: DisambiguatedDefPathData, output: &mut String, ) { match disambiguated_data.data { DefPathData::CrateRoot => { - output.push_str(&tcx.crate_name(def_id.krate).as_str()); + output.push_str(tcx.crate_name(def_id.krate).as_str()); } DefPathData::ClosureExpr if tcx.generator_kind(def_id).is_some() => { + let key = match tcx.generator_kind(def_id).unwrap() { + hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) => "async_block", + hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Closure) => "async_closure", + hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Fn) => "async_fn", + hir::GeneratorKind::Gen => "generator", + }; // Generators look like closures, but we want to treat them differently // in the debug info. - if cpp_like_names(tcx) { - write!(output, "generator${}", disambiguated_data.disambiguator).unwrap(); + if cpp_like_debuginfo(tcx) { + write!(output, "{}${}", key, disambiguated_data.disambiguator).unwrap(); } else { - write!(output, "{{generator#{}}}", disambiguated_data.disambiguator).unwrap(); + write!(output, "{{{}#{}}}", key, disambiguated_data.disambiguator).unwrap(); } } _ => match disambiguated_data.data.name() { DefPathDataName::Named(name) => { - output.push_str(&name.as_str()); + output.push_str(name.as_str()); } DefPathDataName::Anon { namespace } => { - if cpp_like_names(tcx) { + if cpp_like_debuginfo(tcx) { write!(output, "{}${}", namespace, disambiguated_data.disambiguator).unwrap(); } else { write!(output, "{{{}#{}}}", namespace, disambiguated_data.disambiguator) @@ -560,7 +567,7 @@ fn push_generic_params_internal<'tcx>( debug_assert_eq!(substs, tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs)); - let cpp_like_names = cpp_like_names(tcx); + let cpp_like_debuginfo = cpp_like_debuginfo(tcx); output.push('<'); @@ -575,10 +582,10 @@ fn push_generic_params_internal<'tcx>( other => bug!("Unexpected non-erasable generic: {:?}", other), } - push_arg_separator(cpp_like_names, output); + push_arg_separator(cpp_like_debuginfo, output); } pop_arg_separator(output); - push_close_angle_bracket(cpp_like_names, output); + push_close_angle_bracket(cpp_like_debuginfo, output); true } @@ -617,7 +624,7 @@ fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: &'tcx ty::Const<'tcx>, output: // avoiding collisions and will make the emitted type names shorter. let hash: u64 = hasher.finish(); - if cpp_like_names(tcx) { + if cpp_like_debuginfo(tcx) { write!(output, "CONST${:x}", hash) } else { write!(output, "{{CONST#{:x}}}", hash) @@ -634,10 +641,10 @@ pub fn push_generic_params<'tcx>(tcx: TyCtxt<'tcx>, substs: SubstsRef<'tcx>, out push_generic_params_internal(tcx, substs, output, &mut visited); } -fn push_close_angle_bracket(cpp_like_names: bool, output: &mut String) { +fn push_close_angle_bracket(cpp_like_debuginfo: bool, output: &mut String) { // MSVC debugger always treats `>>` as a shift, even when parsing templates, // so add a space to avoid confusion. - if cpp_like_names && output.ends_with('>') { + if cpp_like_debuginfo && output.ends_with('>') { output.push(' ') }; @@ -652,11 +659,11 @@ fn pop_close_angle_bracket(output: &mut String) { } } -fn push_arg_separator(cpp_like_names: bool, output: &mut String) { +fn push_arg_separator(cpp_like_debuginfo: bool, output: &mut String) { // Natvis does not always like having spaces between parts of the type name // and this causes issues when we need to write a typename in natvis, for example // as part of a cast like the `HashMap` visualizer does. - if cpp_like_names { + if cpp_like_debuginfo { output.push(','); } else { output.push_str(", "); @@ -673,6 +680,7 @@ fn pop_arg_separator(output: &mut String) { output.pop(); } -fn cpp_like_names(tcx: TyCtxt<'_>) -> bool { +/// Check if we should generate C++ like names and debug information. +pub fn cpp_like_debuginfo(tcx: TyCtxt<'_>) -> bool { tcx.sess.target.is_like_msvc } diff --git a/compiler/rustc_codegen_ssa/src/glue.rs b/compiler/rustc_codegen_ssa/src/glue.rs index cf217b52c86..2c4e6bbe9a5 100644 --- a/compiler/rustc_codegen_ssa/src/glue.rs +++ b/compiler/rustc_codegen_ssa/src/glue.rs @@ -6,6 +6,7 @@ use crate::common::IntPredicate; use crate::meth; use crate::traits::*; use rustc_middle::ty::{self, Ty}; +use rustc_target::abi::WrappingRange; pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, @@ -21,14 +22,17 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } match t.kind() { ty::Dynamic(..) => { - // load size/align from vtable + // Load size/align from vtable. let vtable = info.unwrap(); - ( - meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_SIZE) - .get_usize(bx, vtable), - meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) - .get_usize(bx, vtable), - ) + let size = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_SIZE) + .get_usize(bx, vtable); + let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) + .get_usize(bx, vtable); + + // Alignment is always nonzero. + bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + + (size, align) } ty::Slice(_) | ty::Str => { let unit = layout.field(bx, 0); diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 4c87d4d896e..350199f4e98 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -2,7 +2,6 @@ #![feature(bool_to_option)] #![feature(box_patterns)] #![feature(try_blocks)] -#![feature(in_band_lifetimes)] #![feature(let_else)] #![feature(once_cell)] #![feature(nll)] diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 8d75b2e7a3d..d768d4920c5 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -73,7 +73,7 @@ struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { locals: IndexVec<mir::Local, LocalKind>, } -impl<Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> { +impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> { fn assign(&mut self, local: mir::Local, location: Location) { let kind = &mut self.locals[local]; match *kind { @@ -275,9 +275,9 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi | TerminatorKind::SwitchInt { .. } | TerminatorKind::Yield { .. } | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } - | TerminatorKind::InlineAsm { .. } => { /* nothing to do */ } + | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ } TerminatorKind::Call { cleanup: unwind, .. } + | TerminatorKind::InlineAsm { cleanup: unwind, .. } | TerminatorKind::Assert { cleanup: unwind, .. } | TerminatorKind::DropAndReplace { unwind, .. } | TerminatorKind::Drop { unwind, .. } => { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index a9471f7b771..4c7a09ca1e9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -10,6 +10,7 @@ use crate::traits::*; use crate::MemFlags; use rustc_ast as ast; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::lang_items::LangItem; use rustc_index::vec::Idx; use rustc_middle::mir::AssertKind; @@ -159,11 +160,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { let llret = bx.call(fn_ty, fn_ptr, &llargs, self.funclet(fx)); bx.apply_attrs_callsite(&fn_abi, llret); if fx.mir[self.bb].is_cleanup { - // Cleanup is always the cold path. Don't inline - // drop glue. Also, when there is a deeply-nested - // struct, there are "symmetry" issues that cause - // exponential inlining - see issue #41696. - bx.do_not_inline(llret); + bx.apply_attrs_to_cleanup_callsite(llret); } if let Some((ret_dest, target)) = destination { @@ -174,6 +171,45 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } } } + + /// Generates inline assembly with optional `destination` and `cleanup`. + fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + bx: &mut Bx, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Bx>], + options: InlineAsmOptions, + line_spans: &[Span], + destination: Option<mir::BasicBlock>, + cleanup: Option<mir::BasicBlock>, + instance: Instance<'_>, + ) { + if let Some(cleanup) = cleanup { + let ret_llbb = if let Some(target) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + + bx.codegen_inline_asm( + template, + &operands, + options, + line_spans, + instance, + Some((ret_llbb, self.llblock(fx, cleanup), self.funclet(fx))), + ); + } else { + bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None); + + if let Some(target) = destination { + self.funclet_br(fx, bx, target); + } else { + bx.unreachable(); + } + } + } } /// Codegen implementations for some terminator variants. @@ -441,6 +477,28 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { helper.do_call(self, &mut bx, fn_abi, llfn, &args, None, cleanup); } + fn codegen_abort_terminator( + &mut self, + helper: TerminatorCodegenHelper<'tcx>, + mut bx: Bx, + terminator: &mir::Terminator<'tcx>, + ) { + let span = terminator.source_info.span; + self.set_debug_loc(&mut bx, terminator.source_info); + + // Get the location information. + let location = self.get_caller_location(&mut bx, terminator.source_info).immediate(); + + // Obtain the panic entry point. + let def_id = common::langcall(bx.tcx(), Some(span), "", LangItem::PanicNoUnwind); + let instance = ty::Instance::mono(bx.tcx(), def_id); + let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); + let llfn = bx.get_fn_addr(instance); + + // Codegen the actual panic invoke/call. + helper.do_call(self, &mut bx, fn_abi, llfn, &[location], None, None); + } + /// Returns `true` if this is indeed a panic intrinsic and codegen is done. fn codegen_panic_intrinsic( &mut self, @@ -877,6 +935,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { options: ast::InlineAsmOptions, line_spans: &[Span], destination: Option<mir::BasicBlock>, + cleanup: Option<mir::BasicBlock>, instance: Instance<'_>, ) { let span = terminator.source_info.span; @@ -931,13 +990,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }) .collect(); - bx.codegen_inline_asm(template, &operands, options, line_spans, instance); - - if let Some(target) = destination { - helper.funclet_br(self, &mut bx, target); - } else { - bx.unreachable(); - } + helper.do_inlineasm( + self, + &mut bx, + template, + &operands, + options, + line_spans, + destination, + cleanup, + instance, + ); } } @@ -973,24 +1036,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::TerminatorKind::Resume => self.codegen_resume_terminator(helper, bx), mir::TerminatorKind::Abort => { - bx.abort(); - // `abort` does not terminate the block, so we still need to generate - // an `unreachable` terminator after it. - bx.unreachable(); + self.codegen_abort_terminator(helper, bx, terminator); } mir::TerminatorKind::Goto { target } => { - if bb == target { - // This is an unconditional branch back to this same basic block. That means we - // have something like a `loop {}` statement. LLVM versions before 12.0 - // miscompile this because they assume forward progress. For older versions - // try to handle just this specific case which comes up commonly in practice - // (e.g., in embedded code). - // - // NB: the `sideeffect` currently checks for the LLVM version used internally. - bx.sideeffect(); - } - helper.funclet_br(self, &mut bx, target); } @@ -1052,6 +1101,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { options, line_spans, destination, + cleanup, } => { self.codegen_asm_terminator( helper, @@ -1062,6 +1112,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { options, line_spans, destination, + cleanup, self.instance, ); } @@ -1295,8 +1346,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mut cp_bx = self.new_block(&format!("cp_funclet{:?}", bb)); ret_llbb = cs_bx.llbb(); - let cs = cs_bx.catch_switch(None, None, 1); - cs_bx.add_handler(cs, cp_bx.llbb()); + let cs = cs_bx.catch_switch(None, None, &[cp_bx.llbb()]); // The "null" here is actually a RTTI type descriptor for the // C++ personality function, but `catch (...)` has no type so @@ -1323,8 +1373,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let llpersonality = self.cx.eh_personality(); let llretty = self.landing_pad_type(); - let lp = bx.landing_pad(llretty, llpersonality, 1); - bx.set_cleanup(lp); + let lp = bx.cleanup_landing_pad(llretty, llpersonality); let slot = self.get_personality_slot(&mut bx); slot.storage_live(&mut bx); @@ -1445,7 +1494,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"), LocalRef::Operand(None) => { let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst.as_ref())); - assert!(!dst_layout.ty.has_erasable_regions(self.cx.tcx())); + assert!(!dst_layout.ty.has_erasable_regions()); let place = PlaceRef::alloca(bx, dst_layout); place.storage_live(bx); self.codegen_transmute_into(bx, src, place); diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index f943157dc66..c654232c10a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -68,7 +68,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let arg_tys = sig.inputs(); let ret_ty = sig.output(); let name = bx.tcx().item_name(def_id); - let name_str = &*name.as_str(); + let name_str = name.as_str(); let llret_ty = bx.backend_type(bx.layout_of(ret_ty)); let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); @@ -369,13 +369,23 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + sym::const_allocate => { + // returns a null pointer at runtime. + bx.const_null(bx.type_i8p()) + } + + sym::const_deallocate => { + // nop at runtime. + return; + } + // This requires that atomic intrinsics follow a specific naming pattern: // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst name if name_str.starts_with("atomic_") => { use crate::common::AtomicOrdering::*; use crate::common::{AtomicRmwBinOp, SynchronizationScope}; - let split: Vec<&str> = name_str.split('_').collect(); + let split: Vec<_> = name_str.split('_').collect(); let is_cxchg = split[1] == "cxchg" || split[1] == "cxchgweak"; let (order, failorder) = match split.len() { diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 1ef863e84af..814e4d626e1 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -209,7 +209,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let mut allocate_local = |local| { let decl = &mir.local_decls[local]; let layout = bx.layout_of(fx.monomorphize(decl.ty)); - assert!(!layout.ty.has_erasable_regions(cx.tcx())); + assert!(!layout.ty.has_erasable_regions()); if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() { debug!("alloc: {:?} (return place) -> place", local); diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index bea55bbc879..0c526ff13f2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -47,7 +47,7 @@ pub struct OperandRef<'tcx, V> { pub layout: TyAndLayout<'tcx>, } -impl<V: CodegenObject> fmt::Debug for OperandRef<'tcx, V> { +impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout) } diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 4b07ed1a1e6..c21d19a6227 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -429,87 +429,78 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let cx = self.cx; let tcx = self.cx.tcx(); - let result = match place_ref { - mir::PlaceRef { local, projection: [] } => match self.locals[local] { - LocalRef::Place(place) => { - return place; - } - LocalRef::UnsizedPlace(place) => { - return bx.load_operand(place).deref(cx); - } - LocalRef::Operand(..) => { + let mut base = 0; + let mut cg_base = match self.locals[place_ref.local] { + LocalRef::Place(place) => place, + LocalRef::UnsizedPlace(place) => bx.load_operand(place).deref(cx), + LocalRef::Operand(..) => { + if let Some(elem) = place_ref + .projection + .iter() + .enumerate() + .find(|elem| matches!(elem.1, mir::ProjectionElem::Deref)) + { + base = elem.0 + 1; + self.codegen_consume( + bx, + mir::PlaceRef { projection: &place_ref.projection[..elem.0], ..place_ref }, + ) + .deref(bx.cx()) + } else { bug!("using operand local {:?} as place", place_ref); } - }, - mir::PlaceRef { local, projection: [proj_base @ .., mir::ProjectionElem::Deref] } => { - // Load the pointer from its location. - self.codegen_consume(bx, mir::PlaceRef { local, projection: proj_base }) - .deref(bx.cx()) } - mir::PlaceRef { local, projection: &[ref proj_base @ .., elem] } => { - // FIXME turn this recursion into iteration - let cg_base = - self.codegen_place(bx, mir::PlaceRef { local, projection: proj_base }); - - match elem { - mir::ProjectionElem::Deref => bug!(), - mir::ProjectionElem::Field(ref field, _) => { - cg_base.project_field(bx, field.index()) - } - mir::ProjectionElem::Index(index) => { - let index = &mir::Operand::Copy(mir::Place::from(index)); - let index = self.codegen_operand(bx, index); - let llindex = index.immediate(); - cg_base.project_index(bx, llindex) - } - mir::ProjectionElem::ConstantIndex { - offset, - from_end: false, - min_length: _, - } => { - let lloffset = bx.cx().const_usize(offset as u64); - cg_base.project_index(bx, lloffset) - } - mir::ProjectionElem::ConstantIndex { - offset, - from_end: true, - min_length: _, - } => { - let lloffset = bx.cx().const_usize(offset as u64); - let lllen = cg_base.len(bx.cx()); - let llindex = bx.sub(lllen, lloffset); - cg_base.project_index(bx, llindex) - } - mir::ProjectionElem::Subslice { from, to, from_end } => { - let mut subslice = - cg_base.project_index(bx, bx.cx().const_usize(from as u64)); - let projected_ty = - PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, elem).ty; - subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); - - if subslice.layout.is_unsized() { - assert!(from_end, "slice subslices should be `from_end`"); - subslice.llextra = Some(bx.sub( - cg_base.llextra.unwrap(), - bx.cx().const_usize((from as u64) + (to as u64)), - )); - } - - // Cast the place pointer type to the new - // array or slice type (`*[%_; new_len]`). - subslice.llval = bx.pointercast( - subslice.llval, - bx.cx().type_ptr_to(bx.cx().backend_type(subslice.layout)), - ); - - subslice + }; + for elem in place_ref.projection[base..].iter() { + cg_base = match elem.clone() { + mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()), + mir::ProjectionElem::Field(ref field, _) => { + cg_base.project_field(bx, field.index()) + } + mir::ProjectionElem::Index(index) => { + let index = &mir::Operand::Copy(mir::Place::from(index)); + let index = self.codegen_operand(bx, index); + let llindex = index.immediate(); + cg_base.project_index(bx, llindex) + } + mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => { + let lloffset = bx.cx().const_usize(offset as u64); + cg_base.project_index(bx, lloffset) + } + mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => { + let lloffset = bx.cx().const_usize(offset as u64); + let lllen = cg_base.len(bx.cx()); + let llindex = bx.sub(lllen, lloffset); + cg_base.project_index(bx, llindex) + } + mir::ProjectionElem::Subslice { from, to, from_end } => { + let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from as u64)); + let projected_ty = + PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, elem.clone()).ty; + subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); + + if subslice.layout.is_unsized() { + assert!(from_end, "slice subslices should be `from_end`"); + subslice.llextra = Some(bx.sub( + cg_base.llextra.unwrap(), + bx.cx().const_usize((from as u64) + (to as u64)), + )); } - mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v), + + // Cast the place pointer type to the new + // array or slice type (`*[%_; new_len]`). + subslice.llval = bx.pointercast( + subslice.llval, + bx.cx().type_ptr_to(bx.cx().backend_type(subslice.layout)), + ); + + subslice } - } - }; - debug!("codegen_place(place={:?}) => {:?}", place_ref, result); - result + mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v), + }; + } + debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base); + cg_base } pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index f087b9f7815..68decce82ab 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -3,12 +3,10 @@ use super::place::PlaceRef; use super::{FunctionCx, LocalRef}; use crate::base; -use crate::common::{self, IntPredicate, RealPredicate}; +use crate::common::{self, IntPredicate}; use crate::traits::*; use crate::MemFlags; -use rustc_apfloat::{ieee, Float, Round, Status}; -use rustc_hir::lang_items::LangItem; use rustc_middle::mir; use rustc_middle::ty::cast::{CastTy, IntTy}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; @@ -112,9 +110,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Aggregate(ref kind, ref operands) => { let (dest, active_field_index) = match **kind { - mir::AggregateKind::Adt(adt_def, variant_index, _, _, active_field_index) => { + mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => { dest.codegen_set_discr(&mut bx, variant_index); - if adt_def.is_enum() { + if bx.tcx().adt_def(adt_did).is_enum() { (dest.project_downcast(&mut bx, variant_index), active_field_index) } else { (dest, active_field_index) @@ -369,10 +367,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.inttoptr(usize_llval, ll_t_out) } (CastTy::Float, CastTy::Int(IntTy::I)) => { - cast_float_to_int(&mut bx, true, llval, ll_t_in, ll_t_out) + bx.cast_float_to_int(true, llval, ll_t_out) } (CastTy::Float, CastTy::Int(_)) => { - cast_float_to_int(&mut bx, false, llval, ll_t_in, ll_t_out) + bx.cast_float_to_int(false, llval, ll_t_out) } _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty), }; @@ -486,31 +484,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ) } - mir::Rvalue::NullaryOp(mir::NullOp::Box, content_ty) => { - let content_ty = self.monomorphize(content_ty); - let content_layout = bx.cx().layout_of(content_ty); - let llsize = bx.cx().const_usize(content_layout.size.bytes()); - let llalign = bx.cx().const_usize(content_layout.align.abi.bytes()); - let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty)); - let llty_ptr = bx.cx().backend_type(box_layout); - - // Allocate space: - let def_id = match bx.tcx().lang_items().require(LangItem::ExchangeMalloc) { - Ok(id) => id, - Err(s) => { - bx.cx().sess().fatal(&format!("allocation of `{}` {}", box_layout.ty, s)); - } - }; - let instance = ty::Instance::mono(bx.tcx(), def_id); - let r = bx.cx().get_fn_addr(instance); - let ty = bx.type_func(&[bx.type_isize(), bx.type_isize()], bx.type_i8p()); - let call = bx.call(ty, r, &[llsize, llalign], None); - let val = bx.pointercast(call, llty_ptr); - - let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout }; - (bx, operand) - } - mir::Rvalue::NullaryOp(null_op, ty) => { let ty = self.monomorphize(ty); assert!(bx.cx().type_is_sized(ty)); @@ -518,7 +491,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let val = match null_op { mir::NullOp::SizeOf => layout.size.bytes(), mir::NullOp::AlignOf => layout.align.abi.bytes(), - mir::NullOp::Box => unreachable!(), }; let val = bx.cx().const_usize(val); let tcx = self.cx.tcx(); @@ -795,146 +767,3 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // (*) this is only true if the type is suitable } } - -fn cast_float_to_int<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - signed: bool, - x: Bx::Value, - float_ty: Bx::Type, - int_ty: Bx::Type, -) -> Bx::Value { - if let Some(false) = bx.cx().sess().opts.debugging_opts.saturating_float_casts { - return if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) }; - } - - let try_sat_result = if signed { bx.fptosi_sat(x, int_ty) } else { bx.fptoui_sat(x, int_ty) }; - if let Some(try_sat_result) = try_sat_result { - return try_sat_result; - } - - let int_width = bx.cx().int_width(int_ty); - let float_width = bx.cx().float_width(float_ty); - // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the - // destination integer type after rounding towards zero. This `undef` value can cause UB in - // safe code (see issue #10184), so we implement a saturating conversion on top of it: - // Semantically, the mathematical value of the input is rounded towards zero to the next - // mathematical integer, and then the result is clamped into the range of the destination - // integer type. Positive and negative infinity are mapped to the maximum and minimum value of - // the destination integer type. NaN is mapped to 0. - // - // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to - // a value representable in int_ty. - // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits. - // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two. - // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly - // representable. Note that this only works if float_ty's exponent range is sufficiently large. - // f16 or 256 bit integers would break this property. Right now the smallest float type is f32 - // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127. - // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because - // we're rounding towards zero, we just get float_ty::MAX (which is always an integer). - // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX. - let int_max = |signed: bool, int_width: u64| -> u128 { - let shift_amount = 128 - int_width; - if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount } - }; - let int_min = |signed: bool, int_width: u64| -> i128 { - if signed { i128::MIN >> (128 - int_width) } else { 0 } - }; - - let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) { - let rounded_min = ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero); - assert_eq!(rounded_min.status, Status::OK); - let rounded_max = ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero); - assert!(rounded_max.value.is_finite()); - (rounded_min.value.to_bits(), rounded_max.value.to_bits()) - }; - let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) { - let rounded_min = ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero); - assert_eq!(rounded_min.status, Status::OK); - let rounded_max = ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero); - assert!(rounded_max.value.is_finite()); - (rounded_min.value.to_bits(), rounded_max.value.to_bits()) - }; - - let mut float_bits_to_llval = |bits| { - let bits_llval = match float_width { - 32 => bx.cx().const_u32(bits as u32), - 64 => bx.cx().const_u64(bits as u64), - n => bug!("unsupported float width {}", n), - }; - bx.bitcast(bits_llval, float_ty) - }; - let (f_min, f_max) = match float_width { - 32 => compute_clamp_bounds_single(signed, int_width), - 64 => compute_clamp_bounds_double(signed, int_width), - n => bug!("unsupported float width {}", n), - }; - let f_min = float_bits_to_llval(f_min); - let f_max = float_bits_to_llval(f_max); - // To implement saturation, we perform the following steps: - // - // 1. Cast x to an integer with fpto[su]i. This may result in undef. - // 2. Compare x to f_min and f_max, and use the comparison results to select: - // a) int_ty::MIN if x < f_min or x is NaN - // b) int_ty::MAX if x > f_max - // c) the result of fpto[su]i otherwise - // 3. If x is NaN, return 0.0, otherwise return the result of step 2. - // - // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the - // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of - // undef does not introduce any non-determinism either. - // More importantly, the above procedure correctly implements saturating conversion. - // Proof (sketch): - // If x is NaN, 0 is returned by definition. - // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max. - // This yields three cases to consider: - // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with - // saturating conversion for inputs in that range. - // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded - // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger - // than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX - // is correct. - // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals - // int_ty::MIN and therefore the return value of int_ty::MIN is correct. - // QED. - - let int_max = bx.cx().const_uint_big(int_ty, int_max(signed, int_width)); - let int_min = bx.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128); - let zero = bx.cx().const_uint(int_ty, 0); - - // Step 1 ... - let fptosui_result = if signed { bx.fptosi(x, int_ty) } else { bx.fptoui(x, int_ty) }; - let less_or_nan = bx.fcmp(RealPredicate::RealULT, x, f_min); - let greater = bx.fcmp(RealPredicate::RealOGT, x, f_max); - - // Step 2: We use two comparisons and two selects, with %s1 being the - // result: - // %less_or_nan = fcmp ult %x, %f_min - // %greater = fcmp olt %x, %f_max - // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result - // %s1 = select %greater, int_ty::MAX, %s0 - // Note that %less_or_nan uses an *unordered* comparison. This - // comparison is true if the operands are not comparable (i.e., if x is - // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if - // x is NaN. - // - // Performance note: Unordered comparison can be lowered to a "flipped" - // comparison and a negation, and the negation can be merged into the - // select. Therefore, it not necessarily any more expensive than an - // ordered ("normal") comparison. Whether these optimizations will be - // performed is ultimately up to the backend, but at least x86 does - // perform them. - let s0 = bx.select(less_or_nan, int_min, fptosui_result); - let s1 = bx.select(greater, int_max, s0); - - // Step 3: NaN replacement. - // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN. - // Therefore we only need to execute this step for signed integer types. - if signed { - // LLVM has no isNaN predicate, so we use (x == x) instead - let cmp = bx.fcmp(RealPredicate::RealOEQ, x, x); - bx.select(cmp, s1, zero) - } else { - s1 - } -} diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 2c96987d339..773dc2adcfa 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -1,9 +1,7 @@ -use rustc_errors::struct_span_err; use rustc_middle::mir; use super::FunctionCx; use super::LocalRef; -use super::OperandValue; use crate::traits::BuilderMethods; use crate::traits::*; @@ -66,51 +64,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } bx } - mir::StatementKind::LlvmInlineAsm(ref asm) => { - let outputs = asm - .outputs - .iter() - .map(|output| self.codegen_place(&mut bx, output.as_ref())) - .collect(); - - let input_vals = asm.inputs.iter().fold( - Vec::with_capacity(asm.inputs.len()), - |mut acc, (span, input)| { - let op = self.codegen_operand(&mut bx, input); - if let OperandValue::Immediate(_) = op.val { - acc.push(op.immediate()); - } else { - struct_span_err!( - bx.sess(), - span.to_owned(), - E0669, - "invalid value for constraint in inline assembly" - ) - .emit(); - } - acc - }, - ); - - if input_vals.len() == asm.inputs.len() { - let res = bx.codegen_llvm_inline_asm( - &asm.asm, - outputs, - input_vals, - statement.source_info.span, - ); - if !res { - struct_span_err!( - bx.sess(), - statement.source_info.span, - E0668, - "malformed inline assembly" - ) - .emit(); - } - } - bx - } mir::StatementKind::Coverage(box ref coverage) => { self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); bx diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index caeeb23feb4..63cc6faf9ec 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -20,6 +20,7 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ ("aes", Some(sym::arm_target_feature)), ("sha2", Some(sym::arm_target_feature)), ("i8mm", Some(sym::arm_target_feature)), + ("dotprod", Some(sym::arm_target_feature)), ("v5te", Some(sym::arm_target_feature)), ("v6", Some(sym::arm_target_feature)), ("v6k", Some(sym::arm_target_feature)), @@ -34,6 +35,8 @@ const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ // since it should be enabled per-function using #[instruction_set], not // #[target_feature]. ("thumb-mode", Some(sym::arm_target_feature)), + ("thumb2", Some(sym::arm_target_feature)), + ("reserve-r9", Some(sym::arm_target_feature)), ]; const AARCH64_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ diff --git a/compiler/rustc_codegen_ssa/src/traits/abi.rs b/compiler/rustc_codegen_ssa/src/traits/abi.rs index dd8495850bd..a00d78daf4d 100644 --- a/compiler/rustc_codegen_ssa/src/traits/abi.rs +++ b/compiler/rustc_codegen_ssa/src/traits/abi.rs @@ -4,5 +4,5 @@ use rustc_target::abi::call::FnAbi; pub trait AbiBuilderMethods<'tcx>: BackendTypes { fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value); - fn get_param(&self, index: usize) -> Self::Value; + fn get_param(&mut self, index: usize) -> Self::Value; } diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index 31f539e1b03..11111a79744 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -3,7 +3,6 @@ use crate::mir::operand::OperandRef; use crate::mir::place::PlaceRef; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::def_id::DefId; -use rustc_hir::LlvmInlineAsmInner; use rustc_middle::ty::Instance; use rustc_span::Span; use rustc_target::asm::InlineAsmRegOrRegClass; @@ -43,15 +42,6 @@ pub enum GlobalAsmOperandRef { pub trait AsmBuilderMethods<'tcx>: BackendTypes { /// Take an inline assembly expression and splat it out via LLVM - fn codegen_llvm_inline_asm( - &mut self, - ia: &LlvmInlineAsmInner, - outputs: Vec<PlaceRef<'tcx, Self::Value>>, - inputs: Vec<Self::Value>, - span: Span, - ) -> bool; - - /// Take an inline assembly expression and splat it out via LLVM fn codegen_inline_asm( &mut self, template: &[InlineAsmTemplatePiece], @@ -59,6 +49,7 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, + dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, ); } diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index 9c8bc3b2109..c6abb3f6eb4 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -97,6 +97,7 @@ pub trait CodegenBackend { &self, ongoing_codegen: Box<dyn Any>, sess: &Session, + outputs: &OutputFilenames, ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorReported>; /// This is called on the returned `Box<dyn Any>` from `join_codegen` @@ -114,12 +115,6 @@ pub trait CodegenBackend { pub trait ExtraBackendMethods: CodegenBackend + WriteBackendMethods + Sized + Send + Sync { fn new_metadata(&self, sess: TyCtxt<'_>, mod_name: &str) -> Self::Module; - fn write_compressed_metadata<'tcx>( - &self, - tcx: TyCtxt<'tcx>, - metadata: &EncodedMetadata, - llvm_module: &mut Self::Module, - ); fn codegen_allocator<'tcx>( &self, tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 158e658301e..53fb21b269a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -1,18 +1,21 @@ use super::abi::AbiBuilderMethods; use super::asm::AsmBuilderMethods; +use super::consts::ConstMethods; use super::coverageinfo::CoverageInfoBuilderMethods; use super::debuginfo::DebugInfoBuilderMethods; use super::intrinsic::IntrinsicCallMethods; -use super::type_::ArgAbiMethods; +use super::misc::MiscMethods; +use super::type_::{ArgAbiMethods, BaseTypeMethods}; use super::{HasCodegen, StaticBuilderMethods}; use crate::common::{ - AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, + AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind, }; use crate::mir::operand::OperandRef; use crate::mir::place::PlaceRef; use crate::MemFlags; +use rustc_apfloat::{ieee, Float, Round, Status}; use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout}; use rustc_middle::ty::Ty; use rustc_span::Span; @@ -202,6 +205,179 @@ pub trait BuilderMethods<'a, 'tcx>: fn intcast(&mut self, val: Self::Value, dest_ty: Self::Type, is_signed: bool) -> Self::Value; fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + fn cast_float_to_int( + &mut self, + signed: bool, + x: Self::Value, + dest_ty: Self::Type, + ) -> Self::Value { + let in_ty = self.cx().val_ty(x); + let (float_ty, int_ty) = if self.cx().type_kind(dest_ty) == TypeKind::Vector + && self.cx().type_kind(in_ty) == TypeKind::Vector + { + (self.cx().element_type(in_ty), self.cx().element_type(dest_ty)) + } else { + (in_ty, dest_ty) + }; + assert!(matches!(self.cx().type_kind(float_ty), TypeKind::Float | TypeKind::Double)); + assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer); + + if let Some(false) = self.cx().sess().opts.debugging_opts.saturating_float_casts { + return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) }; + } + + let try_sat_result = + if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }; + if let Some(try_sat_result) = try_sat_result { + return try_sat_result; + } + + let int_width = self.cx().int_width(int_ty); + let float_width = self.cx().float_width(float_ty); + // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the + // destination integer type after rounding towards zero. This `undef` value can cause UB in + // safe code (see issue #10184), so we implement a saturating conversion on top of it: + // Semantically, the mathematical value of the input is rounded towards zero to the next + // mathematical integer, and then the result is clamped into the range of the destination + // integer type. Positive and negative infinity are mapped to the maximum and minimum value of + // the destination integer type. NaN is mapped to 0. + // + // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to + // a value representable in int_ty. + // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits. + // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two. + // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly + // representable. Note that this only works if float_ty's exponent range is sufficiently large. + // f16 or 256 bit integers would break this property. Right now the smallest float type is f32 + // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127. + // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because + // we're rounding towards zero, we just get float_ty::MAX (which is always an integer). + // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX. + let int_max = |signed: bool, int_width: u64| -> u128 { + let shift_amount = 128 - int_width; + if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount } + }; + let int_min = |signed: bool, int_width: u64| -> i128 { + if signed { i128::MIN >> (128 - int_width) } else { 0 } + }; + + let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) { + let rounded_min = + ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero); + assert_eq!(rounded_min.status, Status::OK); + let rounded_max = + ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero); + assert!(rounded_max.value.is_finite()); + (rounded_min.value.to_bits(), rounded_max.value.to_bits()) + }; + let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) { + let rounded_min = + ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero); + assert_eq!(rounded_min.status, Status::OK); + let rounded_max = + ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero); + assert!(rounded_max.value.is_finite()); + (rounded_min.value.to_bits(), rounded_max.value.to_bits()) + }; + // To implement saturation, we perform the following steps: + // + // 1. Cast x to an integer with fpto[su]i. This may result in undef. + // 2. Compare x to f_min and f_max, and use the comparison results to select: + // a) int_ty::MIN if x < f_min or x is NaN + // b) int_ty::MAX if x > f_max + // c) the result of fpto[su]i otherwise + // 3. If x is NaN, return 0.0, otherwise return the result of step 2. + // + // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the + // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of + // undef does not introduce any non-determinism either. + // More importantly, the above procedure correctly implements saturating conversion. + // Proof (sketch): + // If x is NaN, 0 is returned by definition. + // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max. + // This yields three cases to consider: + // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with + // saturating conversion for inputs in that range. + // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded + // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger + // than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX + // is correct. + // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals + // int_ty::MIN and therefore the return value of int_ty::MIN is correct. + // QED. + + let float_bits_to_llval = |bx: &mut Self, bits| { + let bits_llval = match float_width { + 32 => bx.cx().const_u32(bits as u32), + 64 => bx.cx().const_u64(bits as u64), + n => bug!("unsupported float width {}", n), + }; + bx.bitcast(bits_llval, float_ty) + }; + let (f_min, f_max) = match float_width { + 32 => compute_clamp_bounds_single(signed, int_width), + 64 => compute_clamp_bounds_double(signed, int_width), + n => bug!("unsupported float width {}", n), + }; + let f_min = float_bits_to_llval(self, f_min); + let f_max = float_bits_to_llval(self, f_max); + let int_max = self.cx().const_uint_big(int_ty, int_max(signed, int_width)); + let int_min = self.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128); + let zero = self.cx().const_uint(int_ty, 0); + + // If we're working with vectors, constants must be "splatted": the constant is duplicated + // into each lane of the vector. The algorithm stays the same, we are just using the + // same constant across all lanes. + let maybe_splat = |bx: &mut Self, val| { + if bx.cx().type_kind(dest_ty) == TypeKind::Vector { + bx.vector_splat(bx.vector_length(dest_ty), val) + } else { + val + } + }; + let f_min = maybe_splat(self, f_min); + let f_max = maybe_splat(self, f_max); + let int_max = maybe_splat(self, int_max); + let int_min = maybe_splat(self, int_min); + let zero = maybe_splat(self, zero); + + // Step 1 ... + let fptosui_result = if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) }; + let less_or_nan = self.fcmp(RealPredicate::RealULT, x, f_min); + let greater = self.fcmp(RealPredicate::RealOGT, x, f_max); + + // Step 2: We use two comparisons and two selects, with %s1 being the + // result: + // %less_or_nan = fcmp ult %x, %f_min + // %greater = fcmp olt %x, %f_max + // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result + // %s1 = select %greater, int_ty::MAX, %s0 + // Note that %less_or_nan uses an *unordered* comparison. This + // comparison is true if the operands are not comparable (i.e., if x is + // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if + // x is NaN. + // + // Performance note: Unordered comparison can be lowered to a "flipped" + // comparison and a negation, and the negation can be merged into the + // select. Therefore, it not necessarily any more expensive than an + // ordered ("normal") comparison. Whether these optimizations will be + // performed is ultimately up to the backend, but at least x86 does + // perform them. + let s0 = self.select(less_or_nan, int_min, fptosui_result); + let s1 = self.select(greater, int_max, s0); + + // Step 3: NaN replacement. + // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN. + // Therefore we only need to execute this step for signed integer types. + if signed { + // LLVM has no isNaN predicate, so we use (x == x) instead + let cmp = self.fcmp(RealPredicate::RealOEQ, x, x); + self.select(cmp, s1, zero) + } else { + s1 + } + } + fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; @@ -245,29 +421,22 @@ pub trait BuilderMethods<'a, 'tcx>: fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value; fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value; - fn landing_pad( - &mut self, - ty: Self::Type, - pers_fn: Self::Value, - num_clauses: usize, - ) -> Self::Value; - fn set_cleanup(&mut self, landing_pad: Self::Value); - fn resume(&mut self, exn: Self::Value) -> Self::Value; + fn set_personality_fn(&mut self, personality: Self::Value); + + // These are used by everyone except msvc + fn cleanup_landing_pad(&mut self, ty: Self::Type, pers_fn: Self::Value) -> Self::Value; + fn resume(&mut self, exn: Self::Value); + + // These are used only by msvc fn cleanup_pad(&mut self, parent: Option<Self::Value>, args: &[Self::Value]) -> Self::Funclet; - fn cleanup_ret( - &mut self, - funclet: &Self::Funclet, - unwind: Option<Self::BasicBlock>, - ) -> Self::Value; + fn cleanup_ret(&mut self, funclet: &Self::Funclet, unwind: Option<Self::BasicBlock>); fn catch_pad(&mut self, parent: Self::Value, args: &[Self::Value]) -> Self::Funclet; fn catch_switch( &mut self, parent: Option<Self::Value>, unwind: Option<Self::BasicBlock>, - num_handlers: usize, + handlers: &[Self::BasicBlock], ) -> Self::Value; - fn add_handler(&mut self, catch_switch: Self::Value, handler: Self::BasicBlock); - fn set_personality_fn(&mut self, personality: Self::Value); fn atomic_cmpxchg( &mut self, @@ -311,5 +480,5 @@ pub trait BuilderMethods<'a, 'tcx>: ) -> Self::Value; fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn do_not_inline(&mut self, llret: Self::Value); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value); } diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index 78bf22ef9f2..02be6cd360c 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -20,10 +20,6 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes { fn abort(&mut self); fn assume(&mut self, val: Self::Value); fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; - /// Emits a forced side effect. - /// - /// Currently has any effect only when LLVM versions prior to 12.0 are used as the backend. - fn sideeffect(&mut self); /// Trait method used to test whether a given pointer is associated with a type identifier. fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value; /// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index b94fb1e10db..5d3f07317a3 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -97,7 +97,7 @@ pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> { } } -impl<T> DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {} +impl<'tcx, T> DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {} pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type; @@ -135,4 +135,4 @@ pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> { pub trait TypeMethods<'tcx>: DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> {} -impl<T> TypeMethods<'tcx> for T where Self: DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> {} +impl<'tcx, T> TypeMethods<'tcx> for T where Self: DerivedTypeMethods<'tcx> + LayoutTypeMethods<'tcx> {} |