diff options
| author | Laurențiu Nicola <lnicola@dend.ro> | 2025-02-10 07:49:43 +0200 |
|---|---|---|
| committer | Laurențiu Nicola <lnicola@dend.ro> | 2025-02-10 07:49:43 +0200 |
| commit | 15cd1f011c4f854f7a7cfbe2e727cc4b02b1d254 (patch) | |
| tree | bdd9d79bd54ffe237db9c987d6c1e2ab12349e16 /compiler/rustc_codegen_ssa/src | |
| parent | 879dc387cde7ac4b85aa5a185158c9c3c488c5e7 (diff) | |
| parent | d9a4a47b8b3dc0bdff83360cea2013200d60d49c (diff) | |
| download | rust-15cd1f011c4f854f7a7cfbe2e727cc4b02b1d254.tar.gz rust-15cd1f011c4f854f7a7cfbe2e727cc4b02b1d254.zip | |
Merge from rust-lang/rust
Diffstat (limited to 'compiler/rustc_codegen_ssa/src')
29 files changed, 847 insertions, 274 deletions
diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs index ab65319e3d3..27331ce4ca6 100644 --- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs +++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs @@ -46,8 +46,12 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>, set_reuse: &dyn Fn(&mut CguReuseTr return; } - let available_cgus = - tcx.collect_and_partition_mono_items(()).1.iter().map(|cgu| cgu.name()).collect(); + let available_cgus = tcx + .collect_and_partition_mono_items(()) + .codegen_units + .iter() + .map(|cgu| cgu.name()) + .collect(); let mut ams = AssertModuleSource { tcx, diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs index 58eb137c068..60af462b6b6 100644 --- a/compiler/rustc_codegen_ssa/src/back/archive.rs +++ b/compiler/rustc_codegen_ssa/src/back/archive.rs @@ -414,10 +414,10 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> { let member_path = archive_path.parent().unwrap().join(Path::new(&file_name)); self.entries.push((file_name.into_bytes(), ArchiveEntry::File(member_path))); } else { - self.entries.push((file_name.into_bytes(), ArchiveEntry::FromArchive { - archive_index, - file_range: entry.file_range(), - })); + self.entries.push(( + file_name.into_bytes(), + ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() }, + )); } } } diff --git a/compiler/rustc_codegen_ssa/src/back/command.rs b/compiler/rustc_codegen_ssa/src/back/command.rs index b3c5b86ccf4..63023fdba20 100644 --- a/compiler/rustc_codegen_ssa/src/back/command.rs +++ b/compiler/rustc_codegen_ssa/src/back/command.rs @@ -13,6 +13,7 @@ pub(crate) struct Command { args: Vec<OsString>, env: Vec<(OsString, OsString)>, env_remove: Vec<OsString>, + env_clear: bool, } #[derive(Clone)] @@ -36,7 +37,13 @@ impl Command { } fn _new(program: Program) -> Command { - Command { program, args: Vec::new(), env: Vec::new(), env_remove: Vec::new() } + Command { + program, + args: Vec::new(), + env: Vec::new(), + env_remove: Vec::new(), + env_clear: false, + } } pub(crate) fn arg<P: AsRef<OsStr>>(&mut self, arg: P) -> &mut Command { @@ -79,6 +86,11 @@ impl Command { self } + pub(crate) fn env_clear(&mut self) -> &mut Command { + self.env_clear = true; + self + } + fn _env_remove(&mut self, key: &OsStr) { self.env_remove.push(key.to_owned()); } @@ -106,6 +118,9 @@ impl Command { for k in &self.env_remove { ret.env_remove(k); } + if self.env_clear { + ret.env_clear(); + } ret } diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index df35b5e8426..105a4cb81f0 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -15,12 +15,14 @@ use rustc_ast::CRATE_NODE_ID; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{DiagCtxtHandle, LintDiagnostic}; use rustc_fs_util::{fix_windows_verbatim_for_gcc, try_canonicalize}; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; +use rustc_macros::LintDiagnostic; use rustc_metadata::fs::{METADATA_FILENAME, copy_to_stdout, emit_wrapper_file}; use rustc_metadata::{find_native_static_library, walk_native_lib_search_dirs}; use rustc_middle::bug; +use rustc_middle::lint::lint_level; use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::SymbolExportKind; @@ -29,6 +31,7 @@ use rustc_session::config::{ OutputType, PrintKind, SplitDwarfKind, Strip, }; use rustc_session::cstore::DllImport; +use rustc_session::lint::builtin::LINKER_MESSAGES; use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; use rustc_session::search_paths::PathKind; use rustc_session::utils::NativeLibKind; @@ -749,6 +752,14 @@ fn link_dwarf_object(sess: &Session, cg_results: &CodegenResults, executable_out } } +#[derive(LintDiagnostic)] +#[diag(codegen_ssa_linker_output)] +/// Translating this is kind of useless. We don't pass translation flags to the linker, so we'd just +/// end up with inconsistent languages within the same diagnostic. +struct LinkerOutput { + inner: String, +} + /// Create a dynamic library or executable. /// /// This will invoke the system linker/cc to create the resulting file. This links to all upstream @@ -981,6 +992,11 @@ fn link_natively( match prog { Ok(prog) => { + let is_msvc_link_exe = sess.target.is_like_msvc + && flavor == LinkerFlavor::Msvc(Lld::No) + // Match exactly "link.exe" + && linker_path.to_str() == Some("link.exe"); + if !prog.status.success() { let mut output = prog.stderr.clone(); output.extend_from_slice(&prog.stdout); @@ -991,22 +1007,16 @@ fn link_natively( command: cmd, escaped_output, verbose: sess.opts.verbose, + sysroot_dir: sess.sysroot.clone(), }; sess.dcx().emit_err(err); // If MSVC's `link.exe` was expected but the return code // is not a Microsoft LNK error then suggest a way to fix or // install the Visual Studio build tools. if let Some(code) = prog.status.code() { - if sess.target.is_like_msvc - && flavor == LinkerFlavor::Msvc(Lld::No) - // Respect the command line override - && sess.opts.cg.linker.is_none() - // Match exactly "link.exe" - && linker_path.to_str() == Some("link.exe") - // All Microsoft `link.exe` linking error codes are - // four digit numbers in the range 1000 to 9999 inclusive - && (code < 1000 || code > 9999) - { + // All Microsoft `link.exe` linking ror codes are + // four digit numbers in the range 1000 to 9999 inclusive + if is_msvc_link_exe && (code < 1000 || code > 9999) { let is_vs_installed = windows_registry::find_vs_version().is_ok(); let has_linker = windows_registry::find_tool(&sess.target.arch, "link.exe").is_some(); @@ -1028,8 +1038,49 @@ fn link_natively( sess.dcx().abort_if_errors(); } - info!("linker stderr:\n{}", escape_string(&prog.stderr)); - info!("linker stdout:\n{}", escape_string(&prog.stdout)); + + let stderr = escape_string(&prog.stderr); + let mut stdout = escape_string(&prog.stdout); + info!("linker stderr:\n{}", &stderr); + info!("linker stdout:\n{}", &stdout); + + // Hide some progress messages from link.exe that we don't care about. + // See https://github.com/chromium/chromium/blob/bfa41e41145ffc85f041384280caf2949bb7bd72/build/toolchain/win/tool_wrapper.py#L144-L146 + if is_msvc_link_exe { + if let Ok(str) = str::from_utf8(&prog.stdout) { + let mut output = String::with_capacity(str.len()); + for line in stdout.lines() { + if line.starts_with(" Creating library") + || line.starts_with("Generating code") + || line.starts_with("Finished generating code") + { + continue; + } + output += line; + output += "\r\n" + } + stdout = escape_string(output.trim().as_bytes()) + } + } + + let (level, src) = codegen_results.crate_info.lint_levels.linker_messages; + let lint = |msg| { + lint_level(sess, LINKER_MESSAGES, level, src, None, |diag| { + LinkerOutput { inner: msg }.decorate_lint(diag) + }) + }; + + if !prog.stderr.is_empty() { + // We already print `warning:` at the start of the diagnostic. Remove it from the linker output if present. + let stderr = stderr + .strip_prefix("warning: ") + .unwrap_or(&stderr) + .replace(": warning: ", ": "); + lint(format!("linker stderr: {stderr}")); + } + if !stdout.is_empty() { + lint(format!("linker stdout: {}", stdout)) + } } Err(e) => { let linker_not_found = e.kind() == io::ErrorKind::NotFound; diff --git a/compiler/rustc_codegen_ssa/src/back/linker/tests.rs b/compiler/rustc_codegen_ssa/src/back/linker/tests.rs index bf3e8c90200..143df085dbf 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker/tests.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker/tests.rs @@ -10,23 +10,22 @@ fn test_rpaths_to_args() { #[test] fn test_xlinker() { let mut cmd = Command::new("foo"); - convert_link_args_to_cc_args(&mut cmd, &[ - "arg1", - "arg2", - "arg3,with,comma", - "arg4,with,comma", - "arg5", - "arg6,with,comma", - ]); + convert_link_args_to_cc_args( + &mut cmd, + &["arg1", "arg2", "arg3,with,comma", "arg4,with,comma", "arg5", "arg6,with,comma"], + ); - assert_eq!(cmd.get_args(), [ - OsStr::new("-Wl,arg1,arg2"), - OsStr::new("-Xlinker"), - OsStr::new("arg3,with,comma"), - OsStr::new("-Xlinker"), - OsStr::new("arg4,with,comma"), - OsStr::new("-Wl,arg5"), - OsStr::new("-Xlinker"), - OsStr::new("arg6,with,comma"), - ]); + assert_eq!( + cmd.get_args(), + [ + OsStr::new("-Wl,arg1,arg2"), + OsStr::new("-Xlinker"), + OsStr::new("arg3,with,comma"), + OsStr::new("-Xlinker"), + OsStr::new("arg4,with,comma"), + OsStr::new("-Wl,arg5"), + OsStr::new("-Xlinker"), + OsStr::new("arg6,with,comma"), + ] + ); } diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index efccf7687a1..9fd984b6419 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_data_structures::memmap::Mmap; use rustc_errors::FatalError; -use rustc_middle::ty::TyCtxt; use super::write::CodegenContext; use crate::ModuleCodegen; @@ -89,13 +88,12 @@ impl<B: WriteBackendMethods> LtoModuleCodegen<B> { pub unsafe fn autodiff( self, cgcx: &CodegenContext<B>, - tcx: TyCtxt<'_>, diff_fncs: Vec<AutoDiffItem>, config: &ModuleConfig, ) -> Result<LtoModuleCodegen<B>, FatalError> { match &self { LtoModuleCodegen::Fat(module) => { - B::autodiff(cgcx, tcx, &module, diff_fncs, config)?; + B::autodiff(cgcx, &module, diff_fncs, config)?; } _ => panic!("autodiff called with non-fat LTO module"), } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index ba7b53321a8..d70413b8a47 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -704,13 +704,17 @@ pub fn create_metadata_file_for_wasm(sess: &Session, data: &[u8], section_name: let mut imports = wasm_encoder::ImportSection::new(); if sess.target.pointer_width == 64 { - imports.import("env", "__linear_memory", wasm_encoder::MemoryType { - minimum: 0, - maximum: None, - memory64: true, - shared: false, - page_size_log2: None, - }); + imports.import( + "env", + "__linear_memory", + wasm_encoder::MemoryType { + minimum: 0, + maximum: None, + memory64: true, + shared: false, + page_size_log2: None, + }, + ); } if imports.len() > 0 { diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 60ab2919352..1dbaffaa577 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -140,11 +140,14 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<S .into(); if let Some(id) = tcx.proc_macro_decls_static(()) { - reachable_non_generics.insert(id.to_def_id(), SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - }); + reachable_non_generics.insert( + id.to_def_id(), + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: false, + }, + ); } reachable_non_generics @@ -185,11 +188,14 @@ fn exported_symbols_provider_local( if !tcx.sess.target.dll_tls_export { symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| { tcx.needs_thread_local_shim(def_id).then(|| { - (ExportedSymbol::ThreadLocalShim(def_id), SymbolExportInfo { - level: info.level, - kind: SymbolExportKind::Text, - used: info.used, - }) + ( + ExportedSymbol::ThreadLocalShim(def_id), + SymbolExportInfo { + level: info.level, + kind: SymbolExportKind::Text, + used: info.used, + }, + ) }) })) } @@ -198,11 +204,14 @@ fn exported_symbols_provider_local( let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, tcx.sess.target.entry_name.as_ref())); - symbols.push((exported_symbol, SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Text, - used: false, - })); + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Text, + used: false, + }, + )); } // Mark allocator shim symbols as exported only if they were generated. @@ -214,20 +223,26 @@ fn exported_symbols_provider_local( { let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - symbols.push((exported_symbol, SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - })); + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); } let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, NO_ALLOC_SHIM_IS_UNSTABLE)); - symbols.push((exported_symbol, SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Data, - used: false, - })) + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Data, + used: false, + }, + )) } if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() { @@ -239,11 +254,14 @@ fn exported_symbols_provider_local( symbols.extend(PROFILER_WEAK_SYMBOLS.iter().map(|sym| { let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); - (exported_symbol, SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - }) + ( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: false, + }, + ) })); } @@ -261,11 +279,14 @@ fn exported_symbols_provider_local( symbols.extend(msan_weak_symbols.into_iter().map(|sym| { let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); - (exported_symbol, SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - }) + ( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: false, + }, + ) })); } @@ -275,11 +296,14 @@ fn exported_symbols_provider_local( let symbol_name = metadata_symbol_name(tcx); let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - symbols.push((exported_symbol, SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: true, - })); + symbols.push(( + exported_symbol, + SymbolExportInfo { + level: SymbolExportLevel::C, + kind: SymbolExportKind::Data, + used: true, + }, + )); } if tcx.local_crate_exports_generics() { @@ -293,7 +317,7 @@ fn exported_symbols_provider_local( // external linkage is enough for monomorphization to be linked to. let need_visibility = tcx.sess.target.dynamic_linking && !tcx.sess.target.only_cdylib; - let (_, cgus) = tcx.collect_and_partition_mono_items(()); + let cgus = tcx.collect_and_partition_mono_items(()).codegen_units; // The symbols created in this loop are sorted below it #[allow(rustc::potential_query_instability)] @@ -325,21 +349,27 @@ fn exported_symbols_provider_local( MonoItem::Fn(Instance { def: InstanceKind::Item(def), args }) => { if args.non_erasable_generics().next().is_some() { let symbol = ExportedSymbol::Generic(def, args); - symbols.push((symbol, SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - })); + symbols.push(( + symbol, + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); } } MonoItem::Fn(Instance { def: InstanceKind::DropGlue(_, Some(ty)), args }) => { // A little sanity-check assert_eq!(args.non_erasable_generics().next(), Some(GenericArgKind::Type(ty))); - symbols.push((ExportedSymbol::DropGlue(ty), SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - })); + symbols.push(( + ExportedSymbol::DropGlue(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); } MonoItem::Fn(Instance { def: InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)), @@ -347,11 +377,14 @@ fn exported_symbols_provider_local( }) => { // A little sanity-check assert_eq!(args.non_erasable_generics().next(), Some(GenericArgKind::Type(ty))); - symbols.push((ExportedSymbol::AsyncDropGlueCtorShim(ty), SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - })); + symbols.push(( + ExportedSymbol::AsyncDropGlueCtorShim(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); } _ => { // Any other symbols don't qualify for sharing diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index b40bb4ed5d2..ce2161a07eb 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -7,6 +7,7 @@ use std::sync::mpsc::{Receiver, Sender, channel}; use std::{fs, io, mem, str, thread}; use rustc_ast::attr; +use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_data_structures::jobserver::{self, Acquired}; use rustc_data_structures::memmap::Mmap; @@ -40,7 +41,7 @@ use tracing::debug; use super::link::{self, ensure_removed}; use super::lto::{self, SerializedModule}; use super::symbol_export::symbol_name_for_instance_in_crate; -use crate::errors::ErrorCreatingRemarkDir; +use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir}; use crate::traits::*; use crate::{ CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, @@ -118,6 +119,7 @@ pub struct ModuleConfig { pub merge_functions: bool, pub emit_lifetime_markers: bool, pub llvm_plugins: Vec<String>, + pub autodiff: Vec<config::AutoDiff>, } impl ModuleConfig { @@ -266,6 +268,7 @@ impl ModuleConfig { emit_lifetime_markers: sess.emit_lifetime_markers(), llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]), + autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]), } } @@ -389,6 +392,7 @@ impl<B: WriteBackendMethods> CodegenContext<B> { fn generate_lto_work<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, + autodiff: Vec<AutoDiffItem>, needs_fat_lto: Vec<FatLtoInput<B>>, needs_thin_lto: Vec<(String, B::ThinBuffer)>, import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>, @@ -397,11 +401,19 @@ fn generate_lto_work<B: ExtraBackendMethods>( if !needs_fat_lto.is_empty() { assert!(needs_thin_lto.is_empty()); - let module = + let mut module = B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise()); + if cgcx.lto == Lto::Fat && !autodiff.is_empty() { + let config = cgcx.config(ModuleKind::Regular); + module = unsafe { module.autodiff(cgcx, autodiff, config).unwrap() }; + } // We are adding a single work item, so the cost doesn't matter. vec![(WorkItem::LTO(module), 0)] } else { + if !autodiff.is_empty() { + let dcx = cgcx.create_dcx(); + dcx.handle().emit_fatal(AutodiffWithoutLto {}); + } assert!(needs_fat_lto.is_empty()); let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules) .unwrap_or_else(|e| e.raise()); @@ -1021,6 +1033,9 @@ pub(crate) enum Message<B: WriteBackendMethods> { /// Sent from a backend worker thread. WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize }, + /// A vector containing all the AutoDiff tasks that we have to pass to Enzyme. + AddAutoDiffItems(Vec<AutoDiffItem>), + /// The frontend has finished generating something (backend IR or a /// post-LTO artifact) for a codegen unit, and it should be passed to the /// backend. Sent from the main thread. @@ -1348,6 +1363,7 @@ fn start_executing_work<B: ExtraBackendMethods>( // This is where we collect codegen units that have gone all the way // through codegen and LLVM. + let mut autodiff_items = Vec::new(); let mut compiled_modules = vec![]; let mut compiled_allocator_module = None; let mut needs_link = Vec::new(); @@ -1459,9 +1475,13 @@ fn start_executing_work<B: ExtraBackendMethods>( let needs_thin_lto = mem::take(&mut needs_thin_lto); let import_only_modules = mem::take(&mut lto_import_only_modules); - for (work, cost) in - generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules) - { + for (work, cost) in generate_lto_work( + &cgcx, + autodiff_items.clone(), + needs_fat_lto, + needs_thin_lto, + import_only_modules, + ) { let insertion_index = work_items .binary_search_by_key(&cost, |&(_, cost)| cost) .unwrap_or_else(|e| e); @@ -1596,6 +1616,10 @@ fn start_executing_work<B: ExtraBackendMethods>( main_thread_state = MainThreadState::Idle; } + Message::AddAutoDiffItems(mut items) => { + autodiff_items.append(&mut items); + } + Message::CodegenComplete => { if codegen_state != Aborted { codegen_state = Completed; @@ -2070,6 +2094,10 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> { drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>))); } + pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) { + drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items)))); + } + pub(crate) fn check_for_errors(&self, sess: &Session) { self.shared_emitter_main.check(sess, false); } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 544578b29f1..d9fbf539fd3 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -1,5 +1,6 @@ use std::cmp; use std::collections::BTreeSet; +use std::sync::Arc; use std::time::{Duration, Instant}; use itertools::Itertools; @@ -7,7 +8,7 @@ use rustc_abi::FIRST_VARIANT; use rustc_ast::expand::allocator::{ALLOCATOR_METHODS, AllocatorKind, global_fn_name}; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry}; -use rustc_data_structures::sync::{Lrc, par_map}; +use rustc_data_structures::sync::par_map; use rustc_data_structures::unord::UnordMap; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_hir::lang_items::LangItem; @@ -18,14 +19,13 @@ use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, Debugger use rustc_middle::middle::exported_symbols::SymbolExportKind; use rustc_middle::middle::{exported_symbols, lang_items}; use rustc_middle::mir::BinOp; -use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}; +use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem, MonoItemPartitions}; use rustc_middle::query::Providers; use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_session::Session; use rustc_session::config::{self, CrateType, EntryFnType, OptLevel, OutputType}; use rustc_span::{DUMMY_SP, Symbol, sym}; -use rustc_trait_selection::infer::at::ToTrace; use rustc_trait_selection::infer::{BoundRegionConversionTime, TyCtxtInferExt}; use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt}; use tracing::{debug, info}; @@ -44,7 +44,8 @@ use crate::mir::operand::OperandValue; use crate::mir::place::PlaceRef; use crate::traits::*; use crate::{ - CachedModuleCodegen, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, errors, meth, mir, + CachedModuleCodegen, CodegenLintLevels, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, + errors, meth, mir, }; pub(crate) fn bin_op_to_icmp_predicate(op: BinOp, signed: bool) -> IntPredicate { @@ -128,14 +129,9 @@ pub fn validate_trivial_unsize<'tcx>( BoundRegionConversionTime::HigherRankedType, hr_source_principal, ); - let Ok(()) = ocx.eq_trace( + let Ok(()) = ocx.eq( &ObligationCause::dummy(), param_env, - ToTrace::to_trace( - &ObligationCause::dummy(), - hr_target_principal, - hr_source_principal, - ), target_principal, source_principal, ) else { @@ -210,7 +206,12 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( old_info } } - (_, ty::Dynamic(data, _, _)) => meth::get_vtable(cx, source, data.principal()), + (_, ty::Dynamic(data, _, _)) => meth::get_vtable( + cx, + source, + data.principal() + .map(|principal| bx.tcx().instantiate_bound_regions_with_erased(principal)), + ), _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target), } } @@ -490,8 +491,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let ptr_ty = cx.type_ptr(); let (arg_argc, arg_argv) = get_argc_argv(&mut bx); - let (start_fn, start_ty, args, instance) = if let EntryFnType::Main { sigpipe } = entry_type - { + let EntryFnType::Main { sigpipe } = entry_type; + let (start_fn, start_ty, args, instance) = { let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None); let start_instance = ty::Instance::expect_resolve( cx.tcx(), @@ -512,10 +513,6 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( vec![rust_main, arg_argc, arg_argv, arg_sigpipe], Some(start_instance), ) - } else { - debug!("using user-defined start fn"); - let start_ty = cx.type_func(&[isize_ty, ptr_ty], isize_ty); - (rust_main, start_ty, vec![arg_argc, arg_argv], None) }; let result = bx.call(start_ty, None, None, start_fn, &args, None, instance); @@ -530,7 +527,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } -/// Obtain the `argc` and `argv` values to pass to the rust start function. +/// Obtain the `argc` and `argv` values to pass to the rust start function +/// (i.e., the "start" lang item). fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(bx: &mut Bx) -> (Bx::Value, Bx::Value) { if bx.cx().sess().target.os.contains("uefi") { // Params for UEFI @@ -617,11 +615,18 @@ pub fn codegen_crate<B: ExtraBackendMethods>( return ongoing_codegen; } + if tcx.sess.target.need_explicit_cpu && tcx.sess.opts.cg.target_cpu.is_none() { + // The target has no default cpu, but none is set explicitly + tcx.dcx().emit_fatal(errors::CpuRequired); + } + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); // Run the monomorphization collector and partition the collected items into // codegen units. - let codegen_units = tcx.collect_and_partition_mono_items(()).1; + let MonoItemPartitions { codegen_units, autodiff_items, .. } = + tcx.collect_and_partition_mono_items(()); + let autodiff_fncs = autodiff_items.to_vec(); // Force all codegen_unit queries so they are already either red or green // when compile_codegen_unit accesses them. We are not able to re-execute @@ -630,7 +635,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>( // unnecessarily. if tcx.dep_graph.is_fully_enabled() { for cgu in codegen_units { - tcx.ensure().codegen_unit(cgu.name()); + tcx.ensure_ok().codegen_unit(cgu.name()); } } @@ -692,6 +697,10 @@ pub fn codegen_crate<B: ExtraBackendMethods>( ); } + if !autodiff_fncs.is_empty() { + ongoing_codegen.submit_autodiff_items(autodiff_fncs); + } + // For better throughput during parallel processing by LLVM, we used to sort // CGUs largest to smallest. This would lead to better thread utilization // by, for example, preventing a large CGU from being processed last and @@ -924,9 +933,10 @@ impl CrateInfo { crate_name: UnordMap::with_capacity(n_crates), used_crates, used_crate_source: UnordMap::with_capacity(n_crates), - dependency_formats: Lrc::clone(tcx.dependency_formats(())), + dependency_formats: Arc::clone(tcx.dependency_formats(())), windows_subsystem, natvis_debugger_visualizers: Default::default(), + lint_levels: CodegenLintLevels::from_tcx(tcx), }; info.native_libraries.reserve(n_crates); @@ -937,7 +947,7 @@ impl CrateInfo { info.crate_name.insert(cnum, tcx.crate_name(cnum)); let used_crate_source = tcx.used_crate_source(cnum); - info.used_crate_source.insert(cnum, Lrc::clone(used_crate_source)); + info.used_crate_source.insert(cnum, Arc::clone(used_crate_source)); if tcx.is_profiler_runtime(cnum) { info.profiler_runtime = Some(cnum); } @@ -1052,14 +1062,11 @@ pub(crate) fn provide(providers: &mut Providers) { config::OptLevel::SizeMin => config::OptLevel::Default, }; - let (defids, _) = tcx.collect_and_partition_mono_items(cratenum); + let defids = tcx.collect_and_partition_mono_items(cratenum).all_mono_items; let any_for_speed = defids.items().any(|id| { let CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id); - match optimize { - attr::OptimizeAttr::None | attr::OptimizeAttr::Size => false, - attr::OptimizeAttr::Speed => true, - } + matches!(optimize, attr::OptimizeAttr::Speed) }); if any_for_speed { diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 1daa17fbaf3..7acdbd19993 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -1,5 +1,10 @@ +use std::str::FromStr; + use rustc_ast::attr::list_contains_name; -use rustc_ast::{MetaItemInner, attr}; +use rustc_ast::expand::autodiff_attrs::{ + AutoDiffAttrs, DiffActivity, DiffMode, valid_input_activity, valid_ret_activity, +}; +use rustc_ast::{MetaItem, MetaItemInner, attr}; use rustc_attr_parsing::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::codes::*; @@ -13,6 +18,7 @@ use rustc_middle::middle::codegen_fn_attrs::{ }; use rustc_middle::mir::mono::Linkage; use rustc_middle::query::Providers; +use rustc_middle::span_bug; use rustc_middle::ty::{self as ty, TyCtxt}; use rustc_session::parse::feature_err; use rustc_session::{Session, lint}; @@ -35,7 +41,6 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage { // ghost, dllimport, dllexport and linkonce_odr_autohide are not supported // and don't have to be, LLVM treats them as no-ops. match name { - "appending" => Appending, "available_externally" => AvailableExternally, "common" => Common, "extern_weak" => ExternalWeak, @@ -43,7 +48,6 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage { "internal" => Internal, "linkonce" => LinkOnceAny, "linkonce_odr" => LinkOnceODR, - "private" => Private, "weak" => WeakAny, "weak_odr" => WeakODR, _ => tcx.dcx().span_fatal(tcx.def_span(def_id), "invalid linkage specified"), @@ -65,6 +69,13 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER; } + // If our rustc version supports autodiff/enzyme, then we call our handler + // to check for any `#[rustc_autodiff(...)]` attributes. + if cfg!(llvm_enzyme) { + let ad = autodiff_attrs(tcx, did.into()); + codegen_fn_attrs.autodiff_item = ad; + } + // When `no_builtins` is applied at the crate level, we should add the // `no-builtins` attribute to each function to ensure it takes effect in LTO. let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); @@ -575,7 +586,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs.inline = InlineAttr::Never; } - codegen_fn_attrs.optimize = attrs.iter().fold(OptimizeAttr::None, |ia, attr| { + codegen_fn_attrs.optimize = attrs.iter().fold(OptimizeAttr::Default, |ia, attr| { if !attr.has_name(sym::optimize) { return ia; } @@ -587,17 +598,19 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { inline_span = Some(attr.span); if items.len() != 1 { err(attr.span, "expected one argument"); - OptimizeAttr::None + OptimizeAttr::Default } else if list_contains_name(items, sym::size) { OptimizeAttr::Size } else if list_contains_name(items, sym::speed) { OptimizeAttr::Speed + } else if list_contains_name(items, sym::none) { + OptimizeAttr::DoNotOptimize } else { err(items[0].span(), "invalid argument"); - OptimizeAttr::None + OptimizeAttr::Default } } else { - OptimizeAttr::None + OptimizeAttr::Default } }); @@ -854,6 +867,109 @@ impl<'a> MixedExportNameAndNoMangleState<'a> { } } +/// We now check the #\[rustc_autodiff\] attributes which we generated from the #[autodiff(...)] +/// macros. There are two forms. The pure one without args to mark primal functions (the functions +/// being differentiated). The other form is #[rustc_autodiff(Mode, ActivityList)] on top of the +/// placeholder functions. We wrote the rustc_autodiff attributes ourself, so this should never +/// panic, unless we introduced a bug when parsing the autodiff macro. +fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> { + let attrs = tcx.get_attrs(id, sym::rustc_autodiff); + + let attrs = + attrs.filter(|attr| attr.name_or_empty() == sym::rustc_autodiff).collect::<Vec<_>>(); + + // check for exactly one autodiff attribute on placeholder functions. + // There should only be one, since we generate a new placeholder per ad macro. + // FIXME(ZuseZ4): re-enable this check. Currently we add multiple, which doesn't cause harm but + // looks strange e.g. under cargo-expand. + let attr = match &attrs[..] { + [] => return None, + [attr] => attr, + // These two attributes are the same and unfortunately duplicated due to a previous bug. + [attr, _attr2] => attr, + _ => { + //FIXME(ZuseZ4): Once we fixed our parser, we should also prohibit the two-attribute + //branch above. + span_bug!(attrs[1].span, "cg_ssa: rustc_autodiff should only exist once per source"); + } + }; + + let list = attr.meta_item_list().unwrap_or_default(); + + // empty autodiff attribute macros (i.e. `#[autodiff]`) are used to mark source functions + if list.is_empty() { + return Some(AutoDiffAttrs::source()); + } + + let [mode, input_activities @ .., ret_activity] = &list[..] else { + span_bug!(attr.span, "rustc_autodiff attribute must contain mode and activities"); + }; + let mode = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = mode { + p1.segments.first().unwrap().ident + } else { + span_bug!(attr.span, "rustc_autodiff attribute must contain mode"); + }; + + // parse mode + let mode = match mode.as_str() { + "Forward" => DiffMode::Forward, + "Reverse" => DiffMode::Reverse, + "ForwardFirst" => DiffMode::ForwardFirst, + "ReverseFirst" => DiffMode::ReverseFirst, + _ => { + span_bug!(mode.span, "rustc_autodiff attribute contains invalid mode"); + } + }; + + // First read the ret symbol from the attribute + let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = ret_activity { + p1.segments.first().unwrap().ident + } else { + span_bug!(attr.span, "rustc_autodiff attribute must contain the return activity"); + }; + + // Then parse it into an actual DiffActivity + let Ok(ret_activity) = DiffActivity::from_str(ret_symbol.as_str()) else { + span_bug!(ret_symbol.span, "invalid return activity"); + }; + + // Now parse all the intermediate (input) activities + let mut arg_activities: Vec<DiffActivity> = vec![]; + for arg in input_activities { + let arg_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p2, .. }) = arg { + match p2.segments.first() { + Some(x) => x.ident, + None => { + span_bug!( + arg.span(), + "rustc_autodiff attribute must contain the input activity" + ); + } + } + } else { + span_bug!(arg.span(), "rustc_autodiff attribute must contain the input activity"); + }; + + match DiffActivity::from_str(arg_symbol.as_str()) { + Ok(arg_activity) => arg_activities.push(arg_activity), + Err(_) => { + span_bug!(arg_symbol.span, "invalid input activity"); + } + } + } + + for &input in &arg_activities { + if !valid_input_activity(mode, input) { + span_bug!(attr.span, "Invalid input activity {} for {} mode", input, mode); + } + } + if !valid_ret_activity(mode, ret_activity) { + span_bug!(attr.span, "Invalid return activity {} for {} mode", ret_activity, mode); + } + + Some(AutoDiffAttrs { mode, ret_activity, input_activity: arg_activities }) +} + pub(crate) fn provide(providers: &mut Providers) { *providers = Providers { codegen_fn_attrs, should_inherit_track_caller, ..*providers }; } diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index 869798d8be1..05175371591 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -507,7 +507,7 @@ pub enum VTableNameKind { pub fn compute_debuginfo_vtable_name<'tcx>( tcx: TyCtxt<'tcx>, t: Ty<'tcx>, - trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, kind: VTableNameKind, ) -> String { let cpp_like_debuginfo = cpp_like_debuginfo(tcx); @@ -530,8 +530,8 @@ pub fn compute_debuginfo_vtable_name<'tcx>( } if let Some(trait_ref) = trait_ref { - let trait_ref = tcx - .normalize_erasing_late_bound_regions(ty::TypingEnv::fully_monomorphized(), trait_ref); + let trait_ref = + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), trait_ref); push_item_name(tcx, trait_ref.def_id, true, &mut vtable_name); visited.clear(); push_generic_params_internal(tcx, trait_ref.args, &mut vtable_name, &mut visited); @@ -673,25 +673,23 @@ fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: ty::Const<'tcx>, output: &mut S ty::ConstKind::Param(param) => { write!(output, "{}", param.name) } - ty::ConstKind::Value(ty, valtree) => { - match ty.kind() { + ty::ConstKind::Value(cv) => { + match cv.ty.kind() { ty::Int(ity) => { - // FIXME: directly extract the bits from a valtree instead of evaluating an - // already evaluated `Const` in order to get the bits. - let bits = ct + let bits = cv .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized()) .expect("expected monomorphic const in codegen"); let val = Integer::from_int_ty(&tcx, *ity).size().sign_extend(bits) as i128; write!(output, "{val}") } ty::Uint(_) => { - let val = ct + let val = cv .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized()) .expect("expected monomorphic const in codegen"); write!(output, "{val}") } ty::Bool => { - let val = ct.try_to_bool().expect("expected monomorphic const in codegen"); + let val = cv.try_to_bool().expect("expected monomorphic const in codegen"); write!(output, "{val}") } _ => { @@ -703,9 +701,7 @@ fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: ty::Const<'tcx>, output: &mut S // avoiding collisions and will make the emitted type names shorter. let hash_short = tcx.with_stable_hashing_context(|mut hcx| { let mut hasher = StableHasher::new(); - hcx.while_hashing_spans(false, |hcx| { - (ty, valtree).hash_stable(hcx, &mut hasher) - }); + hcx.while_hashing_spans(false, |hcx| cv.hash_stable(hcx, &mut hasher)); hasher.finish::<Hash64>() }); diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index c7213bbc801..3ddbe4aeeec 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -40,6 +40,10 @@ pub(crate) struct CguNotRecorded<'a> { } #[derive(Diagnostic)] +#[diag(codegen_ssa_autodiff_without_lto)] +pub struct AutodiffWithoutLto; + +#[derive(Diagnostic)] #[diag(codegen_ssa_unknown_reuse_kind)] pub(crate) struct UnknownReuseKind { #[primary_span] @@ -351,6 +355,7 @@ pub(crate) struct LinkingFailed<'a> { pub command: Command, pub escaped_output: String, pub verbose: bool, + pub sysroot_dir: PathBuf, } impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { @@ -364,6 +369,8 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { if self.verbose { diag.note(format!("{:?}", self.command)); } else { + self.command.env_clear(); + enum ArgGroup { Regular(OsString), Objects(usize), @@ -398,26 +405,55 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { args.push(ArgGroup::Regular(arg)); } } - self.command.args(args.into_iter().map(|arg_group| match arg_group { - ArgGroup::Regular(arg) => arg, - ArgGroup::Objects(n) => OsString::from(format!("<{n} object files omitted>")), - ArgGroup::Rlibs(dir, rlibs) => { - let mut arg = dir.into_os_string(); - arg.push("/{"); - let mut first = true; - for rlib in rlibs { - if !first { - arg.push(","); + let crate_hash = regex::bytes::Regex::new(r"-[0-9a-f]+\.rlib$").unwrap(); + self.command.args(args.into_iter().map(|arg_group| { + match arg_group { + // SAFETY: we are only matching on ASCII, not any surrogate pairs, so any replacements we do will still be valid. + ArgGroup::Regular(arg) => unsafe { + use bstr::ByteSlice; + OsString::from_encoded_bytes_unchecked( + arg.as_encoded_bytes().replace( + self.sysroot_dir.as_os_str().as_encoded_bytes(), + b"<sysroot>", + ), + ) + }, + ArgGroup::Objects(n) => OsString::from(format!("<{n} object files omitted>")), + ArgGroup::Rlibs(mut dir, rlibs) => { + let is_sysroot_dir = match dir.strip_prefix(&self.sysroot_dir) { + Ok(short) => { + dir = Path::new("<sysroot>").join(short); + true + } + Err(_) => false, + }; + let mut arg = dir.into_os_string(); + arg.push("/{"); + let mut first = true; + for mut rlib in rlibs { + if !first { + arg.push(","); + } + first = false; + if is_sysroot_dir { + // SAFETY: Regex works one byte at a type, and our regex will not match surrogate pairs (because it only matches ascii). + rlib = unsafe { + OsString::from_encoded_bytes_unchecked( + crate_hash + .replace(rlib.as_encoded_bytes(), b"-*") + .into_owned(), + ) + }; + } + arg.push(rlib); } - first = false; - arg.push(rlib); + arg.push("}.rlib"); + arg } - arg.push("}"); - arg } })); - diag.note(format!("{:?}", self.command)); + diag.note(format!("{:?}", self.command).trim_start_matches("env -i").to_owned()); diag.note("some arguments are omitted. use `--verbose` to show all linker arguments"); } @@ -492,6 +528,10 @@ pub(crate) struct CheckInstalledVisualStudio; pub(crate) struct InsufficientVSCodeProduct; #[derive(Diagnostic)] +#[diag(codegen_ssa_cpu_required)] +pub(crate) struct CpuRequired; + +#[derive(Diagnostic)] #[diag(codegen_ssa_processing_dymutil_failed)] #[note] pub(crate) struct ProcessingDymutilFailed { diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 65c6067c740..428a45975f1 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -24,23 +24,28 @@ use std::collections::BTreeSet; use std::io; use std::path::{Path, PathBuf}; +use std::sync::Arc; use rustc_ast as ast; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; -use rustc_data_structures::sync::Lrc; use rustc_data_structures::unord::UnordMap; +use rustc_hir::CRATE_HIR_ID; use rustc_hir::def_id::CrateNum; use rustc_macros::{Decodable, Encodable, HashStable}; use rustc_middle::dep_graph::WorkProduct; +use rustc_middle::lint::LintLevelSource; use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile; use rustc_middle::middle::dependency_format::Dependencies; use rustc_middle::middle::exported_symbols::SymbolExportKind; +use rustc_middle::ty::TyCtxt; use rustc_middle::util::Providers; use rustc_serialize::opaque::{FileEncoder, MemDecoder}; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_session::Session; use rustc_session::config::{CrateType, OutputFilenames, OutputType, RUST_CGU_EXT}; use rustc_session::cstore::{self, CrateSource}; +use rustc_session::lint::Level; +use rustc_session::lint::builtin::LINKER_MESSAGES; use rustc_session::utils::NativeLibKind; use rustc_span::Symbol; @@ -195,11 +200,12 @@ pub struct CrateInfo { pub native_libraries: FxIndexMap<CrateNum, Vec<NativeLib>>, pub crate_name: UnordMap<CrateNum, Symbol>, pub used_libraries: Vec<NativeLib>, - pub used_crate_source: UnordMap<CrateNum, Lrc<CrateSource>>, + pub used_crate_source: UnordMap<CrateNum, Arc<CrateSource>>, pub used_crates: Vec<CrateNum>, - pub dependency_formats: Lrc<Dependencies>, + pub dependency_formats: Arc<Dependencies>, pub windows_subsystem: Option<String>, pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>, + pub lint_levels: CodegenLintLevels, } #[derive(Encodable, Decodable)] @@ -302,3 +308,19 @@ impl CodegenResults { Ok((codegen_results, outputs)) } } + +/// A list of lint levels used in codegen. +/// +/// When using `-Z link-only`, we don't have access to the tcx and must work +/// solely from the `.rlink` file. `Lint`s are defined too early to be encodeable. +/// Instead, encode exactly the information we need. +#[derive(Copy, Clone, Debug, Encodable, Decodable)] +pub struct CodegenLintLevels { + linker_messages: (Level, LintLevelSource), +} + +impl CodegenLintLevels { + pub fn from_tcx(tcx: TyCtxt<'_>) -> Self { + Self { linker_messages: tcx.lint_level_at_node(LINKER_MESSAGES, CRATE_HIR_ID) } + } +} diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 64cd4c38937..399c592432a 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -1,5 +1,5 @@ use rustc_middle::bug; -use rustc_middle::ty::{self, GenericArgKind, Ty}; +use rustc_middle::ty::{self, GenericArgKind, Ty, TyCtxt}; use rustc_session::config::Lto; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::callconv::FnAbi; @@ -72,12 +72,19 @@ impl<'a, 'tcx> VirtualIndex { /// This takes a valid `self` receiver type and extracts the principal trait /// ref of the type. Return `None` if there is no principal trait. -fn dyn_trait_in_self(ty: Ty<'_>) -> Option<ty::PolyExistentialTraitRef<'_>> { +fn dyn_trait_in_self<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, +) -> Option<ty::ExistentialTraitRef<'tcx>> { for arg in ty.peel_refs().walk() { if let GenericArgKind::Type(ty) = arg.unpack() && let ty::Dynamic(data, _, _) = ty.kind() { - return data.principal(); + // FIXME(arbitrary_self_types): This is likely broken for receivers which + // have a "non-self" trait objects as a generic argument. + return data + .principal() + .map(|principal| tcx.instantiate_bound_regions_with_erased(principal)); } } @@ -96,7 +103,7 @@ fn dyn_trait_in_self(ty: Ty<'_>) -> Option<ty::PolyExistentialTraitRef<'_>> { pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( cx: &Cx, ty: Ty<'tcx>, - trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, ) -> Cx::Value { let tcx = cx.tcx(); @@ -131,7 +138,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if bx.cx().sess().opts.unstable_opts.virtual_function_elimination && bx.cx().sess().lto() == Lto::Fat { - if let Some(trait_ref) = dyn_trait_in_self(ty) { + if let Some(trait_ref) = dyn_trait_in_self(bx.tcx(), ty) { let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref)).unwrap(); let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); return func; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index b0a1dedd646..8c571558717 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1,6 +1,6 @@ use std::cmp; -use rustc_abi::{self as abi, ExternAbi, HasDataLayout, WrappingRange}; +use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange}; use rustc_ast as ast; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::lang_items::LangItem; @@ -14,7 +14,7 @@ use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::source_map::Spanned; use rustc_span::{Span, sym}; -use rustc_target::callconv::{ArgAbi, FnAbi, PassMode, Reg}; +use rustc_target::callconv::{ArgAbi, FnAbi, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -713,6 +713,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // and `#[track_caller]` adds an implicit third argument. (LangItem::PanicMisalignedPointerDereference, vec![required, found, location]) } + AssertKind::NullPointerDereference => { + // It's `fn panic_null_pointer_dereference()`, + // `#[track_caller]` adds an implicit argument. + (LangItem::PanicNullPointerDereference, vec![location]) + } _ => { // It's `pub fn panic_...()` and `#[track_caller]` adds an implicit argument. (msg.panic_function(), vec![location]) @@ -1540,7 +1545,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // the load would just produce `OperandValue::Ref` instead // of the `OperandValue::Immediate` we need for the call. llval = bx.load(bx.backend_type(arg.layout), llval, align); - if let abi::BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { if scalar.is_bool() { bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); } @@ -1599,10 +1604,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if let Some(slot) = self.personality_slot { slot } else { - let layout = cx.layout_of(Ty::new_tup(cx.tcx(), &[ - Ty::new_mut_ptr(cx.tcx(), cx.tcx().types.u8), - cx.tcx().types.i32, - ])); + let layout = cx.layout_of(Ty::new_tup( + cx.tcx(), + &[Ty::new_mut_ptr(cx.tcx(), cx.tcx().types.u8), cx.tcx().types.i32], + )); let slot = PlaceRef::alloca(bx, layout); self.personality_slot = Some(slot); slot @@ -1703,15 +1708,32 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mut cs_bx = Bx::build(self.cx, llbb); let cs = cs_bx.catch_switch(None, None, &[cp_llbb]); - // The "null" here is actually a RTTI type descriptor for the - // C++ personality function, but `catch (...)` has no type so - // it's null. The 64 here is actually a bitfield which - // represents that this is a catch-all block. bx = Bx::build(self.cx, cp_llbb); let null = bx.const_null(bx.type_ptr_ext(bx.cx().data_layout().instruction_address_space)); - let sixty_four = bx.const_i32(64); - funclet = Some(bx.catch_pad(cs, &[null, sixty_four, null])); + + // The `null` in first argument here is actually a RTTI type + // descriptor for the C++ personality function, but `catch (...)` + // has no type so it's null. + let args = if base::wants_msvc_seh(self.cx.sess()) { + // This bitmask is a single `HT_IsStdDotDot` flag, which + // represents that this is a C++-style `catch (...)` block that + // only captures programmatic exceptions, not all SEH + // exceptions. The second `null` points to a non-existent + // `alloca` instruction, which an LLVM pass would inline into + // the initial SEH frame allocation. + let adjectives = bx.const_i32(0x40); + &[null, adjectives, null] as &[_] + } else { + // Specifying more arguments than necessary usually doesn't + // hurt, but the `WasmEHPrepare` LLVM pass does not recognize + // anything other than a single `null` as a `catch (...)` block, + // leading to problems down the line during instruction + // selection. + &[null] as &[_] + }; + + funclet = Some(bx.catch_pad(cs, args)); } else { llbb = Bx::append_block(self.cx, self.llfn, "terminate"); bx = Bx::build(self.cx, llbb); diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index 7676e1e171a..eafc551501c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -43,7 +43,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Const::Ty(_, c) => match c.kind() { // A constant that came from a const generic but was then used as an argument to // old-style simd_shuffle (passing as argument instead of as a generic param). - rustc_type_ir::ConstKind::Value(_, valtree) => return Ok(Ok(valtree)), + rustc_type_ir::ConstKind::Value(cv) => return Ok(Ok(cv.valtree)), other => span_bug!(constant.span, "{other:#?}"), }, // We should never encounter `Const::Val` unless MIR opts (like const prop) evaluate diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 843a996d2bf..5924c8991ad 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -19,9 +19,7 @@ use crate::traits::*; pub struct FunctionDebugContext<'tcx, S, L> { /// Maps from source code to the corresponding debug info scope. - /// May be None if the backend is not capable of representing the scope for - /// some reason. - pub scopes: IndexVec<mir::SourceScope, Option<DebugScope<S, L>>>, + pub scopes: IndexVec<mir::SourceScope, DebugScope<S, L>>, /// Maps from an inlined function to its debug info declaration. pub inlined_function_scopes: FxHashMap<Instance<'tcx>, S>, @@ -232,7 +230,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &self, source_info: mir::SourceInfo, ) -> Option<(Bx::DIScope, Option<Bx::DILocation>, Span)> { - let scope = &self.debug_context.as_ref()?.scopes[source_info.scope]?; + let scope = &self.debug_context.as_ref()?.scopes[source_info.scope]; let span = hygiene::walk_chain_collapsed(source_info.span, self.mir.span); Some((scope.adjust_dbg_scope_for_span(self.cx, span), scope.inlined_at, span)) } diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 304ac4544ee..6e7fbe62c8d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -225,6 +225,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args[1].val.unaligned_volatile_store(bx, dst); return Ok(()); } + sym::disjoint_bitor => { + let a = args[0].immediate(); + let b = args[1].immediate(); + bx.or_disjoint(a, b) + } sym::exact_div => { let ty = arg_tys[0]; match int_type_width_signed(ty, bx.tcx()) { diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index 8df270abc81..eb0711dbb32 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -1,10 +1,14 @@ +use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind}; use rustc_attr_parsing::InstructionSetAttr; +use rustc_hir::def_id::DefId; use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility}; use rustc_middle::mir::{Body, InlineAsmOperand}; -use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf}; -use rustc_middle::ty::{Instance, TyCtxt}; -use rustc_middle::{bug, ty}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; +use rustc_middle::ty::{Instance, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug, ty}; use rustc_span::sym; +use rustc_target::callconv::{ArgAbi, FnAbi, PassMode}; +use rustc_target::spec::WasmCAbi; use crate::common; use crate::traits::{AsmCodegenMethods, BuilderMethods, GlobalAsmOperandRef, MiscCodegenMethods}; @@ -32,7 +36,8 @@ pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let item_data = cx.codegen_unit().items().get(&MonoItem::Fn(instance)).unwrap(); let name = cx.mangled_name(instance); - let (begin, end) = prefix_and_suffix(cx.tcx(), instance, &name, item_data); + let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); + let (begin, end) = prefix_and_suffix(cx.tcx(), instance, &name, item_data, fn_abi); let mut template_vec = Vec::new(); template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(begin.into())); @@ -103,6 +108,7 @@ enum AsmBinaryFormat { Elf, Macho, Coff, + Wasm, } impl AsmBinaryFormat { @@ -111,6 +117,8 @@ impl AsmBinaryFormat { Self::Coff } else if target.is_like_osx { Self::Macho + } else if target.is_like_wasm { + Self::Wasm } else { Self::Elf } @@ -122,6 +130,7 @@ fn prefix_and_suffix<'tcx>( instance: Instance<'tcx>, asm_name: &str, item_data: &MonoItemData, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> (String, String) { use std::fmt::Write; @@ -169,7 +178,7 @@ fn prefix_and_suffix<'tcx>( } Linkage::LinkOnceAny | Linkage::LinkOnceODR | Linkage::WeakAny | Linkage::WeakODR => { match asm_binary_format { - AsmBinaryFormat::Elf | AsmBinaryFormat::Coff => { + AsmBinaryFormat::Elf | AsmBinaryFormat::Coff | AsmBinaryFormat::Wasm => { writeln!(w, ".weak {asm_name}")?; } AsmBinaryFormat::Macho => { @@ -178,10 +187,9 @@ fn prefix_and_suffix<'tcx>( } } } - Linkage::Internal | Linkage::Private => { + Linkage::Internal => { // write nothing } - Linkage::Appending => emit_fatal("Only global variables can have appending linkage!"), Linkage::Common => emit_fatal("Functions may not have common linkage"), Linkage::AvailableExternally => { // this would make the function equal an extern definition @@ -264,7 +272,161 @@ fn prefix_and_suffix<'tcx>( writeln!(end, "{}", arch_suffix).unwrap(); } } + AsmBinaryFormat::Wasm => { + let section = link_section.unwrap_or(format!(".text.{asm_name}")); + + writeln!(begin, ".section {section},\"\",@").unwrap(); + // wasm functions cannot be aligned, so skip + write_linkage(&mut begin).unwrap(); + if let Visibility::Hidden = item_data.visibility { + writeln!(begin, ".hidden {asm_name}").unwrap(); + } + writeln!(begin, ".type {asm_name}, @function").unwrap(); + if !arch_prefix.is_empty() { + writeln!(begin, "{}", arch_prefix).unwrap(); + } + writeln!(begin, "{asm_name}:").unwrap(); + writeln!( + begin, + ".functype {asm_name} {}", + wasm_functype(tcx, fn_abi, instance.def_id()) + ) + .unwrap(); + + writeln!(end).unwrap(); + // .size is ignored for function symbols, so we can skip it + writeln!(end, "end_function").unwrap(); + } } (begin, end) } + +/// The webassembly type signature for the given function. +/// +/// Used by the `.functype` directive on wasm targets. +fn wasm_functype<'tcx>(tcx: TyCtxt<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, def_id: DefId) -> String { + let mut signature = String::with_capacity(64); + + let ptr_type = match tcx.data_layout.pointer_size.bits() { + 32 => "i32", + 64 => "i64", + other => bug!("wasm pointer size cannot be {other} bits"), + }; + + // FIXME: remove this once the wasm32-unknown-unknown ABI is fixed + // please also add `wasm32-unknown-unknown` back in `tests/assembly/wasm32-naked-fn.rs` + // basically the commit introducing this comment should be reverted + if let PassMode::Pair { .. } = fn_abi.ret.mode { + let _ = WasmCAbi::Legacy; + span_bug!( + tcx.def_span(def_id), + "cannot return a pair (the wasm32-unknown-unknown ABI is broken, see https://github.com/rust-lang/rust/issues/115666" + ); + } + + let hidden_return = matches!(fn_abi.ret.mode, PassMode::Indirect { .. }); + + signature.push('('); + + if hidden_return { + signature.push_str(ptr_type); + if !fn_abi.args.is_empty() { + signature.push_str(", "); + } + } + + let mut it = fn_abi.args.iter().peekable(); + while let Some(arg_abi) = it.next() { + wasm_type(tcx, &mut signature, arg_abi, ptr_type, def_id); + if it.peek().is_some() { + signature.push_str(", "); + } + } + + signature.push_str(") -> ("); + + if !hidden_return { + wasm_type(tcx, &mut signature, &fn_abi.ret, ptr_type, def_id); + } + + signature.push(')'); + + signature +} + +fn wasm_type<'tcx>( + tcx: TyCtxt<'tcx>, + signature: &mut String, + arg_abi: &ArgAbi<'_, Ty<'tcx>>, + ptr_type: &'static str, + def_id: DefId, +) { + match arg_abi.mode { + PassMode::Ignore => { /* do nothing */ } + PassMode::Direct(_) => { + let direct_type = match arg_abi.layout.backend_repr { + BackendRepr::Scalar(scalar) => wasm_primitive(scalar.primitive(), ptr_type), + BackendRepr::Vector { .. } => "v128", + BackendRepr::Memory { .. } => { + // FIXME: remove this branch once the wasm32-unknown-unknown ABI is fixed + let _ = WasmCAbi::Legacy; + span_bug!( + tcx.def_span(def_id), + "cannot use memory args (the wasm32-unknown-unknown ABI is broken, see https://github.com/rust-lang/rust/issues/115666" + ); + } + other => unreachable!("unexpected BackendRepr: {:?}", other), + }; + + signature.push_str(direct_type); + } + PassMode::Pair(_, _) => match arg_abi.layout.backend_repr { + BackendRepr::ScalarPair(a, b) => { + signature.push_str(wasm_primitive(a.primitive(), ptr_type)); + signature.push_str(", "); + signature.push_str(wasm_primitive(b.primitive(), ptr_type)); + } + other => unreachable!("{other:?}"), + }, + PassMode::Cast { pad_i32, ref cast } => { + // For wasm, Cast is used for single-field primitive wrappers like `struct Wrapper(i64);` + assert!(!pad_i32, "not currently used by wasm calling convention"); + assert!(cast.prefix[0].is_none(), "no prefix"); + assert_eq!(cast.rest.total, arg_abi.layout.size, "single item"); + + let wrapped_wasm_type = match cast.rest.unit.kind { + RegKind::Integer => match cast.rest.unit.size.bytes() { + ..=4 => "i32", + ..=8 => "i64", + _ => ptr_type, + }, + RegKind::Float => match cast.rest.unit.size.bytes() { + ..=4 => "f32", + ..=8 => "f64", + _ => ptr_type, + }, + RegKind::Vector => "v128", + }; + + signature.push_str(wrapped_wasm_type); + } + PassMode::Indirect { .. } => signature.push_str(ptr_type), + } +} + +fn wasm_primitive(primitive: Primitive, ptr_type: &'static str) -> &'static str { + match primitive { + Primitive::Int(integer, _) => match integer { + Integer::I8 | Integer::I16 | Integer::I32 => "i32", + Integer::I64 => "i64", + Integer::I128 => "i64, i64", + }, + Primitive::Float(float) => match float { + Float::F16 | Float::F32 => "f32", + Float::F64 => "f64", + Float::F128 => "i64, i64", + }, + Primitive::Pointer(_) => ptr_type, + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 19101ec2d1b..9ca7d4f8f00 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -204,14 +204,30 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let alloc_align = alloc.inner().align; assert!(alloc_align >= layout.align.abi); + // Returns `None` when the value is partially undefined or any byte of it has provenance. + // Otherwise returns the value or (if the entire value is undef) returns an undef. let read_scalar = |start, size, s: abi::Scalar, ty| { + let range = alloc_range(start, size); match alloc.0.read_scalar( bx, - alloc_range(start, size), + range, /*read_provenance*/ matches!(s.primitive(), abi::Primitive::Pointer(_)), ) { - Ok(val) => bx.scalar_to_backend(val, s, ty), - Err(_) => bx.const_poison(ty), + Ok(val) => Some(bx.scalar_to_backend(val, s, ty)), + Err(_) => { + // We may have failed due to partial provenance or unexpected provenance, + // continue down the normal code path if so. + if alloc.0.provenance().range_empty(range, &bx.tcx()) + // Since `read_scalar` failed, but there were no relocations involved, the + // bytes must be partially or fully uninitialized. Thus we can now unwrap the + // information about the range of uninit bytes and check if it's the full range. + && alloc.0.init_mask().is_range_initialized(range).unwrap_err() == range + { + Some(bx.const_undef(ty)) + } else { + None + } + } } }; @@ -222,16 +238,14 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { // check that walks over the type of `mplace` to make sure it is truly correct to treat this // like a `Scalar` (or `ScalarPair`). match layout.backend_repr { - BackendRepr::Scalar(s @ abi::Scalar::Initialized { .. }) => { + BackendRepr::Scalar(s) => { let size = s.size(bx); assert_eq!(size, layout.size, "abi::Scalar size does not match layout size"); - let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout)); - OperandRef { val: OperandValue::Immediate(val), layout } + if let Some(val) = read_scalar(offset, size, s, bx.immediate_backend_type(layout)) { + return OperandRef { val: OperandValue::Immediate(val), layout }; + } } - BackendRepr::ScalarPair( - a @ abi::Scalar::Initialized { .. }, - b @ abi::Scalar::Initialized { .. }, - ) => { + BackendRepr::ScalarPair(a, b) => { let (a_size, b_size) = (a.size(bx), b.size(bx)); let b_offset = (offset + a_size).align_to(b.align(bx).abi); assert!(b_offset.bytes() > 0); @@ -247,20 +261,21 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { b, bx.scalar_pair_element_backend_type(layout, 1, true), ); - OperandRef { val: OperandValue::Pair(a_val, b_val), layout } - } - _ if layout.is_zst() => OperandRef::zero_sized(layout), - _ => { - // Neither a scalar nor scalar pair. Load from a place - // FIXME: should we cache `const_data_from_alloc` to avoid repeating this for the - // same `ConstAllocation`? - let init = bx.const_data_from_alloc(alloc); - let base_addr = bx.static_addr_of(init, alloc_align, None); - - let llval = bx.const_ptr_byte_offset(base_addr, offset); - bx.load_operand(PlaceRef::new_sized(llval, layout)) + if let (Some(a_val), Some(b_val)) = (a_val, b_val) { + return OperandRef { val: OperandValue::Pair(a_val, b_val), layout }; + } } + _ if layout.is_zst() => return OperandRef::zero_sized(layout), + _ => {} } + // Neither a scalar nor scalar pair. Load from a place + // FIXME: should we cache `const_data_from_alloc` to avoid repeating this for the + // same `ConstAllocation`? + let init = bx.const_data_from_alloc(alloc); + let base_addr = bx.static_addr_of(init, alloc_align, None); + + let llval = bx.const_ptr_byte_offset(base_addr, offset); + bx.load_operand(PlaceRef::new_sized(llval, layout)) } /// Asserts that this operand refers to a scalar and returns diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index c634f864ffb..eb4270ffe80 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -474,10 +474,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { LocalRef::Operand(..) => { if place_ref.is_indirect_first_projection() { base = 1; - let cg_base = self.codegen_consume(bx, mir::PlaceRef { - projection: &place_ref.projection[..0], - ..place_ref - }); + let cg_base = self.codegen_consume( + bx, + mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref }, + ); cg_base.deref(bx.cx()) } else { bug!("using operand local {:?} as place", place_ref); @@ -502,6 +502,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bug!("encountered OpaqueCast({ty}) in codegen") } mir::ProjectionElem::Subtype(ty) => cg_base.project_type(bx, self.monomorphize(ty)), + mir::ProjectionElem::UnwrapUnsafeBinder(ty) => { + cg_base.project_type(bx, self.monomorphize(ty)) + } mir::ProjectionElem::Index(index) => { let index = &mir::Operand::Copy(mir::Place::from(index)); let index = self.codegen_operand(bx, index); diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index eb4ef599b82..27cb7883b9a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -8,7 +8,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, mir, span_bug}; use rustc_session::config::OptLevel; use rustc_span::{DUMMY_SP, Span}; -use tracing::{debug, instrument}; +use tracing::{debug, instrument, trace}; use super::operand::{OperandRef, OperandValue}; use super::place::PlaceRef; @@ -93,6 +93,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } + // If `v` is an integer constant whose value is just a single byte repeated N times, + // emit a `memset` filling the entire `dest` with that byte. let try_init_all_same = |bx: &mut Bx, v| { let start = dest.val.llval; let size = bx.const_usize(dest.layout.size.bytes()); @@ -117,13 +119,33 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { false }; + trace!(?cg_elem.val); match cg_elem.val { OperandValue::Immediate(v) => { if try_init_all_same(bx, v) { return; } } - _ => (), + OperandValue::Pair(a, b) => { + let a_is_undef = bx.cx().is_undef(a); + match (a_is_undef, bx.cx().is_undef(b)) { + // Can happen for uninit unions + (true, true) => { + // FIXME: can we produce better output here? + } + (false, true) | (true, false) => { + let val = if a_is_undef { b } else { a }; + if try_init_all_same(bx, val) { + return; + } + } + (false, false) => { + // FIXME: if both are the same value, use try_init_all_same + } + } + } + OperandValue::ZeroSized => unreachable!("checked above"), + OperandValue::Ref(..) => {} } let count = self @@ -365,10 +387,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { use abi::Primitive::*; imm = bx.from_immediate(imm); - // When scalars are passed by value, there's no metadata recording their - // valid ranges. For example, `char`s are passed as just `i32`, with no - // way for LLVM to know that they're 0x10FFFF at most. Thus we assume - // the range of the input value too, not just the output range. + // If we have a scalar, we must already know its range. Either + // + // 1) It's a parameter with `range` parameter metadata, + // 2) It's something we `load`ed with `!range` metadata, or + // 3) After a transmute we `assume`d the range (see below). + // + // That said, last time we tried removing this, it didn't actually help + // the rustc-perf results, so might as well keep doing it + // <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182> self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty); imm = match (from_scalar.primitive(), to_scalar.primitive()) { @@ -389,7 +416,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.bitcast(int_imm, to_backend_ty) } }; + + // This `assume` remains important for cases like (a conceptual) + // transmute::<u32, NonZeroU32>(x) == 0 + // since it's never passed to something with parameter metadata (especially + // after MIR inlining) so the only way to tell the backend about the + // constraint that the `transmute` introduced is to `assume` it. self.assume_scalar_range(bx, imm, to_scalar, to_backend_ty); + imm = bx.to_immediate_scalar(imm, to_scalar); imm } @@ -411,31 +445,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - let abi::WrappingRange { start, end } = scalar.valid_range(self.cx); - - if start <= end { - if start > 0 { - let low = bx.const_uint_big(backend_ty, start); - let cmp = bx.icmp(IntPredicate::IntUGE, imm, low); - bx.assume(cmp); - } - - let type_max = scalar.size(self.cx).unsigned_int_max(); - if end < type_max { - let high = bx.const_uint_big(backend_ty, end); - let cmp = bx.icmp(IntPredicate::IntULE, imm, high); - bx.assume(cmp); - } - } else { - let low = bx.const_uint_big(backend_ty, start); - let cmp_low = bx.icmp(IntPredicate::IntUGE, imm, low); - - let high = bx.const_uint_big(backend_ty, end); - let cmp_high = bx.icmp(IntPredicate::IntULE, imm, high); - - let or = bx.or(cmp_low, cmp_high); - bx.assume(or); - } + let range = scalar.valid_range(self.cx); + bx.assume_integer_range(imm, backend_ty, range); } pub(crate) fn codegen_rvalue_unsized( @@ -601,9 +612,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::CopyForDeref(place) => { self.codegen_operand(bx, &mir::Operand::Copy(place)) } - mir::Rvalue::RawPtr(mutability, place) => { - let mk_ptr = - move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| Ty::new_ptr(tcx, ty, mutability); + mir::Rvalue::RawPtr(kind, place) => { + let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { + Ty::new_ptr(tcx, ty, kind.to_mutbl_lossy()) + }; self.codegen_place_to_pointer(bx, place, mk_ptr) } @@ -729,6 +741,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let val = bx.tcx().sess.ub_checks(); bx.cx().const_bool(val) } + mir::NullOp::ContractChecks => { + let val = bx.tcx().sess.contract_checks(); + bx.cx().const_bool(val) + } }; let tcx = self.cx.tcx(); OperandRef { @@ -811,6 +827,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandRef { val: OperandValue::Immediate(val), layout: box_layout } } + mir::Rvalue::WrapUnsafeBinder(ref operand, binder_ty) => { + let operand = self.codegen_operand(bx, operand); + let binder_ty = self.monomorphize(binder_ty); + let layout = bx.cx().layout_of(binder_ty); + OperandRef { val: operand.val, layout } + } } } @@ -1111,7 +1133,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Discriminant(..) | mir::Rvalue::NullaryOp(..) | mir::Rvalue::ThreadLocalRef(_) | - mir::Rvalue::Use(..) => // (*) + mir::Rvalue::Use(..) | + mir::Rvalue::WrapUnsafeBinder(..) => // (*) true, // Arrays are always aggregates, so it's not worth checking anything here. // (If it's really `[(); N]` or `[T; 0]` and we use the place path, fine.) diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 3ee13b19f66..b7dcf16fa2b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -167,6 +167,11 @@ pub trait BuilderMethods<'a, 'tcx>: fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends + /// can emit something more helpful for optimizations. + fn or_disjoint(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.or(lhs, rhs) + } fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn neg(&mut self, v: Self::Value) -> Self::Value; fn fneg(&mut self, v: Self::Value) -> Self::Value; @@ -217,6 +222,27 @@ pub trait BuilderMethods<'a, 'tcx>: dest: PlaceRef<'tcx, Self::Value>, ); + /// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`. + /// + /// This *always* emits the assumption, so you probably want to check the + /// optimization level and `Scalar::is_always_valid` before calling it. + fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) { + let WrappingRange { start, end } = range; + + // Perhaps one day we'll be able to use assume operand bundles for this, + // but for now this encoding with a single icmp+assume is best per + // <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158> + let shifted = if start == 0 { + imm + } else { + let low = self.const_uint_big(ty, start); + self.sub(imm, low) + }; + let width = self.const_uint_big(ty, u128::wrapping_sub(end, start)); + let cmp = self.icmp(IntPredicate::IntULE, shifted, width); + self.assume(cmp); + } + fn range_metadata(&mut self, load: Self::Value, range: WrappingRange); fn nonnull_metadata(&mut self, load: Self::Value); diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index 9af463a691a..5cfb56ebace 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -1,5 +1,5 @@ +use rustc_abi as abi; use rustc_middle::mir::interpret::{ConstAllocation, Scalar}; -use rustc_target::abi; use super::BackendTypes; @@ -9,6 +9,7 @@ pub trait ConstCodegenMethods<'tcx>: BackendTypes { /// Generate an uninitialized value (matching uninitialized memory in MIR). /// Whether memory is initialized or not is tracked byte-for-byte. fn const_undef(&self, t: Self::Type) -> Self::Value; + fn is_undef(&self, v: Self::Value) -> bool; /// Generate a fake value. Poison always affects the entire value, even if just a single byte is /// poison. This can only be used in codepaths that are already UB, i.e., UB-free Rust code /// (including code that e.g. copies uninit memory with `MaybeUninit`) can never encounter a diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs index fe135e911fb..30d77c206a5 100644 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -2,7 +2,7 @@ use std::ops::Range; use rustc_abi::Size; use rustc_middle::mir; -use rustc_middle::ty::{Instance, PolyExistentialTraitRef, Ty}; +use rustc_middle::ty::{ExistentialTraitRef, Instance, Ty}; use rustc_span::{SourceFile, Span, Symbol}; use rustc_target::callconv::FnAbi; @@ -13,7 +13,7 @@ pub trait DebugInfoCodegenMethods<'tcx>: BackendTypes { fn create_vtable_debuginfo( &self, ty: Ty<'tcx>, - trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + trait_ref: Option<ExistentialTraitRef<'tcx>>, vtable: Self::Value, ); diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs index 5b33fd7ab10..4004947b464 100644 --- a/compiler/rustc_codegen_ssa/src/traits/misc.rs +++ b/compiler/rustc_codegen_ssa/src/traits/misc.rs @@ -10,11 +10,11 @@ use super::BackendTypes; pub trait MiscCodegenMethods<'tcx>: BackendTypes { fn vtables( &self, - ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), Self::Value>>; + ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), Self::Value>>; fn apply_vcall_visibility_metadata( &self, _ty: Ty<'tcx>, - _poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, + _poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>, _vtable: Self::Value, ) { } diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 6292d321f6b..c178ebc596e 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -1,8 +1,8 @@ -use rustc_abi::{AddressSpace, Float, Integer}; +use rustc_abi::{AddressSpace, Float, Integer, Reg}; use rustc_middle::bug; use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, TyAndLayout}; use rustc_middle::ty::{self, Ty}; -use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, Reg}; +use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi}; use super::BackendTypes; use super::misc::MiscCodegenMethods; diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index 51e2255efe1..97fe614aa10 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -1,7 +1,6 @@ use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_errors::{DiagCtxtHandle, FatalError}; use rustc_middle::dep_graph::WorkProduct; -use rustc_middle::ty::TyCtxt; use crate::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; use crate::back::write::{CodegenContext, FatLtoInput, ModuleConfig}; @@ -65,7 +64,6 @@ pub trait WriteBackendMethods: 'static + Sized + Clone { fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer); fn autodiff( cgcx: &CodegenContext<Self>, - tcx: TyCtxt<'_>, module: &ModuleCodegen<Self::Module>, diff_fncs: Vec<AutoDiffItem>, config: &ModuleConfig, |