diff options
Diffstat (limited to 'compiler/rustc_codegen_ssa')
34 files changed, 766 insertions, 395 deletions
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index f347a7fb0bb..3771fc6b0a2 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -6,8 +6,10 @@ edition = "2021" [dependencies] # tidy-alphabetical-start ar_archive_writer = "0.2.0" +arrayvec = { version = "0.7", default-features = false } bitflags = "2.4.1" cc = "1.0.90" +either = "1.5.0" itertools = "0.12" jobserver = "0.1.28" pathdiff = "0.2.0" diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl index d159fe58d3e..1a851ad04a1 100644 --- a/compiler/rustc_codegen_ssa/messages.ftl +++ b/compiler/rustc_codegen_ssa/messages.ftl @@ -212,6 +212,8 @@ codegen_ssa_rlib_only_rmeta_found = could not find rlib for: `{$crate_name}`, fo codegen_ssa_select_cpp_build_tool_workload = in the Visual Studio installer, ensure the "C++ build tools" workload is selected +codegen_ssa_self_contained_linker_missing = the self-contained linker was requested, but it wasn't found in the target's sysroot, or in rustc's sysroot + codegen_ssa_shuffle_indices_evaluation = could not evaluate shuffle_indices at compile time codegen_ssa_specify_libraries_to_link = use the `-l` flag to specify native libraries to link diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs index e441aea8400..6e7d736eb29 100644 --- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs +++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs @@ -37,6 +37,7 @@ use rustc_span::{Span, Symbol}; use std::borrow::Cow; use std::fmt; use thin_vec::ThinVec; +use tracing::debug; #[allow(missing_docs)] pub fn assert_module_sources(tcx: TyCtxt<'_>, set_reuse: &dyn Fn(&mut CguReuseTracker)) { diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index b458f325b73..bdb808b1d4f 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -8,6 +8,7 @@ use rustc_fs_util::{fix_windows_verbatim_for_gcc, try_canonicalize}; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_metadata::find_native_static_library; use rustc_metadata::fs::{copy_to_stdout, emit_wrapper_file, METADATA_FILENAME}; +use rustc_middle::bug; use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::SymbolExportKind; @@ -51,6 +52,7 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Output, Stdio}; use std::{env, fmt, fs, io, mem, str}; +use tracing::{debug, info, warn}; pub fn ensure_removed(dcx: &DiagCtxt, path: &Path) { if let Err(e) = fs::remove_file(path) { @@ -785,12 +787,33 @@ fn link_natively( if matches!(flavor, LinkerFlavor::Gnu(Cc::Yes, _)) && unknown_arg_regex.is_match(&out) && out.contains("-no-pie") - && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-no-pie") + && cmd.get_args().iter().any(|e| e == "-no-pie") { info!("linker output: {:?}", out); warn!("Linker does not support -no-pie command line option. Retrying without."); for arg in cmd.take_args() { - if arg.to_string_lossy() != "-no-pie" { + if arg != "-no-pie" { + cmd.arg(arg); + } + } + info!("{:?}", &cmd); + continue; + } + + // Check if linking failed with an error message that indicates the driver didn't recognize + // the `-fuse-ld=lld` option. If so, re-perform the link step without it. This avoids having + // to spawn multiple instances on the happy path to do version checking, and ensures things + // keep working on the tier 1 baseline of GLIBC 2.17+. That is generally understood as GCCs + // circa RHEL/CentOS 7, 4.5 or so, whereas lld support was added in GCC 9. + if matches!(flavor, LinkerFlavor::Gnu(Cc::Yes, Lld::Yes)) + && unknown_arg_regex.is_match(&out) + && out.contains("-fuse-ld=lld") + && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-fuse-ld=lld") + { + info!("linker output: {:?}", out); + warn!("The linker driver does not support `-fuse-ld=lld`. Retrying without it."); + for arg in cmd.take_args() { + if arg.to_string_lossy() != "-fuse-ld=lld" { cmd.arg(arg); } } @@ -803,7 +826,7 @@ fn link_natively( if matches!(flavor, LinkerFlavor::Gnu(Cc::Yes, _)) && unknown_arg_regex.is_match(&out) && (out.contains("-static-pie") || out.contains("--no-dynamic-linker")) - && cmd.get_args().iter().any(|e| e.to_string_lossy() == "-static-pie") + && cmd.get_args().iter().any(|e| e == "-static-pie") { info!("linker output: {:?}", out); warn!( @@ -842,7 +865,7 @@ fn link_natively( assert!(pre_objects_static.is_empty() || !pre_objects_static_pie.is_empty()); assert!(post_objects_static.is_empty() || !post_objects_static_pie.is_empty()); for arg in cmd.take_args() { - if arg.to_string_lossy() == "-static-pie" { + if arg == "-static-pie" { // Replace the output kind. cmd.arg("-static"); } else if pre_objects_static_pie.contains(&arg) { @@ -1229,7 +1252,10 @@ fn add_sanitizer_libraries( if sanitizer.contains(SanitizerSet::DATAFLOW) { link_sanitizer_runtime(sess, flavor, linker, "dfsan"); } - if sanitizer.contains(SanitizerSet::LEAK) { + if sanitizer.contains(SanitizerSet::LEAK) + && !sanitizer.contains(SanitizerSet::ADDRESS) + && !sanitizer.contains(SanitizerSet::HWADDRESS) + { link_sanitizer_runtime(sess, flavor, linker, "lsan"); } if sanitizer.contains(SanitizerSet::MEMORY) { @@ -3115,18 +3141,32 @@ fn add_lld_args( let self_contained_linker = self_contained_cli || self_contained_target; if self_contained_linker && !sess.opts.cg.link_self_contained.is_linker_disabled() { + let mut linker_path_exists = false; for path in sess.get_tools_search_paths(false) { + let linker_path = path.join("gcc-ld"); + linker_path_exists |= linker_path.exists(); cmd.arg({ let mut arg = OsString::from("-B"); - arg.push(path.join("gcc-ld")); + arg.push(linker_path); arg }); } + if !linker_path_exists { + // As a sanity check, we emit an error if none of these paths exist: we want + // self-contained linking and have no linker. + sess.dcx().emit_fatal(errors::SelfContainedLinkerMissing); + } } // 2. Implement the "linker flavor" part of this feature by asking `cc` to use some kind of // `lld` as the linker. - cmd.arg("-fuse-ld=lld"); + // + // Note that wasm targets skip this step since the only option there anyway + // is to use LLD but the `wasm32-wasip2` target relies on a wrapper around + // this, `wasm-component-ld`, which is overridden if this option is passed. + if !sess.target.is_like_wasm { + cmd.arg("-fuse-ld=lld"); + } if !flavor.is_gnu() { // Tell clang to use a non-default LLD flavor. diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index fad6f439441..a82478900b1 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -12,6 +12,7 @@ use std::{env, mem, str}; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_metadata::find_native_static_library; +use rustc_middle::bug; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols; use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; @@ -21,6 +22,7 @@ use rustc_session::Session; use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, Lld}; use cc::windows_registry; +use tracing::{debug, warn}; /// Disables non-English messages from localized linkers. /// Such messages may cause issues with text encoding on Windows (#35785) @@ -902,52 +904,45 @@ impl<'a> Linker for MsvcLinker<'a> { } } - fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]) { - match strip { - Strip::None => { - // This will cause the Microsoft linker to generate a PDB file - // from the CodeView line tables in the object files. - self.cmd.arg("/DEBUG"); - - // Default to emitting only the file name of the PDB file into - // the binary instead of the full path. Emitting the full path - // may leak private information (such as user names). - // See https://github.com/rust-lang/rust/issues/87825. - // - // This default behavior can be overridden by explicitly passing - // `-Clink-arg=/PDBALTPATH:...` to rustc. - self.cmd.arg("/PDBALTPATH:%_PDB%"); - - // This will cause the Microsoft linker to embed .natvis info into the PDB file - let natvis_dir_path = self.sess.sysroot.join("lib\\rustlib\\etc"); - if let Ok(natvis_dir) = fs::read_dir(&natvis_dir_path) { - for entry in natvis_dir { - match entry { - Ok(entry) => { - let path = entry.path(); - if path.extension() == Some("natvis".as_ref()) { - let mut arg = OsString::from("/NATVIS:"); - arg.push(path); - self.cmd.arg(arg); - } - } - Err(error) => { - self.sess.dcx().emit_warn(errors::NoNatvisDirectory { error }); - } + fn debuginfo(&mut self, _strip: Strip, natvis_debugger_visualizers: &[PathBuf]) { + // This will cause the Microsoft linker to generate a PDB file + // from the CodeView line tables in the object files. + self.cmd.arg("/DEBUG"); + + // Default to emitting only the file name of the PDB file into + // the binary instead of the full path. Emitting the full path + // may leak private information (such as user names). + // See https://github.com/rust-lang/rust/issues/87825. + // + // This default behavior can be overridden by explicitly passing + // `-Clink-arg=/PDBALTPATH:...` to rustc. + self.cmd.arg("/PDBALTPATH:%_PDB%"); + + // This will cause the Microsoft linker to embed .natvis info into the PDB file + let natvis_dir_path = self.sess.sysroot.join("lib\\rustlib\\etc"); + if let Ok(natvis_dir) = fs::read_dir(&natvis_dir_path) { + for entry in natvis_dir { + match entry { + Ok(entry) => { + let path = entry.path(); + if path.extension() == Some("natvis".as_ref()) { + let mut arg = OsString::from("/NATVIS:"); + arg.push(path); + self.cmd.arg(arg); } } - } - - // This will cause the Microsoft linker to embed .natvis info for all crates into the PDB file - for path in natvis_debugger_visualizers { - let mut arg = OsString::from("/NATVIS:"); - arg.push(path); - self.cmd.arg(arg); + Err(error) => { + self.sess.dcx().emit_warn(errors::NoNatvisDirectory { error }); + } } } - Strip::Debuginfo | Strip::Symbols => { - self.cmd.arg("/DEBUG:NONE"); - } + } + + // This will cause the Microsoft linker to embed .natvis info for all crates into the PDB file + for path in natvis_debugger_visualizers { + let mut arg = OsString::from("/NATVIS:"); + arg.push(path); + self.cmd.arg(arg); } } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index ab1bc0b6cd2..264a98844ad 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -16,6 +16,7 @@ use rustc_data_structures::owned_slice::{try_slice_owned, OwnedSlice}; use rustc_metadata::creader::MetadataLoader; use rustc_metadata::fs::METADATA_FILENAME; use rustc_metadata::EncodedMetadata; +use rustc_middle::bug; use rustc_session::Session; use rustc_span::sym; use rustc_target::abi::Endian; diff --git a/compiler/rustc_codegen_ssa/src/back/rpath.rs b/compiler/rustc_codegen_ssa/src/back/rpath.rs index ebbf49af184..3114f1c38ae 100644 --- a/compiler/rustc_codegen_ssa/src/back/rpath.rs +++ b/compiler/rustc_codegen_ssa/src/back/rpath.rs @@ -3,6 +3,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_fs_util::try_canonicalize; use std::ffi::OsString; use std::path::{Path, PathBuf}; +use tracing::debug; pub struct RPathConfig<'a> { pub libs: &'a [&'a Path], diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index b19f52182b6..79d6641a0da 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -6,6 +6,7 @@ use rustc_ast::expand::allocator::{ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE} use rustc_data_structures::unord::UnordMap; use rustc_hir::def::DefKind; use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE}; +use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::middle::exported_symbols::{ metadata_symbol_name, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, @@ -17,6 +18,7 @@ use rustc_middle::ty::{GenericArgKind, GenericArgsRef}; use rustc_middle::util::Providers; use rustc_session::config::{CrateType, OomStrategy}; use rustc_target::spec::{SanitizerSet, TlsModel}; +use tracing::debug; pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { crates_export_threshold(tcx.crate_types()) @@ -363,6 +365,24 @@ fn exported_symbols_provider_local( }, )); } + MonoItem::Fn(Instance { + def: InstanceDef::AsyncDropGlueCtorShim(def_id, Some(ty)), + args, + }) => { + // A little sanity-check + debug_assert_eq!( + args.non_erasable_generics(tcx, def_id).skip(1).next(), + Some(GenericArgKind::Type(ty)) + ); + symbols.push(( + ExportedSymbol::AsyncDropGlueCtorShim(ty), + SymbolExportInfo { + level: SymbolExportLevel::Rust, + kind: SymbolExportKind::Text, + used: false, + }, + )); + } _ => { // Any other symbols don't qualify for sharing } @@ -380,11 +400,12 @@ fn upstream_monomorphizations_provider( tcx: TyCtxt<'_>, (): (), ) -> DefIdMap<UnordMap<GenericArgsRef<'_>, CrateNum>> { - let cnums = tcx.crates(()); + let cnums = tcx.used_crates(()); let mut instances: DefIdMap<UnordMap<_, _>> = Default::default(); let drop_in_place_fn_def_id = tcx.lang_items().drop_in_place_fn(); + let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn(); for &cnum in cnums.iter() { for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() { @@ -399,6 +420,18 @@ fn upstream_monomorphizations_provider( continue; } } + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id { + ( + async_drop_in_place_fn_def_id, + tcx.mk_args(&[tcx.lifetimes.re_erased.into(), ty.into()]), + ) + } else { + // `drop_in_place` in place does not exist, don't try + // to use it. + continue; + } + } ExportedSymbol::NonGeneric(..) | ExportedSymbol::ThreadLocalShim(..) | ExportedSymbol::NoDefId(..) => { @@ -534,6 +567,13 @@ pub fn symbol_name_for_instance_in_crate<'tcx>( Instance::resolve_drop_in_place(tcx, ty), instantiating_crate, ), + ExportedSymbol::AsyncDropGlueCtorShim(ty) => { + rustc_symbol_mangling::symbol_name_for_instance_in_crate( + tcx, + Instance::resolve_async_drop_in_place(tcx, ty), + instantiating_crate, + ) + } ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(), } } @@ -582,6 +622,9 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>( // DropGlue always use the Rust calling convention and thus follow the target's default // symbol decoration scheme. ExportedSymbol::DropGlue(..) => None, + // AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the + // target's default symbol decoration scheme. + ExportedSymbol::AsyncDropGlueCtorShim(..) => None, // NoDefId always follow the target's default symbol decoration scheme. ExportedSymbol::NoDefId(..) => None, // ThreadLocalShim always follow the target's default symbol decoration scheme. diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index c4f062405bb..d57f4ddf8aa 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -26,6 +26,7 @@ use rustc_incremental::{ }; use rustc_metadata::fs::copy_to_stdout; use rustc_metadata::EncodedMetadata; +use rustc_middle::bug; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::exported_symbols::SymbolExportInfo; use rustc_middle::ty::TyCtxt; @@ -48,6 +49,7 @@ use std::str; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::Arc; use std::thread; +use tracing::debug; const PRE_LTO_BC_EXT: &str = "pre-lto.bc"; @@ -106,6 +108,7 @@ pub struct ModuleConfig { pub emit_asm: bool, pub emit_obj: EmitObj, pub emit_thin_lto: bool, + pub emit_thin_lto_summary: bool, pub bc_cmdline: String, // Miscellaneous flags. These are mostly copied from command-line @@ -230,6 +233,10 @@ impl ModuleConfig { ), emit_obj, emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto, + emit_thin_lto_summary: if_regular!( + sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode), + false + ), bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(), verify_llvm_ir: sess.verify_llvm_ir(), @@ -281,6 +288,7 @@ impl ModuleConfig { pub fn bitcode_needed(&self) -> bool { self.emit_bc + || self.emit_thin_lto_summary || self.emit_obj == EmitObj::Bitcode || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) } @@ -628,6 +636,9 @@ fn produce_final_output_artifacts( // them for making an rlib. copy_if_one_unit(OutputType::Bitcode, true); } + OutputType::ThinLinkBitcode => { + copy_if_one_unit(OutputType::ThinLinkBitcode, false); + } OutputType::LlvmAssembly => { copy_if_one_unit(OutputType::LlvmAssembly, false); } @@ -881,7 +892,7 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>( match lto_type { ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config), ComputedLtoType::Thin => { - let (name, thin_buffer) = B::prepare_thin(module); + let (name, thin_buffer) = B::prepare_thin(module, false); if let Some(path) = bitcode { fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 930b9b8c0db..f372d3a0522 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -20,16 +20,17 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry}; use rustc_data_structures::sync::par_map; use rustc_data_structures::unord::UnordMap; -use rustc_hir as hir; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_hir::lang_items::LangItem; use rustc_metadata::EncodedMetadata; +use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, DebuggerVisualizerType}; use rustc_middle::middle::exported_symbols; use rustc_middle::middle::exported_symbols::SymbolExportKind; use rustc_middle::middle::lang_items; use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem}; +use rustc_middle::mir::BinOp; use rustc_middle::query::Providers; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; @@ -44,33 +45,34 @@ use std::collections::BTreeSet; use std::time::{Duration, Instant}; use itertools::Itertools; +use tracing::{debug, info}; -pub fn bin_op_to_icmp_predicate(op: hir::BinOpKind, signed: bool) -> IntPredicate { +pub fn bin_op_to_icmp_predicate(op: BinOp, signed: bool) -> IntPredicate { match op { - hir::BinOpKind::Eq => IntPredicate::IntEQ, - hir::BinOpKind::Ne => IntPredicate::IntNE, - hir::BinOpKind::Lt => { + BinOp::Eq => IntPredicate::IntEQ, + BinOp::Ne => IntPredicate::IntNE, + BinOp::Lt => { if signed { IntPredicate::IntSLT } else { IntPredicate::IntULT } } - hir::BinOpKind::Le => { + BinOp::Le => { if signed { IntPredicate::IntSLE } else { IntPredicate::IntULE } } - hir::BinOpKind::Gt => { + BinOp::Gt => { if signed { IntPredicate::IntSGT } else { IntPredicate::IntUGT } } - hir::BinOpKind::Ge => { + BinOp::Ge => { if signed { IntPredicate::IntSGE } else { @@ -85,14 +87,14 @@ pub fn bin_op_to_icmp_predicate(op: hir::BinOpKind, signed: bool) -> IntPredicat } } -pub fn bin_op_to_fcmp_predicate(op: hir::BinOpKind) -> RealPredicate { +pub fn bin_op_to_fcmp_predicate(op: BinOp) -> RealPredicate { match op { - hir::BinOpKind::Eq => RealPredicate::RealOEQ, - hir::BinOpKind::Ne => RealPredicate::RealUNE, - hir::BinOpKind::Lt => RealPredicate::RealOLT, - hir::BinOpKind::Le => RealPredicate::RealOLE, - hir::BinOpKind::Gt => RealPredicate::RealOGT, - hir::BinOpKind::Ge => RealPredicate::RealOGE, + BinOp::Eq => RealPredicate::RealOEQ, + BinOp::Ne => RealPredicate::RealUNE, + BinOp::Lt => RealPredicate::RealOLT, + BinOp::Le => RealPredicate::RealOLE, + BinOp::Gt => RealPredicate::RealOGT, + BinOp::Ge => RealPredicate::RealOGE, op => { bug!( "comparison_op_to_fcmp_predicate: expected comparison operator, \ @@ -109,7 +111,7 @@ pub fn compare_simd_types<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( rhs: Bx::Value, t: Ty<'tcx>, ret_ty: Bx::Type, - op: hir::BinOpKind, + op: BinOp, ) -> Bx::Value { let signed = match t.kind() { ty::Float(_) => { @@ -282,7 +284,7 @@ pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } if src_f.layout.ty == dst_f.layout.ty { - bx.typed_place_copy(dst_f, src_f); + bx.typed_place_copy(dst_f.val, src_f.val, src_f.layout); } else { coerce_unsized_into(bx, src_f, dst_f); } @@ -292,12 +294,13 @@ pub fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } -/// Returns `rhs` sufficiently masked, truncated, and/or extended so that -/// it can be used to shift `lhs`. +/// Returns `rhs` sufficiently masked, truncated, and/or extended so that it can be used to shift +/// `lhs`: it has the same size as `lhs`, and the value, when interpreted unsigned (no matter its +/// type), will not exceed the size of `lhs`. /// -/// Shifts in MIR are all allowed to have mismatched LHS & RHS types. +/// Shifts in MIR are all allowed to have mismatched LHS & RHS types, and signed RHS. /// The shift methods in `BuilderMethods`, however, are fully homogeneous -/// (both parameters and the return type are all the same type). +/// (both parameters and the return type are all the same size) and assume an unsigned RHS. /// /// If `is_unchecked` is false, this masks the RHS to ensure it stays in-bounds, /// as the `BuilderMethods` shifts are UB for out-of-bounds shift amounts. @@ -508,7 +511,7 @@ fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let ptr_size = bx.tcx().data_layout.pointer_size; let ptr_align = bx.tcx().data_layout.pointer_align.abi; let arg_argc = bx.const_int(cx.type_isize(), 2); - let arg_argv = bx.alloca(cx.type_array(cx.type_ptr(), 2), ptr_align); + let arg_argv = bx.alloca(2 * ptr_size, ptr_align); bx.store(param_handle, arg_argv, ptr_align); let arg_argv_el1 = bx.inbounds_ptradd(arg_argv, bx.const_usize(ptr_size.bytes())); bx.store(param_system_table, arg_argv_el1, ptr_align); @@ -538,7 +541,7 @@ pub fn collect_debugger_visualizers_transitive( tcx.debugger_visualizers(LOCAL_CRATE) .iter() .chain( - tcx.crates(()) + tcx.used_crates(()) .iter() .filter(|&cnum| { let used_crate_source = tcx.used_crate_source(*cnum); @@ -848,7 +851,7 @@ impl CrateInfo { // `compiler_builtins` are always placed last to ensure that they're linked correctly. used_crates.extend(compiler_builtins); - let crates = tcx.crates(()); + let crates = tcx.used_crates(()); let n_crates = crates.len(); let mut info = CrateInfo { target_cpu, diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 9c9e134f033..5d7257b15c4 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -276,7 +276,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { sym::target_feature => { if !tcx.is_closure_like(did.to_def_id()) && let Some(fn_sig) = fn_sig() - && fn_sig.skip_binder().unsafety() == hir::Unsafety::Normal + && fn_sig.skip_binder().safety() == hir::Safety::Safe { if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { // The `#[target_feature]` attribute is allowed on @@ -327,6 +327,18 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } else { codegen_fn_attrs.linkage = linkage; } + if tcx.is_mutable_static(did.into()) { + let mut diag = tcx.dcx().struct_span_err( + attr.span, + "mutable statics are not allowed with `#[linkage]`", + ); + diag.note( + "making the static mutable would allow changing which symbol the \ + static references rather than make the target of the symbol \ + mutable", + ); + diag.emit(); + } } } sym::link_section => { @@ -435,7 +447,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { sym::repr => { codegen_fn_attrs.alignment = if let Some(items) = attr.meta_item_list() && let [item] = items.as_slice() - && let Some((sym::align, literal)) = item.name_value_literal() + && let Some((sym::align, literal)) = item.singleton_lit_list() { rustc_attr::parse_alignment(&literal.kind) .map_err(|msg| { @@ -564,8 +576,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { lint::builtin::INLINE_NO_SANITIZE, hir_id, no_sanitize_span, - "`no_sanitize` will have no effect after inlining", |lint| { + lint.primary_message("`no_sanitize` will have no effect after inlining"); lint.span_note(inline_span, "inlining requested here"); }, ) diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs index b41739867c7..e4a36b3f591 100644 --- a/compiler/rustc_codegen_ssa/src/common.rs +++ b/compiler/rustc_codegen_ssa/src/common.rs @@ -4,6 +4,7 @@ use rustc_hir::LangItem; use rustc_middle::mir; use rustc_middle::ty::Instance; use rustc_middle::ty::{self, layout::TyAndLayout, TyCtxt}; +use rustc_middle::{bug, span_bug}; use rustc_span::Span; use crate::traits::*; diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index 5f0dcf9510f..07473ee476b 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -16,6 +16,7 @@ use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher}; use rustc_hir::def_id::DefId; use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData}; use rustc_hir::{CoroutineDesugaring, CoroutineKind, CoroutineSource, Mutability}; +use rustc_middle::bug; use rustc_middle::ty::layout::{IntegerExt, TyAndLayout}; use rustc_middle::ty::{self, ExistentialProjection, ParamEnv, Ty, TyCtxt}; use rustc_middle::ty::{GenericArgKind, GenericArgsRef}; @@ -364,7 +365,7 @@ fn push_debuginfo_type_name<'tcx>( } output.push_str(" (*)("); } else { - output.push_str(sig.unsafety.prefix_str()); + output.push_str(sig.safety.prefix_str()); if sig.abi != rustc_target::spec::abi::Abi::Rust { output.push_str("extern \""); diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index b843d1bdf23..3641e7842cf 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -414,6 +414,10 @@ pub struct UnableToExeLinker { pub struct MsvcMissingLinker; #[derive(Diagnostic)] +#[diag(codegen_ssa_self_contained_linker_missing)] +pub struct SelfContainedLinkerMissing; + +#[derive(Diagnostic)] #[diag(codegen_ssa_check_installed_visual_studio)] pub struct CheckInstalledVisualStudio; @@ -431,7 +435,6 @@ pub struct ProcessingDymutilFailed { #[derive(Diagnostic)] #[diag(codegen_ssa_unable_to_run_dsymutil)] -#[note] pub struct UnableToRunDsymutil { pub error: Error, } diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 80fe7e0bb78..1668104d7e2 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -4,7 +4,6 @@ #![allow(internal_features)] #![allow(rustc::diagnostic_outside_of_impl)] #![allow(rustc::untranslatable_diagnostic)] -#![cfg_attr(bootstrap, feature(associated_type_bounds))] #![feature(box_patterns)] #![feature(if_let_guard)] #![feature(let_chains)] @@ -16,19 +15,13 @@ //! The backend-agnostic functions of this crate use functions defined in various traits that //! have to be implemented by each backend. -#[macro_use] -extern crate rustc_macros; -#[macro_use] -extern crate tracing; -#[macro_use] -extern crate rustc_middle; - use rustc_ast as ast; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::Lrc; use rustc_data_structures::unord::UnordMap; use rustc_hir::def_id::CrateNum; +use rustc_macros::{Decodable, Encodable, HashStable}; use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile; use rustc_middle::middle::dependency_format::Dependencies; @@ -199,6 +192,7 @@ pub enum CodegenErrors { EmptyVersionNumber, EncodingVersionMismatch { version_array: String, rlink_version: u32 }, RustcVersionMismatch { rustc_version: String }, + CorruptFile, } pub fn provide(providers: &mut Providers) { @@ -269,7 +263,9 @@ impl CodegenResults { }); } - let mut decoder = MemDecoder::new(&data[4..], 0); + let Ok(mut decoder) = MemDecoder::new(&data[4..], 0) else { + return Err(CodegenErrors::CorruptFile); + }; let rustc_version = decoder.read_str(); if rustc_version != sess.cfg_version { return Err(CodegenErrors::RustcVersionMismatch { diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 4f7dc9968a1..ddc6797388e 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -1,9 +1,11 @@ use crate::traits::*; +use rustc_middle::bug; use rustc_middle::ty::{self, GenericArgKind, Ty}; use rustc_session::config::Lto; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::call::FnAbi; +use tracing::{debug, instrument}; #[derive(Copy, Clone, Debug)] pub struct VirtualIndex(u64); @@ -13,12 +15,13 @@ impl<'a, 'tcx> VirtualIndex { VirtualIndex(index as u64) } - pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>( + fn get_fn_inner<Bx: BuilderMethods<'a, 'tcx>>( self, bx: &mut Bx, llvtable: Bx::Value, ty: Ty<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + nonnull: bool, ) -> Bx::Value { // Load the function pointer from the object. debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); @@ -39,13 +42,35 @@ impl<'a, 'tcx> VirtualIndex { } else { let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); let ptr = bx.load(llty, gep, ptr_align); - bx.nonnull_metadata(ptr); // VTable loads are invariant. bx.set_invariant_load(ptr); + if nonnull { + bx.nonnull_metadata(ptr); + } ptr } } + pub fn get_optional_fn<Bx: BuilderMethods<'a, 'tcx>>( + self, + bx: &mut Bx, + llvtable: Bx::Value, + ty: Ty<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + ) -> Bx::Value { + self.get_fn_inner(bx, llvtable, ty, fn_abi, false) + } + + pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>( + self, + bx: &mut Bx, + llvtable: Bx::Value, + ty: Ty<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + ) -> Bx::Value { + self.get_fn_inner(bx, llvtable, ty, fn_abi, true) + } + pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>( self, bx: &mut Bx, diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index c1de9b76fe7..0577ba32ffd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -10,6 +10,8 @@ use rustc_middle::mir::traversal; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::{bug, span_bug}; +use tracing::debug; pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( fx: &FunctionCx<'a, 'tcx, Bx>, diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 452398e6d82..bd9704b37ae 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -17,12 +17,14 @@ use rustc_middle::mir::{self, AssertKind, BasicBlock, SwitchTargets, UnwindTermi use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::{bug, span_bug}; use rustc_monomorphize::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_session::config::OptLevel; use rustc_span::{source_map::Spanned, sym, Span}; use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode, Reg}; use rustc_target::abi::{self, HasDataLayout, WrappingRange}; use rustc_target::spec::abi::Abi; +use tracing::{debug, info}; use std::cmp; @@ -498,6 +500,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx, + source_info: &mir::SourceInfo, location: mir::Place<'tcx>, target: mir::BasicBlock, unwind: mir::UnwindAction, @@ -521,90 +524,106 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args1 = [place.val.llval]; &args1[..] }; - let (drop_fn, fn_abi, drop_instance) = - match ty.kind() { - // FIXME(eddyb) perhaps move some of this logic into - // `Instance::resolve_drop_in_place`? - ty::Dynamic(_, _, ty::Dyn) => { - // IN THIS ARM, WE HAVE: - // ty = *mut (dyn Trait) - // which is: exists<T> ( *mut T, Vtable<T: Trait> ) - // args[0] args[1] - // - // args = ( Data, Vtable ) - // | - // v - // /-------\ - // | ... | - // \-------/ - // - let virtual_drop = Instance { - def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), - args: drop_fn.args, - }; - debug!("ty = {:?}", ty); - debug!("drop_fn = {:?}", drop_fn); - debug!("args = {:?}", args); - let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); - let vtable = args[1]; - // Truncate vtable off of args list - args = &args[..1]; - ( - meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) - .get_fn(bx, vtable, ty, fn_abi), - fn_abi, - virtual_drop, - ) - } - ty::Dynamic(_, _, ty::DynStar) => { - // IN THIS ARM, WE HAVE: - // ty = *mut (dyn* Trait) - // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) - // - // args = [ * ] - // | - // v - // ( Data, Vtable ) - // | - // v - // /-------\ - // | ... | - // \-------/ - // - // - // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING - // - // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) - // vtable = (*args[0]).1 // loads the vtable out - // (data, vtable) // an equivalent Rust `*mut dyn Trait` - // - // SO THEN WE CAN USE THE ABOVE CODE. - let virtual_drop = Instance { - def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), - args: drop_fn.args, - }; - debug!("ty = {:?}", ty); - debug!("drop_fn = {:?}", drop_fn); - debug!("args = {:?}", args); - let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); - let meta_ptr = place.project_field(bx, 1); - let meta = bx.load_operand(meta_ptr); - // Truncate vtable off of args list - args = &args[..1]; - debug!("args' = {:?}", args); - ( - meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) - .get_fn(bx, meta.immediate(), ty, fn_abi), - fn_abi, - virtual_drop, - ) - } - _ => ( - bx.get_fn_addr(drop_fn), - bx.fn_abi_of_instance(drop_fn, ty::List::empty()), - drop_fn, - ), - }; + let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() { + // FIXME(eddyb) perhaps move some of this logic into + // `Instance::resolve_drop_in_place`? + ty::Dynamic(_, _, ty::Dyn) => { + // IN THIS ARM, WE HAVE: + // ty = *mut (dyn Trait) + // which is: exists<T> ( *mut T, Vtable<T: Trait> ) + // args[0] args[1] + // + // args = ( Data, Vtable ) + // | + // v + // /-------\ + // | ... | + // \-------/ + // + let virtual_drop = Instance { + def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function + args: drop_fn.args, + }; + debug!("ty = {:?}", ty); + debug!("drop_fn = {:?}", drop_fn); + debug!("args = {:?}", args); + let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); + let vtable = args[1]; + // Truncate vtable off of args list + args = &args[..1]; + ( + true, + meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) + .get_optional_fn(bx, vtable, ty, fn_abi), + fn_abi, + virtual_drop, + ) + } + ty::Dynamic(_, _, ty::DynStar) => { + // IN THIS ARM, WE HAVE: + // ty = *mut (dyn* Trait) + // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) + // + // args = [ * ] + // | + // v + // ( Data, Vtable ) + // | + // v + // /-------\ + // | ... | + // \-------/ + // + // + // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING + // + // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) + // vtable = (*args[0]).1 // loads the vtable out + // (data, vtable) // an equivalent Rust `*mut dyn Trait` + // + // SO THEN WE CAN USE THE ABOVE CODE. + let virtual_drop = Instance { + def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function + args: drop_fn.args, + }; + debug!("ty = {:?}", ty); + debug!("drop_fn = {:?}", drop_fn); + debug!("args = {:?}", args); + let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); + let meta_ptr = place.project_field(bx, 1); + let meta = bx.load_operand(meta_ptr); + // Truncate vtable off of args list + args = &args[..1]; + debug!("args' = {:?}", args); + ( + true, + meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) + .get_optional_fn(bx, meta.immediate(), ty, fn_abi), + fn_abi, + virtual_drop, + ) + } + _ => ( + false, + bx.get_fn_addr(drop_fn), + bx.fn_abi_of_instance(drop_fn, ty::List::empty()), + drop_fn, + ), + }; + + // We generate a null check for the drop_fn. This saves a bunch of relocations being + // generated for no-op drops. + if maybe_null { + let is_not_null = bx.append_sibling_block("is_not_null"); + let llty = bx.fn_ptr_backend_type(fn_abi); + let null = bx.const_null(llty); + let non_null = + bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null); + bx.cond_br(non_null, is_not_null, helper.llbb_with_cleanup(self, target)); + bx.switch_to_block(is_not_null); + self.set_debug_loc(bx, *source_info); + } + helper.do_call( self, bx, @@ -615,7 +634,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { unwind, &[], Some(drop_instance), - mergeable_succ, + !maybe_null && mergeable_succ, ) } @@ -648,8 +667,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return helper.funclet_br(self, bx, target, mergeable_succ); } - // Pass the condition through llvm.expect for branch hinting. - let cond = bx.expect(cond, expected); + // Because we're branching to a panic block (either a `#[cold]` one + // or an inlined abort), there's no need to `expect` it. // Create the failure block and the conditional branch to it. let lltarget = helper.llbb_with_cleanup(self, target); @@ -835,7 +854,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let def = instance.map(|i| i.def); - if let Some(ty::InstanceDef::DropGlue(_, None)) = def { + if let Some( + ty::InstanceDef::DropGlue(_, None) | ty::InstanceDef::AsyncDropGlueCtorShim(_, None), + ) = def + { // Empty drop glue; a no-op. let target = target.unwrap(); return helper.funclet_br(self, bx, target, mergeable_succ); @@ -1056,7 +1078,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Make sure that we've actually unwrapped the rcvr down // to a pointer or ref to `dyn* Trait`. - if !op.layout.ty.builtin_deref(true).unwrap().ty.is_dyn_star() { + if !op.layout.ty.builtin_deref(true).unwrap().is_dyn_star() { span_bug!(span, "can't codegen a virtual call on {:#?}", op); } let place = op.deref(bx.cx()); @@ -1341,9 +1363,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { MergingSucc::False } - mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => { - self.codegen_drop_terminator(helper, bx, place, target, unwind, mergeable_succ()) - } + mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => self + .codegen_drop_terminator( + helper, + bx, + &terminator.source_info, + place, + target, + unwind, + mergeable_succ(), + ), mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self .codegen_assert_terminator( @@ -1450,9 +1479,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { Some(pointee_align) => cmp::max(pointee_align, arg.layout.align.abi), None => arg.layout.align.abi, }; - let scratch = PlaceRef::alloca_aligned(bx, arg.layout, required_align); - op.val.store(bx, scratch); - (scratch.val.llval, scratch.val.align, true) + let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align); + op.val.store(bx, scratch.with_type(arg.layout)); + (scratch.llval, scratch.align, true) } PassMode::Cast { .. } => { let scratch = PlaceRef::alloca(bx, arg.layout); @@ -1471,10 +1500,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // For `foo(packed.large_field)`, and types with <4 byte alignment on x86, // alignment requirements may be higher than the type's alignment, so copy // to a higher-aligned alloca. - let scratch = PlaceRef::alloca_aligned(bx, arg.layout, required_align); - let op_place = PlaceRef { val: op_place_val, layout: op.layout }; - bx.typed_place_copy(scratch, op_place); - (scratch.val.llval, scratch.val.align, true) + let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align); + bx.typed_place_copy(scratch, op_place_val, op.layout); + (scratch.llval, scratch.align, true) } else { (op_place_val.llval, op_place_val.align, true) } @@ -1514,7 +1542,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // when passed by value, making it larger. let copy_bytes = cmp::min(scratch_size.bytes(), arg.layout.size.bytes()); // Allocate some scratch space... - let llscratch = bx.alloca(bx.cast_backend_type(cast), scratch_align); + let llscratch = bx.alloca(scratch_size, scratch_align); bx.lifetime_start(llscratch, scratch_size); // ...memcpy the value... bx.memcpy( @@ -1563,7 +1591,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if place_val.llextra.is_some() { bug!("closure arguments must be sized"); } - let tuple_ptr = PlaceRef { val: place_val, layout: tuple.layout }; + let tuple_ptr = place_val.with_type(tuple.layout); for i in 0..tuple.layout.fields.count() { let field_ptr = tuple_ptr.project_field(bx, i); let field = bx.load_operand(field_ptr); diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index c6260d35916..dba5fbefd8a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -5,6 +5,7 @@ use rustc_middle::mir; use rustc_middle::mir::interpret::ErrorHandled; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::{self, Ty}; +use rustc_middle::{bug, span_bug}; use rustc_target::abi::Abi; use super::FunctionCx; diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index a5fd82a3054..6b89636b654 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -1,16 +1,16 @@ use crate::traits::*; use rustc_data_structures::fx::FxHashMap; use rustc_index::IndexVec; +use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir; use rustc_middle::ty; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Instance; use rustc_middle::ty::Ty; use rustc_session::config::DebugInfo; use rustc_span::symbol::{kw, Symbol}; -use rustc_span::{BytePos, Span}; +use rustc_span::{hygiene, BytePos, Span}; use rustc_target::abi::{Abi, FieldIdx, FieldsShape, Size, VariantIdx}; use super::operand::{OperandRef, OperandValue}; @@ -120,7 +120,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> DebugInfoOffsetLocation<'tcx, Bx> { fn deref(&self, bx: &mut Bx) -> Self { bx.cx().layout_of( - self.ty.builtin_deref(true).unwrap_or_else(|| bug!("cannot deref `{}`", self.ty)).ty, + self.ty.builtin_deref(true).unwrap_or_else(|| bug!("cannot deref `{}`", self.ty)), ) } @@ -220,26 +220,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &self, source_info: mir::SourceInfo, ) -> Option<(Bx::DIScope, Option<Bx::DILocation>, Span)> { - let span = self.adjust_span_for_debugging(source_info.span); let scope = &self.debug_context.as_ref()?.scopes[source_info.scope]; + let span = hygiene::walk_chain_collapsed(source_info.span, self.mir.span); Some((scope.adjust_dbg_scope_for_span(self.cx, span), scope.inlined_at, span)) } - /// In order to have a good line stepping behavior in debugger, we overwrite debug - /// locations of macro expansions with that of the outermost expansion site (when the macro is - /// annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided). - fn adjust_span_for_debugging(&self, span: Span) -> Span { - // Bail out if debug info emission is not enabled. - if self.debug_context.is_none() { - return span; - } - // Walk up the macro expansion chain until we reach a non-expanded span. - // We also stop at the function body level because no line stepping can occur - // at the level above that. - // Use span of the outermost expansion site, while keeping the original lexical scope. - self.cx.tcx().collapsed_debuginfo(span, self.mir.span) - } - fn spill_operand_to_stack( operand: OperandRef<'tcx, Bx::Value>, name: Option<String>, diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index eb14a90412d..f88deaa7abc 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,4 +1,4 @@ -use super::operand::{OperandRef, OperandValue}; +use super::operand::OperandRef; use super::place::PlaceRef; use super::FunctionCx; use crate::errors; @@ -9,6 +9,7 @@ use crate::traits::*; use crate::MemFlags; use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::{sym, Span}; use rustc_target::abi::{ @@ -92,9 +93,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // into the (unoptimized) direct swapping implementation, so we disable it. || bx.sess().target.arch == "spirv" { - let x_place = PlaceRef::new_sized(args[0].immediate(), pointee_layout); - let y_place = PlaceRef::new_sized(args[1].immediate(), pointee_layout); - bx.typed_place_swap(x_place, y_place); + let align = pointee_layout.align.abi; + let x_place = args[0].val.deref(align); + let y_place = args[1].val.deref(align); + bx.typed_place_swap(x_place, y_place, pointee_layout); return Ok(()); } } @@ -112,15 +114,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { sym::va_end => bx.va_end(args[0].immediate()), sym::size_of_val => { let tp_ty = fn_args.type_at(0); - let meta = - if let OperandValue::Pair(_, meta) = args[0].val { Some(meta) } else { None }; + let (_, meta) = args[0].val.pointer_parts(); let (llsize, _) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta); llsize } sym::min_align_of_val => { let tp_ty = fn_args.type_at(0); - let meta = - if let OperandValue::Pair(_, meta) = args[0].val { Some(meta) } else { None }; + let (_, meta) = args[0].val.pointer_parts(); let (_, llalign) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta); llalign } diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs index 7db260c9f5b..c7f63eab829 100644 --- a/compiler/rustc_codegen_ssa/src/mir/locals.rs +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -7,6 +7,8 @@ use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::print::with_no_trimmed_paths; use std::ops::{Index, IndexMut}; +use tracing::{debug, warn}; + pub(super) struct Locals<'tcx, V> { values: IndexVec<mir::Local, LocalRef<'tcx, V>>, } diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index b98e90b5cde..e8da9842882 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -8,7 +8,9 @@ use rustc_middle::mir::traversal; use rustc_middle::mir::UnwindTerminateReason; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; +use rustc_middle::{bug, span_bug}; use rustc_target::abi::call::{FnAbi, PassMode}; +use tracing::{debug, instrument}; use std::iter; @@ -258,6 +260,10 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // Apply debuginfo to the newly allocated locals. fx.debug_introduce_locals(&mut start_bx); + // If the backend supports coverage, and coverage is enabled for this function, + // do any necessary start-of-function codegen (e.g. locals for MC/DC bitmaps). + start_bx.init_coverage(instance); + // The builders will be created separately for each basic block at `codegen_block`. // So drop the builder of `start_llbb` to avoid having two at the same time. drop(start_bx); diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 9cf64e2d676..cc0e9139650 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -5,6 +5,7 @@ use crate::size_of_val; use crate::traits::*; use crate::MemFlags; +use rustc_middle::bug; use rustc_middle::mir::interpret::{alloc_range, Pointer, Scalar}; use rustc_middle::mir::{self, ConstValue}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -13,6 +14,10 @@ use rustc_target::abi::{self, Abi, Align, Size}; use std::fmt; +use arrayvec::ArrayVec; +use either::Either; +use tracing::debug; + /// The representation of a Rust value. The enum variant is in fact /// uniquely determined by the value's type, but is kept as a /// safety check. @@ -57,6 +62,70 @@ pub enum OperandValue<V> { ZeroSized, } +impl<V: CodegenObject> OperandValue<V> { + /// If this is ZeroSized/Immediate/Pair, return an array of the 0/1/2 values. + /// If this is Ref, return the place. + #[inline] + pub fn immediates_or_place(self) -> Either<ArrayVec<V, 2>, PlaceValue<V>> { + match self { + OperandValue::ZeroSized => Either::Left(ArrayVec::new()), + OperandValue::Immediate(a) => Either::Left(ArrayVec::from_iter([a])), + OperandValue::Pair(a, b) => Either::Left([a, b].into()), + OperandValue::Ref(p) => Either::Right(p), + } + } + + /// Given an array of 0/1/2 immediate values, return ZeroSized/Immediate/Pair. + #[inline] + pub fn from_immediates(immediates: ArrayVec<V, 2>) -> Self { + let mut it = immediates.into_iter(); + let Some(a) = it.next() else { + return OperandValue::ZeroSized; + }; + let Some(b) = it.next() else { + return OperandValue::Immediate(a); + }; + OperandValue::Pair(a, b) + } + + /// Treat this value as a pointer and return the data pointer and + /// optional metadata as backend values. + /// + /// If you're making a place, use [`Self::deref`] instead. + pub fn pointer_parts(self) -> (V, Option<V>) { + match self { + OperandValue::Immediate(llptr) => (llptr, None), + OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), + _ => bug!("OperandValue cannot be a pointer: {self:?}"), + } + } + + /// Treat this value as a pointer and return the place to which it points. + /// + /// The pointer immediate doesn't inherently know its alignment, + /// so you need to pass it in. If you want to get it from a type's ABI + /// alignment, then maybe you want [`OperandRef::deref`] instead. + /// + /// This is the inverse of [`PlaceValue::address`]. + pub fn deref(self, align: Align) -> PlaceValue<V> { + let (llval, llextra) = self.pointer_parts(); + PlaceValue { llval, llextra, align } + } + + pub(crate) fn is_expected_variant_for_type<'tcx, Cx: LayoutTypeMethods<'tcx>>( + &self, + cx: &Cx, + ty: TyAndLayout<'tcx>, + ) -> bool { + match self { + OperandValue::ZeroSized => ty.is_zst(), + OperandValue::Immediate(_) => cx.is_backend_immediate(ty), + OperandValue::Pair(_, _) => cx.is_backend_scalar_pair(ty), + OperandValue::Ref(_) => cx.is_backend_ref(ty), + } + } +} + /// An `OperandRef` is an "SSA" reference to a Rust value, along with /// its type. /// @@ -204,6 +273,15 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { } } + /// Asserts that this operand is a pointer (or reference) and returns + /// the place to which it points. (This requires no code to be emitted + /// as we represent places using the pointer to the place.) + /// + /// This uses [`Ty::builtin_deref`] to include the type of the place and + /// assumes the place is aligned to the pointee's usual ABI alignment. + /// + /// If you don't need the type, see [`OperandValue::pointer_parts`] + /// or [`OperandValue::deref`]. pub fn deref<Cx: LayoutTypeMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> { if self.layout.ty.is_box() { // Derefer should have removed all Box derefs @@ -214,18 +292,10 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { .layout .ty .builtin_deref(true) - .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self)) - .ty; + .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self)); - let (llptr, llextra) = match self.val { - OperandValue::Immediate(llptr) => (llptr, None), - OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), - OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self), - OperandValue::ZeroSized => bug!("Deref of ZST operand {:?}", self), - }; let layout = cx.layout_of(projected_ty); - let val = PlaceValue { llval: llptr, llextra, align: layout.align.abi }; - PlaceRef { val, layout } + self.val.deref(layout.align.abi).with_type(layout) } /// If this operand is a `Pair`, we return an aggregate with the two values. @@ -327,7 +397,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let llfield_ty = bx.cx().backend_type(field); // Can't bitcast an aggregate, so round trip through memory. - let llptr = bx.alloca(llfield_ty, field.align.abi); + let llptr = bx.alloca(field.size, field.align.abi); bx.store(*llval, llptr, field.align.abi); *llval = bx.load(llfield_ty, llptr, field.align.abi); } @@ -418,8 +488,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { if val.llextra.is_some() { bug!("cannot directly store unsized values"); } - let source_place = PlaceRef { val, layout: dest.layout }; - bx.typed_place_copy_with_flags(dest, source_place, flags); + bx.typed_place_copy_with_flags(dest.val, val, dest.layout, flags); } OperandValue::Immediate(s) => { let val = bx.from_immediate(s); @@ -454,8 +523,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { .layout .ty .builtin_deref(true) - .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest)) - .ty; + .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest)); let OperandValue::Ref(PlaceValue { llval: llptr, llextra: Some(llextra), .. }) = self else { @@ -470,7 +538,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> { let align_minus_1 = bx.sub(align, one); let size_extra = bx.add(size, align_minus_1); let min_align = Align::ONE; - let alloca = bx.byte_array_alloca(size_extra, min_align); + let alloca = bx.dynamic_alloca(size_extra, min_align); let address = bx.ptrtoint(alloca, bx.type_isize()); let neg_address = bx.neg(address); let offset = bx.and(neg_address, align_minus_1); @@ -497,6 +565,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { for elem in place_ref.projection.iter() { match elem { mir::ProjectionElem::Field(ref f, _) => { + debug_assert!( + !o.layout.ty.is_any_ptr(), + "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \ + but tried to access field {f:?} of pointer {o:?}", + ); o = o.extract_field(bx, f.index()); } mir::ProjectionElem::Index(_) diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 90627da579e..449fd9ae0db 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -5,16 +5,21 @@ use crate::common::IntPredicate; use crate::size_of_val; use crate::traits::*; +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::mir::tcx::PlaceTy; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Ty}; -use rustc_target::abi::{Align, FieldsShape, Int, Pointer, TagEncoding}; +use rustc_target::abi::{Align, FieldsShape, Int, Pointer, Size, TagEncoding}; use rustc_target::abi::{VariantIdx, Variants}; +use tracing::{debug, instrument}; /// The location and extra runtime properties of the place. /// /// Typically found in a [`PlaceRef`] or an [`OperandValue::Ref`]. +/// +/// As a location in memory, this has no specific type. If you want to +/// load or store it using a typed operation, use [`Self::with_type`]. #[derive(Copy, Clone, Debug)] pub struct PlaceValue<V> { /// A pointer to the contents of the place. @@ -34,6 +39,41 @@ impl<V: CodegenObject> PlaceValue<V> { pub fn new_sized(llval: V, align: Align) -> PlaceValue<V> { PlaceValue { llval, llextra: None, align } } + + /// Allocates a stack slot in the function for a value + /// of the specified size and alignment. + /// + /// The allocation itself is untyped. + pub fn alloca<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx, Value = V>>( + bx: &mut Bx, + size: Size, + align: Align, + ) -> PlaceValue<V> { + let llval = bx.alloca(size, align); + PlaceValue::new_sized(llval, align) + } + + /// Creates a `PlaceRef` to this location with the given type. + pub fn with_type<'tcx>(self, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { + debug_assert!( + layout.is_unsized() || layout.abi.is_uninhabited() || self.llextra.is_none(), + "Had pointer metadata {:?} for sized type {layout:?}", + self.llextra, + ); + PlaceRef { val: self, layout } + } + + /// Gets the pointer to this place as an [`OperandValue::Immediate`] + /// or, for those needing metadata, an [`OperandValue::Pair`]. + /// + /// This is the inverse of [`OperandValue::deref`]. + pub fn address(self) -> OperandValue<V> { + if let Some(llextra) = self.llextra { + OperandValue::Pair(self.llval, llextra) + } else { + OperandValue::Immediate(self.llval) + } + } } #[derive(Copy, Clone, Debug)] @@ -51,9 +91,7 @@ pub struct PlaceRef<'tcx, V> { impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { - assert!(layout.is_sized()); - let val = PlaceValue::new_sized(llval, layout.align.abi); - PlaceRef { val, layout } + PlaceRef::new_sized_aligned(llval, layout, layout.align.abi) } pub fn new_sized_aligned( @@ -62,8 +100,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { align: Align, ) -> PlaceRef<'tcx, V> { assert!(layout.is_sized()); - let val = PlaceValue::new_sized(llval, align); - PlaceRef { val, layout } + PlaceValue::new_sized(llval, align).with_type(layout) } // FIXME(eddyb) pass something else for the name so no work is done @@ -72,17 +109,8 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { bx: &mut Bx, layout: TyAndLayout<'tcx>, ) -> Self { - Self::alloca_aligned(bx, layout, layout.align.abi) - } - - pub fn alloca_aligned<Bx: BuilderMethods<'a, 'tcx, Value = V>>( - bx: &mut Bx, - layout: TyAndLayout<'tcx>, - align: Align, - ) -> Self { assert!(layout.is_sized(), "tried to statically allocate unsized place"); - let tmp = bx.alloca(bx.cx().backend_type(layout), align); - Self::new_sized_aligned(tmp, layout, align) + PlaceValue::alloca(bx, layout.size, layout.align.abi).with_type(layout) } /// Returns a place for an indirect reference to an unsized place. @@ -131,18 +159,12 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { } else { bx.inbounds_ptradd(self.val.llval, bx.const_usize(offset.bytes())) }; - PlaceRef { - val: PlaceValue { - llval, - llextra: if bx.cx().type_has_metadata(field.ty) { - self.val.llextra - } else { - None - }, - align: effective_field_align, - }, - layout: field, - } + let val = PlaceValue { + llval, + llextra: if bx.cx().type_has_metadata(field.ty) { self.val.llextra } else { None }, + align: effective_field_align, + }; + val.with_type(field) }; // Simple cases, which don't need DST adjustment: @@ -197,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { let ptr = bx.inbounds_ptradd(self.val.llval, offset); let val = PlaceValue { llval: ptr, llextra: self.val.llextra, align: effective_field_align }; - PlaceRef { val, layout: field } + val.with_type(field) } /// Obtain the actual discriminant of a value. @@ -386,18 +408,13 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { layout.size }; - PlaceRef { - val: PlaceValue { - llval: bx.inbounds_gep( - bx.cx().backend_type(self.layout), - self.val.llval, - &[bx.cx().const_usize(0), llindex], - ), - llextra: None, - align: self.val.align.restrict_for_offset(offset), - }, - layout, - } + let llval = bx.inbounds_gep( + bx.cx().backend_type(self.layout), + self.val.llval, + &[bx.cx().const_usize(0), llindex], + ); + let align = self.val.align.restrict_for_offset(offset); + PlaceValue::new_sized(llval, align).with_type(layout) } pub fn project_downcast<Bx: BuilderMethods<'a, 'tcx, Value = V>>( @@ -463,6 +480,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { cg_base = match *elem { mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()), mir::ProjectionElem::Field(ref field, _) => { + debug_assert!( + !cg_base.layout.ty.is_any_ptr(), + "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \ + but tried to access field {field:?} of pointer {cg_base:?}", + ); cg_base.project_field(bx, field.index()) } mir::ProjectionElem::OpaqueCast(ty) => { diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 6725a6d9e38..c23867be3a1 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -7,15 +7,17 @@ use crate::common::IntPredicate; use crate::traits::*; use crate::MemFlags; -use rustc_hir as hir; use rustc_middle::mir; -use rustc_middle::mir::Operand; use rustc_middle::ty::cast::{CastTy, IntTy}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::{Span, DUMMY_SP}; -use rustc_target::abi::{self, FIRST_VARIANT}; +use rustc_target::abi::{self, FieldIdx, FIRST_VARIANT}; + +use arrayvec::ArrayVec; +use tracing::{debug, instrument}; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] @@ -72,8 +74,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if val.llextra.is_some() { bug!("unsized coercion on an unsized rvalue"); } - let source = PlaceRef { val, layout: operand.layout }; - base::coerce_unsized_into(bx, source, dest); + base::coerce_unsized_into(bx, val.with_type(operand.layout), dest); } OperandValue::ZeroSized => { bug!("unsized coercion on a ZST rvalue"); @@ -120,7 +121,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.write_operand_repeatedly(cg_elem, count, dest); } - mir::Rvalue::Aggregate(ref kind, ref operands) => { + // This implementation does field projection, so never use it for `RawPtr`, + // which will always be fine with the `codegen_rvalue_operand` path below. + mir::Rvalue::Aggregate(ref kind, ref operands) + if !matches!(**kind, mir::AggregateKind::RawPtr(..)) => + { let (variant_index, variant_dest, active_field_index) = match **kind { mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { let variant_dest = dest.project_downcast(bx, variant_index); @@ -182,10 +187,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Immediate(..) | OperandValue::Pair(..) => { // When we have immediate(s), the alignment of the source is irrelevant, // so we can store them using the destination's alignment. - src.val.store( - bx, - PlaceRef::new_sized_aligned(dst.val.llval, src.layout, dst.val.align), - ); + src.val.store(bx, dst.val.with_type(src.layout)); } } } @@ -223,8 +225,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Ref(source_place_val) => { debug_assert_eq!(source_place_val.llextra, None); debug_assert!(matches!(operand_kind, OperandValueKind::Ref)); - let fake_place = PlaceRef { val: source_place_val, layout: cast }; - Some(bx.load_operand(fake_place).val) + Some(bx.load_operand(source_place_val.with_type(cast)).val) } OperandValue::ZeroSized => { let OperandValueKind::ZeroSized = operand_kind else { @@ -306,17 +307,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty); imm = match (from_scalar.primitive(), to_scalar.primitive()) { - (Int(..) | F16 | F32 | F64 | F128, Int(..) | F16 | F32 | F64 | F128) => { - bx.bitcast(imm, to_backend_ty) - } + (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty), (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty), (Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm), (Pointer(..), Int(..)) => bx.ptrtoint(imm, to_backend_ty), - (F16 | F32 | F64 | F128, Pointer(..)) => { + (Float(_), Pointer(..)) => { let int_imm = bx.bitcast(imm, bx.cx().type_isize()); bx.ptradd(bx.const_null(bx.type_ptr()), int_imm) } - (Pointer(..), F16 | F32 | F64 | F128) => { + (Pointer(..), Float(_)) => { let int_imm = bx.ptrtoint(imm, bx.cx().type_isize()); bx.bitcast(int_imm, to_backend_ty) } @@ -452,23 +451,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } mir::CastKind::PointerCoercion(PointerCoercion::Unsize) => { assert!(bx.cx().is_backend_scalar_pair(cast)); - let (lldata, llextra) = match operand.val { - OperandValue::Pair(lldata, llextra) => { - // unsize from a fat pointer -- this is a - // "trait-object-to-supertrait" coercion. - (lldata, Some(llextra)) - } - OperandValue::Immediate(lldata) => { - // "standard" unsize - (lldata, None) - } - OperandValue::Ref(..) => { - bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand); - } - OperandValue::ZeroSized => { - bug!("zero-sized operand {:?} in `codegen_rvalue_operand`", operand); - } - }; + let (lldata, llextra) = operand.val.pointer_parts(); let (lldata, llextra) = base::unsize_ptr(bx, lldata, operand.layout.ty, cast.ty, llextra); OperandValue::Pair(lldata, llextra) @@ -489,12 +472,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } mir::CastKind::DynStar => { - let (lldata, llextra) = match operand.val { - OperandValue::Ref(..) => todo!(), - OperandValue::Immediate(v) => (v, None), - OperandValue::Pair(v, l) => (v, Some(l)), - OperandValue::ZeroSized => bug!("ZST -- which is not PointerLike -- in DynStar"), - }; + let (lldata, llextra) = operand.val.pointer_parts(); let (lldata, llextra) = base::cast_to_dyn_star(bx, lldata, operand.layout, cast.ty, llextra); OperandValue::Pair(lldata, llextra) @@ -581,7 +559,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_place_to_pointer(bx, place, mk_ref) } - mir::Rvalue::CopyForDeref(place) => self.codegen_operand(bx, &Operand::Copy(place)), + mir::Rvalue::CopyForDeref(place) => { + self.codegen_operand(bx, &mir::Operand::Copy(place)) + } mir::Rvalue::AddressOf(mutability, place) => { let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| Ty::new_ptr(tcx, ty, mutability); @@ -596,6 +576,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + mir::Rvalue::BinaryOp(op_with_overflow, box (ref lhs, ref rhs)) + if let Some(op) = op_with_overflow.overflowing_to_wrapping() => + { + let lhs = self.codegen_operand(bx, lhs); + let rhs = self.codegen_operand(bx, rhs); + let result = self.codegen_scalar_checked_binop( + bx, + op, + lhs.immediate(), + rhs.immediate(), + lhs.layout.ty, + ); + let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); + let operand_ty = Ty::new_tup(bx.tcx(), &[val_ty, bx.tcx().types.bool]); + OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) } + } mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => { let lhs = self.codegen_operand(bx, lhs); let rhs = self.codegen_operand(bx, rhs); @@ -624,36 +620,39 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)), } } - mir::Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => { - let lhs = self.codegen_operand(bx, lhs); - let rhs = self.codegen_operand(bx, rhs); - let result = self.codegen_scalar_checked_binop( - bx, - op, - lhs.immediate(), - rhs.immediate(), - lhs.layout.ty, - ); - let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); - let operand_ty = Ty::new_tup(bx.tcx(), &[val_ty, bx.tcx().types.bool]); - OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) } - } mir::Rvalue::UnaryOp(op, ref operand) => { let operand = self.codegen_operand(bx, operand); - let lloperand = operand.immediate(); let is_float = operand.layout.ty.is_floating_point(); - let llval = match op { - mir::UnOp::Not => bx.not(lloperand), + let (val, layout) = match op { + mir::UnOp::Not => { + let llval = bx.not(operand.immediate()); + (OperandValue::Immediate(llval), operand.layout) + } mir::UnOp::Neg => { - if is_float { - bx.fneg(lloperand) + let llval = if is_float { + bx.fneg(operand.immediate()) } else { - bx.neg(lloperand) + bx.neg(operand.immediate()) + }; + (OperandValue::Immediate(llval), operand.layout) + } + mir::UnOp::PtrMetadata => { + debug_assert!(operand.layout.ty.is_unsafe_ptr()); + let (_, meta) = operand.val.pointer_parts(); + assert_eq!(operand.layout.fields.count() > 1, meta.is_some()); + if let Some(meta) = meta { + (OperandValue::Immediate(meta), operand.layout.field(self.cx, 1)) + } else { + (OperandValue::ZeroSized, bx.cx().layout_of(bx.tcx().types.unit)) } } }; - OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout } + debug_assert!( + val.is_expected_variant_for_type(self.cx, layout), + "Made wrong variant {val:?} for type {layout:?}", + ); + OperandRef { val, layout } } mir::Rvalue::Discriminant(ref place) => { @@ -720,11 +719,46 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandRef { val: OperandValue::Immediate(static_), layout } } mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand), - mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => { - // According to `rvalue_creates_operand`, only ZST - // aggregate rvalues are allowed to be operands. + mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"), + mir::Rvalue::Aggregate(_, ref fields) => { let ty = rvalue.ty(self.mir, self.cx.tcx()); - OperandRef::zero_sized(self.cx.layout_of(self.monomorphize(ty))) + let ty = self.monomorphize(ty); + let layout = self.cx.layout_of(ty); + + // `rvalue_creates_operand` has arranged that we only get here if + // we can build the aggregate immediate from the field immediates. + let mut inputs = ArrayVec::<Bx::Value, 2>::new(); + let mut input_scalars = ArrayVec::<abi::Scalar, 2>::new(); + for field_idx in layout.fields.index_by_increasing_offset() { + let field_idx = FieldIdx::from_usize(field_idx); + let op = self.codegen_operand(bx, &fields[field_idx]); + let values = op.val.immediates_or_place().left_or_else(|p| { + bug!("Field {field_idx:?} is {p:?} making {layout:?}"); + }); + let scalars = self.value_kind(op.layout).scalars().unwrap(); + debug_assert_eq!(values.len(), scalars.len()); + inputs.extend(values); + input_scalars.extend(scalars); + } + + let output_scalars = self.value_kind(layout).scalars().unwrap(); + itertools::izip!(&mut inputs, input_scalars, output_scalars).for_each( + |(v, in_s, out_s)| { + if in_s != out_s { + // We have to be really careful about bool here, because + // `(bool,)` stays i1 but `Cell<bool>` becomes i8. + *v = bx.from_immediate(*v); + *v = bx.to_immediate_scalar(*v, out_s); + } + }, + ); + + let val = OperandValue::from_immediates(inputs); + debug_assert!( + val.is_expected_variant_for_type(self.cx, layout), + "Made wrong variant {val:?} for type {layout:?}", + ); + OperandRef { val, layout } } mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { let operand = self.codegen_operand(bx, operand); @@ -762,16 +796,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mk_ptr_ty: impl FnOnce(TyCtxt<'tcx>, Ty<'tcx>) -> Ty<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { let cg_place = self.codegen_place(bx, place.as_ref()); + let val = cg_place.val.address(); let ty = cg_place.layout.ty; + debug_assert!( + if bx.cx().type_has_metadata(ty) { + matches!(val, OperandValue::Pair(..)) + } else { + matches!(val, OperandValue::Immediate(..)) + }, + "Address of place was unexpectedly {val:?} for pointee type {ty:?}", + ); - // Note: places are indirect, so storing the `llval` into the - // destination effectively creates a reference. - let val = if !bx.cx().type_has_metadata(ty) { - OperandValue::Immediate(cg_place.val.llval) - } else { - OperandValue::Pair(cg_place.val.llval, cg_place.val.llextra.unwrap()) - }; OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) } } @@ -852,8 +888,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::BinOp::Offset => { let pointee_type = input_ty .builtin_deref(true) - .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty)) - .ty; + .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty)); let pointee_layout = bx.cx().layout_of(pointee_type); if pointee_layout.is_zst() { // `Offset` works in terms of the size of pointee, @@ -879,9 +914,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | mir::BinOp::Le | mir::BinOp::Ge => { if is_float { - bx.fcmp(base::bin_op_to_fcmp_predicate(op.to_hir_binop()), lhs, rhs) + bx.fcmp(base::bin_op_to_fcmp_predicate(op), lhs, rhs) } else { - bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs) + bx.icmp(base::bin_op_to_icmp_predicate(op, is_signed), lhs, rhs) } } mir::BinOp::Cmp => { @@ -895,16 +930,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // `PartialOrd`, so only use it in debug for now. Once LLVM can handle it // better (see <https://github.com/llvm/llvm-project/issues/73417>), it'll // be worth trying it in optimized builds as well. - let is_gt = bx.icmp(pred(hir::BinOpKind::Gt), lhs, rhs); + let is_gt = bx.icmp(pred(mir::BinOp::Gt), lhs, rhs); let gtext = bx.zext(is_gt, bx.type_i8()); - let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs); + let is_lt = bx.icmp(pred(mir::BinOp::Lt), lhs, rhs); let ltext = bx.zext(is_lt, bx.type_i8()); bx.unchecked_ssub(gtext, ltext) } else { // These operations are those expected by `tests/codegen/integer-cmp.rs`, // from <https://github.com/rust-lang/rust/pull/63767>. - let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs); - let is_ne = bx.icmp(pred(hir::BinOpKind::Ne), lhs, rhs); + let is_lt = bx.icmp(pred(mir::BinOp::Lt), lhs, rhs); + let is_ne = bx.icmp(pred(mir::BinOp::Ne), lhs, rhs); let ge = bx.select( is_ne, bx.cx().const_i8(Ordering::Greater as i8), @@ -913,6 +948,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge) } } + mir::BinOp::AddWithOverflow + | mir::BinOp::SubWithOverflow + | mir::BinOp::MulWithOverflow => { + bug!("{op:?} needs to return a pair, so call codegen_scalar_checked_binop instead") + } } } @@ -1025,19 +1065,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Cast(..) | // (*) mir::Rvalue::ShallowInitBox(..) | // (*) mir::Rvalue::BinaryOp(..) | - mir::Rvalue::CheckedBinaryOp(..) | mir::Rvalue::UnaryOp(..) | mir::Rvalue::Discriminant(..) | mir::Rvalue::NullaryOp(..) | mir::Rvalue::ThreadLocalRef(_) | mir::Rvalue::Use(..) => // (*) true, - mir::Rvalue::Repeat(..) | - mir::Rvalue::Aggregate(..) => { + // Arrays are always aggregates, so it's not worth checking anything here. + // (If it's really `[(); N]` or `[T; 0]` and we use the place path, fine.) + mir::Rvalue::Repeat(..) => false, + mir::Rvalue::Aggregate(ref kind, _) => { + let allowed_kind = match **kind { + // This always produces a `ty::RawPtr`, so will be Immediate or Pair + mir::AggregateKind::RawPtr(..) => true, + mir::AggregateKind::Array(..) => false, + mir::AggregateKind::Tuple => true, + mir::AggregateKind::Adt(def_id, ..) => { + let adt_def = self.cx.tcx().adt_def(def_id); + adt_def.is_struct() && !adt_def.repr().simd() + } + mir::AggregateKind::Closure(..) => true, + // FIXME: Can we do this for simple coroutines too? + mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false, + }; + allowed_kind && { let ty = rvalue.ty(self.mir, self.cx.tcx()); let ty = self.monomorphize(ty); - // For ZST this can be `OperandValueKind::ZeroSized`. - self.cx.spanned_layout_of(ty, span).is_zst() + let layout = self.cx.spanned_layout_of(ty, span); + !self.cx.is_backend_ref(layout) + } } } @@ -1079,3 +1135,14 @@ enum OperandValueKind { Pair(abi::Scalar, abi::Scalar), ZeroSized, } + +impl OperandValueKind { + fn scalars(self) -> Option<ArrayVec<abi::Scalar, 2>> { + Some(match self { + OperandValueKind::ZeroSized => ArrayVec::new(), + OperandValueKind::Immediate(a) => ArrayVec::from_iter([a]), + OperandValueKind::Pair(a, b) => [a, b].into(), + OperandValueKind::Ref => return None, + }) + } +} diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 2188eeae426..27494f48b09 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -1,6 +1,7 @@ -use rustc_middle::mir; -use rustc_middle::mir::NonDivergingIntrinsic; +use rustc_middle::mir::{self, NonDivergingIntrinsic}; +use rustc_middle::span_bug; use rustc_session::config::OptLevel; +use tracing::instrument; use super::FunctionCx; use super::LocalRef; diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index df564f705bc..0fbcb938d1a 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -5,9 +5,11 @@ use rustc_hir as hir; use rustc_middle::mir::interpret::ErrorHandled; use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::span_bug; use rustc_middle::ty; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::Instance; +use tracing::debug; pub trait MonoItemExt<'a, 'tcx> { fn define<Bx: BuilderMethods<'a, 'tcx>>(&self, cx: &'a Bx::CodegenCx); diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index c250cc26823..130fe2eaf2f 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -5,9 +5,11 @@ use crate::common::IntPredicate; use crate::meth; use crate::traits::*; use rustc_hir::LangItem; +use rustc_middle::bug; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty}; use rustc_target::abi::WrappingRange; +use tracing::{debug, trace}; pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index 241b0a15f78..bcddfe9fb9c 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -8,6 +8,7 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_hir::def_id::LocalDefId; use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::bug; use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::parse::feature_err; diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 9191618c064..2f0daefa46a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -110,8 +110,16 @@ pub trait BuilderMethods<'a, 'tcx>: fn frem(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn frem_fast(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn frem_algebraic(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + /// Generate a left-shift. Both operands must have the same size. The right operand must be + /// interpreted as unsigned and can be assumed to be less than the size of the left operand. fn shl(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + /// Generate a logical right-shift. Both operands must have the same size. The right operand + /// must be interpreted as unsigned and can be assumed to be less than the size of the left + /// operand. fn lshr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + /// Generate an arithmetic right-shift. Both operands must have the same size. The right operand + /// must be interpreted as unsigned and can be assumed to be less than the size of the left + /// operand. fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; @@ -144,8 +152,8 @@ pub trait BuilderMethods<'a, 'tcx>: } fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value; - fn alloca(&mut self, ty: Self::Type, align: Align) -> Self::Value; - fn byte_array_alloca(&mut self, len: Self::Value, align: Align) -> Self::Value; + fn alloca(&mut self, size: Size, align: Align) -> Self::Value; + fn dynamic_alloca(&mut self, size: Self::Value, align: Align) -> Self::Value; fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value; fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value; @@ -186,6 +194,15 @@ pub trait BuilderMethods<'a, 'tcx>: align: Align, flags: MemFlags, ) -> Self::Value; + fn store_to_place_with_flags( + &mut self, + val: Self::Value, + place: PlaceValue<Self::Value>, + flags: MemFlags, + ) -> Self::Value { + debug_assert_eq!(place.llextra, None); + self.store_with_flags(val, place.llval, place.align, flags) + } fn atomic_store( &mut self, val: Self::Value, @@ -238,7 +255,10 @@ pub trait BuilderMethods<'a, 'tcx>: } else { (in_ty, dest_ty) }; - assert!(matches!(self.cx().type_kind(float_ty), TypeKind::Float | TypeKind::Double)); + assert!(matches!( + self.cx().type_kind(float_ty), + TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128 + )); assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer); if let Some(false) = self.cx().sess().opts.unstable_opts.saturating_float_casts { @@ -286,35 +306,36 @@ pub trait BuilderMethods<'a, 'tcx>: /// (For example, typed load-stores with alias metadata.) fn typed_place_copy( &mut self, - dst: PlaceRef<'tcx, Self::Value>, - src: PlaceRef<'tcx, Self::Value>, + dst: PlaceValue<Self::Value>, + src: PlaceValue<Self::Value>, + layout: TyAndLayout<'tcx>, ) { - self.typed_place_copy_with_flags(dst, src, MemFlags::empty()); + self.typed_place_copy_with_flags(dst, src, layout, MemFlags::empty()); } fn typed_place_copy_with_flags( &mut self, - dst: PlaceRef<'tcx, Self::Value>, - src: PlaceRef<'tcx, Self::Value>, + dst: PlaceValue<Self::Value>, + src: PlaceValue<Self::Value>, + layout: TyAndLayout<'tcx>, flags: MemFlags, ) { - debug_assert!(src.val.llextra.is_none(), "cannot directly copy from unsized values"); - debug_assert!(dst.val.llextra.is_none(), "cannot directly copy into unsized values"); - debug_assert_eq!(dst.layout.size, src.layout.size); + debug_assert!(layout.is_sized(), "cannot typed-copy an unsigned type"); + debug_assert!(src.llextra.is_none(), "cannot directly copy from unsized values"); + debug_assert!(dst.llextra.is_none(), "cannot directly copy into unsized values"); if flags.contains(MemFlags::NONTEMPORAL) { // HACK(nox): This is inefficient but there is no nontemporal memcpy. - let ty = self.backend_type(dst.layout); - let val = self.load_from_place(ty, src.val); - self.store_with_flags(val, dst.val.llval, dst.val.align, flags); - } else if self.sess().opts.optimize == OptLevel::No && self.is_backend_immediate(dst.layout) - { + let ty = self.backend_type(layout); + let val = self.load_from_place(ty, src); + self.store_to_place_with_flags(val, dst, flags); + } else if self.sess().opts.optimize == OptLevel::No && self.is_backend_immediate(layout) { // If we're not optimizing, the aliasing information from `memcpy` // isn't useful, so just load-store the value for smaller code. - let temp = self.load_operand(src); - temp.val.store_with_flags(self, dst, flags); - } else if !dst.layout.is_zst() { - let bytes = self.const_usize(dst.layout.size.bytes()); - self.memcpy(dst.val.llval, dst.val.align, src.val.llval, src.val.align, bytes, flags); + let temp = self.load_operand(src.with_type(layout)); + temp.val.store_with_flags(self, dst.with_type(layout), flags); + } else if !layout.is_zst() { + let bytes = self.const_usize(layout.size.bytes()); + self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags); } } @@ -327,18 +348,19 @@ pub trait BuilderMethods<'a, 'tcx>: /// cases (in non-debug), preferring the fallback body instead. fn typed_place_swap( &mut self, - left: PlaceRef<'tcx, Self::Value>, - right: PlaceRef<'tcx, Self::Value>, + left: PlaceValue<Self::Value>, + right: PlaceValue<Self::Value>, + layout: TyAndLayout<'tcx>, ) { - let mut temp = self.load_operand(left); + let mut temp = self.load_operand(left.with_type(layout)); if let OperandValue::Ref(..) = temp.val { // The SSA value isn't stand-alone, so we need to copy it elsewhere - let alloca = PlaceRef::alloca(self, left.layout); - self.typed_place_copy(alloca, left); + let alloca = PlaceRef::alloca(self, layout); + self.typed_place_copy(alloca.val, left, layout); temp = self.load_operand(alloca); } - self.typed_place_copy(left, right); - temp.val.store(self, right); + self.typed_place_copy(left, right, layout); + temp.val.store(self, right.with_type(layout)); } fn select( diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs index d1d813bd389..906d8b87d3b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs @@ -3,6 +3,11 @@ use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { + /// Performs any start-of-function codegen needed for coverage instrumentation. + /// + /// Can be a no-op in backends that don't support coverage instrumentation. + fn init_coverage(&mut self, _instance: Instance<'tcx>) {} + /// Handle the MIR coverage info in a backend-specific way. /// /// This can potentially be a no-op in backends that don't support diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 34d9e75036f..403f6a73277 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -3,10 +3,11 @@ use super::Backend; use super::HasCodegen; use crate::common::TypeKind; use crate::mir::place::PlaceRef; +use rustc_middle::bug; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty}; use rustc_target::abi::call::{ArgAbi, CastTarget, FnAbi, Reg}; -use rustc_target::abi::{AddressSpace, Integer}; +use rustc_target::abi::{AddressSpace, Float, Integer}; // This depends on `Backend` and not `BackendTypes`, because consumers will probably want to use // `LayoutOf` or `HasTyCtxt`. This way, they don't have to add a constraint on it themselves. @@ -64,6 +65,16 @@ pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> { } } + fn type_from_float(&self, f: Float) -> Self::Type { + use Float::*; + match f { + F16 => self.type_f16(), + F32 => self.type_f32(), + F64 => self.type_f64(), + F128 => self.type_f128(), + } + } + fn type_needs_drop(&self, ty: Ty<'tcx>) -> bool { ty.needs_drop(self.tcx(), ty::ParamEnv::reveal_all()) } diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index 048540894ac..f83e34ab01b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -56,12 +56,16 @@ pub trait WriteBackendMethods: 'static + Sized + Clone { module: ModuleCodegen<Self::Module>, config: &ModuleConfig, ) -> Result<CompiledModule, FatalError>; - fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer); + fn prepare_thin( + module: ModuleCodegen<Self::Module>, + want_summary: bool, + ) -> (String, Self::ThinBuffer); fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer); } pub trait ThinBufferMethods: Send + Sync { fn data(&self) -> &[u8]; + fn thin_link_data(&self) -> &[u8]; } pub trait ModuleBufferMethods: Send + Sync { |