diff options
Diffstat (limited to 'compiler/rustc_codegen_ssa/src')
22 files changed, 694 insertions, 469 deletions
diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs index 27331ce4ca6..da9f8d69297 100644 --- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs +++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs @@ -94,7 +94,7 @@ impl<'tcx> AssertModuleSource<'tcx> { other => { self.tcx .dcx() - .emit_fatal(errors::UnknownReuseKind { span: attr.span, kind: other }); + .emit_fatal(errors::UnknownReuseKind { span: attr.span(), kind: other }); } } } else { @@ -102,7 +102,7 @@ impl<'tcx> AssertModuleSource<'tcx> { }; if !self.tcx.sess.opts.unstable_opts.query_dep_graph { - self.tcx.dcx().emit_fatal(errors::MissingQueryDepGraph { span: attr.span }); + self.tcx.dcx().emit_fatal(errors::MissingQueryDepGraph { span: attr.span() }); } if !self.check_config(attr) { @@ -115,7 +115,7 @@ impl<'tcx> AssertModuleSource<'tcx> { if !user_path.starts_with(&crate_name) { self.tcx.dcx().emit_fatal(errors::MalformedCguName { - span: attr.span, + span: attr.span(), user_path, crate_name, }); @@ -145,7 +145,7 @@ impl<'tcx> AssertModuleSource<'tcx> { let cgu_names: Vec<&str> = self.available_cgus.items().map(|cgu| cgu.as_str()).into_sorted_stable_ord(); self.tcx.dcx().emit_err(errors::NoModuleNamed { - span: attr.span, + span: attr.span(), user_path, cgu_name, cgu_names: cgu_names.join(", "), @@ -155,7 +155,7 @@ impl<'tcx> AssertModuleSource<'tcx> { self.cgu_reuse_tracker.set_expectation( cgu_name, user_path, - attr.span, + attr.span(), expected_reuse, comp_kind, ); @@ -175,7 +175,7 @@ impl<'tcx> AssertModuleSource<'tcx> { } } - self.tcx.dcx().emit_fatal(errors::NoField { span: attr.span, name }); + self.tcx.dcx().emit_fatal(errors::NoField { span: attr.span(), name }); } /// Scan for a `cfg="foo"` attribute and check whether we have a diff --git a/compiler/rustc_codegen_ssa/src/back/apple.rs b/compiler/rustc_codegen_ssa/src/back/apple.rs index d9c5c3e5af9..bfa7635a869 100644 --- a/compiler/rustc_codegen_ssa/src/back/apple.rs +++ b/compiler/rustc_codegen_ssa/src/back/apple.rs @@ -2,6 +2,7 @@ use std::env; use std::fmt::{Display, from_fn}; use std::num::ParseIntError; +use rustc_middle::middle::exported_symbols::SymbolExportKind; use rustc_session::Session; use rustc_target::spec::Target; @@ -26,6 +27,89 @@ pub(super) fn macho_platform(target: &Target) -> u32 { } } +/// Add relocation and section data needed for a symbol to be considered +/// undefined by ld64. +/// +/// The relocation must be valid, and hence must point to a valid piece of +/// machine code, and hence this is unfortunately very architecture-specific. +/// +/// +/// # New architectures +/// +/// The values here are basically the same as emitted by the following program: +/// +/// ```c +/// // clang -c foo.c -target $CLANG_TARGET +/// void foo(void); +/// +/// extern int bar; +/// +/// void* foobar[2] = { +/// (void*)foo, +/// (void*)&bar, +/// // ... +/// }; +/// ``` +/// +/// Can be inspected with: +/// ```console +/// objdump --macho --reloc foo.o +/// objdump --macho --full-contents foo.o +/// ``` +pub(super) fn add_data_and_relocation( + file: &mut object::write::Object<'_>, + section: object::write::SectionId, + symbol: object::write::SymbolId, + target: &Target, + kind: SymbolExportKind, +) -> object::write::Result<()> { + let authenticated_pointer = + kind == SymbolExportKind::Text && target.llvm_target.starts_with("arm64e"); + + let data: &[u8] = match target.pointer_width { + _ if authenticated_pointer => &[0, 0, 0, 0, 0, 0, 0, 0x80], + 32 => &[0; 4], + 64 => &[0; 8], + pointer_width => unimplemented!("unsupported Apple pointer width {pointer_width:?}"), + }; + + if target.arch == "x86_64" { + // Force alignment for the entire section to be 16 on x86_64. + file.section_mut(section).append_data(&[], 16); + } else { + // Elsewhere, the section alignment is the same as the pointer width. + file.section_mut(section).append_data(&[], target.pointer_width as u64); + } + + let offset = file.section_mut(section).append_data(data, data.len() as u64); + + let flags = if authenticated_pointer { + object::write::RelocationFlags::MachO { + r_type: object::macho::ARM64_RELOC_AUTHENTICATED_POINTER, + r_pcrel: false, + r_length: 3, + } + } else if target.arch == "arm" { + // FIXME(madsmtm): Remove once `object` supports 32-bit ARM relocations: + // https://github.com/gimli-rs/object/pull/757 + object::write::RelocationFlags::MachO { + r_type: object::macho::ARM_RELOC_VANILLA, + r_pcrel: false, + r_length: 2, + } + } else { + object::write::RelocationFlags::Generic { + kind: object::RelocationKind::Absolute, + encoding: object::RelocationEncoding::Generic, + size: target.pointer_width as u8, + } + }; + + file.add_relocation(section, object::write::Relocation { offset, addend: 0, symbol, flags })?; + + Ok(()) +} + /// Deployment target or SDK version. /// /// The size of the numbers in here are limited by Mach-O's `LC_BUILD_VERSION`. diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 105a4cb81f0..b090730ac6b 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -244,22 +244,17 @@ pub fn each_linked_rlib( fmts } else { - for combination in info.dependency_formats.iter().combinations(2) { - let (ty1, list1) = &combination[0]; - let (ty2, list2) = &combination[1]; - if list1 != list2 { - return Err(errors::LinkRlibError::IncompatibleDependencyFormats { - ty1: format!("{ty1:?}"), - ty2: format!("{ty2:?}"), - list1: format!("{list1:?}"), - list2: format!("{list2:?}"), - }); - } - } - if info.dependency_formats.is_empty() { - return Err(errors::LinkRlibError::MissingFormat); + let mut dep_formats = info.dependency_formats.iter(); + let (ty1, list1) = dep_formats.next().ok_or(errors::LinkRlibError::MissingFormat)?; + if let Some((ty2, list2)) = dep_formats.find(|(_, list2)| list1 != *list2) { + return Err(errors::LinkRlibError::IncompatibleDependencyFormats { + ty1: format!("{ty1:?}"), + ty2: format!("{ty2:?}"), + list1: format!("{list1:?}"), + list2: format!("{list2:?}"), + }); } - info.dependency_formats.first().unwrap().1 + list1 }; let used_dep_crates = info.used_crates.iter(); @@ -626,10 +621,9 @@ fn link_staticlib( let mut all_rust_dylibs = vec![]; for &cnum in crates { - match fmts.get(cnum) { - Some(&Linkage::Dynamic) => {} - _ => continue, - } + let Some(Linkage::Dynamic) = fmts.get(cnum) else { + continue; + }; let crate_name = codegen_results.crate_info.crate_name[&cnum]; let used_crate_source = &codegen_results.crate_info.used_crate_source[&cnum]; if let Some((path, _)) = &used_crate_source.dylib { @@ -1990,6 +1984,7 @@ fn add_pre_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) if let Some(args) = sess.target.pre_link_args.get(&flavor) { cmd.verbatim_args(args.iter().map(Deref::deref)); } + cmd.verbatim_args(&sess.opts.unstable_opts.pre_link_args); } @@ -2063,8 +2058,8 @@ fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor /// linker, and since they never participate in the linking, using `KEEP` in the linker scripts /// can't keep them either. This causes #47384. /// -/// To keep them around, we could use `--whole-archive` and equivalents to force rlib to -/// participate in linking like object files, but this proves to be expensive (#93791). Therefore +/// To keep them around, we could use `--whole-archive`, `-force_load` and equivalents to force rlib +/// to participate in linking like object files, but this proves to be expensive (#93791). Therefore /// we instead just introduce an undefined reference to them. This could be done by `-u` command /// line option to the linker or `EXTERN(...)` in linker scripts, however they does not only /// introduce an undefined reference, but also make them the GC roots, preventing `--gc-sections` @@ -2106,8 +2101,20 @@ fn add_linked_symbol_object( file.set_mangling(object::write::Mangling::None); } + // ld64 requires a relocation to load undefined symbols, see below. + // Not strictly needed if linking with lld, but might as well do it there too. + let ld64_section_helper = if file.format() == object::BinaryFormat::MachO { + Some(file.add_section( + file.segment_name(object::write::StandardSegment::Data).to_vec(), + "__data".into(), + object::SectionKind::Data, + )) + } else { + None + }; + for (sym, kind) in symbols.iter() { - file.add_symbol(object::write::Symbol { + let symbol = file.add_symbol(object::write::Symbol { name: sym.clone().into(), value: 0, size: 0, @@ -2121,6 +2128,47 @@ fn add_linked_symbol_object( section: object::write::SymbolSection::Undefined, flags: object::SymbolFlags::None, }); + + // The linker shipped with Apple's Xcode, ld64, works a bit differently from other linkers. + // + // Code-wise, the relevant parts of ld64 are roughly: + // 1. Find the `ArchiveLoadMode` based on commandline options, default to `parseObjects`. + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/Options.cpp#L924-L932 + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/Options.h#L55 + // + // 2. Read the archive table of contents (__.SYMDEF file). + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/parsers/archive_file.cpp#L294-L325 + // + // 3. Begin linking by loading "atoms" from input files. + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/doc/design/linker.html + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/InputFiles.cpp#L1349 + // + // a. Directly specified object files (`.o`) are parsed immediately. + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/parsers/macho_relocatable_file.cpp#L4611-L4627 + // + // - Undefined symbols are not atoms (`n_value > 0` denotes a common symbol). + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/parsers/macho_relocatable_file.cpp#L2455-L2468 + // https://maskray.me/blog/2022-02-06-all-about-common-symbols + // + // - Relocations/fixups are atoms. + // https://github.com/apple-oss-distributions/ld64/blob/ce6341ae966b3451aa54eeb049f2be865afbd578/src/ld/parsers/macho_relocatable_file.cpp#L2088-L2114 + // + // b. Archives are not parsed yet. + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/parsers/archive_file.cpp#L467-L577 + // + // 4. When a symbol is needed by an atom, parse the object file that contains the symbol. + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/InputFiles.cpp#L1417-L1491 + // https://github.com/apple-oss-distributions/ld64/blob/ld64-954.16/src/ld/parsers/archive_file.cpp#L579-L597 + // + // All of the steps above are fairly similar to other linkers, except that **it completely + // ignores undefined symbols**. + // + // So to make this trick work on ld64, we need to do something else to load the relevant + // object files. We do this by inserting a relocation (fixup) for each symbol. + if let Some(section) = ld64_section_helper { + apple::add_data_and_relocation(&mut file, section, symbol, &sess.target, *kind) + .expect("failed adding relocation"); + } } let path = tmpdir.join("symbols.o"); @@ -2518,6 +2566,12 @@ fn add_order_independent_options( "--target-cpu", &codegen_results.crate_info.target_cpu, ]); + if codegen_results.crate_info.target_features.len() > 0 { + cmd.link_arg(&format!( + "--target-feature={}", + &codegen_results.crate_info.target_features.join(",") + )); + } } else if flavor == LinkerFlavor::Ptx { cmd.link_args(&["--fallback-arch", &codegen_results.crate_info.target_cpu]); } else if flavor == LinkerFlavor::Bpf { diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 8fb831471a9..8900405c1b8 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -153,6 +153,7 @@ pub(crate) fn get_linker<'a>( hinted_static: None, is_ld: cc == Cc::No, is_gnu: flavor.is_gnu(), + uses_lld: flavor.uses_lld(), }) as Box<dyn Linker>, LinkerFlavor::Msvc(..) => Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>, LinkerFlavor::EmCc => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>, @@ -361,6 +362,7 @@ struct GccLinker<'a> { // Link as ld is_ld: bool, is_gnu: bool, + uses_lld: bool, } impl<'a> GccLinker<'a> { @@ -410,7 +412,7 @@ impl<'a> GccLinker<'a> { let opt_level = match self.sess.opts.optimize { config::OptLevel::No => "O0", config::OptLevel::Less => "O1", - config::OptLevel::Default | config::OptLevel::Size | config::OptLevel::SizeMin => "O2", + config::OptLevel::More | config::OptLevel::Size | config::OptLevel::SizeMin => "O2", config::OptLevel::Aggressive => "O3", }; @@ -552,6 +554,7 @@ impl<'a> Linker for GccLinker<'a> { self.link_args(&["--entry", "_initialize"]); } } + // VxWorks compiler driver introduced `--static-crt` flag specifically for rustc, // it switches linking for libc and similar system libraries to static without using // any `#[link]` attributes in the `libc` crate, see #72782 for details. @@ -567,6 +570,15 @@ impl<'a> Linker for GccLinker<'a> { { self.cc_arg("--static-crt"); } + + // avr-none doesn't have default ISA, users must specify which specific + // CPU (well, microcontroller) they are targetting using `-Ctarget-cpu`. + // + // Currently this makes sense only when using avr-gcc as a linker, since + // it brings a couple of hand-written important intrinsics from libgcc. + if self.sess.target.arch == "avr" && !self.uses_lld { + self.verbatim_arg(format!("-mmcu={}", self.target_cpu)); + } } fn link_dylib_by_name(&mut self, name: &str, verbatim: bool, as_needed: bool) { @@ -685,7 +697,7 @@ impl<'a> Linker for GccLinker<'a> { // GNU-style linkers support optimization with -O. GNU ld doesn't // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::Default + if self.sess.opts.optimize == config::OptLevel::More || self.sess.opts.optimize == config::OptLevel::Aggressive { self.link_arg("-O1"); @@ -1213,7 +1225,7 @@ impl<'a> Linker for EmLinker<'a> { self.cc_arg(match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", OptLevel::Size => "-Os", OptLevel::SizeMin => "-Oz", @@ -1384,7 +1396,7 @@ impl<'a> Linker for WasmLd<'a> { self.link_arg(match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", // Currently LLD doesn't support `Os` and `Oz`, so pass through `O2` // instead. @@ -1451,7 +1463,7 @@ impl<'a> WasmLd<'a> { let opt_level = match self.sess.opts.optimize { config::OptLevel::No => "O0", config::OptLevel::Less => "O1", - config::OptLevel::Default => "O2", + config::OptLevel::More => "O2", config::OptLevel::Aggressive => "O3", // wasm-ld only handles integer LTO opt levels. Use O2 config::OptLevel::Size | config::OptLevel::SizeMin => "O2", @@ -1525,7 +1537,7 @@ impl<'a> Linker for L4Bender<'a> { fn optimize(&mut self) { // GNU-style linkers support optimization with -O. GNU ld doesn't // need a numeric argument, but other linkers do. - if self.sess.opts.optimize == config::OptLevel::Default + if self.sess.opts.optimize == config::OptLevel::More || self.sess.opts.optimize == config::OptLevel::Aggressive { self.link_arg("-O1"); @@ -1776,6 +1788,7 @@ fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) - symbols.push(symbol_export::exporting_symbol_name_for_instance_in_crate( tcx, symbol, cnum, )); + symbol_export::extend_exported_symbols(&mut symbols, tcx, symbol, cnum); } }); @@ -1929,7 +1942,7 @@ impl<'a> Linker for LlbcLinker<'a> { match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", OptLevel::Size => "-Os", OptLevel::SizeMin => "-Oz", @@ -2006,7 +2019,7 @@ impl<'a> Linker for BpfLinker<'a> { self.link_arg(match self.sess.opts.optimize { OptLevel::No => "-O0", OptLevel::Less => "-O1", - OptLevel::Default => "-O2", + OptLevel::More => "-O2", OptLevel::Aggressive => "-O3", OptLevel::Size => "-Os", OptLevel::SizeMin => "-Oz", diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index d70413b8a47..236507ac0cd 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -252,15 +252,7 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static // Unsupported architecture. _ => return None, }; - let binary_format = if sess.target.is_like_osx { - BinaryFormat::MachO - } else if sess.target.is_like_windows { - BinaryFormat::Coff - } else if sess.target.is_like_aix { - BinaryFormat::Xcoff - } else { - BinaryFormat::Elf - }; + let binary_format = sess.target.binary_format.to_object(); let mut file = write::Object::new(binary_format, architecture, endianness); file.set_sub_architecture(sub_architecture); diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 1dbaffaa577..459f4329d6e 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -10,9 +10,10 @@ use rustc_middle::middle::exported_symbols::{ ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, metadata_symbol_name, }; use rustc_middle::query::LocalCrate; -use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, SymbolName, TyCtxt}; +use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, SymbolName, Ty, TyCtxt}; use rustc_middle::util::Providers; use rustc_session::config::{CrateType, OomStrategy}; +use rustc_target::callconv::Conv; use rustc_target::spec::{SanitizerSet, TlsModel}; use tracing::debug; @@ -182,11 +183,11 @@ fn exported_symbols_provider_local( }); let mut symbols: Vec<_> = - sorted.iter().map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect(); + sorted.iter().map(|&(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect(); // Export TLS shims if !tcx.sess.target.dll_tls_export { - symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| { + symbols.extend(sorted.iter().filter_map(|&(&def_id, &info)| { tcx.needs_thread_local_shim(def_id).then(|| { ( ExportedSymbol::ThreadLocalShim(def_id), @@ -584,6 +585,42 @@ pub(crate) fn symbol_name_for_instance_in_crate<'tcx>( } } +fn calling_convention_for_symbol<'tcx>( + tcx: TyCtxt<'tcx>, + symbol: ExportedSymbol<'tcx>, +) -> (Conv, &'tcx [rustc_target::callconv::ArgAbi<'tcx, Ty<'tcx>>]) { + let instance = match symbol { + ExportedSymbol::NonGeneric(def_id) | ExportedSymbol::Generic(def_id, _) + if tcx.is_static(def_id) => + { + None + } + ExportedSymbol::NonGeneric(def_id) => Some(Instance::mono(tcx, def_id)), + ExportedSymbol::Generic(def_id, args) => Some(Instance::new(def_id, args)), + // DropGlue always use the Rust calling convention and thus follow the target's default + // symbol decoration scheme. + ExportedSymbol::DropGlue(..) => None, + // AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the + // target's default symbol decoration scheme. + ExportedSymbol::AsyncDropGlueCtorShim(..) => None, + // NoDefId always follow the target's default symbol decoration scheme. + ExportedSymbol::NoDefId(..) => None, + // ThreadLocalShim always follow the target's default symbol decoration scheme. + ExportedSymbol::ThreadLocalShim(..) => None, + }; + + instance + .map(|i| { + tcx.fn_abi_of_instance( + ty::TypingEnv::fully_monomorphized().as_query_input((i, ty::List::empty())), + ) + .unwrap_or_else(|_| bug!("fn_abi_of_instance({i:?}) failed")) + }) + .map(|fnabi| (fnabi.conv, &fnabi.args[..])) + // FIXME(workingjubilee): why don't we know the convention here? + .unwrap_or((Conv::Rust, &[])) +} + /// This is the symbol name of the given instance as seen by the linker. /// /// On 32-bit Windows symbols are decorated according to their calling conventions. @@ -592,8 +629,6 @@ pub(crate) fn linking_symbol_name_for_instance_in_crate<'tcx>( symbol: ExportedSymbol<'tcx>, instantiating_crate: CrateNum, ) -> String { - use rustc_target::callconv::Conv; - let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate); // thread local will not be a function call, @@ -617,35 +652,7 @@ pub(crate) fn linking_symbol_name_for_instance_in_crate<'tcx>( _ => return undecorated, }; - let instance = match symbol { - ExportedSymbol::NonGeneric(def_id) | ExportedSymbol::Generic(def_id, _) - if tcx.is_static(def_id) => - { - None - } - ExportedSymbol::NonGeneric(def_id) => Some(Instance::mono(tcx, def_id)), - ExportedSymbol::Generic(def_id, args) => Some(Instance::new(def_id, args)), - // DropGlue always use the Rust calling convention and thus follow the target's default - // symbol decoration scheme. - ExportedSymbol::DropGlue(..) => None, - // AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the - // target's default symbol decoration scheme. - ExportedSymbol::AsyncDropGlueCtorShim(..) => None, - // NoDefId always follow the target's default symbol decoration scheme. - ExportedSymbol::NoDefId(..) => None, - // ThreadLocalShim always follow the target's default symbol decoration scheme. - ExportedSymbol::ThreadLocalShim(..) => None, - }; - - let (conv, args) = instance - .map(|i| { - tcx.fn_abi_of_instance( - ty::TypingEnv::fully_monomorphized().as_query_input((i, ty::List::empty())), - ) - .unwrap_or_else(|_| bug!("fn_abi_of_instance({i:?}) failed")) - }) - .map(|fnabi| (fnabi.conv, &fnabi.args[..])) - .unwrap_or((Conv::Rust, &[])); + let (conv, args) = calling_convention_for_symbol(tcx, symbol); // Decorate symbols with prefixes, suffixes and total number of bytes of arguments. // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170 @@ -677,6 +684,27 @@ pub(crate) fn exporting_symbol_name_for_instance_in_crate<'tcx>( maybe_emutls_symbol_name(tcx, symbol, &undecorated).unwrap_or(undecorated) } +/// On amdhsa, `gpu-kernel` functions have an associated metadata object with a `.kd` suffix. +/// Add it to the symbols list for all kernel functions, so that it is exported in the linked +/// object. +pub(crate) fn extend_exported_symbols<'tcx>( + symbols: &mut Vec<String>, + tcx: TyCtxt<'tcx>, + symbol: ExportedSymbol<'tcx>, + instantiating_crate: CrateNum, +) { + let (conv, _) = calling_convention_for_symbol(tcx, symbol); + + if conv != Conv::GpuKernel || tcx.sess.target.os != "amdhsa" { + return; + } + + let undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate); + + // Add the symbol for the kernel descriptor (with .kd suffix) + symbols.push(format!("{undecorated}.kd")); +} + fn maybe_emutls_symbol_name<'tcx>( tcx: TyCtxt<'tcx>, symbol: ExportedSymbol<'tcx>, diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index ce2161a07eb..f008bd12ed8 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -236,7 +236,7 @@ impl ModuleConfig { // Copy what clang does by turning on loop vectorization at O2 and // slp vectorization at O3. vectorize_loop: !sess.opts.cg.no_vectorize_loops - && (sess.opts.optimize == config::OptLevel::Default + && (sess.opts.optimize == config::OptLevel::More || sess.opts.optimize == config::OptLevel::Aggressive), vectorize_slp: !sess.opts.cg.no_vectorize_slp && sess.opts.optimize == config::OptLevel::Aggressive, @@ -260,7 +260,7 @@ impl ModuleConfig { MergeFunctions::Trampolines | MergeFunctions::Aliases => { use config::OptLevel::*; match sess.opts.optimize { - Aggressive | Default | SizeMin | Size => true, + Aggressive | More | SizeMin | Size => true, Less | No => false, } } @@ -405,7 +405,8 @@ fn generate_lto_work<B: ExtraBackendMethods>( B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise()); if cgcx.lto == Lto::Fat && !autodiff.is_empty() { let config = cgcx.config(ModuleKind::Regular); - module = unsafe { module.autodiff(cgcx, autodiff, config).unwrap() }; + module = + unsafe { module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise()) }; } // We are adding a single work item, so the cost doesn't matter. vec![(WorkItem::LTO(module), 0)] @@ -572,10 +573,10 @@ fn produce_final_output_artifacts( }; let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| { - if compiled_modules.modules.len() == 1 { + if let [module] = &compiled_modules.modules[..] { // 1) Only one codegen unit. In this case it's no difficulty // to copy `foo.0.x` to `foo.x`. - let module_name = Some(&compiled_modules.modules[0].name[..]); + let module_name = Some(&module.name[..]); let path = crate_output.temp_path(output_type, module_name); let output = crate_output.path(output_type); if !output_type.is_text_output() && output.is_tty() { @@ -707,8 +708,8 @@ fn produce_final_output_artifacts( } if sess.opts.json_artifact_notifications { - if compiled_modules.modules.len() == 1 { - compiled_modules.modules[0].for_each_output(|_path, ty| { + if let [module] = &compiled_modules.modules[..] { + module.for_each_output(|_path, ty| { if sess.opts.output_types.contains_key(&ty) { let descr = ty.shorthand(); // for single cgu file is renamed to drop cgu specific suffix @@ -864,7 +865,7 @@ pub(crate) fn compute_per_cgu_lto_type( // require LTO so the request for LTO is always unconditionally // passed down to the backend, but we don't actually want to do // anything about it yet until we've got a final product. - let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib; + let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]); match sess_lto { Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin, @@ -1537,8 +1538,9 @@ fn start_executing_work<B: ExtraBackendMethods>( // Spin up what work we can, only doing this while we've got available // parallelism slots and work left to spawn. if codegen_state != Aborted { - while !work_items.is_empty() && running_with_own_token < tokens.len() { - let (item, _) = work_items.pop().unwrap(); + while running_with_own_token < tokens.len() + && let Some((item, _)) = work_items.pop() + { spawn_work( &cgcx, &mut llvm_start_time, diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index d9fbf539fd3..73a97d32c2d 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -5,7 +5,9 @@ use std::time::{Duration, Instant}; use itertools::Itertools; use rustc_abi::FIRST_VARIANT; +use rustc_ast as ast; use rustc_ast::expand::allocator::{ALLOCATOR_METHODS, AllocatorKind, global_fn_name}; +use rustc_attr_parsing::OptimizeAttr; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry}; use rustc_data_structures::sync::par_map; @@ -24,12 +26,11 @@ use rustc_middle::query::Providers; use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_session::Session; -use rustc_session::config::{self, CrateType, EntryFnType, OptLevel, OutputType}; +use rustc_session::config::{self, CrateType, EntryFnType, OutputType}; use rustc_span::{DUMMY_SP, Symbol, sym}; use rustc_trait_selection::infer::{BoundRegionConversionTime, TyCtxtInferExt}; use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt}; use tracing::{debug, info}; -use {rustc_ast as ast, rustc_attr_parsing as attr}; use crate::assert_module_sources::CguReuse; use crate::back::link::are_upstream_rust_objects_already_included; @@ -364,13 +365,7 @@ pub(crate) fn build_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let rhs_sz = bx.cx().int_width(rhs_llty); let lhs_sz = bx.cx().int_width(lhs_llty); if lhs_sz < rhs_sz { - if is_unchecked && bx.sess().opts.optimize != OptLevel::No { - // FIXME: Use `trunc nuw` once that's available - let inrange = bx.icmp(IntPredicate::IntULE, rhs, mask); - bx.assume(inrange); - } - - bx.trunc(rhs, lhs_llty) + if is_unchecked { bx.unchecked_utrunc(rhs, lhs_llty) } else { bx.trunc(rhs, lhs_llty) } } else if lhs_sz > rhs_sz { // We zero-extend even if the RHS is signed. So e.g. `(x: i32) << -1i8` will zero-extend the // RHS to `255i32`. But then we mask the shift amount to be within the size of the LHS @@ -921,6 +916,7 @@ impl CrateInfo { let n_crates = crates.len(); let mut info = CrateInfo { target_cpu, + target_features: tcx.global_backend_features(()).clone(), crate_types, exported_symbols, linked_symbols, @@ -1054,19 +1050,19 @@ pub(crate) fn provide(providers: &mut Providers) { config::OptLevel::No => return config::OptLevel::No, // If globally optimise-speed is already specified, just use that level. config::OptLevel::Less => return config::OptLevel::Less, - config::OptLevel::Default => return config::OptLevel::Default, + config::OptLevel::More => return config::OptLevel::More, config::OptLevel::Aggressive => return config::OptLevel::Aggressive, // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size) // are present). - config::OptLevel::Size => config::OptLevel::Default, - config::OptLevel::SizeMin => config::OptLevel::Default, + config::OptLevel::Size => config::OptLevel::More, + config::OptLevel::SizeMin => config::OptLevel::More, }; let defids = tcx.collect_and_partition_mono_items(cratenum).all_mono_items; let any_for_speed = defids.items().any(|id| { let CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id); - matches!(optimize, attr::OptimizeAttr::Speed) + matches!(optimize, OptimizeAttr::Speed) }); if any_for_speed { diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 7acdbd19993..673740b4aab 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -1,11 +1,12 @@ use std::str::FromStr; -use rustc_ast::attr::list_contains_name; +use rustc_abi::ExternAbi; use rustc_ast::expand::autodiff_attrs::{ AutoDiffAttrs, DiffActivity, DiffMode, valid_input_activity, valid_ret_activity, }; use rustc_ast::{MetaItem, MetaItemInner, attr}; -use rustc_attr_parsing::{InlineAttr, InstructionSetAttr, OptimizeAttr}; +use rustc_attr_parsing::ReprAttr::ReprAlign; +use rustc_attr_parsing::{AttributeKind, InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::codes::*; use rustc_errors::{DiagMessage, SubdiagMessage, struct_span_code_err}; @@ -23,7 +24,7 @@ use rustc_middle::ty::{self as ty, TyCtxt}; use rustc_session::parse::feature_err; use rustc_session::{Session, lint}; use rustc_span::{Ident, Span, sym}; -use rustc_target::spec::{SanitizerSet, abi}; +use rustc_target::spec::SanitizerSet; use tracing::debug; use crate::errors; @@ -104,12 +105,26 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if let Fn | AssocFn | Variant | Ctor(..) = def_kind { Some(tcx.fn_sig(did)) } else { - tcx.dcx() - .span_delayed_bug(attr.span, "this attribute can only be applied to functions"); + tcx.dcx().span_delayed_bug( + attr.span(), + "this attribute can only be applied to functions", + ); None } }; + if let hir::Attribute::Parsed(p) = attr { + match p { + AttributeKind::Repr(reprs) => { + codegen_fn_attrs.alignment = reprs + .iter() + .find_map(|(r, _)| if let ReprAlign(x) = r { Some(*x) } else { None }); + } + + _ => {} + } + } + let Some(Ident { name, .. }) = attr.ident() else { continue; }; @@ -130,14 +145,14 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if tcx.opt_item_name(did.to_def_id()).is_some() { codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE; mixed_export_name_no_mangle_lint_state.track_no_mangle( - attr.span, + attr.span(), tcx.local_def_id_to_hir_id(did), attr, ); } else { tcx.dcx() .struct_span_err( - attr.span, + attr.span(), format!( "`#[no_mangle]` cannot be used on {} {} as it has no name", tcx.def_descr_article(did.to_def_id()), @@ -158,7 +173,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { feature_err( &tcx.sess, sym::used_with_arg, - attr.span, + attr.span(), "`#[used(linker)]` is currently unstable", ) .emit(); @@ -170,7 +185,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { feature_err( &tcx.sess, sym::used_with_arg, - attr.span, + attr.span(), "`#[used(compiler)]` is currently unstable", ) .emit(); @@ -178,7 +193,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED; } Some(_) => { - tcx.dcx().emit_err(errors::ExpectedUsedSymbol { span: attr.span }); + tcx.dcx().emit_err(errors::ExpectedUsedSymbol { span: attr.span() }); } None => { // Unfortunately, unconditionally using `llvm.used` causes @@ -219,11 +234,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if !is_closure && let Some(fn_sig) = fn_sig() - && fn_sig.skip_binder().abi() != abi::Abi::Rust + && fn_sig.skip_binder().abi() != ExternAbi::Rust { struct_span_code_err!( tcx.dcx(), - attr.span, + attr.span(), E0737, "`#[track_caller]` requires Rust ABI" ) @@ -231,12 +246,12 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } if is_closure && !tcx.features().closure_track_caller() - && !attr.span.allows_unstable(sym::closure_track_caller) + && !attr.span().allows_unstable(sym::closure_track_caller) { feature_err( &tcx.sess, sym::closure_track_caller, - attr.span, + attr.span(), "`#[track_caller]` on closures is currently unstable", ) .emit(); @@ -250,19 +265,19 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // so it may not contain any null characters. struct_span_code_err!( tcx.dcx(), - attr.span, + attr.span(), E0648, "`export_name` may not contain null characters" ) .emit(); } codegen_fn_attrs.export_name = Some(s); - mixed_export_name_no_mangle_lint_state.track_export_name(attr.span); + mixed_export_name_no_mangle_lint_state.track_export_name(attr.span()); } } sym::target_feature => { let Some(sig) = tcx.hir_node_by_def_id(did).fn_sig() else { - tcx.dcx().span_delayed_bug(attr.span, "target_feature applied to non-fn"); + tcx.dcx().span_delayed_bug(attr.span(), "target_feature applied to non-fn"); continue; }; let safe_target_features = @@ -271,10 +286,9 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if safe_target_features { if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { // The `#[target_feature]` attribute is allowed on - // WebAssembly targets on all functions, including safe - // ones. Other targets require that `#[target_feature]` is - // only applied to unsafe functions (pending the - // `target_feature_11` feature) because on most targets + // WebAssembly targets on all functions. Prior to stabilizing + // the `target_feature_11` feature, `#[target_feature]` was + // only permitted on unsafe functions because on most targets // execution of instructions that are not supported is // considered undefined behavior. For WebAssembly which is a // 100% safe target at execution time it's not possible to @@ -288,19 +302,12 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // if a target is documenting some wasm-specific code then // it's not spuriously denied. // - // This exception needs to be kept in sync with allowing - // `#[target_feature]` on `main` and `start`. - } else if !tcx.features().target_feature_11() { - feature_err( - &tcx.sess, - sym::target_feature_11, - attr.span, - "`#[target_feature(..)]` can only be applied to `unsafe` functions", - ) - .with_span_label(tcx.def_span(did), "not an `unsafe` function") - .emit(); + // Now that `#[target_feature]` is permitted on safe functions, + // this exception must still exist for allowing the attribute on + // `main`, `start`, and other functions that are not usually + // allowed. } else { - check_target_feature_trait_unsafe(tcx, did, attr.span); + check_target_feature_trait_unsafe(tcx, did, attr.span()); } } from_target_feature_attr( @@ -318,7 +325,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if tcx.is_mutable_static(did.into()) { let mut diag = tcx.dcx().struct_span_err( - attr.span, + attr.span(), "extern mutable statics are not allowed with `#[linkage]`", ); diag.note( @@ -337,7 +344,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if let Some(val) = attr.value_str() { if val.as_str().bytes().any(|b| b == 0) { let msg = format!("illegal null byte in link_section value: `{val}`"); - tcx.dcx().span_err(attr.span, msg); + tcx.dcx().span_err(attr.span(), msg); } else { codegen_fn_attrs.link_section = Some(val); } @@ -345,13 +352,13 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } sym::link_name => codegen_fn_attrs.link_name = attr.value_str(), sym::link_ordinal => { - link_ordinal_span = Some(attr.span); + link_ordinal_span = Some(attr.span()); if let ordinal @ Some(_) = check_link_ordinal(tcx, attr) { codegen_fn_attrs.link_ordinal = ordinal; } } sym::no_sanitize => { - no_sanitize_span = Some(attr.span); + no_sanitize_span = Some(attr.span()); if let Some(list) = attr.meta_item_list() { for item in list.iter() { match item.name_or_empty() { @@ -384,28 +391,24 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { let segments = set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>(); match segments.as_slice() { - [sym::arm, sym::a32] | [sym::arm, sym::t32] => { - if !tcx.sess.target.has_thumb_interworking { - struct_span_code_err!( - tcx.dcx(), - attr.span, - E0779, - "target does not support `#[instruction_set]`" - ) - .emit(); - None - } else if segments[1] == sym::a32 { - Some(InstructionSetAttr::ArmA32) - } else if segments[1] == sym::t32 { - Some(InstructionSetAttr::ArmT32) - } else { - unreachable!() - } + [sym::arm, sym::a32 | sym::t32] + if !tcx.sess.target.has_thumb_interworking => + { + struct_span_code_err!( + tcx.dcx(), + attr.span(), + E0779, + "target does not support `#[instruction_set]`" + ) + .emit(); + None } + [sym::arm, sym::a32] => Some(InstructionSetAttr::ArmA32), + [sym::arm, sym::t32] => Some(InstructionSetAttr::ArmT32), _ => { struct_span_code_err!( tcx.dcx(), - attr.span, + attr.span(), E0779, "invalid instruction set specified", ) @@ -417,7 +420,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { [] => { struct_span_code_err!( tcx.dcx(), - attr.span, + attr.span(), E0778, "`#[instruction_set]` requires an argument" ) @@ -427,7 +430,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { _ => { struct_span_code_err!( tcx.dcx(), - attr.span, + attr.span(), E0779, "cannot specify more than one instruction set" ) @@ -436,27 +439,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } }) } - sym::repr => { - codegen_fn_attrs.alignment = if let Some(items) = attr.meta_item_list() - && let [item] = items.as_slice() - && let Some((sym::align, literal)) = item.singleton_lit_list() - { - rustc_attr_parsing::parse_alignment(&literal.kind) - .map_err(|msg| { - struct_span_code_err!( - tcx.dcx(), - literal.span, - E0589, - "invalid `repr(align)` attribute: {}", - msg - ) - .emit(); - }) - .ok() - } else { - None - }; - } sym::patchable_function_entry => { codegen_fn_attrs.patchable_function_entry = attr.meta_item_list().and_then(|l| { let mut prefix = None; @@ -522,7 +504,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } if let (None, None) = (prefix, entry) { - tcx.dcx().span_err(attr.span, "must specify at least one parameter"); + tcx.dcx().span_err(attr.span(), "must specify at least one parameter"); } Some(PatchableFunctionEntry::from_prefix_and_entry( @@ -543,25 +525,28 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } if attr.is_word() { - InlineAttr::Hint - } else if let Some(ref items) = attr.meta_item_list() { - inline_span = Some(attr.span); - if items.len() != 1 { - struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit(); - InlineAttr::None - } else if list_contains_name(items, sym::always) { - InlineAttr::Always - } else if list_contains_name(items, sym::never) { - InlineAttr::Never - } else { - struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") - .with_help("valid inline arguments are `always` and `never`") - .emit(); + return InlineAttr::Hint; + } + let Some(ref items) = attr.meta_item_list() else { + return ia; + }; + inline_span = Some(attr.span()); - InlineAttr::None - } + let [item] = &items[..] else { + struct_span_code_err!(tcx.dcx(), attr.span(), E0534, "expected one argument").emit(); + return InlineAttr::None; + }; + + if item.has_name(sym::always) { + InlineAttr::Always + } else if item.has_name(sym::never) { + InlineAttr::Never } else { - ia + struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") + .with_help("valid inline arguments are `always` and `never`") + .emit(); + + InlineAttr::None } }); codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| { @@ -570,9 +555,9 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } if attr.is_word() { - InlineAttr::Force { attr_span: attr.span, reason: None } + InlineAttr::Force { attr_span: attr.span(), reason: None } } else if let Some(val) = attr.value_str() { - InlineAttr::Force { attr_span: attr.span, reason: Some(val) } + InlineAttr::Force { attr_span: attr.span(), reason: Some(val) } } else { debug!("`rustc_force_inline` not checked by attribute validation"); ia @@ -592,24 +577,26 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } let err = |sp, s| struct_span_code_err!(tcx.dcx(), sp, E0722, "{}", s).emit(); if attr.is_word() { - err(attr.span, "expected one argument"); - ia - } else if let Some(ref items) = attr.meta_item_list() { - inline_span = Some(attr.span); - if items.len() != 1 { - err(attr.span, "expected one argument"); - OptimizeAttr::Default - } else if list_contains_name(items, sym::size) { - OptimizeAttr::Size - } else if list_contains_name(items, sym::speed) { - OptimizeAttr::Speed - } else if list_contains_name(items, sym::none) { - OptimizeAttr::DoNotOptimize - } else { - err(items[0].span(), "invalid argument"); - OptimizeAttr::Default - } + err(attr.span(), "expected one argument"); + return ia; + } + let Some(ref items) = attr.meta_item_list() else { + return OptimizeAttr::Default; + }; + + inline_span = Some(attr.span()); + let [item] = &items[..] else { + err(attr.span(), "expected one argument"); + return OptimizeAttr::Default; + }; + if item.has_name(sym::size) { + OptimizeAttr::Size + } else if item.has_name(sym::speed) { + OptimizeAttr::Speed + } else if item.has_name(sym::none) { + OptimizeAttr::DoNotOptimize } else { + err(item.span(), "invalid argument"); OptimizeAttr::Default } }); @@ -627,10 +614,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // its parent function, which effectively inherits the features anyway. Boxing this closure // would result in this closure being compiled without the inherited target features, but this // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute. - if tcx.features().target_feature_11() - && tcx.is_closure_like(did.to_def_id()) - && !codegen_fn_attrs.inline.always() - { + if tcx.is_closure_like(did.to_def_id()) && codegen_fn_attrs.inline != InlineAttr::Always { let owner_id = tcx.parent(did.to_def_id()); if tcx.def_kind(owner_id).has_codegen_attrs() { codegen_fn_attrs @@ -654,25 +638,20 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // llvm/llvm-project#70563). if !codegen_fn_attrs.target_features.is_empty() && matches!(codegen_fn_attrs.inline, InlineAttr::Always) + && let Some(span) = inline_span { - if let Some(span) = inline_span { - tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`"); - } + tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`"); } - if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() { - if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) { - let hir_id = tcx.local_def_id_to_hir_id(did); - tcx.node_span_lint( - lint::builtin::INLINE_NO_SANITIZE, - hir_id, - no_sanitize_span, - |lint| { - lint.primary_message("`no_sanitize` will have no effect after inlining"); - lint.span_note(inline_span, "inlining requested here"); - }, - ) - } + if !codegen_fn_attrs.no_sanitize.is_empty() + && codegen_fn_attrs.inline.always() + && let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) + { + let hir_id = tcx.local_def_id_to_hir_id(did); + tcx.node_span_lint(lint::builtin::INLINE_NO_SANITIZE, hir_id, no_sanitize_span, |lint| { + lint.primary_message("`no_sanitize` will have no effect after inlining"); + lint.span_note(inline_span, "inlining requested here"); + }) } // Weak lang items have the same semantics as "std internal" symbols in the @@ -702,10 +681,10 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // Any linkage to LLVM intrinsics for now forcibly marks them all as never // unwinds since LLVM sometimes can't handle codegen which `invoke`s // intrinsic functions. - if let Some(name) = &codegen_fn_attrs.link_name { - if name.as_str().starts_with("llvm.") { - codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND; - } + if let Some(name) = &codegen_fn_attrs.link_name + && name.as_str().starts_with("llvm.") + { + codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND; } if let Some(features) = check_tied_features( @@ -719,7 +698,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { let span = tcx .get_attrs(did, sym::target_feature) .next() - .map_or_else(|| tcx.def_span(did), |a| a.span); + .map_or_else(|| tcx.def_span(did), |a| a.span()); tcx.dcx() .create_err(errors::TargetFeatureDisableOrEnable { features, @@ -766,18 +745,13 @@ fn should_inherit_track_caller(tcx: TyCtxt<'_>, def_id: DefId) -> bool { fn check_link_ordinal(tcx: TyCtxt<'_>, attr: &hir::Attribute) -> Option<u16> { use rustc_ast::{LitIntType, LitKind, MetaItemLit}; - let meta_item_list = attr.meta_item_list(); - let meta_item_list = meta_item_list.as_deref(); - let sole_meta_list = match meta_item_list { - Some([item]) => item.lit(), - Some(_) => { - tcx.dcx().emit_err(errors::InvalidLinkOrdinalNargs { span: attr.span }); - return None; - } - _ => None, + let meta_item_list = attr.meta_item_list()?; + let [sole_meta_list] = &meta_item_list[..] else { + tcx.dcx().emit_err(errors::InvalidLinkOrdinalNargs { span: attr.span() }); + return None; }; if let Some(MetaItemLit { kind: LitKind::Int(ordinal, LitIntType::Unsuffixed), .. }) = - sole_meta_list + sole_meta_list.lit() { // According to the table at // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#import-header, the @@ -797,13 +771,13 @@ fn check_link_ordinal(tcx: TyCtxt<'_>, attr: &hir::Attribute) -> Option<u16> { } else { let msg = format!("ordinal value in `link_ordinal` is too large: `{ordinal}`"); tcx.dcx() - .struct_span_err(attr.span, msg) + .struct_span_err(attr.span(), msg) .with_note("the value may not exceed `u16::MAX`") .emit(); None } } else { - tcx.dcx().emit_err(errors::InvalidLinkOrdinalFormat { span: attr.span }); + tcx.dcx().emit_err(errors::InvalidLinkOrdinalFormat { span: attr.span() }); None } } @@ -849,7 +823,7 @@ impl<'a> MixedExportNameAndNoMangleState<'a> { export_name: Some(export_name), no_mangle: Some(no_mangle), hir_id: Some(hir_id), - no_mangle_attr: Some(no_mangle_attr), + no_mangle_attr: Some(_), } = self { tcx.emit_node_span_lint( @@ -858,7 +832,7 @@ impl<'a> MixedExportNameAndNoMangleState<'a> { no_mangle, errors::MixedExportNameAndNoMangle { no_mangle, - no_mangle_attr: rustc_hir_pretty::attribute_to_string(&tcx, no_mangle_attr), + no_mangle_attr: "#[unsafe(no_mangle)]".to_string(), export_name, removal_span: no_mangle, }, @@ -890,7 +864,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> { _ => { //FIXME(ZuseZ4): Once we fixed our parser, we should also prohibit the two-attribute //branch above. - span_bug!(attrs[1].span, "cg_ssa: rustc_autodiff should only exist once per source"); + span_bug!(attrs[1].span(), "cg_ssa: rustc_autodiff should only exist once per source"); } }; @@ -902,30 +876,28 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> { } let [mode, input_activities @ .., ret_activity] = &list[..] else { - span_bug!(attr.span, "rustc_autodiff attribute must contain mode and activities"); + span_bug!(attr.span(), "rustc_autodiff attribute must contain mode and activities"); }; - let mode = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = mode { + let mode = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = mode { p1.segments.first().unwrap().ident } else { - span_bug!(attr.span, "rustc_autodiff attribute must contain mode"); + span_bug!(attr.span(), "rustc_autodiff attribute must contain mode"); }; // parse mode let mode = match mode.as_str() { "Forward" => DiffMode::Forward, "Reverse" => DiffMode::Reverse, - "ForwardFirst" => DiffMode::ForwardFirst, - "ReverseFirst" => DiffMode::ReverseFirst, _ => { span_bug!(mode.span, "rustc_autodiff attribute contains invalid mode"); } }; // First read the ret symbol from the attribute - let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = ret_activity { + let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = ret_activity { p1.segments.first().unwrap().ident } else { - span_bug!(attr.span, "rustc_autodiff attribute must contain the return activity"); + span_bug!(attr.span(), "rustc_autodiff attribute must contain the return activity"); }; // Then parse it into an actual DiffActivity @@ -936,7 +908,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> { // Now parse all the intermediate (input) activities let mut arg_activities: Vec<DiffActivity> = vec![]; for arg in input_activities { - let arg_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p2, .. }) = arg { + let arg_symbol = if let MetaItemInner::MetaItem(MetaItem { path: p2, .. }) = arg { match p2.segments.first() { Some(x) => x.ident, None => { @@ -960,11 +932,11 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> { for &input in &arg_activities { if !valid_input_activity(mode, input) { - span_bug!(attr.span, "Invalid input activity {} for {} mode", input, mode); + span_bug!(attr.span(), "Invalid input activity {} for {} mode", input, mode); } } if !valid_ret_activity(mode, ret_activity) { - span_bug!(attr.span, "Invalid return activity {} for {} mode", ret_activity, mode); + span_bug!(attr.span(), "Invalid return activity {} for {} mode", ret_activity, mode); } Some(AutoDiffAttrs { mode, ret_activity, input_activity: arg_activities }) diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index 05175371591..84703a0a156 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -15,7 +15,8 @@ use std::fmt::Write; use rustc_abi::Integer; use rustc_data_structures::fx::FxHashSet; -use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_hashes::Hash64; use rustc_hir::def_id::DefId; use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData}; use rustc_hir::{CoroutineDesugaring, CoroutineKind, CoroutineSource, Mutability}; @@ -367,9 +368,7 @@ fn push_debuginfo_type_name<'tcx>( output.push_str(sig.safety.prefix_str()); if sig.abi != rustc_abi::ExternAbi::Rust { - output.push_str("extern \""); - output.push_str(sig.abi.name()); - output.push_str("\" "); + let _ = write!(output, "extern {} ", sig.abi); } output.push_str("fn("); diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index 3ddbe4aeeec..5e25de02a77 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -1180,6 +1180,8 @@ pub(crate) struct ErrorCreatingRemarkDir { pub struct CompilerBuiltinsCannotCall { pub caller: String, pub callee: String, + #[primary_span] + pub span: Span, } #[derive(Diagnostic)] diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 428a45975f1..9d2ac219d59 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -190,6 +190,7 @@ impl From<&cstore::NativeLib> for NativeLib { #[derive(Debug, Encodable, Decodable)] pub struct CrateInfo { pub target_cpu: String, + pub target_features: Vec<String>, pub crate_types: Vec<CrateType>, pub exported_symbols: UnordMap<CrateType, Vec<String>>, pub linked_symbols: FxIndexMap<CrateType, Vec<(String, SymbolExportKind)>>, @@ -230,6 +231,7 @@ pub fn provide(providers: &mut Providers) { crate::base::provide(providers); crate::target_features::provide(providers); crate::codegen_attrs::provide(providers); + providers.queries.global_backend_features = |_tcx: TyCtxt<'_>, ()| vec![]; } /// Checks if the given filename ends with the `.rcgu.o` extension that `rustc` diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 8c571558717..e2a9b540d30 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -4,9 +4,7 @@ use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange}; use rustc_ast as ast; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::lang_items::LangItem; -use rustc_middle::mir::{ - self, AssertKind, BasicBlock, InlineAsmMacro, SwitchTargets, UnwindTerminateReason, -}; +use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Instance, Ty}; @@ -167,9 +165,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { if let Some(instance) = instance { if is_call_from_compiler_builtins_to_upstream_monomorphization(tcx, instance) { if destination.is_some() { - let caller = with_no_trimmed_paths!(tcx.def_path_str(fx.instance.def_id())); - let callee = with_no_trimmed_paths!(tcx.def_path_str(instance.def_id())); - tcx.dcx().emit_err(CompilerBuiltinsCannotCall { caller, callee }); + let caller_def = fx.instance.def_id(); + let e = CompilerBuiltinsCannotCall { + span: tcx.def_span(caller_def), + caller: with_no_trimmed_paths!(tcx.def_path_str(caller_def)), + callee: with_no_trimmed_paths!(tcx.def_path_str(instance.def_id())), + }; + tcx.dcx().emit_err(e); } else { info!( "compiler_builtins call to diverging function {:?} replaced with abort", @@ -429,11 +431,34 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval); bx.cond_br(cmp, ll1, ll2); } else { - bx.switch( - discr_value, - helper.llbb_with_cleanup(self, targets.otherwise()), - target_iter.map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))), - ); + let otherwise = targets.otherwise(); + let otherwise_cold = self.cold_blocks[otherwise]; + let otherwise_unreachable = self.mir[otherwise].is_empty_unreachable(); + let cold_count = targets.iter().filter(|(_, target)| self.cold_blocks[*target]).count(); + let none_cold = cold_count == 0; + let all_cold = cold_count == targets.iter().len(); + if (none_cold && (!otherwise_cold || otherwise_unreachable)) + || (all_cold && (otherwise_cold || otherwise_unreachable)) + { + // All targets have the same weight, + // or `otherwise` is unreachable and it's the only target with a different weight. + bx.switch( + discr_value, + helper.llbb_with_cleanup(self, targets.otherwise()), + target_iter + .map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))), + ); + } else { + // Targets have different weights + bx.switch_with_weights( + discr_value, + helper.llbb_with_cleanup(self, targets.otherwise()), + otherwise_cold, + target_iter.map(|(value, target)| { + (value, helper.llbb_with_cleanup(self, target), self.cold_blocks[target]) + }), + ); + } } } @@ -699,14 +724,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Put together the arguments to the panic entry point. let (lang_item, args) = match msg { - AssertKind::BoundsCheck { ref len, ref index } => { + AssertKind::BoundsCheck { len, index } => { let len = self.codegen_operand(bx, len).immediate(); let index = self.codegen_operand(bx, index).immediate(); // It's `fn panic_bounds_check(index: usize, len: usize)`, // and `#[track_caller]` adds an implicit third argument. (LangItem::PanicBoundsCheck, vec![index, len, location]) } - AssertKind::MisalignedPointerDereference { ref required, ref found } => { + AssertKind::MisalignedPointerDereference { required, found } => { let required = self.codegen_operand(bx, required).immediate(); let found = self.codegen_operand(bx, found).immediate(); // It's `fn panic_misaligned_pointer_dereference(required: usize, found: usize)`, @@ -919,7 +944,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &fn_abi.ret, &mut llargs, Some(intrinsic), - target, ); let dest = match ret_dest { _ if fn_abi.ret.is_indirect() => llargs[0], @@ -975,23 +999,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let mut llargs = Vec::with_capacity(arg_count); - let destination = target.as_ref().map(|&target| { - ( - self.make_return_dest( - bx, - destination, - &fn_abi.ret, - &mut llargs, - None, - Some(target), - ), - target, - ) - }); + + // We still need to call `make_return_dest` even if there's no `target`, since + // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, + // and `make_return_dest` adds the return-place indirect pointer to `llargs`. + let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, None); + let destination = target.map(|target| (return_dest, target)); // Split the rust-call tupled arguments off. - let (first_args, untuple) = if abi == ExternAbi::RustCall && !args.is_empty() { - let (tup, args) = args.split_last().unwrap(); + let (first_args, untuple) = if abi == ExternAbi::RustCall + && let Some((tup, args)) = args.split_last() + { (args, Some(tup)) } else { (args, None) @@ -1013,11 +1031,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // // This is also relevant for `Pin<&mut Self>`, where we need to peel the // `Pin`. - while !op.layout.ty.is_unsafe_ptr() && !op.layout.ty.is_ref() { + while !op.layout.ty.is_raw_ptr() && !op.layout.ty.is_ref() { let (idx, _) = op.layout.non_1zst_field(bx).expect( "not exactly one non-1-ZST field in a `DispatchFromDyn` type", ); - op = op.extract_field(bx, idx); + op = op.extract_field(self, bx, idx); } // Now that we have `*dyn Trait` or `&dyn Trait`, split it up into its @@ -1045,11 +1063,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } Immediate(_) => { // See comment above explaining why we peel these newtypes - while !op.layout.ty.is_unsafe_ptr() && !op.layout.ty.is_ref() { + while !op.layout.ty.is_raw_ptr() && !op.layout.ty.is_ref() { let (idx, _) = op.layout.non_1zst_field(bx).expect( "not exactly one non-1-ZST field in a `DispatchFromDyn` type", ); - op = op.extract_field(bx, idx); + op = op.extract_field(self, bx, idx); } // Make sure that we've actually unwrapped the rcvr down @@ -1549,9 +1567,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if scalar.is_bool() { bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); } + // We store bools as `i8` so we need to truncate to `i1`. + llval = bx.to_immediate_scalar(llval, scalar); } - // We store bools as `i8` so we need to truncate to `i1`. - llval = bx.to_immediate(llval, arg.layout); } } @@ -1581,7 +1599,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { // If the tuple is immediate, the elements are as well. for i in 0..tuple.layout.fields.count() { - let op = tuple.extract_field(bx, i); + let op = tuple.extract_field(self, bx, i); self.codegen_argument(bx, op, llargs, &args[i]); } } @@ -1790,11 +1808,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, llargs: &mut Vec<Bx::Value>, intrinsic: Option<ty::IntrinsicDef>, - target: Option<BasicBlock>, ) -> ReturnDest<'tcx, Bx::Value> { - if target.is_none() { - return ReturnDest::Nothing; - } // If the return is ignored, we can just return a do-nothing `ReturnDest`. if fn_ret.is_ignore() { return ReturnDest::Nothing; diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 6e7fbe62c8d..b34e966ba6c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -367,7 +367,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.sess().dcx().emit_fatal(errors::AtomicCompareExchange); }; let ty = fn_args.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { let weak = instruction == "cxchgweak"; let dst = args[0].immediate(); let cmp = args[1].immediate(); @@ -395,7 +395,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { "load" => { let ty = fn_args.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { let layout = bx.layout_of(ty); let size = layout.size; let source = args[0].immediate(); @@ -413,7 +413,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { "store" => { let ty = fn_args.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { let size = bx.layout_of(ty).size; let val = args[1].immediate(); let ptr = args[0].immediate(); @@ -458,7 +458,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let ty = fn_args.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { let ptr = args[0].immediate(); let val = args[1].immediate(); bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering)) diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 3a896071bc6..0758e5d0456 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -3,7 +3,7 @@ use std::iter; use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::{UnwindTerminateReason, traversal}; +use rustc_middle::mir::{Local, UnwindTerminateReason, traversal}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; use rustc_middle::{bug, mir, span_bug}; @@ -240,7 +240,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let local_values = { let args = arg_local_refs(&mut start_bx, &mut fx, &memory_locals); - let mut allocate_local = |local| { + let mut allocate_local = |local: Local| { let decl = &mir.local_decls[local]; let layout = start_bx.layout_of(fx.monomorphize(decl.ty)); assert!(!layout.ty.has_erasable_regions()); @@ -502,14 +502,25 @@ fn find_cold_blocks<'tcx>( for (bb, bb_data) in traversal::postorder(mir) { let terminator = bb_data.terminator(); - // If a BB ends with a call to a cold function, mark it as cold. - if let mir::TerminatorKind::Call { ref func, .. } = terminator.kind - && let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind() - && let attrs = tcx.codegen_fn_attrs(def_id) - && attrs.flags.contains(CodegenFnAttrFlags::COLD) - { - cold_blocks[bb] = true; - continue; + match terminator.kind { + // If a BB ends with a call to a cold function, mark it as cold. + mir::TerminatorKind::Call { ref func, .. } + | mir::TerminatorKind::TailCall { ref func, .. } + if let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind() + && let attrs = tcx.codegen_fn_attrs(def_id) + && attrs.flags.contains(CodegenFnAttrFlags::COLD) => + { + cold_blocks[bb] = true; + continue; + } + + // If a BB ends with an `unreachable`, also mark it as cold. + mir::TerminatorKind::Unreachable => { + cold_blocks[bb] = true; + continue; + } + + _ => {} } // If all successors of a BB are cold and there's at least one of them, mark this BB as cold diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index eb0711dbb32..0593fb420c3 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -8,7 +8,7 @@ use rustc_middle::ty::{Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug, ty}; use rustc_span::sym; use rustc_target::callconv::{ArgAbi, FnAbi, PassMode}; -use rustc_target::spec::WasmCAbi; +use rustc_target::spec::{BinaryFormat, WasmCAbi}; use crate::common; use crate::traits::{AsmCodegenMethods, BuilderMethods, GlobalAsmOperandRef, MiscCodegenMethods}; @@ -104,27 +104,6 @@ fn inline_to_global_operand<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } -enum AsmBinaryFormat { - Elf, - Macho, - Coff, - Wasm, -} - -impl AsmBinaryFormat { - fn from_target(target: &rustc_target::spec::Target) -> Self { - if target.is_like_windows { - Self::Coff - } else if target.is_like_osx { - Self::Macho - } else if target.is_like_wasm { - Self::Wasm - } else { - Self::Elf - } - } -} - fn prefix_and_suffix<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, @@ -134,7 +113,7 @@ fn prefix_and_suffix<'tcx>( ) -> (String, String) { use std::fmt::Write; - let asm_binary_format = AsmBinaryFormat::from_target(&tcx.sess.target); + let asm_binary_format = &tcx.sess.target.binary_format; let is_arm = tcx.sess.target.arch == "arm"; let is_thumb = tcx.sess.unstable_target_features.contains(&sym::thumb_mode); @@ -178,10 +157,13 @@ fn prefix_and_suffix<'tcx>( } Linkage::LinkOnceAny | Linkage::LinkOnceODR | Linkage::WeakAny | Linkage::WeakODR => { match asm_binary_format { - AsmBinaryFormat::Elf | AsmBinaryFormat::Coff | AsmBinaryFormat::Wasm => { + BinaryFormat::Elf + | BinaryFormat::Coff + | BinaryFormat::Wasm + | BinaryFormat::Xcoff => { writeln!(w, ".weak {asm_name}")?; } - AsmBinaryFormat::Macho => { + BinaryFormat::MachO => { writeln!(w, ".globl {asm_name}")?; writeln!(w, ".weak_definition {asm_name}")?; } @@ -207,7 +189,7 @@ fn prefix_and_suffix<'tcx>( let mut begin = String::new(); let mut end = String::new(); match asm_binary_format { - AsmBinaryFormat::Elf => { + BinaryFormat::Elf | BinaryFormat::Xcoff => { let section = link_section.unwrap_or(format!(".text.{asm_name}")); let progbits = match is_arm { @@ -239,7 +221,7 @@ fn prefix_and_suffix<'tcx>( writeln!(end, "{}", arch_suffix).unwrap(); } } - AsmBinaryFormat::Macho => { + BinaryFormat::MachO => { let section = link_section.unwrap_or("__TEXT,__text".to_string()); writeln!(begin, ".pushsection {},regular,pure_instructions", section).unwrap(); writeln!(begin, ".balign {align}").unwrap(); @@ -255,7 +237,7 @@ fn prefix_and_suffix<'tcx>( writeln!(end, "{}", arch_suffix).unwrap(); } } - AsmBinaryFormat::Coff => { + BinaryFormat::Coff => { let section = link_section.unwrap_or(format!(".text.{asm_name}")); writeln!(begin, ".pushsection {},\"xr\"", section).unwrap(); writeln!(begin, ".balign {align}").unwrap(); @@ -272,7 +254,7 @@ fn prefix_and_suffix<'tcx>( writeln!(end, "{}", arch_suffix).unwrap(); } } - AsmBinaryFormat::Wasm => { + BinaryFormat::Wasm => { let section = link_section.unwrap_or(format!(".text.{asm_name}")); writeln!(begin, ".section {section},\"\",@").unwrap(); diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 9ca7d4f8f00..461cf1b8acd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -1,15 +1,14 @@ -use std::assert_matches::assert_matches; use std::fmt; use arrayvec::ArrayVec; use either::Either; use rustc_abi as abi; use rustc_abi::{Align, BackendRepr, Size}; -use rustc_middle::bug; use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range}; use rustc_middle::mir::{self, ConstValue}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::{bug, span_bug}; use tracing::debug; use super::place::{PlaceRef, PlaceValue}; @@ -352,79 +351,83 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { pub(crate) fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>( &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, bx: &mut Bx, i: usize, ) -> Self { let field = self.layout.field(bx.cx(), i); let offset = self.layout.fields.offset(i); - let mut val = match (self.val, self.layout.backend_repr) { - // If the field is ZST, it has no data. - _ if field.is_zst() => OperandValue::ZeroSized, - - // Newtype of a scalar, scalar pair or vector. - (OperandValue::Immediate(_) | OperandValue::Pair(..), _) - if field.size == self.layout.size => + if !bx.is_backend_ref(self.layout) && bx.is_backend_ref(field) { + if let BackendRepr::Vector { count, .. } = self.layout.backend_repr + && let BackendRepr::Memory { sized: true } = field.backend_repr + && count.is_power_of_two() { - assert_eq!(offset.bytes(), 0); - self.val + assert_eq!(field.size, self.layout.size); + // This is being deprecated, but for now stdarch still needs it for + // Newtype vector of array, e.g. #[repr(simd)] struct S([i32; 4]); + let place = PlaceRef::alloca(bx, field); + self.val.store(bx, place.val.with_type(self.layout)); + return bx.load_operand(place); + } else { + // Part of https://github.com/rust-lang/compiler-team/issues/838 + bug!("Non-ref type {self:?} cannot project to ref field type {field:?}"); } + } - // Extract a scalar component from a pair. - (OperandValue::Pair(a_llval, b_llval), BackendRepr::ScalarPair(a, b)) => { - if offset.bytes() == 0 { - assert_eq!(field.size, a.size(bx.cx())); - OperandValue::Immediate(a_llval) - } else { - assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi)); - assert_eq!(field.size, b.size(bx.cx())); - OperandValue::Immediate(b_llval) + let val = if field.is_zst() { + OperandValue::ZeroSized + } else if field.size == self.layout.size { + assert_eq!(offset.bytes(), 0); + fx.codegen_transmute_operand(bx, *self, field).unwrap_or_else(|| { + bug!( + "Expected `codegen_transmute_operand` to handle equal-size \ + field {i:?} projection from {self:?} to {field:?}" + ) + }) + } else { + let (in_scalar, imm) = match (self.val, self.layout.backend_repr) { + // Extract a scalar component from a pair. + (OperandValue::Pair(a_llval, b_llval), BackendRepr::ScalarPair(a, b)) => { + if offset.bytes() == 0 { + assert_eq!(field.size, a.size(bx.cx())); + (Some(a), a_llval) + } else { + assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi)); + assert_eq!(field.size, b.size(bx.cx())); + (Some(b), b_llval) + } } - } - - // `#[repr(simd)]` types are also immediate. - (OperandValue::Immediate(llval), BackendRepr::Vector { .. }) => { - OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64))) - } - _ => bug!("OperandRef::extract_field({:?}): not applicable", self), + _ => { + span_bug!(fx.mir.span, "OperandRef::extract_field({:?}): not applicable", self) + } + }; + OperandValue::Immediate(match field.backend_repr { + BackendRepr::Vector { .. } => imm, + BackendRepr::Scalar(out_scalar) => { + let Some(in_scalar) = in_scalar else { + span_bug!( + fx.mir.span, + "OperandRef::extract_field({:?}): missing input scalar for output scalar", + self + ) + }; + if in_scalar != out_scalar { + // If the backend and backend_immediate types might differ, + // flip back to the backend type then to the new immediate. + // This avoids nop truncations, but still handles things like + // Bools in union fields needs to be truncated. + let backend = bx.from_immediate(imm); + bx.to_immediate_scalar(backend, out_scalar) + } else { + imm + } + } + BackendRepr::ScalarPair(_, _) | BackendRepr::Memory { .. } => bug!(), + }) }; - match (&mut val, field.backend_repr) { - (OperandValue::ZeroSized, _) => {} - ( - OperandValue::Immediate(llval), - BackendRepr::Scalar(_) | BackendRepr::ScalarPair(..) | BackendRepr::Vector { .. }, - ) => { - // Bools in union fields needs to be truncated. - *llval = bx.to_immediate(*llval, field); - } - (OperandValue::Pair(a, b), BackendRepr::ScalarPair(a_abi, b_abi)) => { - // Bools in union fields needs to be truncated. - *a = bx.to_immediate_scalar(*a, a_abi); - *b = bx.to_immediate_scalar(*b, b_abi); - } - // Newtype vector of array, e.g. #[repr(simd)] struct S([i32; 4]); - (OperandValue::Immediate(llval), BackendRepr::Memory { sized: true }) => { - assert_matches!(self.layout.backend_repr, BackendRepr::Vector { .. }); - - let llfield_ty = bx.cx().backend_type(field); - - // Can't bitcast an aggregate, so round trip through memory. - let llptr = bx.alloca(field.size, field.align.abi); - bx.store(*llval, llptr, field.align.abi); - *llval = bx.load(llfield_ty, llptr, field.align.abi); - } - ( - OperandValue::Immediate(_), - BackendRepr::Uninhabited | BackendRepr::Memory { sized: false }, - ) => { - bug!() - } - (OperandValue::Pair(..), _) => bug!(), - (OperandValue::Ref(..), _) => bug!(), - } - OperandRef { val, layout: field } } } @@ -581,13 +584,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Moves out of scalar and scalar pair fields are trivial. for elem in place_ref.projection.iter() { match elem { - mir::ProjectionElem::Field(ref f, _) => { + mir::ProjectionElem::Field(f, _) => { assert!( !o.layout.ty.is_any_ptr(), "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \ but tried to access field {f:?} of pointer {o:?}", ); - o = o.extract_field(bx, f.index()); + o = o.extract_field(self, bx, f.index()); } mir::ProjectionElem::Index(_) | mir::ProjectionElem::ConstantIndex { .. } => { diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index eb4270ffe80..edd09b9c3c5 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -1,7 +1,7 @@ use rustc_abi::Primitive::{Int, Pointer}; use rustc_abi::{Align, BackendRepr, FieldsShape, Size, TagEncoding, VariantIdx, Variants}; +use rustc_middle::mir::PlaceTy; use rustc_middle::mir::interpret::Scalar; -use rustc_middle::mir::tcx::PlaceTy; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, mir}; @@ -423,7 +423,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { layout.size }; - let llval = bx.inbounds_gep(bx.cx().backend_type(layout), self.val.llval, &[llindex]); + let llval = bx.inbounds_nuw_gep(bx.cx().backend_type(layout), self.val.llval, &[llindex]); let align = self.val.align.restrict_for_offset(offset); PlaceValue::new_sized(llval, align).with_type(layout) } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 27cb7883b9a..1eebe04225b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -231,7 +231,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { /// /// Returns `None` for cases that can't work in that framework, such as for /// `Immediate`->`Ref` that needs an `alloc` to get the location. - fn codegen_transmute_operand( + pub(crate) fn codegen_transmute_operand( &mut self, bx: &mut Bx, operand: OperandRef<'tcx, Bx::Value>, @@ -260,6 +260,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Ref(source_place_val) => { assert_eq!(source_place_val.llextra, None); assert_matches!(operand_kind, OperandValueKind::Ref); + // The existing alignment is part of `source_place_val`, + // so that alignment will be used, not `cast`'s. Some(bx.load_operand(source_place_val.with_type(cast)).val) } OperandValue::ZeroSized => { @@ -435,18 +437,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { scalar: abi::Scalar, backend_ty: Bx::Type, ) { - if matches!(self.cx.sess().opts.optimize, OptLevel::No) - // For now, the critical niches are all over `Int`eger values. - // Should floating-point values or pointers ever get more complex - // niches, then this code will probably want to handle them too. - || !matches!(scalar.primitive(), abi::Primitive::Int(..)) - || scalar.is_always_valid(self.cx) - { + if matches!(self.cx.sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(self.cx) { return; } - let range = scalar.valid_range(self.cx); - bx.assume_integer_range(imm, backend_ty, range); + match scalar.primitive() { + abi::Primitive::Int(..) => { + let range = scalar.valid_range(self.cx); + bx.assume_integer_range(imm, backend_ty, range); + } + abi::Primitive::Pointer(abi::AddressSpace::DATA) + if !scalar.valid_range(self.cx).contains(0) => + { + bx.assume_nonnull(imm); + } + abi::Primitive::Pointer(..) | abi::Primitive::Float(..) => {} + } } pub(crate) fn codegen_rvalue_unsized( @@ -660,9 +666,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { lhs.layout.ty, ), - (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => { - self.codegen_scalar_binop(bx, op, lhs_val, rhs_val, lhs.layout.ty) - } + (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => self + .codegen_scalar_binop( + bx, + op, + lhs_val, + rhs_val, + lhs.layout.ty, + rhs.layout.ty, + ), _ => bug!(), }; @@ -689,7 +701,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { (OperandValue::Immediate(llval), operand.layout) } mir::UnOp::PtrMetadata => { - assert!(operand.layout.ty.is_unsafe_ptr() || operand.layout.ty.is_ref(),); + assert!(operand.layout.ty.is_raw_ptr() || operand.layout.ty.is_ref(),); let (_, meta) = operand.val.pointer_parts(); assert_eq!(operand.layout.fields.count() > 1, meta.is_some()); if let Some(meta) = meta { @@ -883,10 +895,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { op: mir::BinOp, lhs: Bx::Value, rhs: Bx::Value, - input_ty: Ty<'tcx>, + lhs_ty: Ty<'tcx>, + rhs_ty: Ty<'tcx>, ) -> Bx::Value { - let is_float = input_ty.is_floating_point(); - let is_signed = input_ty.is_signed(); + let is_float = lhs_ty.is_floating_point(); + let is_signed = lhs_ty.is_signed(); match op { mir::BinOp::Add => { if is_float { @@ -952,9 +965,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::BinOp::BitAnd => bx.and(lhs, rhs), mir::BinOp::BitXor => bx.xor(lhs, rhs), mir::BinOp::Offset => { - let pointee_type = input_ty + let pointee_type = lhs_ty .builtin_deref(true) - .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty)); + .unwrap_or_else(|| bug!("deref of non-pointer {:?}", lhs_ty)); let pointee_layout = bx.cx().layout_of(pointee_type); if pointee_layout.is_zst() { // `Offset` works in terms of the size of pointee, @@ -962,7 +975,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { lhs } else { let llty = bx.cx().backend_type(pointee_layout); - bx.inbounds_gep(llty, lhs, &[rhs]) + if !rhs_ty.is_signed() { + bx.inbounds_nuw_gep(llty, lhs, &[rhs]) + } else { + bx.inbounds_gep(llty, lhs, &[rhs]) + } } } mir::BinOp::Shl | mir::BinOp::ShlUnchecked => { diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index 6749bc63327..f6af889fd6e 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -35,8 +35,8 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { cx.codegen_static(def_id); } MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx().hir().item(item_id); - if let hir::ItemKind::GlobalAsm(asm) = item.kind { + let item = cx.tcx().hir_item(item_id); + if let hir::ItemKind::GlobalAsm { asm, .. } = item.kind { let operands: Vec<_> = asm .operands .iter() @@ -71,11 +71,8 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { } } } - hir::InlineAsmOperand::SymFn { ref anon_const } => { - let ty = cx - .tcx() - .typeck_body(anon_const.body) - .node_type(anon_const.hir_id); + hir::InlineAsmOperand::SymFn { expr } => { + let ty = cx.tcx().typeck(item_id.owner_id).expr_ty(expr); let instance = match ty.kind() { &ty::FnDef(def_id, args) => Instance::new(def_id, args), _ => span_bug!(*op_sp, "asm sym is not a function"), diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 71a2f916db5..ac2366340fb 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -45,14 +45,9 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // The info in this case is the length of the str, so the size is that // times the unit size. ( - // All slice sizes must fit into `isize`, so this multiplication cannot (signed) - // wrap. - // NOTE: ideally, we want the effects of both `unchecked_smul` and `unchecked_umul` - // (resulting in `mul nsw nuw` in LLVM IR), since we know that the multiplication - // cannot signed wrap, and that both operands are non-negative. But at the time of - // writing, the `LLVM-C` binding can't do this, and it doesn't seem to enable any - // further optimizations. - bx.unchecked_smul(info.unwrap(), bx.const_usize(unit.size.bytes())), + // All slice sizes must fit into `isize`, so this multiplication cannot + // wrap -- neither signed nor unsigned. + bx.unchecked_sumul(info.unwrap(), bx.const_usize(unit.size.bytes())), bx.const_usize(unit.align.abi.bytes()), ) } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index b7dcf16fa2b..99fd6b6510f 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -1,7 +1,7 @@ use std::assert_matches::assert_matches; use std::ops::Deref; -use rustc_abi::{Align, BackendRepr, Scalar, Size, WrappingRange}; +use rustc_abi::{Align, Scalar, Size, WrappingRange}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::{Instance, Ty}; @@ -110,6 +110,20 @@ pub trait BuilderMethods<'a, 'tcx>: else_llbb: Self::BasicBlock, cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock)>, ); + + // This is like `switch()`, but every case has a bool flag indicating whether it's cold. + // + // Default implementation throws away the cold flags and calls `switch()`. + fn switch_with_weights( + &mut self, + v: Self::Value, + else_llbb: Self::BasicBlock, + _else_is_cold: bool, + cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>, + ) { + self.switch(v, else_llbb, cases.map(|(val, bb, _)| (val, bb))) + } + fn invoke( &mut self, llty: Self::Type, @@ -159,12 +173,35 @@ pub trait BuilderMethods<'a, 'tcx>: /// must be interpreted as unsigned and can be assumed to be less than the size of the left /// operand. fn ashr(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; - fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + fn unchecked_sadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.add(lhs, rhs) + } + fn unchecked_uadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.add(lhs, rhs) + } + fn unchecked_suadd(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.unchecked_sadd(lhs, rhs) + } + fn unchecked_ssub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.sub(lhs, rhs) + } + fn unchecked_usub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.sub(lhs, rhs) + } + fn unchecked_susub(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.unchecked_ssub(lhs, rhs) + } + fn unchecked_smul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.mul(lhs, rhs) + } + fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.mul(lhs, rhs) + } + fn unchecked_sumul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + // Which to default to is a fairly arbitrary choice, + // but this is what slice layout was using before. + self.unchecked_smul(lhs, rhs) + } fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends @@ -186,13 +223,6 @@ pub trait BuilderMethods<'a, 'tcx>: ) -> (Self::Value, Self::Value); fn from_immediate(&mut self, val: Self::Value) -> Self::Value; - fn to_immediate(&mut self, val: Self::Value, layout: TyAndLayout<'_>) -> Self::Value { - if let BackendRepr::Scalar(scalar) = layout.backend_repr { - self.to_immediate_scalar(val, scalar) - } else { - val - } - } fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value; fn alloca(&mut self, size: Size, align: Align) -> Self::Value; @@ -243,6 +273,19 @@ pub trait BuilderMethods<'a, 'tcx>: self.assume(cmp); } + /// Emits an `assume` that the `val` of pointer type is non-null. + /// + /// You may want to check the optimization level before bothering calling this. + fn assume_nonnull(&mut self, val: Self::Value) { + // Arguably in LLVM it'd be better to emit an assume operand bundle instead + // <https://llvm.org/docs/LangRef.html#assume-operand-bundles> + // but this works fine for all backends. + + let null = self.const_null(self.type_ptr()); + let is_null = self.icmp(IntPredicate::IntNE, val, null); + self.assume(is_null); + } + fn range_metadata(&mut self, load: Self::Value, range: WrappingRange); fn nonnull_metadata(&mut self, load: Self::Value); @@ -282,6 +325,14 @@ pub trait BuilderMethods<'a, 'tcx>: ptr: Self::Value, indices: &[Self::Value], ) -> Self::Value; + fn inbounds_nuw_gep( + &mut self, + ty: Self::Type, + ptr: Self::Value, + indices: &[Self::Value], + ) -> Self::Value { + self.inbounds_gep(ty, ptr, indices) + } fn ptradd(&mut self, ptr: Self::Value, offset: Self::Value) -> Self::Value { self.gep(self.cx().type_i8(), ptr, &[offset]) } @@ -290,6 +341,17 @@ pub trait BuilderMethods<'a, 'tcx>: } fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; + /// Produces the same value as [`Self::trunc`] (and defaults to that), + /// but is UB unless the *zero*-extending the result can reproduce `val`. + fn unchecked_utrunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value { + self.trunc(val, dest_ty) + } + /// Produces the same value as [`Self::trunc`] (and defaults to that), + /// but is UB unless the *sign*-extending the result can reproduce `val`. + fn unchecked_strunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value { + self.trunc(val, dest_ty) + } + fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; |