diff options
| author | Jakub Beránek <berykubik@gmail.com> | 2025-07-08 16:18:56 +0200 |
|---|---|---|
| committer | Jakub Beránek <berykubik@gmail.com> | 2025-07-08 16:18:56 +0200 |
| commit | bdead41b65ed10d6c947b20ddb724f1ae59e2b48 (patch) | |
| tree | 05a5849ce18ed88207f14d23da57a06dfc1e8126 /compiler/rustc_codegen_ssa/src | |
| parent | 3c9297c845c513a2609dd915ca9346664fe82663 (diff) | |
| parent | 040e2f8b9ff2d76fbe2146d6003e297ed4532088 (diff) | |
| download | rust-bdead41b65ed10d6c947b20ddb724f1ae59e2b48.tar.gz rust-bdead41b65ed10d6c947b20ddb724f1ae59e2b48.zip | |
Merge ref '040e2f8b9ff2' from rust-lang/rust
Pull recent changes from https://github.com/rust-lang/rust via Josh. Upstream ref: 040e2f8b9ff2d76fbe2146d6003e297ed4532088 Filtered ref: cf4d00a666607e356e410a820ae07eeba28a9b53 This merge was created using https://github.com/rust-lang/josh-sync.
Diffstat (limited to 'compiler/rustc_codegen_ssa/src')
18 files changed, 516 insertions, 566 deletions
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 4a2425967e4..343cb0eeca9 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -2767,7 +2767,7 @@ fn add_upstream_rust_crates( if sess.target.is_like_aix { // Unlike ELF linkers, AIX doesn't feature `DT_SONAME` to override - // the dependency name when outputing a shared library. Thus, `ld` will + // the dependency name when outputting a shared library. Thus, `ld` will // use the full path to shared libraries as the dependency if passed it // by default unless `noipath` is passed. // https://www.ibm.com/docs/en/aix/7.3?topic=l-ld-command. @@ -3051,7 +3051,7 @@ fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavo // Supported architecture names can be found in the source: // https://github.com/apple-oss-distributions/ld64/blob/ld64-951.9/src/abstraction/MachOFileAbstraction.hpp#L578-L648 // - // Intentially verbose to ensure that the list always matches correctly + // Intentionally verbose to ensure that the list always matches correctly // with the list in the source above. let ld64_arch = match llvm_arch { "armv7k" => "armv7k", @@ -3118,7 +3118,7 @@ fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavo // We do not currently know the actual SDK version though, so we have a few options: // 1. Use the minimum version supported by rustc. // 2. Use the same as the deployment target. - // 3. Use an arbitary recent version. + // 3. Use an arbitrary recent version. // 4. Omit the version. // // The first option is too low / too conservative, and means that users will not get the diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index fba84dec097..1896f63bd2d 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -1794,7 +1794,10 @@ fn for_each_exported_symbols_include_dep<'tcx>( for (cnum, dep_format) in deps.iter_enumerated() { // For each dependency that we are linking to statically ... if *dep_format == Linkage::Static { - for &(symbol, info) in tcx.exported_symbols(cnum).iter() { + for &(symbol, info) in tcx.exported_non_generic_symbols(cnum).iter() { + callback(symbol, info, cnum); + } + for &(symbol, info) in tcx.exported_generic_symbols(cnum).iter() { callback(symbol, info, cnum); } } @@ -1870,8 +1873,13 @@ pub(crate) fn linked_symbols( crate_type: CrateType, ) -> Vec<(String, SymbolExportKind)> { match crate_type { - CrateType::Executable | CrateType::Cdylib | CrateType::Dylib | CrateType::Sdylib => (), - CrateType::Staticlib | CrateType::ProcMacro | CrateType::Rlib => { + CrateType::Executable + | CrateType::ProcMacro + | CrateType::Cdylib + | CrateType::Dylib + | CrateType::Sdylib => (), + CrateType::Staticlib | CrateType::Rlib => { + // These are not linked, so no need to generate symbols.o for them. return Vec::new(); } } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index d091c46d9c1..bf38c02e908 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -301,7 +301,7 @@ pub(super) fn elf_e_flags(architecture: Architecture, sess: &Session) -> u32 { "n32" if !is_32bit => e_flags |= elf::EF_MIPS_ABI2, "n64" if !is_32bit => {} "" if is_32bit => e_flags |= elf::EF_MIPS_ABI_O32, - "" => sess.dcx().fatal("LLVM ABI must be specifed for 64-bit MIPS targets"), + "" => sess.dcx().fatal("LLVM ABI must be specified for 64-bit MIPS targets"), s if is_32bit => { sess.dcx().fatal(format!("invalid LLVM ABI `{}` for 32-bit MIPS target", s)) } diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 75f7a463556..2f5eca2d6b2 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -168,7 +168,7 @@ fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> b tcx.reachable_non_generics(def_id.krate).contains_key(&def_id) } -fn exported_symbols_provider_local<'tcx>( +fn exported_non_generic_symbols_provider_local<'tcx>( tcx: TyCtxt<'tcx>, _: LocalCrate, ) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] { @@ -306,6 +306,22 @@ fn exported_symbols_provider_local<'tcx>( )); } + // Sort so we get a stable incr. comp. hash. + symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); + + tcx.arena.alloc_from_iter(symbols) +} + +fn exported_generic_symbols_provider_local<'tcx>( + tcx: TyCtxt<'tcx>, + _: LocalCrate, +) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] { + if !tcx.sess.opts.output_types.should_codegen() && !tcx.is_sdylib_interface_build() { + return &[]; + } + + let mut symbols: Vec<_> = vec![]; + if tcx.local_crate_exports_generics() { use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; use rustc_middle::ty::InstanceKind; @@ -474,7 +490,7 @@ fn upstream_monomorphizations_provider( let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn(); for &cnum in cnums.iter() { - for (exported_symbol, _) in tcx.exported_symbols(cnum).iter() { + for (exported_symbol, _) in tcx.exported_generic_symbols(cnum).iter() { let (def_id, args) = match *exported_symbol { ExportedSymbol::Generic(def_id, args) => (def_id, args), ExportedSymbol::DropGlue(ty) => { @@ -496,10 +512,7 @@ fn upstream_monomorphizations_provider( ExportedSymbol::AsyncDropGlue(def_id, ty) => (def_id, tcx.mk_args(&[ty.into()])), ExportedSymbol::NonGeneric(..) | ExportedSymbol::ThreadLocalShim(..) - | ExportedSymbol::NoDefId(..) => { - // These are no monomorphizations - continue; - } + | ExportedSymbol::NoDefId(..) => unreachable!("{exported_symbol:?}"), }; let args_map = instances.entry(def_id).or_default(); @@ -554,7 +567,8 @@ fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) pub(crate) fn provide(providers: &mut Providers) { providers.reachable_non_generics = reachable_non_generics_provider; providers.is_reachable_non_generic = is_reachable_non_generic_provider_local; - providers.exported_symbols = exported_symbols_provider_local; + providers.exported_non_generic_symbols = exported_non_generic_symbols_provider_local; + providers.exported_generic_symbols = exported_generic_symbols_provider_local; providers.upstream_monomorphizations = upstream_monomorphizations_provider; providers.is_unreachable_local_definition = is_unreachable_local_definition_provider; providers.upstream_drop_glue_for = upstream_drop_glue_for_provider; diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index c3bfe4c13cd..d2a64ec2993 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -1124,8 +1124,9 @@ fn start_executing_work<B: ExtraBackendMethods>( let copy_symbols = |cnum| { let symbols = tcx - .exported_symbols(cnum) + .exported_non_generic_symbols(cnum) .iter() + .chain(tcx.exported_generic_symbols(cnum)) .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl)) .collect(); Arc::new(symbols) @@ -1207,7 +1208,7 @@ fn start_executing_work<B: ExtraBackendMethods>( split_debuginfo: tcx.sess.split_debuginfo(), split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind, parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend, - pointer_size: tcx.data_layout.pointer_size, + pointer_size: tcx.data_layout.pointer_size(), invocation_temp: sess.invocation_temp.clone(), }; diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b06cfd1e473..18581f854b6 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -200,7 +200,7 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let vptr_entry_idx = cx.tcx().supertrait_vtable_slot((source, target)); if let Some(entry_idx) = vptr_entry_idx { - let ptr_size = bx.data_layout().pointer_size; + let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = u64::try_from(entry_idx).unwrap() * ptr_size.bytes(); load_vtable(bx, old_info, bx.type_ptr(), vtable_byte_offset, source, true) } else { @@ -262,28 +262,6 @@ pub(crate) fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } -/// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type. -pub(crate) fn cast_to_dyn_star<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - bx: &mut Bx, - src: Bx::Value, - src_ty_and_layout: TyAndLayout<'tcx>, - dst_ty: Ty<'tcx>, - old_info: Option<Bx::Value>, -) -> (Bx::Value, Bx::Value) { - debug!("cast_to_dyn_star: {:?} => {:?}", src_ty_and_layout.ty, dst_ty); - assert!( - matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)), - "destination type must be a dyn*" - ); - let src = match bx.cx().type_kind(bx.cx().backend_type(src_ty_and_layout)) { - TypeKind::Pointer => src, - TypeKind::Integer => bx.inttoptr(src, bx.type_ptr()), - // FIXME(dyn-star): We probably have to do a bitcast first, then inttoptr. - kind => bug!("unexpected TypeKind for left-hand side of `dyn*` cast: {kind:?}"), - }; - (src, unsized_info(bx, src_ty_and_layout.ty, dst_ty, old_info)) -} - /// Coerces `src`, which is a reference to a value of type `src_ty`, /// to a value of type `dst_ty`, and stores the result in `dst`. pub(crate) fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( @@ -599,8 +577,8 @@ fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(bx: &mut Bx) -> (Bx::Va // Params for UEFI let param_handle = bx.get_param(0); let param_system_table = bx.get_param(1); - let ptr_size = bx.tcx().data_layout.pointer_size; - let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let ptr_size = bx.tcx().data_layout.pointer_size(); + let ptr_align = bx.tcx().data_layout.pointer_align().abi; let arg_argc = bx.const_int(bx.cx().type_isize(), 2); let arg_argv = bx.alloca(2 * ptr_size, ptr_align); bx.store(param_handle, arg_argv, ptr_align); diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index acdda32d58a..ff454427871 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -1,10 +1,10 @@ use std::str::FromStr; -use rustc_abi::ExternAbi; +use rustc_abi::{Align, ExternAbi}; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; use rustc_ast::{LitKind, MetaItem, MetaItemInner, attr}; use rustc_attr_data_structures::{ - AttributeKind, InlineAttr, InstructionSetAttr, OptimizeAttr, ReprAttr, UsedBy, find_attr, + AttributeKind, InlineAttr, InstructionSetAttr, OptimizeAttr, UsedBy, find_attr, }; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; @@ -109,20 +109,16 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { if let hir::Attribute::Parsed(p) = attr { match p { - AttributeKind::Repr(reprs) => { - codegen_fn_attrs.alignment = reprs - .iter() - .filter_map( - |(r, _)| if let ReprAttr::ReprAlign(x) = r { Some(*x) } else { None }, - ) - .max(); - } AttributeKind::Cold(_) => codegen_fn_attrs.flags |= CodegenFnAttrFlags::COLD, AttributeKind::ExportName { name, .. } => { codegen_fn_attrs.export_name = Some(*name); } AttributeKind::Naked(_) => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NAKED, AttributeKind::Align { align, .. } => codegen_fn_attrs.alignment = Some(*align), + AttributeKind::LinkName { name, .. } => codegen_fn_attrs.link_name = Some(*name), + AttributeKind::LinkSection { name, .. } => { + codegen_fn_attrs.link_section = Some(*name) + } AttributeKind::NoMangle(attr_span) => { if tcx.opt_item_name(did.to_def_id()).is_some() { codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE; @@ -137,6 +133,49 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { }); } } + AttributeKind::TargetFeature(features, attr_span) => { + let Some(sig) = tcx.hir_node_by_def_id(did).fn_sig() else { + tcx.dcx().span_delayed_bug(*attr_span, "target_feature applied to non-fn"); + continue; + }; + let safe_target_features = + matches!(sig.header.safety, hir::HeaderSafety::SafeTargetFeatures); + codegen_fn_attrs.safe_target_features = safe_target_features; + if safe_target_features { + if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { + // The `#[target_feature]` attribute is allowed on + // WebAssembly targets on all functions. Prior to stabilizing + // the `target_feature_11` feature, `#[target_feature]` was + // only permitted on unsafe functions because on most targets + // execution of instructions that are not supported is + // considered undefined behavior. For WebAssembly which is a + // 100% safe target at execution time it's not possible to + // execute undefined instructions, and even if a future + // feature was added in some form for this it would be a + // deterministic trap. There is no undefined behavior when + // executing WebAssembly so `#[target_feature]` is allowed + // on safe functions (but again, only for WebAssembly) + // + // Note that this is also allowed if `actually_rustdoc` so + // if a target is documenting some wasm-specific code then + // it's not spuriously denied. + // + // Now that `#[target_feature]` is permitted on safe functions, + // this exception must still exist for allowing the attribute on + // `main`, `start`, and other functions that are not usually + // allowed. + } else { + check_target_feature_trait_unsafe(tcx, did, *attr_span); + } + } + from_target_feature_attr( + tcx, + did, + features, + rust_target_features, + &mut codegen_fn_attrs.target_features, + ); + } AttributeKind::TrackCaller(attr_span) => { let is_closure = tcx.is_closure_like(did.to_def_id()); @@ -186,49 +225,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL } sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL, - sym::target_feature => { - let Some(sig) = tcx.hir_node_by_def_id(did).fn_sig() else { - tcx.dcx().span_delayed_bug(attr.span(), "target_feature applied to non-fn"); - continue; - }; - let safe_target_features = - matches!(sig.header.safety, hir::HeaderSafety::SafeTargetFeatures); - codegen_fn_attrs.safe_target_features = safe_target_features; - if safe_target_features { - if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { - // The `#[target_feature]` attribute is allowed on - // WebAssembly targets on all functions. Prior to stabilizing - // the `target_feature_11` feature, `#[target_feature]` was - // only permitted on unsafe functions because on most targets - // execution of instructions that are not supported is - // considered undefined behavior. For WebAssembly which is a - // 100% safe target at execution time it's not possible to - // execute undefined instructions, and even if a future - // feature was added in some form for this it would be a - // deterministic trap. There is no undefined behavior when - // executing WebAssembly so `#[target_feature]` is allowed - // on safe functions (but again, only for WebAssembly) - // - // Note that this is also allowed if `actually_rustdoc` so - // if a target is documenting some wasm-specific code then - // it's not spuriously denied. - // - // Now that `#[target_feature]` is permitted on safe functions, - // this exception must still exist for allowing the attribute on - // `main`, `start`, and other functions that are not usually - // allowed. - } else { - check_target_feature_trait_unsafe(tcx, did, attr.span()); - } - } - from_target_feature_attr( - tcx, - did, - attr, - rust_target_features, - &mut codegen_fn_attrs.target_features, - ); - } sym::linkage => { if let Some(val) = attr.value_str() { let linkage = Some(linkage_by_name(tcx, did, val.as_str())); @@ -252,17 +248,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } } - sym::link_section => { - if let Some(val) = attr.value_str() { - if val.as_str().bytes().any(|b| b == 0) { - let msg = format!("illegal null byte in link_section value: `{val}`"); - tcx.dcx().span_err(attr.span(), msg); - } else { - codegen_fn_attrs.link_section = Some(val); - } - } - } - sym::link_name => codegen_fn_attrs.link_name = attr.value_str(), sym::link_ordinal => { link_ordinal_span = Some(attr.span()); if let ordinal @ Some(_) = check_link_ordinal(tcx, attr) { @@ -402,6 +387,9 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs.alignment = Ord::max(codegen_fn_attrs.alignment, tcx.sess.opts.unstable_opts.min_function_alignment); + // On trait methods, inherit the `#[align]` of the trait's method prototype. + codegen_fn_attrs.alignment = Ord::max(codegen_fn_attrs.alignment, tcx.inherited_align(did)); + let inline_span; (codegen_fn_attrs.inline, inline_span) = if let Some((inline_attr, span)) = find_attr!(attrs, AttributeKind::Inline(i, span) => (*i, *span)) @@ -540,10 +528,10 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { .map(|features| (features.name.as_str(), true)) .collect(), ) { - let span = tcx - .get_attrs(did, sym::target_feature) - .next() - .map_or_else(|| tcx.def_span(did), |a| a.span()); + let span = + find_attr!(tcx.get_all_attrs(did), AttributeKind::TargetFeature(_, span) => *span) + .unwrap_or_else(|| tcx.def_span(did)); + tcx.dcx() .create_err(errors::TargetFeatureDisableOrEnable { features, @@ -556,17 +544,26 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs } +/// If the provided DefId is a method in a trait impl, return the DefId of the method prototype. +fn opt_trait_item(tcx: TyCtxt<'_>, def_id: DefId) -> Option<DefId> { + let impl_item = tcx.opt_associated_item(def_id)?; + match impl_item.container { + ty::AssocItemContainer::Impl => impl_item.trait_item_def_id, + _ => None, + } +} + /// Checks if the provided DefId is a method in a trait impl for a trait which has track_caller /// applied to the method prototype. fn should_inherit_track_caller(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - if let Some(impl_item) = tcx.opt_associated_item(def_id) - && let ty::AssocItemContainer::Impl = impl_item.container - && let Some(trait_item) = impl_item.trait_item_def_id - { - return tcx.codegen_fn_attrs(trait_item).flags.intersects(CodegenFnAttrFlags::TRACK_CALLER); - } + let Some(trait_item) = opt_trait_item(tcx, def_id) else { return false }; + tcx.codegen_fn_attrs(trait_item).flags.intersects(CodegenFnAttrFlags::TRACK_CALLER) +} - false +/// If the provided DefId is a method in a trait impl, return the value of the `#[align]` +/// attribute on the method prototype (if any). +fn inherited_align<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<Align> { + tcx.codegen_fn_attrs(opt_trait_item(tcx, def_id)?).alignment } fn check_link_ordinal(tcx: TyCtxt<'_>, attr: &hir::Attribute) -> Option<u16> { @@ -734,5 +731,6 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> { } pub(crate) fn provide(providers: &mut Providers) { - *providers = Providers { codegen_fn_attrs, should_inherit_track_caller, ..*providers }; + *providers = + Providers { codegen_fn_attrs, should_inherit_track_caller, inherited_align, ..*providers }; } diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index 1950a35b364..e042fe1f819 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -457,7 +457,7 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { } else if arg.as_encoded_bytes().ends_with(b".rlib") { let rlib_path = Path::new(&arg); let dir = rlib_path.parent().unwrap(); - let filename = rlib_path.file_name().unwrap().to_owned(); + let filename = rlib_path.file_stem().unwrap().to_owned(); if let Some(ArgGroup::Rlibs(parent, rlibs)) = args.last_mut() { if parent == dir { rlibs.push(filename); @@ -471,7 +471,7 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { args.push(ArgGroup::Regular(arg)); } } - let crate_hash = regex::bytes::Regex::new(r"-[0-9a-f]+\.rlib$").unwrap(); + let crate_hash = regex::bytes::Regex::new(r"-[0-9a-f]+").unwrap(); self.command.args(args.into_iter().map(|arg_group| { match arg_group { // SAFETY: we are only matching on ASCII, not any surrogate pairs, so any replacements we do will still be valid. @@ -494,7 +494,11 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { Err(_) => false, }; let mut arg = dir.into_os_string(); - arg.push("/{"); + arg.push("/"); + let needs_braces = rlibs.len() >= 2; + if needs_braces { + arg.push("{"); + } let mut first = true; for mut rlib in rlibs { if !first { @@ -513,7 +517,10 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for LinkingFailed<'_> { } arg.push(rlib); } - arg.push("}.rlib"); + if needs_braces { + arg.push("}"); + } + arg.push(".rlib"); arg } } @@ -1292,3 +1299,14 @@ pub(crate) struct NoMangleNameless { pub span: Span, pub definition: String, } + +#[derive(Diagnostic)] +#[diag(codegen_ssa_feature_not_valid)] +pub(crate) struct FeatureNotValid<'a> { + pub feature: &'a str, + #[primary_span] + #[label] + pub span: Span, + #[help] + pub plus_hint: bool, +} diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 3a11ce6befb..2aa5c3c27ea 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -27,7 +27,7 @@ impl<'a, 'tcx> VirtualIndex { debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); let llty = bx.fn_ptr_backend_type(fn_abi); - let ptr_size = bx.data_layout().pointer_size; + let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, nonnull) @@ -63,7 +63,7 @@ impl<'a, 'tcx> VirtualIndex { debug!("get_int({:?}, {:?})", llvtable, self); let llty = bx.type_isize(); - let ptr_size = bx.data_layout().pointer_size; + let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, false) @@ -115,7 +115,7 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); let vtable_const = cx.const_data_from_alloc(vtable_allocation); - let align = cx.data_layout().pointer_align.abi; + let align = cx.data_layout().pointer_align().abi; let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); cx.apply_vcall_visibility_metadata(ty, trait_ref, vtable); @@ -133,7 +133,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ty: Ty<'tcx>, nonnull: bool, ) -> Bx::Value { - let ptr_align = bx.data_layout().pointer_align.abi; + let ptr_align = bx.data_layout().pointer_align().abi; if bx.cx().sess().opts.unstable_opts.virtual_function_elimination && bx.cx().sess().lto() == Lto::Fat diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 3df97429e09..bde63fd501a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -628,50 +628,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { virtual_drop, ) } - ty::Dynamic(_, _, ty::DynStar) => { - // IN THIS ARM, WE HAVE: - // ty = *mut (dyn* Trait) - // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) - // - // args = [ * ] - // | - // v - // ( Data, Vtable ) - // | - // v - // /-------\ - // | ... | - // \-------/ - // - // - // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING - // - // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) - // vtable = (*args[0]).1 // loads the vtable out - // (data, vtable) // an equivalent Rust `*mut dyn Trait` - // - // SO THEN WE CAN USE THE ABOVE CODE. - let virtual_drop = Instance { - def: ty::InstanceKind::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function - args: drop_fn.args, - }; - debug!("ty = {:?}", ty); - debug!("drop_fn = {:?}", drop_fn); - debug!("args = {:?}", args); - let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); - let meta_ptr = place.project_field(bx, 1); - let meta = bx.load_operand(meta_ptr); - // Truncate vtable off of args list - args = &args[..1]; - debug!("args' = {:?}", args); - ( - true, - meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) - .get_optional_fn(bx, meta.immediate(), ty, fn_abi), - fn_abi, - virtual_drop, - ) - } _ => ( false, bx.get_fn_addr(drop_fn), @@ -776,6 +732,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // `#[track_caller]` adds an implicit argument. (LangItem::PanicNullPointerDereference, vec![location]) } + AssertKind::InvalidEnumConstruction(source) => { + let source = self.codegen_operand(bx, source).immediate(); + // It's `fn panic_invalid_enum_construction(source: u128)`, + // `#[track_caller]` adds an implicit argument. + (LangItem::PanicInvalidEnumConstruction, vec![source, location]) + } _ => { // It's `pub fn panic_...()` and `#[track_caller]` adds an implicit argument. (msg.panic_function(), vec![location]) @@ -1102,33 +1064,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { llargs.push(data_ptr); continue; } - Immediate(_) => { - // See comment above explaining why we peel these newtypes - while !op.layout.ty.is_raw_ptr() && !op.layout.ty.is_ref() { - let (idx, _) = op.layout.non_1zst_field(bx).expect( - "not exactly one non-1-ZST field in a `DispatchFromDyn` type", - ); - op = op.extract_field(self, bx, idx.as_usize()); - } - - // Make sure that we've actually unwrapped the rcvr down - // to a pointer or ref to `dyn* Trait`. - if !op.layout.ty.builtin_deref(true).unwrap().is_dyn_star() { - span_bug!(fn_span, "can't codegen a virtual call on {:#?}", op); - } - let place = op.deref(bx.cx()); - let data_place = place.project_field(bx, 0); - let meta_place = place.project_field(bx, 1); - let meta = bx.load_operand(meta_place); - llfn = Some(meth::VirtualIndex::from_index(idx).get_fn( - bx, - meta.immediate(), - op.layout.ty, - fn_abi, - )); - llargs.push(data_place.val.llval); - continue; - } _ => { span_bug!(fn_span, "can't codegen a virtual call on {:#?}", op); } diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 27fcab8ed2d..fc95f62b4a4 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,7 +1,7 @@ use rustc_abi::WrappingRange; -use rustc_middle::bug; use rustc_middle::mir::SourceInfo; use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::sym; @@ -98,6 +98,27 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { discr.to_atomic_ordering() }; + if args.is_empty() { + match name { + sym::abort + | sym::unreachable + | sym::cold_path + | sym::breakpoint + | sym::assert_zero_valid + | sym::assert_mem_uninitialized_valid + | sym::assert_inhabited + | sym::ub_checks + | sym::contract_checks + | sym::atomic_fence + | sym::atomic_singlethreadfence + | sym::caller_location => {} + _ => { + span_bug!(span, "nullary intrinsic {name} must either be in a const block or explicitly opted out because it is inherently a runtime intrinsic +"); + } + } + } + let llval = match name { sym::abort => { bx.abort(); @@ -150,10 +171,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } value } - sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => { - let value = bx.tcx().const_eval_instance(bx.typing_env(), instance, span).unwrap(); - OperandRef::from_const(bx, value, result.layout.ty).immediate_or_packed_pair(bx) - } sym::arith_offset => { let ty = fn_args.type_at(0); let layout = bx.layout_of(ty); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 66c4af4c935..10b44a1faf0 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -354,15 +354,15 @@ fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let destination_block = target.unwrap(); - bb.statements.push(mir::Statement { - source_info: bb.terminator().source_info, - kind: mir::StatementKind::Assign(Box::new(( + bb.statements.push(mir::Statement::new( + bb.terminator().source_info, + mir::StatementKind::Assign(Box::new(( *destination, mir::Rvalue::Use(mir::Operand::Copy( arg_place.project_deeper(&[mir::ProjectionElem::Deref], tcx), )), ))), - }); + )); bb.terminator_mut().kind = mir::TerminatorKind::Goto { target: destination_block }; } diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index 9da4b8cc8fd..beaf8950978 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -326,7 +326,7 @@ fn prefix_and_suffix<'tcx>( fn wasm_functype<'tcx>(tcx: TyCtxt<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> String { let mut signature = String::with_capacity(64); - let ptr_type = match tcx.data_layout.pointer_size.bits() { + let ptr_type = match tcx.data_layout.pointer_size().bits() { 32 => "i32", 64 => "i64", other => bug!("wasm pointer size cannot be {other} bits"), diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 99957c67708..2896dfd5463 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -1,5 +1,6 @@ use std::fmt; +use itertools::Either; use rustc_abi as abi; use rustc_abi::{ Align, BackendRepr, FIRST_VARIANT, FieldIdx, Primitive, Size, TagEncoding, VariantIdx, Variants, @@ -13,7 +14,7 @@ use rustc_session::config::OptLevel; use tracing::{debug, instrument}; use super::place::{PlaceRef, PlaceValue}; -use super::rvalue::transmute_immediate; +use super::rvalue::transmute_scalar; use super::{FunctionCx, LocalRef}; use crate::common::IntPredicate; use crate::traits::*; @@ -346,14 +347,16 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let val = if field.is_zst() { OperandValue::ZeroSized + } else if let BackendRepr::SimdVector { .. } = self.layout.backend_repr { + // codegen_transmute_operand doesn't support SIMD, but since the previous + // check handled ZSTs, the only possible field access into something SIMD + // is to the `non_1zst_field` that's the same SIMD. (Other things, even + // just padding, would change the wrapper's representation type.) + assert_eq!(field.size, self.layout.size); + self.val } else if field.size == self.layout.size { assert_eq!(offset.bytes(), 0); - fx.codegen_transmute_operand(bx, *self, field).unwrap_or_else(|| { - bug!( - "Expected `codegen_transmute_operand` to handle equal-size \ - field {i:?} projection from {self:?} to {field:?}" - ) - }) + fx.codegen_transmute_operand(bx, *self, field) } else { let (in_scalar, imm) = match (self.val, self.layout.backend_repr) { // Extract a scalar component from a pair. @@ -479,17 +482,8 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)), }; - // Layout ensures that we only get here for cases where the discriminant + // `layout_sanity_check` ensures that we only get here for cases where the discriminant // value and the variant index match, since that's all `Niche` can encode. - // But for emphasis and debugging, let's double-check one anyway. - debug_assert_eq!( - self.layout - .ty - .discriminant_for_variant(bx.tcx(), untagged_variant) - .unwrap() - .val, - u128::from(untagged_variant.as_u32()), - ); let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); @@ -574,23 +568,30 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { /// Creates an incomplete operand containing the [`abi::Scalar`]s expected based /// on the `layout` passed. This is for use with [`OperandRef::insert_field`] - /// later to set the necessary immediate(s). + /// later to set the necessary immediate(s), one-by-one converting all the `Right` to `Left`. /// /// Returns `None` for `layout`s which cannot be built this way. pub(crate) fn builder( layout: TyAndLayout<'tcx>, - ) -> Option<OperandRef<'tcx, Result<V, abi::Scalar>>> { + ) -> Option<OperandRef<'tcx, Either<V, abi::Scalar>>> { + // Uninhabited types are weird, because for example `Result<!, !>` + // shows up as `FieldsShape::Primitive` and we need to be able to write + // a field into `(u32, !)`. We'll do that in an `alloca` instead. + if layout.uninhabited { + return None; + } + let val = match layout.backend_repr { BackendRepr::Memory { .. } if layout.is_zst() => OperandValue::ZeroSized, - BackendRepr::Scalar(s) => OperandValue::Immediate(Err(s)), - BackendRepr::ScalarPair(a, b) => OperandValue::Pair(Err(a), Err(b)), + BackendRepr::Scalar(s) => OperandValue::Immediate(Either::Right(s)), + BackendRepr::ScalarPair(a, b) => OperandValue::Pair(Either::Right(a), Either::Right(b)), BackendRepr::Memory { .. } | BackendRepr::SimdVector { .. } => return None, }; Some(OperandRef { val, layout }) } } -impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> { +impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Either<V, abi::Scalar>> { pub(crate) fn insert_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>( &mut self, bx: &mut Bx, @@ -614,31 +615,29 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> { (field_layout.is_zst(), field_offset == Size::ZERO) }; - let mut update = |tgt: &mut Result<V, abi::Scalar>, src, from_scalar| { - let from_bty = bx.cx().type_from_scalar(from_scalar); - let to_scalar = tgt.unwrap_err(); - let to_bty = bx.cx().type_from_scalar(to_scalar); - let imm = transmute_immediate(bx, src, from_scalar, from_bty, to_scalar, to_bty); - *tgt = Ok(imm); + let mut update = |tgt: &mut Either<V, abi::Scalar>, src, from_scalar| { + let to_scalar = tgt.unwrap_right(); + let imm = transmute_scalar(bx, src, from_scalar, to_scalar); + *tgt = Either::Left(imm); }; match (operand.val, operand.layout.backend_repr) { (OperandValue::ZeroSized, _) if expect_zst => {} (OperandValue::Immediate(v), BackendRepr::Scalar(from_scalar)) => match &mut self.val { - OperandValue::Immediate(val @ Err(_)) if is_zero_offset => { + OperandValue::Immediate(val @ Either::Right(_)) if is_zero_offset => { update(val, v, from_scalar); } - OperandValue::Pair(fst @ Err(_), _) if is_zero_offset => { + OperandValue::Pair(fst @ Either::Right(_), _) if is_zero_offset => { update(fst, v, from_scalar); } - OperandValue::Pair(_, snd @ Err(_)) if !is_zero_offset => { + OperandValue::Pair(_, snd @ Either::Right(_)) if !is_zero_offset => { update(snd, v, from_scalar); } _ => bug!("Tried to insert {operand:?} into {v:?}.{f:?} of {self:?}"), }, (OperandValue::Pair(a, b), BackendRepr::ScalarPair(from_sa, from_sb)) => { match &mut self.val { - OperandValue::Pair(fst @ Err(_), snd @ Err(_)) => { + OperandValue::Pair(fst @ Either::Right(_), snd @ Either::Right(_)) => { update(fst, a, from_sa); update(snd, b, from_sb); } @@ -649,17 +648,47 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, Result<V, abi::Scalar>> { } } + /// Insert the immediate value `imm` for field `f` in the *type itself*, + /// rather than into one of the variants. + /// + /// Most things want [`OperandRef::insert_field`] instead, but this one is + /// necessary for writing things like enum tags that aren't in any variant. + pub(super) fn insert_imm(&mut self, f: FieldIdx, imm: V) { + let field_offset = self.layout.fields.offset(f.as_usize()); + let is_zero_offset = field_offset == Size::ZERO; + match &mut self.val { + OperandValue::Immediate(val @ Either::Right(_)) if is_zero_offset => { + *val = Either::Left(imm); + } + OperandValue::Pair(fst @ Either::Right(_), _) if is_zero_offset => { + *fst = Either::Left(imm); + } + OperandValue::Pair(_, snd @ Either::Right(_)) if !is_zero_offset => { + *snd = Either::Left(imm); + } + _ => bug!("Tried to insert {imm:?} into field {f:?} of {self:?}"), + } + } + /// After having set all necessary fields, this converts the - /// `OperandValue<Result<V, _>>` (as obtained from [`OperandRef::builder`]) + /// `OperandValue<Either<V, _>>` (as obtained from [`OperandRef::builder`]) /// to the normal `OperandValue<V>`. /// /// ICEs if any required fields were not set. - pub fn build(&self) -> OperandRef<'tcx, V> { + pub fn build(&self, cx: &impl CodegenMethods<'tcx, Value = V>) -> OperandRef<'tcx, V> { let OperandRef { val, layout } = *self; - let unwrap = |r: Result<V, abi::Scalar>| match r { - Ok(v) => v, - Err(_) => bug!("OperandRef::build called while fields are missing {self:?}"), + // For something like `Option::<u32>::None`, it's expected that the + // payload scalar will not actually have been set, so this converts + // unset scalars to corresponding `undef` values so long as the scalar + // from the layout allows uninit. + let unwrap = |r: Either<V, abi::Scalar>| match r { + Either::Left(v) => v, + Either::Right(s) if s.is_uninit_valid() => { + let bty = cx.type_from_scalar(s); + cx.const_undef(bty) + } + Either::Right(_) => bug!("OperandRef::build called while fields are missing {self:?}"), }; let val = match val { diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 937063c24a6..0090be9fdef 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -1,4 +1,6 @@ -use rustc_abi::{Align, BackendRepr, FieldsShape, Size, TagEncoding, VariantIdx, Variants}; +use rustc_abi::{ + Align, BackendRepr, FieldIdx, FieldsShape, Size, TagEncoding, VariantIdx, Variants, +}; use rustc_middle::mir::PlaceTy; use rustc_middle::mir::interpret::Scalar; use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; @@ -239,53 +241,17 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { bx: &mut Bx, variant_index: VariantIdx, ) { - if self.layout.for_variant(bx.cx(), variant_index).is_uninhabited() { - // We play it safe by using a well-defined `abort`, but we could go for immediate UB - // if that turns out to be helpful. - bx.abort(); - return; - } - match self.layout.variants { - Variants::Empty => unreachable!("we already handled uninhabited types"), - Variants::Single { index } => assert_eq!(index, variant_index), - - Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => { - let ptr = self.project_field(bx, tag_field.as_usize()); - let to = - self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val; - bx.store_to_place( - bx.cx().const_uint_big(bx.cx().backend_type(ptr.layout), to), - ptr.val, - ); + match codegen_tag_value(bx.cx(), variant_index, self.layout) { + Err(UninhabitedVariantError) => { + // We play it safe by using a well-defined `abort`, but we could go for immediate UB + // if that turns out to be helpful. + bx.abort(); } - Variants::Multiple { - tag_encoding: - TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, - tag_field, - .. - } => { - if variant_index != untagged_variant { - let niche = self.project_field(bx, tag_field.as_usize()); - let niche_llty = bx.cx().immediate_backend_type(niche.layout); - let BackendRepr::Scalar(scalar) = niche.layout.backend_repr else { - bug!("expected a scalar placeref for the niche"); - }; - // We are supposed to compute `niche_value.wrapping_add(niche_start)` wrapping - // around the `niche`'s type. - // The easiest way to do that is to do wrapping arithmetic on `u128` and then - // masking off any extra bits that occur because we did the arithmetic with too many bits. - let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); - let niche_value = (niche_value as u128).wrapping_add(niche_start); - let niche_value = niche_value & niche.layout.size.unsigned_int_max(); - - let niche_llval = bx.cx().scalar_to_backend( - Scalar::from_uint(niche_value, niche.layout.size), - scalar, - niche_llty, - ); - OperandValue::Immediate(niche_llval).store(bx, niche); - } + Ok(Some((tag_field, imm))) => { + let tag_place = self.project_field(bx, tag_field.as_usize()); + OperandValue::Immediate(imm).store(bx, tag_place); } + Ok(None) => {} } } @@ -471,3 +437,73 @@ fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let offset = bx.and(neg_value, align_minus_1); bx.add(value, offset) } + +/// Calculates the value that needs to be stored to mark the discriminant. +/// +/// This might be `None` for a `struct` or a niched variant (like `Some(&3)`). +/// +/// If it's `Some`, it returns the value to store and the field in which to +/// store it. Note that this value is *not* the same as the discriminant, in +/// general, as it might be a niche value or have a different size. +/// +/// It might also be an `Err` because the variant is uninhabited. +pub(super) fn codegen_tag_value<'tcx, V>( + cx: &impl CodegenMethods<'tcx, Value = V>, + variant_index: VariantIdx, + layout: TyAndLayout<'tcx>, +) -> Result<Option<(FieldIdx, V)>, UninhabitedVariantError> { + // By checking uninhabited-ness first we don't need to worry about types + // like `(u32, !)` which are single-variant but weird. + if layout.for_variant(cx, variant_index).is_uninhabited() { + return Err(UninhabitedVariantError); + } + + Ok(match layout.variants { + Variants::Empty => unreachable!("we already handled uninhabited types"), + Variants::Single { index } => { + assert_eq!(index, variant_index); + None + } + + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => { + let discr = layout.ty.discriminant_for_variant(cx.tcx(), variant_index); + let to = discr.unwrap().val; + let tag_layout = layout.field(cx, tag_field.as_usize()); + let tag_llty = cx.immediate_backend_type(tag_layout); + let imm = cx.const_uint_big(tag_llty, to); + Some((tag_field, imm)) + } + Variants::Multiple { + tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, + tag_field, + .. + } => { + if variant_index != untagged_variant { + let niche_layout = layout.field(cx, tag_field.as_usize()); + let niche_llty = cx.immediate_backend_type(niche_layout); + let BackendRepr::Scalar(scalar) = niche_layout.backend_repr else { + bug!("expected a scalar placeref for the niche"); + }; + // We are supposed to compute `niche_value.wrapping_add(niche_start)` wrapping + // around the `niche`'s type. + // The easiest way to do that is to do wrapping arithmetic on `u128` and then + // masking off any extra bits that occur because we did the arithmetic with too many bits. + let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); + let niche_value = (niche_value as u128).wrapping_add(niche_start); + let niche_value = niche_value & niche_layout.size.unsigned_int_max(); + + let niche_llval = cx.scalar_to_backend( + Scalar::from_uint(niche_value, niche_layout.size), + scalar, + niche_llty, + ); + Some((tag_field, niche_llval)) + } else { + None + } + } + }) +} + +#[derive(Debug)] +pub(super) struct UninhabitedVariantError; diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index db5ac6a514f..4af4e1e6a61 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -1,18 +1,16 @@ -use std::assert_matches::assert_matches; - use rustc_abi::{self as abi, FIRST_VARIANT}; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; -use rustc_middle::{bug, mir, span_bug}; +use rustc_middle::{bug, mir}; use rustc_session::config::OptLevel; use rustc_span::{DUMMY_SP, Span}; use tracing::{debug, instrument}; use super::operand::{OperandRef, OperandValue}; -use super::place::PlaceRef; +use super::place::{PlaceRef, codegen_tag_value}; use super::{FunctionCx, LocalRef}; -use crate::common::IntPredicate; +use crate::common::{IntPredicate, TypeKind}; use crate::traits::*; use crate::{MemFlags, base}; @@ -190,6 +188,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + /// Transmutes the `src` value to the destination type by writing it to `dst`. + /// + /// See also [`Self::codegen_transmute_operand`] for cases that can be done + /// without needing a pre-allocated place for the destination. fn codegen_transmute( &mut self, bx: &mut Bx, @@ -200,37 +202,36 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { assert!(src.layout.is_sized()); assert!(dst.layout.is_sized()); - if let Some(val) = self.codegen_transmute_operand(bx, src, dst.layout) { - val.store(bx, dst); - return; - } - - match src.val { - OperandValue::Ref(..) | OperandValue::ZeroSized => { - span_bug!( - self.mir.span, - "Operand path should have handled transmute \ - from {src:?} to place {dst:?}" - ); - } - OperandValue::Immediate(..) | OperandValue::Pair(..) => { - // When we have immediate(s), the alignment of the source is irrelevant, - // so we can store them using the destination's alignment. - src.val.store(bx, dst.val.with_type(src.layout)); - } + if src.layout.size != dst.layout.size + || src.layout.is_uninhabited() + || dst.layout.is_uninhabited() + { + // These cases are all UB to actually hit, so don't emit code for them. + // (The size mismatches are reachable via `transmute_unchecked`.) + // We can't use unreachable because that's a terminator, and we + // need something that can be in the middle of a basic block. + bx.assume(bx.cx().const_bool(false)) + } else { + // Since in this path we have a place anyway, we can store or copy to it, + // making sure we use the destination place's alignment even if the + // source would normally have a higher one. + src.val.store(bx, dst.val.with_type(src.layout)); } } - /// Attempts to transmute an `OperandValue` to another `OperandValue`. + /// Transmutes an `OperandValue` to another `OperandValue`. /// - /// Returns `None` for cases that can't work in that framework, such as for - /// `Immediate`->`Ref` that needs an `alloc` to get the location. + /// This is supported only for cases where [`Self::rvalue_creates_operand`] + /// returns `true`, and will ICE otherwise. (In particular, anything that + /// would need to `alloca` in order to return a `PlaceValue` will ICE, + /// expecting those to go via [`Self::codegen_transmute`] instead where + /// the destination place is already allocated.) pub(crate) fn codegen_transmute_operand( &mut self, bx: &mut Bx, operand: OperandRef<'tcx, Bx::Value>, cast: TyAndLayout<'tcx>, - ) -> Option<OperandValue<Bx::Value>> { + ) -> OperandValue<Bx::Value> { // Check for transmutes that are always UB. if operand.layout.size != cast.size || operand.layout.is_uninhabited() @@ -244,71 +245,34 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Because this transmute is UB, return something easy to generate, // since it's fine that later uses of the value are probably UB. - return Some(OperandValue::poison(bx, cast)); + return OperandValue::poison(bx, cast); } - let operand_kind = self.value_kind(operand.layout); - let cast_kind = self.value_kind(cast); - - match operand.val { - OperandValue::Ref(source_place_val) => { + match (operand.val, operand.layout.backend_repr, cast.backend_repr) { + _ if cast.is_zst() => OperandValue::ZeroSized, + (_, _, abi::BackendRepr::Memory { .. }) => { + bug!("Cannot `codegen_transmute_operand` to non-ZST memory-ABI output {cast:?}"); + } + (OperandValue::Ref(source_place_val), abi::BackendRepr::Memory { .. }, _) => { assert_eq!(source_place_val.llextra, None); - assert_matches!(operand_kind, OperandValueKind::Ref); // The existing alignment is part of `source_place_val`, // so that alignment will be used, not `cast`'s. - Some(bx.load_operand(source_place_val.with_type(cast)).val) - } - OperandValue::ZeroSized => { - let OperandValueKind::ZeroSized = operand_kind else { - bug!("Found {operand_kind:?} for operand {operand:?}"); - }; - if let OperandValueKind::ZeroSized = cast_kind { - Some(OperandValue::ZeroSized) - } else { - None - } - } - OperandValue::Immediate(imm) => { - let OperandValueKind::Immediate(from_scalar) = operand_kind else { - bug!("Found {operand_kind:?} for operand {operand:?}"); - }; - if let OperandValueKind::Immediate(to_scalar) = cast_kind - && from_scalar.size(self.cx) == to_scalar.size(self.cx) - { - let from_backend_ty = bx.backend_type(operand.layout); - let to_backend_ty = bx.backend_type(cast); - Some(OperandValue::Immediate(transmute_immediate( - bx, - imm, - from_scalar, - from_backend_ty, - to_scalar, - to_backend_ty, - ))) - } else { - None - } - } - OperandValue::Pair(imm_a, imm_b) => { - let OperandValueKind::Pair(in_a, in_b) = operand_kind else { - bug!("Found {operand_kind:?} for operand {operand:?}"); - }; - if let OperandValueKind::Pair(out_a, out_b) = cast_kind - && in_a.size(self.cx) == out_a.size(self.cx) - && in_b.size(self.cx) == out_b.size(self.cx) - { - let in_a_ibty = bx.scalar_pair_element_backend_type(operand.layout, 0, false); - let in_b_ibty = bx.scalar_pair_element_backend_type(operand.layout, 1, false); - let out_a_ibty = bx.scalar_pair_element_backend_type(cast, 0, false); - let out_b_ibty = bx.scalar_pair_element_backend_type(cast, 1, false); - Some(OperandValue::Pair( - transmute_immediate(bx, imm_a, in_a, in_a_ibty, out_a, out_a_ibty), - transmute_immediate(bx, imm_b, in_b, in_b_ibty, out_b, out_b_ibty), - )) - } else { - None - } + bx.load_operand(source_place_val.with_type(cast)).val } + ( + OperandValue::Immediate(imm), + abi::BackendRepr::Scalar(from_scalar), + abi::BackendRepr::Scalar(to_scalar), + ) => OperandValue::Immediate(transmute_scalar(bx, imm, from_scalar, to_scalar)), + ( + OperandValue::Pair(imm_a, imm_b), + abi::BackendRepr::ScalarPair(in_a, in_b), + abi::BackendRepr::ScalarPair(out_a, out_b), + ) => OperandValue::Pair( + transmute_scalar(bx, imm_a, in_a, out_a), + transmute_scalar(bx, imm_b, in_b, out_b), + ), + _ => bug!("Cannot `codegen_transmute_operand` {operand:?} to {cast:?}"), } } @@ -468,12 +432,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bug!("unexpected non-pair operand"); } } - mir::CastKind::PointerCoercion(PointerCoercion::DynStar, _) => { - let (lldata, llextra) = operand.val.pointer_parts(); - let (lldata, llextra) = - base::cast_to_dyn_star(bx, lldata, operand.layout, cast.ty, llextra); - OperandValue::Pair(lldata, llextra) - } | mir::CastKind::IntToInt | mir::CastKind::FloatToInt | mir::CastKind::FloatToFloat @@ -485,9 +443,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // path as the other integer-to-X casts. | mir::CastKind::PointerWithExposedProvenance => { let imm = operand.immediate(); - let operand_kind = self.value_kind(operand.layout); - let OperandValueKind::Immediate(from_scalar) = operand_kind else { - bug!("Found {operand_kind:?} for operand {operand:?}"); + let abi::BackendRepr::Scalar(from_scalar) = operand.layout.backend_repr else { + bug!("Found non-scalar for operand {operand:?}"); }; let from_backend_ty = bx.cx().immediate_backend_type(operand.layout); @@ -497,9 +454,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let val = OperandValue::Immediate(bx.cx().const_poison(to_backend_ty)); return OperandRef { val, layout: cast }; } - let cast_kind = self.value_kind(cast); - let OperandValueKind::Immediate(to_scalar) = cast_kind else { - bug!("Found {cast_kind:?} for operand {cast:?}"); + let abi::BackendRepr::Scalar(to_scalar) = cast.layout.backend_repr else { + bug!("Found non-scalar for cast {cast:?}"); }; self.cast_immediate(bx, imm, from_scalar, from_backend_ty, to_scalar, to_backend_ty) @@ -509,9 +465,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }) } mir::CastKind::Transmute => { - self.codegen_transmute_operand(bx, operand, cast).unwrap_or_else(|| { - bug!("Unsupported transmute-as-operand of {operand:?} to {cast:?}"); - }) + self.codegen_transmute_operand(bx, operand, cast) } }; OperandRef { val, layout: cast } @@ -700,7 +654,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand), mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"), - mir::Rvalue::Aggregate(_, ref fields) => { + mir::Rvalue::Aggregate(ref kind, ref fields) => { + let (variant_index, active_field_index) = match **kind { + mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { + (variant_index, active_field_index) + } + _ => (FIRST_VARIANT, None), + }; + let ty = rvalue.ty(self.mir, self.cx.tcx()); let ty = self.monomorphize(ty); let layout = self.cx.layout_of(ty); @@ -712,10 +673,27 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; for (field_idx, field) in fields.iter_enumerated() { let op = self.codegen_operand(bx, field); - builder.insert_field(bx, FIRST_VARIANT, field_idx, op); + let fi = active_field_index.unwrap_or(field_idx); + builder.insert_field(bx, variant_index, fi, op); } - builder.build() + let tag_result = codegen_tag_value(self.cx, variant_index, layout); + match tag_result { + Err(super::place::UninhabitedVariantError) => { + // Like codegen_set_discr we use a sound abort, but could + // potentially `unreachable` or just return the poison for + // more optimizability, if that turns out to be helpful. + bx.abort(); + let val = OperandValue::poison(bx, layout); + OperandRef { val, layout } + } + Ok(maybe_tag_value) => { + if let Some((tag_field, tag_imm)) = maybe_tag_value { + builder.insert_imm(tag_field, tag_imm); + } + builder.build(bx.cx()) + } + } } mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { let operand = self.codegen_operand(bx, operand); @@ -993,37 +971,46 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::Pair(val, of) } + /// Returns `true` if the `rvalue` can be computed into an [`OperandRef`], + /// rather than needing a full `PlaceRef` for the assignment destination. + /// + /// This is used by the [`super::analyze`] code to decide which MIR locals + /// can stay as SSA values (as opposed to generating `alloca` slots for them). + /// As such, some paths here return `true` even where the specific rvalue + /// will not actually take the operand path because the result type is such + /// that it always gets an `alloca`, but where it's not worth re-checking the + /// layout in this code when the right thing will happen anyway. pub(crate) fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool { match *rvalue { mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => { let operand_ty = operand.ty(self.mir, self.cx.tcx()); let cast_layout = self.cx.layout_of(self.monomorphize(cast_ty)); let operand_layout = self.cx.layout_of(self.monomorphize(operand_ty)); + match (operand_layout.backend_repr, cast_layout.backend_repr) { + // When the output will be in memory anyway, just use its place + // (instead of the operand path) unless it's the trivial ZST case. + (_, abi::BackendRepr::Memory { .. }) => cast_layout.is_zst(), - match (self.value_kind(operand_layout), self.value_kind(cast_layout)) { - // Can always load from a pointer as needed - (OperandValueKind::Ref, _) => true, - - // ZST-to-ZST is the easiest thing ever - (OperandValueKind::ZeroSized, OperandValueKind::ZeroSized) => true, - - // But if only one of them is a ZST the sizes can't match - (OperandValueKind::ZeroSized, _) | (_, OperandValueKind::ZeroSized) => false, - - // Need to generate an `alloc` to get a pointer from an immediate - (OperandValueKind::Immediate(..) | OperandValueKind::Pair(..), OperandValueKind::Ref) => false, + // Otherwise (for a non-memory output) if the input is memory + // then we can just read the value from the place. + (abi::BackendRepr::Memory { .. }, _) => true, // When we have scalar immediates, we can only convert things // where the sizes match, to avoid endianness questions. - (OperandValueKind::Immediate(a), OperandValueKind::Immediate(b)) => + (abi::BackendRepr::Scalar(a), abi::BackendRepr::Scalar(b)) => a.size(self.cx) == b.size(self.cx), - (OperandValueKind::Pair(a0, a1), OperandValueKind::Pair(b0, b1)) => + (abi::BackendRepr::ScalarPair(a0, a1), abi::BackendRepr::ScalarPair(b0, b1)) => a0.size(self.cx) == b0.size(self.cx) && a1.size(self.cx) == b1.size(self.cx), - // Send mixings between scalars and pairs through the memory route - // FIXME: Maybe this could use insertvalue/extractvalue instead? - (OperandValueKind::Immediate(..), OperandValueKind::Pair(..)) | - (OperandValueKind::Pair(..), OperandValueKind::Immediate(..)) => false, + // Mixing Scalars and ScalarPairs can get quite complicated when + // padding and undef get involved, so leave that to the memory path. + (abi::BackendRepr::Scalar(_), abi::BackendRepr::ScalarPair(_, _)) | + (abi::BackendRepr::ScalarPair(_, _), abi::BackendRepr::Scalar(_)) => false, + + // SIMD vectors aren't worth the trouble of dealing with complex + // cases like from vectors of f32 to vectors of pointers or + // from fat pointers to vectors of u16. (See #143194 #110021 ...) + (abi::BackendRepr::SimdVector { .. }, _) | (_, abi::BackendRepr::SimdVector { .. }) => false, } } mir::Rvalue::Ref(..) | @@ -1043,82 +1030,38 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Arrays are always aggregates, so it's not worth checking anything here. // (If it's really `[(); N]` or `[T; 0]` and we use the place path, fine.) mir::Rvalue::Repeat(..) => false, - mir::Rvalue::Aggregate(ref kind, _) => { - let allowed_kind = match **kind { - // This always produces a `ty::RawPtr`, so will be Immediate or Pair - mir::AggregateKind::RawPtr(..) => true, - mir::AggregateKind::Array(..) => false, - mir::AggregateKind::Tuple => true, - mir::AggregateKind::Adt(def_id, ..) => { - let adt_def = self.cx.tcx().adt_def(def_id); - adt_def.is_struct() && !adt_def.repr().simd() - } - mir::AggregateKind::Closure(..) => true, - // FIXME: Can we do this for simple coroutines too? - mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false, - }; - allowed_kind && { + mir::Rvalue::Aggregate(..) => { let ty = rvalue.ty(self.mir, self.cx.tcx()); let ty = self.monomorphize(ty); let layout = self.cx.spanned_layout_of(ty, span); OperandRef::<Bx::Value>::builder(layout).is_some() } } - } // (*) this is only true if the type is suitable } - - /// Gets which variant of [`OperandValue`] is expected for a particular type. - fn value_kind(&self, layout: TyAndLayout<'tcx>) -> OperandValueKind { - if layout.is_zst() { - OperandValueKind::ZeroSized - } else if self.cx.is_backend_immediate(layout) { - assert!(!self.cx.is_backend_scalar_pair(layout)); - OperandValueKind::Immediate(match layout.backend_repr { - abi::BackendRepr::Scalar(s) => s, - abi::BackendRepr::SimdVector { element, .. } => element, - x => span_bug!(self.mir.span, "Couldn't translate {x:?} as backend immediate"), - }) - } else if self.cx.is_backend_scalar_pair(layout) { - let abi::BackendRepr::ScalarPair(s1, s2) = layout.backend_repr else { - span_bug!( - self.mir.span, - "Couldn't translate {:?} as backend scalar pair", - layout.backend_repr, - ); - }; - OperandValueKind::Pair(s1, s2) - } else { - OperandValueKind::Ref - } - } } -/// The variants of this match [`OperandValue`], giving details about the -/// backend values that will be held in that other type. -#[derive(Debug, Copy, Clone)] -enum OperandValueKind { - Ref, - Immediate(abi::Scalar), - Pair(abi::Scalar, abi::Scalar), - ZeroSized, -} - -/// Transmutes one of the immediates from an [`OperandValue::Immediate`] -/// or an [`OperandValue::Pair`] to an immediate of the target type. +/// Transmutes a single scalar value `imm` from `from_scalar` to `to_scalar`. +/// +/// This is expected to be in *immediate* form, as seen in [`OperandValue::Immediate`] +/// or [`OperandValue::Pair`] (so `i1` for bools, not `i8`, for example). /// -/// `to_backend_ty` must be the *non*-immediate backend type (so it will be -/// `i8`, not `i1`, for `bool`-like types.) -pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( +/// ICEs if the passed-in `imm` is not a value of the expected type for +/// `from_scalar`, such as if it's a vector or a pair. +pub(super) fn transmute_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, mut imm: Bx::Value, from_scalar: abi::Scalar, - from_backend_ty: Bx::Type, to_scalar: abi::Scalar, - to_backend_ty: Bx::Type, ) -> Bx::Value { assert_eq!(from_scalar.size(bx.cx()), to_scalar.size(bx.cx())); + let imm_ty = bx.cx().val_ty(imm); + assert_ne!( + bx.cx().type_kind(imm_ty), + TypeKind::Vector, + "Vector type {imm_ty:?} not allowed in transmute_scalar {from_scalar:?} -> {to_scalar:?}" + ); // While optimizations will remove no-op transmutes, they might still be // there in debug or things that aren't no-op in MIR because they change @@ -1130,6 +1073,10 @@ pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( use abi::Primitive::*; imm = bx.from_immediate(imm); + let from_backend_ty = bx.cx().type_from_scalar(from_scalar); + debug_assert_eq!(bx.cx().val_ty(imm), from_backend_ty); + let to_backend_ty = bx.cx().type_from_scalar(to_scalar); + // If we have a scalar, we must already know its range. Either // // 1) It's a parameter with `range` parameter metadata, @@ -1142,13 +1089,7 @@ pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( assume_scalar_range(bx, imm, from_scalar, from_backend_ty); imm = match (from_scalar.primitive(), to_scalar.primitive()) { - (Int(..) | Float(_), Int(..) | Float(_)) => { - if from_backend_ty == to_backend_ty { - imm - } else { - bx.bitcast(imm, to_backend_ty) - } - } + (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty), (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty), (Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm), (Pointer(..), Int(..)) => { @@ -1166,6 +1107,8 @@ pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } }; + debug_assert_eq!(bx.cx().val_ty(imm), to_backend_ty); + // This `assume` remains important for cases like (a conceptual) // transmute::<u32, NonZeroU32>(x) == 0 // since it's never passed to something with parameter metadata (especially @@ -1192,7 +1135,7 @@ fn assume_scalar_range<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let range = scalar.valid_range(bx.cx()); bx.assume_integer_range(imm, backend_ty, range); } - abi::Primitive::Pointer(abi::AddressSpace::DATA) + abi::Primitive::Pointer(abi::AddressSpace::ZERO) if !scalar.valid_range(bx.cx()).contains(0) => { bx.assume_nonnull(imm); diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index 67ac619091b..53df99993f0 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -1,8 +1,6 @@ use rustc_attr_data_structures::InstructionSetAttr; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_data_structures::unord::{UnordMap, UnordSet}; -use rustc_errors::Applicability; -use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; use rustc_middle::middle::codegen_fn_attrs::TargetFeature; @@ -12,110 +10,85 @@ use rustc_session::Session; use rustc_session::lint::builtin::AARCH64_SOFTFLOAT_NEON; use rustc_session::parse::feature_err; use rustc_span::{Span, Symbol, sym}; -use rustc_target::target_features::{self, RUSTC_SPECIFIC_FEATURES, Stability}; +use rustc_target::target_features::{RUSTC_SPECIFIC_FEATURES, Stability}; use smallvec::SmallVec; -use crate::errors; +use crate::errors::FeatureNotValid; +use crate::{errors, target_features}; /// Compute the enabled target features from the `#[target_feature]` function attribute. /// Enabled target features are added to `target_features`. pub(crate) fn from_target_feature_attr( tcx: TyCtxt<'_>, did: LocalDefId, - attr: &hir::Attribute, + features: &[(Symbol, Span)], rust_target_features: &UnordMap<String, target_features::Stability>, target_features: &mut Vec<TargetFeature>, ) { - let Some(list) = attr.meta_item_list() else { return }; - let bad_item = |span| { - let msg = "malformed `target_feature` attribute input"; - let code = "enable = \"..\""; - tcx.dcx() - .struct_span_err(span, msg) - .with_span_suggestion(span, "must be of the form", code, Applicability::HasPlaceholders) - .emit(); - }; let rust_features = tcx.features(); let abi_feature_constraints = tcx.sess.target.abi_required_features(); - for item in list { - // Only `enable = ...` is accepted in the meta-item list. - if !item.has_name(sym::enable) { - bad_item(item.span()); - continue; - } - - // Must be of the form `enable = "..."` (a string). - let Some(value) = item.value_str() else { - bad_item(item.span()); + for &(feature, feature_span) in features { + let feature_str = feature.as_str(); + let Some(stability) = rust_target_features.get(feature_str) else { + let plus_hint = feature_str + .strip_prefix('+') + .is_some_and(|stripped| rust_target_features.contains_key(stripped)); + tcx.dcx().emit_err(FeatureNotValid { + feature: feature_str, + span: feature_span, + plus_hint, + }); continue; }; - // We allow comma separation to enable multiple features. - for feature in value.as_str().split(',') { - let Some(stability) = rust_target_features.get(feature) else { - let msg = format!("the feature named `{feature}` is not valid for this target"); - let mut err = tcx.dcx().struct_span_err(item.span(), msg); - err.span_label(item.span(), format!("`{feature}` is not valid for this target")); - if let Some(stripped) = feature.strip_prefix('+') { - let valid = rust_target_features.contains_key(stripped); - if valid { - err.help("consider removing the leading `+` in the feature name"); - } - } - err.emit(); - continue; - }; - - // Only allow target features whose feature gates have been enabled - // and which are permitted to be toggled. - if let Err(reason) = stability.toggle_allowed() { - tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr { - span: item.span(), - feature, - reason, - }); - } else if let Some(nightly_feature) = stability.requires_nightly() - && !rust_features.enabled(nightly_feature) - { - feature_err( - &tcx.sess, - nightly_feature, - item.span(), - format!("the target feature `{feature}` is currently unstable"), - ) - .emit(); - } else { - // Add this and the implied features. - let feature_sym = Symbol::intern(feature); - for &name in tcx.implied_target_features(feature_sym) { - // But ensure the ABI does not forbid enabling this. - // Here we do assume that the backend doesn't add even more implied features - // we don't know about, at least no features that would have ABI effects! - // We skip this logic in rustdoc, where we want to allow all target features of - // all targets, so we can't check their ABI compatibility and anyway we are not - // generating code so "it's fine". - if !tcx.sess.opts.actually_rustdoc { - if abi_feature_constraints.incompatible.contains(&name.as_str()) { - // For "neon" specifically, we emit an FCW instead of a hard error. - // See <https://github.com/rust-lang/rust/issues/134375>. - if tcx.sess.target.arch == "aarch64" && name.as_str() == "neon" { - tcx.emit_node_span_lint( - AARCH64_SOFTFLOAT_NEON, - tcx.local_def_id_to_hir_id(did), - item.span(), - errors::Aarch64SoftfloatNeon, - ); - } else { - tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr { - span: item.span(), - feature: name.as_str(), - reason: "this feature is incompatible with the target ABI", - }); - } + // Only allow target features whose feature gates have been enabled + // and which are permitted to be toggled. + if let Err(reason) = stability.toggle_allowed() { + tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr { + span: feature_span, + feature: feature_str, + reason, + }); + } else if let Some(nightly_feature) = stability.requires_nightly() + && !rust_features.enabled(nightly_feature) + { + feature_err( + &tcx.sess, + nightly_feature, + feature_span, + format!("the target feature `{feature}` is currently unstable"), + ) + .emit(); + } else { + // Add this and the implied features. + for &name in tcx.implied_target_features(feature) { + // But ensure the ABI does not forbid enabling this. + // Here we do assume that the backend doesn't add even more implied features + // we don't know about, at least no features that would have ABI effects! + // We skip this logic in rustdoc, where we want to allow all target features of + // all targets, so we can't check their ABI compatibility and anyway we are not + // generating code so "it's fine". + if !tcx.sess.opts.actually_rustdoc { + if abi_feature_constraints.incompatible.contains(&name.as_str()) { + // For "neon" specifically, we emit an FCW instead of a hard error. + // See <https://github.com/rust-lang/rust/issues/134375>. + if tcx.sess.target.arch == "aarch64" && name.as_str() == "neon" { + tcx.emit_node_span_lint( + AARCH64_SOFTFLOAT_NEON, + tcx.local_def_id_to_hir_id(did), + feature_span, + errors::Aarch64SoftfloatNeon, + ); + } else { + tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr { + span: feature_span, + feature: name.as_str(), + reason: "this feature is incompatible with the target ABI", + }); } } - target_features.push(TargetFeature { name, implied: name != feature_sym }) } + target_features.push(TargetFeature { name, implied: name != feature }) } } } @@ -457,7 +430,7 @@ pub(crate) fn provide(providers: &mut Providers) { // one, just keep it. } _ => { - // Overwrite stabilite. + // Overwrite stability. occupied_entry.insert(stability); } } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index d19de6f5d26..9d367748c2a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -87,7 +87,7 @@ pub trait BuilderMethods<'a, 'tcx>: // // This function is opt-in for back ends. // - // The default implementation calls `self.expect()` before emiting the branch + // The default implementation calls `self.expect()` before emitting the branch // by calling `self.cond_br()` fn cond_br_with_expect( &mut self, |