From 2bf27e09beb4cd1c5f01369e7086e36b3de04f5c Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 26 Dec 2024 18:32:22 +0100 Subject: explicitly model that certain ABIs require/forbid certain target features --- compiler/rustc_codegen_ssa/src/target_features.rs | 52 +++++++++++------------ 1 file changed, 25 insertions(+), 27 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index 7e80d014ea2..af9ff311061 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -32,7 +32,7 @@ pub(crate) fn from_target_feature_attr( .emit(); }; let rust_features = tcx.features(); - let mut added_target_features = Vec::new(); + let (_abi_enable, abi_disable) = tcx.sess.target.abi_required_features(); for item in list { // Only `enable = ...` is accepted in the meta-item list. if !item.has_name(sym::enable) { @@ -47,7 +47,7 @@ pub(crate) fn from_target_feature_attr( }; // We allow comma separation to enable multiple features. - added_target_features.extend(value.as_str().split(',').filter_map(|feature| { + for feature in value.as_str().split(',') { let Some(stability) = rust_target_features.get(feature) else { let msg = format!("the feature named `{feature}` is not valid for this target"); let mut err = tcx.dcx().struct_span_err(item.span(), msg); @@ -59,7 +59,7 @@ pub(crate) fn from_target_feature_attr( } } err.emit(); - return None; + continue; }; // Only allow target features whose feature gates have been enabled @@ -80,31 +80,25 @@ pub(crate) fn from_target_feature_attr( format!("the target feature `{feature}` is currently unstable"), ) .emit(); + } else { + // Add this and the implied features. + let feature_sym = Symbol::intern(feature); + for &name in tcx.implied_target_features(feature_sym) { + // But ensure the ABI does not forbid enabling this. + // Here we do assume that LLVM doesn't add even more implied features + // we don't know about, at least no features that would have ABI effects! + if abi_disable.contains(&name.as_str()) { + tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr { + span: item.span(), + feature: name.as_str(), + reason: "this feature is incompatible with the target ABI", + }); + } + target_features.push(TargetFeature { name, implied: name != feature_sym }) + } } - Some(Symbol::intern(feature)) - })); - } - - // Add explicit features - target_features.extend( - added_target_features.iter().copied().map(|name| TargetFeature { name, implied: false }), - ); - - // Add implied features - let mut implied_target_features = UnordSet::new(); - for feature in added_target_features.iter() { - implied_target_features.extend(tcx.implied_target_features(*feature).clone()); - } - for feature in added_target_features.iter() { - implied_target_features.remove(feature); + } } - target_features.extend( - implied_target_features - .into_sorted_stable_ord() - .iter() - .copied() - .map(|name| TargetFeature { name, implied: true }), - ) } /// Computes the set of target features used in a function for the purposes of @@ -163,9 +157,13 @@ pub(crate) fn provide(providers: &mut Providers) { .collect() } }, - implied_target_features: |tcx, feature| { + implied_target_features: |tcx, feature: Symbol| { + let feature = feature.as_str(); UnordSet::from(tcx.sess.target.implied_target_features(std::iter::once(feature))) .into_sorted_stable_ord() + .into_iter() + .map(|s| Symbol::intern(s)) + .collect() }, asm_target_features, ..*providers -- cgit 1.4.1-3-g733a5 From cfae43d6380279fb8b6995f9bf29deb490dc2ff4 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 26 Dec 2024 18:59:19 +0100 Subject: clean up target feature system; most of the toggleability is now handled by the ABI target feature check --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 2 +- compiler/rustc_codegen_ssa/src/target_features.rs | 9 +- compiler/rustc_middle/src/query/mod.rs | 2 +- compiler/rustc_target/src/target_features.rs | 774 ++++++++++------------ 4 files changed, 349 insertions(+), 438 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 0486ae84b82..d4b2260123b 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -728,7 +728,7 @@ pub(crate) fn global_llvm_features( sess.dcx().emit_warn(unknown_feature); } Some((_, stability, _)) => { - if let Err(reason) = stability.toggle_allowed(&sess.target, enable) { + if let Err(reason) = stability.toggle_allowed() { sess.dcx().emit_warn(ForbiddenCTargetFeature { feature, enabled: if enable { "enabled" } else { "disabled" }, diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index af9ff311061..61eeae630ed 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -19,7 +19,7 @@ use crate::errors; pub(crate) fn from_target_feature_attr( tcx: TyCtxt<'_>, attr: &hir::Attribute, - rust_target_features: &UnordMap, + rust_target_features: &UnordMap, target_features: &mut Vec, ) { let Some(list) = attr.meta_item_list() else { return }; @@ -64,7 +64,7 @@ pub(crate) fn from_target_feature_attr( // Only allow target features whose feature gates have been enabled // and which are permitted to be toggled. - if let Err(reason) = stability.toggle_allowed(/*enable*/ true) { + if let Err(reason) = stability.toggle_allowed() { tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr { span: item.span(), feature, @@ -141,19 +141,18 @@ pub(crate) fn provide(providers: &mut Providers) { *providers = Providers { rust_target_features: |tcx, cnum| { assert_eq!(cnum, LOCAL_CRATE); - let target = &tcx.sess.target; if tcx.sess.opts.actually_rustdoc { // rustdoc needs to be able to document functions that use all the features, so // whitelist them all rustc_target::target_features::all_rust_features() - .map(|(a, b)| (a.to_string(), b.compute_toggleability(target))) + .map(|(a, b)| (a.to_string(), b)) .collect() } else { tcx.sess .target .rust_target_features() .iter() - .map(|(a, b, _)| (a.to_string(), b.compute_toggleability(target))) + .map(|(a, b, _)| (a.to_string(), *b)) .collect() } }, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 7e7b602c560..5f6a1a06a6a 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -2355,7 +2355,7 @@ rustc_queries! { } /// Returns the Rust target features for the current target. These are not always the same as LLVM target features! - query rust_target_features(_: CrateNum) -> &'tcx UnordMap { + query rust_target_features(_: CrateNum) -> &'tcx UnordMap { arena_cache eval_always desc { "looking up Rust target features" } diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs index dd9c3921fc2..31e2cabed62 100644 --- a/compiler/rustc_target/src/target_features.rs +++ b/compiler/rustc_target/src/target_features.rs @@ -17,60 +17,34 @@ pub const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; pub const RUSTC_SPECIAL_FEATURES: &[&str] = &["backchain"]; /// Stability information for target features. -/// `Toggleability` is the type storing whether (un)stable features can be toggled: -/// this is initially a function since it can depend on `Target`, but for stable hashing -/// it needs to be something hashable to we have to make the type generic. -#[derive(Debug, Clone)] -pub enum Stability { +#[derive(Debug, Copy, Clone)] +pub enum Stability { /// This target feature is stable, it can be used in `#[target_feature]` and /// `#[cfg(target_feature)]`. - Stable { - /// When enabling/disabling the feature via `-Ctarget-feature` or `#[target_feature]`, - /// determine if that is allowed. - allow_toggle: Toggleability, - }, + Stable, /// This target feature is unstable. It is only present in `#[cfg(target_feature)]` on /// nightly and using it in `#[target_feature]` requires enabling the given nightly feature. - Unstable { + Unstable( /// This must be a *language* feature, or else rustc will ICE when reporting a missing /// feature gate! - nightly_feature: Symbol, - /// See `Stable::allow_toggle` comment above. - allow_toggle: Toggleability, - }, + Symbol, + ), /// This feature can not be set via `-Ctarget-feature` or `#[target_feature]`, it can only be /// set in the target spec. It is never set in `cfg(target_feature)`. Used in - /// particular for features that change the floating-point ABI. + /// particular for features are actually ABI configuration flags (not all targets are as nice as + /// RISC-V and have an explicit way to set the ABI separate from target features). Forbidden { reason: &'static str }, } +use Stability::*; -/// Returns `Ok` if the toggle is allowed, `Err` with an explanation of not. -/// The `bool` indicates whether the feature is being enabled (`true`) or disabled. -pub type AllowToggleUncomputed = fn(&Target, bool) -> Result<(), &'static str>; - -/// The computed result of whether a feature can be enabled/disabled on the current target. -#[derive(Debug, Clone)] -pub struct AllowToggleComputed { - enable: Result<(), &'static str>, - disable: Result<(), &'static str>, -} - -/// `Stability` where `allow_toggle` has not been computed yet. -pub type StabilityUncomputed = Stability; -/// `Stability` where `allow_toggle` has already been computed. -pub type StabilityComputed = Stability; - -impl> HashStable for Stability { +impl HashStable for Stability { #[inline] fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { std::mem::discriminant(self).hash_stable(hcx, hasher); match self { - Stability::Stable { allow_toggle } => { - allow_toggle.hash_stable(hcx, hasher); - } - Stability::Unstable { nightly_feature, allow_toggle } => { + Stability::Stable => {} + Stability::Unstable(nightly_feature) => { nightly_feature.hash_stable(hcx, hasher); - allow_toggle.hash_stable(hcx, hasher); } Stability::Forbidden { reason } => { reason.hash_stable(hcx, hasher); @@ -79,16 +53,7 @@ impl> HashStable for Stability HashStable for AllowToggleComputed { - #[inline] - fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { - let AllowToggleComputed { enable, disable } = self; - enable.hash_stable(hcx, hasher); - disable.hash_stable(hcx, hasher); - } -} - -impl Stability { +impl Stability { /// Returns whether the feature can be used in `cfg(target_feature)` ever. /// (It might still be nightly-only even if this returns `true`, so make sure to also check /// `requires_nightly`.) @@ -106,59 +71,23 @@ impl Stability { /// - for `cfg(target_feature)`, check `in_cfg` pub fn requires_nightly(&self) -> Option { match *self { - Stability::Unstable { nightly_feature, .. } => Some(nightly_feature), + Stability::Unstable(nightly_feature) => Some(nightly_feature), Stability::Stable { .. } => None, Stability::Forbidden { .. } => panic!("forbidden features should not reach this far"), } } -} - -impl StabilityUncomputed { - pub fn compute_toggleability(&self, target: &Target) -> StabilityComputed { - use Stability::*; - let compute = |f: AllowToggleUncomputed| AllowToggleComputed { - enable: f(target, true), - disable: f(target, false), - }; - match *self { - Stable { allow_toggle } => Stable { allow_toggle: compute(allow_toggle) }, - Unstable { nightly_feature, allow_toggle } => { - Unstable { nightly_feature, allow_toggle: compute(allow_toggle) } - } - Forbidden { reason } => Forbidden { reason }, - } - } - - pub fn toggle_allowed(&self, target: &Target, enable: bool) -> Result<(), &'static str> { - use Stability::*; - match *self { - Stable { allow_toggle } => allow_toggle(target, enable), - Unstable { allow_toggle, .. } => allow_toggle(target, enable), - Forbidden { reason } => Err(reason), - } - } -} -impl StabilityComputed { /// Returns whether the feature may be toggled via `#[target_feature]` or `-Ctarget-feature`. /// (It might still be nightly-only even if this returns `true`, so make sure to also check /// `requires_nightly`.) - pub fn toggle_allowed(&self, enable: bool) -> Result<(), &'static str> { - let allow_toggle = match self { - Stability::Stable { allow_toggle } => allow_toggle, - Stability::Unstable { allow_toggle, .. } => allow_toggle, - Stability::Forbidden { reason } => return Err(reason), - }; - if enable { allow_toggle.enable } else { allow_toggle.disable } + pub fn toggle_allowed(&self) -> Result<(), &'static str> { + match self { + Stability::Forbidden { reason } => Err(reason), + _ => Ok(()), + } } } -// Constructors for the list below, defaulting to "always allow toggle". -const STABLE: StabilityUncomputed = Stability::Stable { allow_toggle: |_target, _enable| Ok(()) }; -const fn unstable(nightly_feature: Symbol) -> StabilityUncomputed { - Stability::Unstable { nightly_feature, allow_toggle: |_target, _enable| Ok(()) } -} - // Here we list target features that rustc "understands": they can be used in `#[target_feature]` // and `#[cfg(target_feature)]`. They also do not trigger any warnings when used with // `-Ctarget-feature`. @@ -204,182 +133,182 @@ const fn unstable(nightly_feature: Symbol) -> StabilityUncomputed { // Both of these are also applied transitively. type ImpliedFeatures = &'static [&'static str]; -const ARM_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const ARM_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("aclass", unstable(sym::arm_target_feature), &[]), - ("aes", unstable(sym::arm_target_feature), &["neon"]), - ("crc", unstable(sym::arm_target_feature), &[]), - ("d32", unstable(sym::arm_target_feature), &[]), - ("dotprod", unstable(sym::arm_target_feature), &["neon"]), - ("dsp", unstable(sym::arm_target_feature), &[]), - ("fp-armv8", unstable(sym::arm_target_feature), &["vfp4"]), - ("fpregs", unstable(sym::arm_target_feature), &[]), - ("i8mm", unstable(sym::arm_target_feature), &["neon"]), - ("mclass", unstable(sym::arm_target_feature), &[]), - ("neon", unstable(sym::arm_target_feature), &["vfp3"]), - ("rclass", unstable(sym::arm_target_feature), &[]), - ("sha2", unstable(sym::arm_target_feature), &["neon"]), + ("aclass", Unstable(sym::arm_target_feature), &[]), + ("aes", Unstable(sym::arm_target_feature), &["neon"]), + ("crc", Unstable(sym::arm_target_feature), &[]), + ("d32", Unstable(sym::arm_target_feature), &[]), + ("dotprod", Unstable(sym::arm_target_feature), &["neon"]), + ("dsp", Unstable(sym::arm_target_feature), &[]), + ("fp-armv8", Unstable(sym::arm_target_feature), &["vfp4"]), + ("fpregs", Unstable(sym::arm_target_feature), &[]), + ("i8mm", Unstable(sym::arm_target_feature), &["neon"]), + ("mclass", Unstable(sym::arm_target_feature), &[]), + ("neon", Unstable(sym::arm_target_feature), &["vfp3"]), + ("rclass", Unstable(sym::arm_target_feature), &[]), + ("sha2", Unstable(sym::arm_target_feature), &["neon"]), ("soft-float", Stability::Forbidden { reason: "unsound because it changes float ABI" }, &[]), // This is needed for inline assembly, but shouldn't be stabilized as-is // since it should be enabled per-function using #[instruction_set], not // #[target_feature]. - ("thumb-mode", unstable(sym::arm_target_feature), &[]), - ("thumb2", unstable(sym::arm_target_feature), &[]), - ("trustzone", unstable(sym::arm_target_feature), &[]), - ("v5te", unstable(sym::arm_target_feature), &[]), - ("v6", unstable(sym::arm_target_feature), &["v5te"]), - ("v6k", unstable(sym::arm_target_feature), &["v6"]), - ("v6t2", unstable(sym::arm_target_feature), &["v6k", "thumb2"]), - ("v7", unstable(sym::arm_target_feature), &["v6t2"]), - ("v8", unstable(sym::arm_target_feature), &["v7"]), - ("vfp2", unstable(sym::arm_target_feature), &[]), - ("vfp3", unstable(sym::arm_target_feature), &["vfp2", "d32"]), - ("vfp4", unstable(sym::arm_target_feature), &["vfp3"]), - ("virtualization", unstable(sym::arm_target_feature), &[]), + ("thumb-mode", Unstable(sym::arm_target_feature), &[]), + ("thumb2", Unstable(sym::arm_target_feature), &[]), + ("trustzone", Unstable(sym::arm_target_feature), &[]), + ("v5te", Unstable(sym::arm_target_feature), &[]), + ("v6", Unstable(sym::arm_target_feature), &["v5te"]), + ("v6k", Unstable(sym::arm_target_feature), &["v6"]), + ("v6t2", Unstable(sym::arm_target_feature), &["v6k", "thumb2"]), + ("v7", Unstable(sym::arm_target_feature), &["v6t2"]), + ("v8", Unstable(sym::arm_target_feature), &["v7"]), + ("vfp2", Unstable(sym::arm_target_feature), &[]), + ("vfp3", Unstable(sym::arm_target_feature), &["vfp2", "d32"]), + ("vfp4", Unstable(sym::arm_target_feature), &["vfp3"]), + ("virtualization", Unstable(sym::arm_target_feature), &[]), // tidy-alphabetical-end ]; -const AARCH64_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const AARCH64_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start // FEAT_AES & FEAT_PMULL - ("aes", STABLE, &["neon"]), + ("aes", Stable, &["neon"]), // FEAT_BF16 - ("bf16", STABLE, &[]), + ("bf16", Stable, &[]), // FEAT_BTI - ("bti", STABLE, &[]), + ("bti", Stable, &[]), // FEAT_CRC - ("crc", STABLE, &[]), + ("crc", Stable, &[]), // FEAT_CSSC - ("cssc", unstable(sym::aarch64_unstable_target_feature), &[]), + ("cssc", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_DIT - ("dit", STABLE, &[]), + ("dit", Stable, &[]), // FEAT_DotProd - ("dotprod", STABLE, &["neon"]), + ("dotprod", Stable, &["neon"]), // FEAT_DPB - ("dpb", STABLE, &[]), + ("dpb", Stable, &[]), // FEAT_DPB2 - ("dpb2", STABLE, &["dpb"]), + ("dpb2", Stable, &["dpb"]), // FEAT_ECV - ("ecv", unstable(sym::aarch64_unstable_target_feature), &[]), + ("ecv", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_F32MM - ("f32mm", STABLE, &["sve"]), + ("f32mm", Stable, &["sve"]), // FEAT_F64MM - ("f64mm", STABLE, &["sve"]), + ("f64mm", Stable, &["sve"]), // FEAT_FAMINMAX - ("faminmax", unstable(sym::aarch64_unstable_target_feature), &[]), + ("faminmax", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_FCMA - ("fcma", STABLE, &["neon"]), + ("fcma", Stable, &["neon"]), // FEAT_FHM - ("fhm", STABLE, &["fp16"]), + ("fhm", Stable, &["fp16"]), // FEAT_FLAGM - ("flagm", STABLE, &[]), + ("flagm", Stable, &[]), // FEAT_FLAGM2 - ("flagm2", unstable(sym::aarch64_unstable_target_feature), &[]), + ("flagm2", Unstable(sym::aarch64_unstable_target_feature), &[]), ("fp-armv8", Stability::Forbidden { reason: "Rust ties `fp-armv8` to `neon`" }, &[]), // FEAT_FP16 // Rust ties FP and Neon: https://github.com/rust-lang/rust/pull/91608 - ("fp16", STABLE, &["neon"]), + ("fp16", Stable, &["neon"]), // FEAT_FP8 - ("fp8", unstable(sym::aarch64_unstable_target_feature), &["faminmax", "lut", "bf16"]), + ("fp8", Unstable(sym::aarch64_unstable_target_feature), &["faminmax", "lut", "bf16"]), // FEAT_FP8DOT2 - ("fp8dot2", unstable(sym::aarch64_unstable_target_feature), &["fp8dot4"]), + ("fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["fp8dot4"]), // FEAT_FP8DOT4 - ("fp8dot4", unstable(sym::aarch64_unstable_target_feature), &["fp8fma"]), + ("fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["fp8fma"]), // FEAT_FP8FMA - ("fp8fma", unstable(sym::aarch64_unstable_target_feature), &["fp8"]), + ("fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["fp8"]), // FEAT_FRINTTS - ("frintts", STABLE, &[]), + ("frintts", Stable, &[]), // FEAT_HBC - ("hbc", unstable(sym::aarch64_unstable_target_feature), &[]), + ("hbc", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_I8MM - ("i8mm", STABLE, &[]), + ("i8mm", Stable, &[]), // FEAT_JSCVT // Rust ties FP and Neon: https://github.com/rust-lang/rust/pull/91608 - ("jsconv", STABLE, &["neon"]), + ("jsconv", Stable, &["neon"]), // FEAT_LOR - ("lor", STABLE, &[]), + ("lor", Stable, &[]), // FEAT_LSE - ("lse", STABLE, &[]), + ("lse", Stable, &[]), // FEAT_LSE128 - ("lse128", unstable(sym::aarch64_unstable_target_feature), &["lse"]), + ("lse128", Unstable(sym::aarch64_unstable_target_feature), &["lse"]), // FEAT_LSE2 - ("lse2", unstable(sym::aarch64_unstable_target_feature), &[]), + ("lse2", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_LUT - ("lut", unstable(sym::aarch64_unstable_target_feature), &[]), + ("lut", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_MOPS - ("mops", unstable(sym::aarch64_unstable_target_feature), &[]), + ("mops", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_MTE & FEAT_MTE2 - ("mte", STABLE, &[]), + ("mte", Stable, &[]), // FEAT_AdvSimd & FEAT_FP - ("neon", STABLE, &[]), + ("neon", Stable, &[]), // FEAT_PAUTH (address authentication) - ("paca", STABLE, &[]), + ("paca", Stable, &[]), // FEAT_PAUTH (generic authentication) - ("pacg", STABLE, &[]), + ("pacg", Stable, &[]), // FEAT_PAN - ("pan", STABLE, &[]), + ("pan", Stable, &[]), // FEAT_PAuth_LR - ("pauth-lr", unstable(sym::aarch64_unstable_target_feature), &[]), + ("pauth-lr", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_PMUv3 - ("pmuv3", STABLE, &[]), + ("pmuv3", Stable, &[]), // FEAT_RNG - ("rand", STABLE, &[]), + ("rand", Stable, &[]), // FEAT_RAS & FEAT_RASv1p1 - ("ras", STABLE, &[]), + ("ras", Stable, &[]), // FEAT_LRCPC - ("rcpc", STABLE, &[]), + ("rcpc", Stable, &[]), // FEAT_LRCPC2 - ("rcpc2", STABLE, &["rcpc"]), + ("rcpc2", Stable, &["rcpc"]), // FEAT_LRCPC3 - ("rcpc3", unstable(sym::aarch64_unstable_target_feature), &["rcpc2"]), + ("rcpc3", Unstable(sym::aarch64_unstable_target_feature), &["rcpc2"]), // FEAT_RDM - ("rdm", STABLE, &["neon"]), + ("rdm", Stable, &["neon"]), // This is needed for inline assembly, but shouldn't be stabilized as-is // since it should be enabled globally using -Zfixed-x18, not // #[target_feature]. // Note that cfg(target_feature = "reserve-x18") is currently not set for // targets that reserve x18 by default. - ("reserve-x18", unstable(sym::aarch64_unstable_target_feature), &[]), + ("reserve-x18", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_SB - ("sb", STABLE, &[]), + ("sb", Stable, &[]), // FEAT_SHA1 & FEAT_SHA256 - ("sha2", STABLE, &["neon"]), + ("sha2", Stable, &["neon"]), // FEAT_SHA512 & FEAT_SHA3 - ("sha3", STABLE, &["sha2"]), + ("sha3", Stable, &["sha2"]), // FEAT_SM3 & FEAT_SM4 - ("sm4", STABLE, &["neon"]), + ("sm4", Stable, &["neon"]), // FEAT_SME - ("sme", unstable(sym::aarch64_unstable_target_feature), &["bf16"]), + ("sme", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]), // FEAT_SME_B16B16 - ("sme-b16b16", unstable(sym::aarch64_unstable_target_feature), &["bf16", "sme2", "sve-b16b16"]), + ("sme-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16", "sme2", "sve-b16b16"]), // FEAT_SME_F16F16 - ("sme-f16f16", unstable(sym::aarch64_unstable_target_feature), &["sme2"]), + ("sme-f16f16", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]), // FEAT_SME_F64F64 - ("sme-f64f64", unstable(sym::aarch64_unstable_target_feature), &["sme"]), + ("sme-f64f64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]), // FEAT_SME_F8F16 - ("sme-f8f16", unstable(sym::aarch64_unstable_target_feature), &["sme-f8f32"]), + ("sme-f8f16", Unstable(sym::aarch64_unstable_target_feature), &["sme-f8f32"]), // FEAT_SME_F8F32 - ("sme-f8f32", unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]), + ("sme-f8f32", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]), // FEAT_SME_FA64 - ("sme-fa64", unstable(sym::aarch64_unstable_target_feature), &["sme", "sve2"]), + ("sme-fa64", Unstable(sym::aarch64_unstable_target_feature), &["sme", "sve2"]), // FEAT_SME_I16I64 - ("sme-i16i64", unstable(sym::aarch64_unstable_target_feature), &["sme"]), + ("sme-i16i64", Unstable(sym::aarch64_unstable_target_feature), &["sme"]), // FEAT_SME_LUTv2 - ("sme-lutv2", unstable(sym::aarch64_unstable_target_feature), &[]), + ("sme-lutv2", Unstable(sym::aarch64_unstable_target_feature), &[]), // FEAT_SME2 - ("sme2", unstable(sym::aarch64_unstable_target_feature), &["sme"]), + ("sme2", Unstable(sym::aarch64_unstable_target_feature), &["sme"]), // FEAT_SME2p1 - ("sme2p1", unstable(sym::aarch64_unstable_target_feature), &["sme2"]), + ("sme2p1", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]), // FEAT_SPE - ("spe", STABLE, &[]), + ("spe", Stable, &[]), // FEAT_SSBS & FEAT_SSBS2 - ("ssbs", STABLE, &[]), + ("ssbs", Stable, &[]), // FEAT_SSVE_FP8FDOT2 - ("ssve-fp8dot2", unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8dot4"]), + ("ssve-fp8dot2", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8dot4"]), // FEAT_SSVE_FP8FDOT4 - ("ssve-fp8dot4", unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8fma"]), + ("ssve-fp8dot4", Unstable(sym::aarch64_unstable_target_feature), &["ssve-fp8fma"]), // FEAT_SSVE_FP8FMA - ("ssve-fp8fma", unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]), + ("ssve-fp8fma", Unstable(sym::aarch64_unstable_target_feature), &["sme2", "fp8"]), // FEAT_SVE // It was decided that SVE requires Neon: https://github.com/rust-lang/rust/pull/91608 // @@ -387,46 +316,46 @@ const AARCH64_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ // exist together: https://developer.arm.com/documentation/102340/0100/New-features-in-SVE2 // // "For backwards compatibility, Neon and VFP are required in the latest architectures." - ("sve", STABLE, &["neon"]), + ("sve", Stable, &["neon"]), // FEAT_SVE_B16B16 (SVE or SME Z-targeting instructions) - ("sve-b16b16", unstable(sym::aarch64_unstable_target_feature), &["bf16"]), + ("sve-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]), // FEAT_SVE2 - ("sve2", STABLE, &["sve"]), + ("sve2", Stable, &["sve"]), // FEAT_SVE_AES & FEAT_SVE_PMULL128 - ("sve2-aes", STABLE, &["sve2", "aes"]), + ("sve2-aes", Stable, &["sve2", "aes"]), // FEAT_SVE2_BitPerm - ("sve2-bitperm", STABLE, &["sve2"]), + ("sve2-bitperm", Stable, &["sve2"]), // FEAT_SVE2_SHA3 - ("sve2-sha3", STABLE, &["sve2", "sha3"]), + ("sve2-sha3", Stable, &["sve2", "sha3"]), // FEAT_SVE2_SM4 - ("sve2-sm4", STABLE, &["sve2", "sm4"]), + ("sve2-sm4", Stable, &["sve2", "sm4"]), // FEAT_SVE2p1 - ("sve2p1", unstable(sym::aarch64_unstable_target_feature), &["sve2"]), + ("sve2p1", Unstable(sym::aarch64_unstable_target_feature), &["sve2"]), // FEAT_TME - ("tme", STABLE, &[]), - ("v8.1a", unstable(sym::aarch64_ver_target_feature), &[ + ("tme", Stable, &[]), + ("v8.1a", Unstable(sym::aarch64_ver_target_feature), &[ "crc", "lse", "rdm", "pan", "lor", "vh", ]), - ("v8.2a", unstable(sym::aarch64_ver_target_feature), &["v8.1a", "ras", "dpb"]), - ("v8.3a", unstable(sym::aarch64_ver_target_feature), &[ + ("v8.2a", Unstable(sym::aarch64_ver_target_feature), &["v8.1a", "ras", "dpb"]), + ("v8.3a", Unstable(sym::aarch64_ver_target_feature), &[ "v8.2a", "rcpc", "paca", "pacg", "jsconv", ]), - ("v8.4a", unstable(sym::aarch64_ver_target_feature), &["v8.3a", "dotprod", "dit", "flagm"]), - ("v8.5a", unstable(sym::aarch64_ver_target_feature), &["v8.4a", "ssbs", "sb", "dpb2", "bti"]), - ("v8.6a", unstable(sym::aarch64_ver_target_feature), &["v8.5a", "bf16", "i8mm"]), - ("v8.7a", unstable(sym::aarch64_ver_target_feature), &["v8.6a", "wfxt"]), - ("v8.8a", unstable(sym::aarch64_ver_target_feature), &["v8.7a", "hbc", "mops"]), - ("v8.9a", unstable(sym::aarch64_ver_target_feature), &["v8.8a", "cssc"]), - ("v9.1a", unstable(sym::aarch64_ver_target_feature), &["v9a", "v8.6a"]), - ("v9.2a", unstable(sym::aarch64_ver_target_feature), &["v9.1a", "v8.7a"]), - ("v9.3a", unstable(sym::aarch64_ver_target_feature), &["v9.2a", "v8.8a"]), - ("v9.4a", unstable(sym::aarch64_ver_target_feature), &["v9.3a", "v8.9a"]), - ("v9.5a", unstable(sym::aarch64_ver_target_feature), &["v9.4a"]), - ("v9a", unstable(sym::aarch64_ver_target_feature), &["v8.5a", "sve2"]), + ("v8.4a", Unstable(sym::aarch64_ver_target_feature), &["v8.3a", "dotprod", "dit", "flagm"]), + ("v8.5a", Unstable(sym::aarch64_ver_target_feature), &["v8.4a", "ssbs", "sb", "dpb2", "bti"]), + ("v8.6a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "bf16", "i8mm"]), + ("v8.7a", Unstable(sym::aarch64_ver_target_feature), &["v8.6a", "wfxt"]), + ("v8.8a", Unstable(sym::aarch64_ver_target_feature), &["v8.7a", "hbc", "mops"]), + ("v8.9a", Unstable(sym::aarch64_ver_target_feature), &["v8.8a", "cssc"]), + ("v9.1a", Unstable(sym::aarch64_ver_target_feature), &["v9a", "v8.6a"]), + ("v9.2a", Unstable(sym::aarch64_ver_target_feature), &["v9.1a", "v8.7a"]), + ("v9.3a", Unstable(sym::aarch64_ver_target_feature), &["v9.2a", "v8.8a"]), + ("v9.4a", Unstable(sym::aarch64_ver_target_feature), &["v9.3a", "v8.9a"]), + ("v9.5a", Unstable(sym::aarch64_ver_target_feature), &["v9.4a"]), + ("v9a", Unstable(sym::aarch64_ver_target_feature), &["v8.5a", "sve2"]), // FEAT_VHE - ("vh", STABLE, &[]), + ("vh", Stable, &[]), // FEAT_WFxT - ("wfxt", unstable(sym::aarch64_unstable_target_feature), &[]), + ("wfxt", Unstable(sym::aarch64_unstable_target_feature), &[]), // tidy-alphabetical-end ]; @@ -434,275 +363,260 @@ const AARCH64_TIED_FEATURES: &[&[&str]] = &[ &["paca", "pacg"], // Together these represent `pauth` in LLVM ]; -const X86_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const X86_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("adx", STABLE, &[]), - ("aes", STABLE, &["sse2"]), - ("amx-bf16", unstable(sym::x86_amx_intrinsics), &["amx-tile"]), - ("amx-complex", unstable(sym::x86_amx_intrinsics), &["amx-tile"]), - ("amx-fp16", unstable(sym::x86_amx_intrinsics), &["amx-tile"]), - ("amx-int8", unstable(sym::x86_amx_intrinsics), &["amx-tile"]), - ("amx-tile", unstable(sym::x86_amx_intrinsics), &[]), - ("avx", STABLE, &["sse4.2"]), - ("avx2", STABLE, &["avx"]), - ("avx512bf16", unstable(sym::avx512_target_feature), &["avx512bw"]), - ("avx512bitalg", unstable(sym::avx512_target_feature), &["avx512bw"]), - ("avx512bw", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512cd", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512dq", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512f", unstable(sym::avx512_target_feature), &["avx2", "fma", "f16c"]), - ("avx512fp16", unstable(sym::avx512_target_feature), &["avx512bw", "avx512vl", "avx512dq"]), - ("avx512ifma", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512vbmi", unstable(sym::avx512_target_feature), &["avx512bw"]), - ("avx512vbmi2", unstable(sym::avx512_target_feature), &["avx512bw"]), - ("avx512vl", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512vnni", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512vp2intersect", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avx512vpopcntdq", unstable(sym::avx512_target_feature), &["avx512f"]), - ("avxifma", unstable(sym::avx512_target_feature), &["avx2"]), - ("avxneconvert", unstable(sym::avx512_target_feature), &["avx2"]), - ("avxvnni", unstable(sym::avx512_target_feature), &["avx2"]), - ("avxvnniint16", unstable(sym::avx512_target_feature), &["avx2"]), - ("avxvnniint8", unstable(sym::avx512_target_feature), &["avx2"]), - ("bmi1", STABLE, &[]), - ("bmi2", STABLE, &[]), - ("cmpxchg16b", STABLE, &[]), - ("ermsb", unstable(sym::ermsb_target_feature), &[]), - ("f16c", STABLE, &["avx"]), - ("fma", STABLE, &["avx"]), - ("fxsr", STABLE, &[]), - ("gfni", unstable(sym::avx512_target_feature), &["sse2"]), - ("lahfsahf", unstable(sym::lahfsahf_target_feature), &[]), - ("lzcnt", STABLE, &[]), - ("movbe", STABLE, &[]), - ("pclmulqdq", STABLE, &["sse2"]), - ("popcnt", STABLE, &[]), - ("prfchw", unstable(sym::prfchw_target_feature), &[]), - ("rdrand", STABLE, &[]), - ("rdseed", STABLE, &[]), - ("rtm", unstable(sym::rtm_target_feature), &[]), - ("sha", STABLE, &["sse2"]), - ("sha512", unstable(sym::sha512_sm_x86), &["avx2"]), - ("sm3", unstable(sym::sha512_sm_x86), &["avx"]), - ("sm4", unstable(sym::sha512_sm_x86), &["avx2"]), + ("adx", Stable, &[]), + ("aes", Stable, &["sse2"]), + ("amx-bf16", Unstable(sym::x86_amx_intrinsics), &["amx-tile"]), + ("amx-complex", Unstable(sym::x86_amx_intrinsics), &["amx-tile"]), + ("amx-fp16", Unstable(sym::x86_amx_intrinsics), &["amx-tile"]), + ("amx-int8", Unstable(sym::x86_amx_intrinsics), &["amx-tile"]), + ("amx-tile", Unstable(sym::x86_amx_intrinsics), &[]), + ("avx", Stable, &["sse4.2"]), + ("avx2", Stable, &["avx"]), + ("avx512bf16", Unstable(sym::avx512_target_feature), &["avx512bw"]), + ("avx512bitalg", Unstable(sym::avx512_target_feature), &["avx512bw"]), + ("avx512bw", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512cd", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512dq", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512f", Unstable(sym::avx512_target_feature), &["avx2", "fma", "f16c"]), + ("avx512fp16", Unstable(sym::avx512_target_feature), &["avx512bw", "avx512vl", "avx512dq"]), + ("avx512ifma", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512vbmi", Unstable(sym::avx512_target_feature), &["avx512bw"]), + ("avx512vbmi2", Unstable(sym::avx512_target_feature), &["avx512bw"]), + ("avx512vl", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512vnni", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512vp2intersect", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avx512vpopcntdq", Unstable(sym::avx512_target_feature), &["avx512f"]), + ("avxifma", Unstable(sym::avx512_target_feature), &["avx2"]), + ("avxneconvert", Unstable(sym::avx512_target_feature), &["avx2"]), + ("avxvnni", Unstable(sym::avx512_target_feature), &["avx2"]), + ("avxvnniint16", Unstable(sym::avx512_target_feature), &["avx2"]), + ("avxvnniint8", Unstable(sym::avx512_target_feature), &["avx2"]), + ("bmi1", Stable, &[]), + ("bmi2", Stable, &[]), + ("cmpxchg16b", Stable, &[]), + ("ermsb", Unstable(sym::ermsb_target_feature), &[]), + ("f16c", Stable, &["avx"]), + ("fma", Stable, &["avx"]), + ("fxsr", Stable, &[]), + ("gfni", Unstable(sym::avx512_target_feature), &["sse2"]), + ("lahfsahf", Unstable(sym::lahfsahf_target_feature), &[]), + ("lzcnt", Stable, &[]), + ("movbe", Stable, &[]), + ("pclmulqdq", Stable, &["sse2"]), + ("popcnt", Stable, &[]), + ("prfchw", Unstable(sym::prfchw_target_feature), &[]), + ("rdrand", Stable, &[]), + ("rdseed", Stable, &[]), + ("rtm", Unstable(sym::rtm_target_feature), &[]), + ("sha", Stable, &["sse2"]), + ("sha512", Unstable(sym::sha512_sm_x86), &["avx2"]), + ("sm3", Unstable(sym::sha512_sm_x86), &["avx"]), + ("sm4", Unstable(sym::sha512_sm_x86), &["avx2"]), ("soft-float", Stability::Forbidden { reason: "unsound because it changes float ABI" }, &[]), - ("sse", STABLE, &[]), - ("sse2", STABLE, &["sse"]), - ("sse3", STABLE, &["sse2"]), - ("sse4.1", STABLE, &["ssse3"]), - ("sse4.2", STABLE, &["sse4.1"]), - ("sse4a", unstable(sym::sse4a_target_feature), &["sse3"]), - ("ssse3", STABLE, &["sse3"]), - ("tbm", unstable(sym::tbm_target_feature), &[]), - ("vaes", unstable(sym::avx512_target_feature), &["avx2", "aes"]), - ("vpclmulqdq", unstable(sym::avx512_target_feature), &["avx", "pclmulqdq"]), - ("x87", unstable(sym::x87_target_feature), &[]), - ("xop", unstable(sym::xop_target_feature), &[/*"fma4", */ "avx", "sse4a"]), - ("xsave", STABLE, &[]), - ("xsavec", STABLE, &["xsave"]), - ("xsaveopt", STABLE, &["xsave"]), - ("xsaves", STABLE, &["xsave"]), + ("sse", Stable, &[]), + ("sse2", Stable, &["sse"]), + ("sse3", Stable, &["sse2"]), + ("sse4.1", Stable, &["ssse3"]), + ("sse4.2", Stable, &["sse4.1"]), + ("sse4a", Unstable(sym::sse4a_target_feature), &["sse3"]), + ("ssse3", Stable, &["sse3"]), + ("tbm", Unstable(sym::tbm_target_feature), &[]), + ("vaes", Unstable(sym::avx512_target_feature), &["avx2", "aes"]), + ("vpclmulqdq", Unstable(sym::avx512_target_feature), &["avx", "pclmulqdq"]), + ("x87", Unstable(sym::x87_target_feature), &[]), + ("xop", Unstable(sym::xop_target_feature), &[/*"fma4", */ "avx", "sse4a"]), + ("xsave", Stable, &[]), + ("xsavec", Stable, &["xsave"]), + ("xsaveopt", Stable, &["xsave"]), + ("xsaves", Stable, &["xsave"]), // tidy-alphabetical-end ]; -const HEXAGON_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const HEXAGON_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("hvx", unstable(sym::hexagon_target_feature), &[]), - ("hvx-length128b", unstable(sym::hexagon_target_feature), &["hvx"]), + ("hvx", Unstable(sym::hexagon_target_feature), &[]), + ("hvx-length128b", Unstable(sym::hexagon_target_feature), &["hvx"]), // tidy-alphabetical-end ]; -const POWERPC_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const POWERPC_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("altivec", unstable(sym::powerpc_target_feature), &[]), - ("partword-atomics", unstable(sym::powerpc_target_feature), &[]), - ("power10-vector", unstable(sym::powerpc_target_feature), &["power9-vector"]), - ("power8-altivec", unstable(sym::powerpc_target_feature), &["altivec"]), - ("power8-crypto", unstable(sym::powerpc_target_feature), &["power8-altivec"]), - ("power8-vector", unstable(sym::powerpc_target_feature), &["vsx", "power8-altivec"]), - ("power9-altivec", unstable(sym::powerpc_target_feature), &["power8-altivec"]), - ("power9-vector", unstable(sym::powerpc_target_feature), &["power8-vector", "power9-altivec"]), - ("quadword-atomics", unstable(sym::powerpc_target_feature), &[]), - ("vsx", unstable(sym::powerpc_target_feature), &["altivec"]), + ("altivec", Unstable(sym::powerpc_target_feature), &[]), + ("partword-atomics", Unstable(sym::powerpc_target_feature), &[]), + ("power10-vector", Unstable(sym::powerpc_target_feature), &["power9-vector"]), + ("power8-altivec", Unstable(sym::powerpc_target_feature), &["altivec"]), + ("power8-crypto", Unstable(sym::powerpc_target_feature), &["power8-altivec"]), + ("power8-vector", Unstable(sym::powerpc_target_feature), &["vsx", "power8-altivec"]), + ("power9-altivec", Unstable(sym::powerpc_target_feature), &["power8-altivec"]), + ("power9-vector", Unstable(sym::powerpc_target_feature), &["power8-vector", "power9-altivec"]), + ("quadword-atomics", Unstable(sym::powerpc_target_feature), &[]), + ("vsx", Unstable(sym::powerpc_target_feature), &["altivec"]), // tidy-alphabetical-end ]; -const MIPS_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const MIPS_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("fp64", unstable(sym::mips_target_feature), &[]), - ("msa", unstable(sym::mips_target_feature), &[]), - ("virt", unstable(sym::mips_target_feature), &[]), + ("fp64", Unstable(sym::mips_target_feature), &[]), + ("msa", Unstable(sym::mips_target_feature), &[]), + ("virt", Unstable(sym::mips_target_feature), &[]), // tidy-alphabetical-end ]; -const RISCV_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const RISCV_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("a", STABLE, &["zaamo", "zalrsc"]), - ("c", STABLE, &[]), - ("d", unstable(sym::riscv_target_feature), &["f"]), - ("e", unstable(sym::riscv_target_feature), &[]), - ( - "f", - Stability::Unstable { - nightly_feature: sym::riscv_target_feature, - allow_toggle: |target, enable| { - match &*target.llvm_abiname { - "ilp32f" | "ilp32d" | "lp64f" | "lp64d" if !enable => { - // The ABI requires the `f` feature, so it cannot be disabled. - Err("feature is required by ABI") - } - _ => Ok(()), - } - }, - }, - &[], - ), + ("a", Stable, &["zaamo", "zalrsc"]), + ("c", Stable, &[]), + ("d", Unstable(sym::riscv_target_feature), &["f"]), + ("e", Unstable(sym::riscv_target_feature), &[]), + ("f", Unstable(sym::riscv_target_feature), &[]), ( "forced-atomics", Stability::Forbidden { reason: "unsound because it changes the ABI of atomic operations" }, &[], ), - ("m", STABLE, &[]), - ("relax", unstable(sym::riscv_target_feature), &[]), - ("unaligned-scalar-mem", unstable(sym::riscv_target_feature), &[]), - ("v", unstable(sym::riscv_target_feature), &[]), - ("zaamo", unstable(sym::riscv_target_feature), &[]), - ("zabha", unstable(sym::riscv_target_feature), &["zaamo"]), - ("zalrsc", unstable(sym::riscv_target_feature), &[]), - ("zba", STABLE, &[]), - ("zbb", STABLE, &[]), - ("zbc", STABLE, &[]), - ("zbkb", STABLE, &[]), - ("zbkc", STABLE, &[]), - ("zbkx", STABLE, &[]), - ("zbs", STABLE, &[]), - ("zdinx", unstable(sym::riscv_target_feature), &["zfinx"]), - ("zfh", unstable(sym::riscv_target_feature), &["zfhmin"]), - ("zfhmin", unstable(sym::riscv_target_feature), &["f"]), - ("zfinx", unstable(sym::riscv_target_feature), &[]), - ("zhinx", unstable(sym::riscv_target_feature), &["zhinxmin"]), - ("zhinxmin", unstable(sym::riscv_target_feature), &["zfinx"]), - ("zk", STABLE, &["zkn", "zkr", "zkt"]), - ("zkn", STABLE, &["zbkb", "zbkc", "zbkx", "zkne", "zknd", "zknh"]), - ("zknd", STABLE, &[]), - ("zkne", STABLE, &[]), - ("zknh", STABLE, &[]), - ("zkr", STABLE, &[]), - ("zks", STABLE, &["zbkb", "zbkc", "zbkx", "zksed", "zksh"]), - ("zksed", STABLE, &[]), - ("zksh", STABLE, &[]), - ("zkt", STABLE, &[]), + ("m", Stable, &[]), + ("relax", Unstable(sym::riscv_target_feature), &[]), + ("unaligned-scalar-mem", Unstable(sym::riscv_target_feature), &[]), + ("v", Unstable(sym::riscv_target_feature), &[]), + ("zaamo", Unstable(sym::riscv_target_feature), &[]), + ("zabha", Unstable(sym::riscv_target_feature), &["zaamo"]), + ("zalrsc", Unstable(sym::riscv_target_feature), &[]), + ("zba", Stable, &[]), + ("zbb", Stable, &[]), + ("zbc", Stable, &[]), + ("zbkb", Stable, &[]), + ("zbkc", Stable, &[]), + ("zbkx", Stable, &[]), + ("zbs", Stable, &[]), + ("zdinx", Unstable(sym::riscv_target_feature), &["zfinx"]), + ("zfh", Unstable(sym::riscv_target_feature), &["zfhmin"]), + ("zfhmin", Unstable(sym::riscv_target_feature), &["f"]), + ("zfinx", Unstable(sym::riscv_target_feature), &[]), + ("zhinx", Unstable(sym::riscv_target_feature), &["zhinxmin"]), + ("zhinxmin", Unstable(sym::riscv_target_feature), &["zfinx"]), + ("zk", Stable, &["zkn", "zkr", "zkt"]), + ("zkn", Stable, &["zbkb", "zbkc", "zbkx", "zkne", "zknd", "zknh"]), + ("zknd", Stable, &[]), + ("zkne", Stable, &[]), + ("zknh", Stable, &[]), + ("zkr", Stable, &[]), + ("zks", Stable, &["zbkb", "zbkc", "zbkx", "zksed", "zksh"]), + ("zksed", Stable, &[]), + ("zksh", Stable, &[]), + ("zkt", Stable, &[]), // tidy-alphabetical-end ]; -const WASM_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const WASM_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("atomics", unstable(sym::wasm_target_feature), &[]), - ("bulk-memory", STABLE, &[]), - ("exception-handling", unstable(sym::wasm_target_feature), &[]), - ("extended-const", STABLE, &[]), - ("multivalue", STABLE, &[]), - ("mutable-globals", STABLE, &[]), - ("nontrapping-fptoint", STABLE, &[]), - ("reference-types", STABLE, &[]), - ("relaxed-simd", STABLE, &["simd128"]), - ("sign-ext", STABLE, &[]), - ("simd128", STABLE, &[]), - ("tail-call", STABLE, &[]), - ("wide-arithmetic", unstable(sym::wasm_target_feature), &[]), + ("atomics", Unstable(sym::wasm_target_feature), &[]), + ("bulk-memory", Stable, &[]), + ("exception-handling", Unstable(sym::wasm_target_feature), &[]), + ("extended-const", Stable, &[]), + ("multivalue", Stable, &[]), + ("mutable-globals", Stable, &[]), + ("nontrapping-fptoint", Stable, &[]), + ("reference-types", Stable, &[]), + ("relaxed-simd", Stable, &["simd128"]), + ("sign-ext", Stable, &[]), + ("simd128", Stable, &[]), + ("tail-call", Stable, &[]), + ("wide-arithmetic", Unstable(sym::wasm_target_feature), &[]), // tidy-alphabetical-end ]; -const BPF_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = - &[("alu32", unstable(sym::bpf_target_feature), &[])]; +const BPF_FEATURES: &[(&str, Stability, ImpliedFeatures)] = + &[("alu32", Unstable(sym::bpf_target_feature), &[])]; -const CSKY_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const CSKY_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("10e60", unstable(sym::csky_target_feature), &["7e10"]), - ("2e3", unstable(sym::csky_target_feature), &["e2"]), - ("3e3r1", unstable(sym::csky_target_feature), &[]), - ("3e3r2", unstable(sym::csky_target_feature), &["3e3r1", "doloop"]), - ("3e3r3", unstable(sym::csky_target_feature), &["doloop"]), - ("3e7", unstable(sym::csky_target_feature), &["2e3"]), - ("7e10", unstable(sym::csky_target_feature), &["3e7"]), - ("cache", unstable(sym::csky_target_feature), &[]), - ("doloop", unstable(sym::csky_target_feature), &[]), - ("dsp1e2", unstable(sym::csky_target_feature), &[]), - ("dspe60", unstable(sym::csky_target_feature), &[]), - ("e1", unstable(sym::csky_target_feature), &["elrw"]), - ("e2", unstable(sym::csky_target_feature), &["e2"]), - ("edsp", unstable(sym::csky_target_feature), &[]), - ("elrw", unstable(sym::csky_target_feature), &[]), - ("float1e2", unstable(sym::csky_target_feature), &[]), - ("float1e3", unstable(sym::csky_target_feature), &[]), - ("float3e4", unstable(sym::csky_target_feature), &[]), - ("float7e60", unstable(sym::csky_target_feature), &[]), - ("floate1", unstable(sym::csky_target_feature), &[]), - ("hard-tp", unstable(sym::csky_target_feature), &[]), - ("high-registers", unstable(sym::csky_target_feature), &[]), - ("hwdiv", unstable(sym::csky_target_feature), &[]), - ("mp", unstable(sym::csky_target_feature), &["2e3"]), - ("mp1e2", unstable(sym::csky_target_feature), &["3e7"]), - ("nvic", unstable(sym::csky_target_feature), &[]), - ("trust", unstable(sym::csky_target_feature), &[]), - ("vdsp2e60f", unstable(sym::csky_target_feature), &[]), - ("vdspv1", unstable(sym::csky_target_feature), &[]), - ("vdspv2", unstable(sym::csky_target_feature), &[]), + ("10e60", Unstable(sym::csky_target_feature), &["7e10"]), + ("2e3", Unstable(sym::csky_target_feature), &["e2"]), + ("3e3r1", Unstable(sym::csky_target_feature), &[]), + ("3e3r2", Unstable(sym::csky_target_feature), &["3e3r1", "doloop"]), + ("3e3r3", Unstable(sym::csky_target_feature), &["doloop"]), + ("3e7", Unstable(sym::csky_target_feature), &["2e3"]), + ("7e10", Unstable(sym::csky_target_feature), &["3e7"]), + ("cache", Unstable(sym::csky_target_feature), &[]), + ("doloop", Unstable(sym::csky_target_feature), &[]), + ("dsp1e2", Unstable(sym::csky_target_feature), &[]), + ("dspe60", Unstable(sym::csky_target_feature), &[]), + ("e1", Unstable(sym::csky_target_feature), &["elrw"]), + ("e2", Unstable(sym::csky_target_feature), &["e2"]), + ("edsp", Unstable(sym::csky_target_feature), &[]), + ("elrw", Unstable(sym::csky_target_feature), &[]), + ("float1e2", Unstable(sym::csky_target_feature), &[]), + ("float1e3", Unstable(sym::csky_target_feature), &[]), + ("float3e4", Unstable(sym::csky_target_feature), &[]), + ("float7e60", Unstable(sym::csky_target_feature), &[]), + ("floate1", Unstable(sym::csky_target_feature), &[]), + ("hard-tp", Unstable(sym::csky_target_feature), &[]), + ("high-registers", Unstable(sym::csky_target_feature), &[]), + ("hwdiv", Unstable(sym::csky_target_feature), &[]), + ("mp", Unstable(sym::csky_target_feature), &["2e3"]), + ("mp1e2", Unstable(sym::csky_target_feature), &["3e7"]), + ("nvic", Unstable(sym::csky_target_feature), &[]), + ("trust", Unstable(sym::csky_target_feature), &[]), + ("vdsp2e60f", Unstable(sym::csky_target_feature), &[]), + ("vdspv1", Unstable(sym::csky_target_feature), &[]), + ("vdspv2", Unstable(sym::csky_target_feature), &[]), // tidy-alphabetical-end //fpu // tidy-alphabetical-start - ("fdivdu", unstable(sym::csky_target_feature), &[]), - ("fpuv2_df", unstable(sym::csky_target_feature), &[]), - ("fpuv2_sf", unstable(sym::csky_target_feature), &[]), - ("fpuv3_df", unstable(sym::csky_target_feature), &[]), - ("fpuv3_hf", unstable(sym::csky_target_feature), &[]), - ("fpuv3_hi", unstable(sym::csky_target_feature), &[]), - ("fpuv3_sf", unstable(sym::csky_target_feature), &[]), - ("hard-float", unstable(sym::csky_target_feature), &[]), - ("hard-float-abi", unstable(sym::csky_target_feature), &[]), + ("fdivdu", Unstable(sym::csky_target_feature), &[]), + ("fpuv2_df", Unstable(sym::csky_target_feature), &[]), + ("fpuv2_sf", Unstable(sym::csky_target_feature), &[]), + ("fpuv3_df", Unstable(sym::csky_target_feature), &[]), + ("fpuv3_hf", Unstable(sym::csky_target_feature), &[]), + ("fpuv3_hi", Unstable(sym::csky_target_feature), &[]), + ("fpuv3_sf", Unstable(sym::csky_target_feature), &[]), + ("hard-float", Unstable(sym::csky_target_feature), &[]), + ("hard-float-abi", Unstable(sym::csky_target_feature), &[]), // tidy-alphabetical-end ]; -const LOONGARCH_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const LOONGARCH_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("d", unstable(sym::loongarch_target_feature), &["f"]), - ("f", unstable(sym::loongarch_target_feature), &[]), - ("frecipe", unstable(sym::loongarch_target_feature), &[]), - ("lasx", unstable(sym::loongarch_target_feature), &["lsx"]), - ("lbt", unstable(sym::loongarch_target_feature), &[]), - ("lsx", unstable(sym::loongarch_target_feature), &["d"]), - ("lvz", unstable(sym::loongarch_target_feature), &[]), - ("relax", unstable(sym::loongarch_target_feature), &[]), - ("ual", unstable(sym::loongarch_target_feature), &[]), + ("d", Unstable(sym::loongarch_target_feature), &["f"]), + ("f", Unstable(sym::loongarch_target_feature), &[]), + ("frecipe", Unstable(sym::loongarch_target_feature), &[]), + ("lasx", Unstable(sym::loongarch_target_feature), &["lsx"]), + ("lbt", Unstable(sym::loongarch_target_feature), &[]), + ("lsx", Unstable(sym::loongarch_target_feature), &["d"]), + ("lvz", Unstable(sym::loongarch_target_feature), &[]), + ("relax", Unstable(sym::loongarch_target_feature), &[]), + ("ual", Unstable(sym::loongarch_target_feature), &[]), // tidy-alphabetical-end ]; -const IBMZ_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const IBMZ_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("backchain", unstable(sym::s390x_target_feature), &[]), - ("vector", unstable(sym::s390x_target_feature), &[]), + ("backchain", Unstable(sym::s390x_target_feature), &[]), + ("vector", Unstable(sym::s390x_target_feature), &[]), // tidy-alphabetical-end ]; -const SPARC_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const SPARC_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("leoncasa", unstable(sym::sparc_target_feature), &[]), - ("v8plus", unstable(sym::sparc_target_feature), &[]), - ("v9", unstable(sym::sparc_target_feature), &[]), + ("leoncasa", Unstable(sym::sparc_target_feature), &[]), + ("v8plus", Unstable(sym::sparc_target_feature), &[]), + ("v9", Unstable(sym::sparc_target_feature), &[]), // tidy-alphabetical-end ]; -const M68K_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ +const M68K_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // tidy-alphabetical-start - ("isa-68000", unstable(sym::m68k_target_feature), &[]), - ("isa-68010", unstable(sym::m68k_target_feature), &["isa-68000"]), - ("isa-68020", unstable(sym::m68k_target_feature), &["isa-68010"]), - ("isa-68030", unstable(sym::m68k_target_feature), &["isa-68020"]), - ("isa-68040", unstable(sym::m68k_target_feature), &["isa-68030", "isa-68882"]), - ("isa-68060", unstable(sym::m68k_target_feature), &["isa-68040"]), + ("isa-68000", Unstable(sym::m68k_target_feature), &[]), + ("isa-68010", Unstable(sym::m68k_target_feature), &["isa-68000"]), + ("isa-68020", Unstable(sym::m68k_target_feature), &["isa-68010"]), + ("isa-68030", Unstable(sym::m68k_target_feature), &["isa-68020"]), + ("isa-68040", Unstable(sym::m68k_target_feature), &["isa-68030", "isa-68882"]), + ("isa-68060", Unstable(sym::m68k_target_feature), &["isa-68040"]), // FPU - ("isa-68881", unstable(sym::m68k_target_feature), &[]), - ("isa-68882", unstable(sym::m68k_target_feature), &["isa-68881"]), + ("isa-68881", Unstable(sym::m68k_target_feature), &[]), + ("isa-68882", Unstable(sym::m68k_target_feature), &["isa-68881"]), // tidy-alphabetical-end ]; @@ -710,7 +624,7 @@ const M68K_FEATURES: &[(&str, StabilityUncomputed, ImpliedFeatures)] = &[ /// primitives may be documented. /// /// IMPORTANT: If you're adding another feature list above, make sure to add it to this iterator! -pub fn all_rust_features() -> impl Iterator { +pub fn all_rust_features() -> impl Iterator { std::iter::empty() .chain(ARM_FEATURES.iter()) .chain(AARCH64_FEATURES.iter()) @@ -756,9 +670,7 @@ const LOONGARCH_FEATURES_FOR_CORRECT_VECTOR_ABI: &'static [(u64, &'static str)] &[(128, "lsx"), (256, "lasx")]; impl Target { - pub fn rust_target_features( - &self, - ) -> &'static [(&'static str, StabilityUncomputed, ImpliedFeatures)] { + pub fn rust_target_features(&self) -> &'static [(&'static str, Stability, ImpliedFeatures)] { match &*self.arch { "arm" => ARM_FEATURES, "aarch64" | "arm64ec" => AARCH64_FEATURES, -- cgit 1.4.1-3-g733a5 From 2e64b5352be9ec8b2a7b956c2be108394b85f4b0 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 5 Jan 2025 10:34:33 +0100 Subject: add dedicated type for ABI target feature constraints --- compiler/rustc_codegen_gcc/src/gcc_util.rs | 19 ++++++++------- compiler/rustc_codegen_llvm/src/llvm_util.rs | 17 ++++++++------ compiler/rustc_codegen_ssa/src/target_features.rs | 4 ++-- compiler/rustc_target/src/spec/mod.rs | 8 +++---- compiler/rustc_target/src/target_features.rs | 28 +++++++++++++++-------- 5 files changed, 45 insertions(+), 31 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_gcc/src/gcc_util.rs b/compiler/rustc_codegen_gcc/src/gcc_util.rs index e9dfb4e4cf4..1994a2a3c53 100644 --- a/compiler/rustc_codegen_gcc/src/gcc_util.rs +++ b/compiler/rustc_codegen_gcc/src/gcc_util.rs @@ -47,9 +47,9 @@ pub(crate) fn global_gcc_features(sess: &Session, diagnostics: bool) -> Vec Vec Vec Vec Vec &'static [(&'static str, Stability, ImpliedFeatures)] { match &*self.arch { @@ -749,8 +757,8 @@ impl Target { /// All features enabled/disabled via `-Ctarget-features` and `#[target_features]` are checked /// against this. We also check any implied features, based on the information above. If LLVM /// implicitly enables more implied features than we do, that could bypass this check! - pub fn abi_required_features(&self) -> (&'static [&'static str], &'static [&'static str]) { - const NOTHING: (&'static [&'static str], &'static [&'static str]) = (&[], &[]); + pub fn abi_required_features(&self) -> FeatureConstraints { + const NOTHING: FeatureConstraints = FeatureConstraints { required: &[], incompatible: &[] }; // Some architectures don't have a clean explicit ABI designation; instead, the ABI is // defined by target features. When that is the case, those target features must be // "forbidden" in the list above to ensure that there is a consistent answer to the @@ -763,7 +771,7 @@ impl Target { NOTHING } else { // Hardfloat ABI. x87 must be enabled. - (&["x87"], &[]) + FeatureConstraints { required: &["x87"], incompatible: &[] } } } "x86_64" => { @@ -773,7 +781,7 @@ impl Target { NOTHING } else { // Hardfloat ABI. x87 and SSE2 must be enabled. - (&["x87", "sse2"], &[]) + FeatureConstraints { required: &["x87", "sse2"], incompatible: &[] } } } "arm" => { @@ -786,7 +794,7 @@ impl Target { } FloatAbi::Hard => { // Must have `fpregs` and must not have `soft-float`. - (&["fpregs"], &["soft-float"]) + FeatureConstraints { required: &["fpregs"], incompatible: &["soft-float"] } } } } @@ -803,7 +811,7 @@ impl Target { _ => { // Everything else is assumed to use a hardfloat ABI. neon and fp-armv8 must be enabled. // These are Rust feature names and we use "neon" to control both of them. - (&["neon"], &[]) + FeatureConstraints { required: &["neon"], incompatible: &[] } } } } @@ -813,15 +821,15 @@ impl Target { match &*self.llvm_abiname { "ilp32d" | "lp64d" => { // Requires d (which implies f), incompatible with e. - (&["d"], &["e"]) + FeatureConstraints { required: &["d"], incompatible: &["e"] } } "ilp32f" | "lp64f" => { // Requires f, incompatible with e. - (&["f"], &["e"]) + FeatureConstraints { required: &["f"], incompatible: &["e"] } } "ilp32" | "lp64" => { // Requires nothing, incompatible with e. - (&[], &["e"]) + FeatureConstraints { required: &[], incompatible: &["e"] } } "ilp32e" => { // ilp32e is documented to be incompatible with features that need aligned @@ -832,7 +840,7 @@ impl Target { // Note that the `e` feature is not required: the ABI treats the extra // registers as caller-save, so it is safe to use them only in some parts of // a program while the rest doesn't know they even exist. - (&[], &["d"]) + FeatureConstraints { required: &[], incompatible: &["d"] } } "lp64e" => { // As above, `e` is not required. -- cgit 1.4.1-3-g733a5 From 49c74234a79107afa7f86ca947708c3abef07674 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Thu, 17 Oct 2024 13:00:35 +0200 Subject: Add support for wasm exception handling to Emscripten target Gated behind an unstable `-Z emscripten-wasm-eh` flag --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 5 +- compiler/rustc_codegen_ssa/src/back/link.rs | 8 ++- compiler/rustc_codegen_ssa/src/base.rs | 3 +- compiler/rustc_feature/src/builtin_attrs.rs | 1 + compiler/rustc_feature/src/unstable.rs | 2 + compiler/rustc_interface/src/tests.rs | 1 + compiler/rustc_passes/src/weak_lang_items.rs | 5 +- compiler/rustc_session/src/config/cfg.rs | 5 ++ compiler/rustc_session/src/options.rs | 2 + compiler/rustc_span/src/symbol.rs | 2 + library/panic_unwind/Cargo.toml | 2 +- library/panic_unwind/src/lib.rs | 3 +- library/unwind/Cargo.toml | 2 +- library/unwind/src/lib.rs | 3 +- .../src/compiler-flags/emscripten-wasm-eh.md | 6 ++ tests/codegen/emcripten-catch-unwind.rs | 66 ---------------------- tests/codegen/emscripten-catch-unwind-js-eh.rs | 66 ++++++++++++++++++++++ tests/codegen/emscripten-catch-unwind-wasm-eh.rs | 65 +++++++++++++++++++++ .../disallowed-cli-cfgs.emscripten_wasm_eh_.stderr | 8 +++ tests/ui/cfg/disallowed-cli-cfgs.rs | 2 + .../feature-gate-cfg-emscripten-wasm-eh.rs | 4 ++ .../feature-gate-cfg-emscripten-wasm-eh.stderr | 12 ++++ 22 files changed, 197 insertions(+), 76 deletions(-) create mode 100644 src/doc/unstable-book/src/compiler-flags/emscripten-wasm-eh.md delete mode 100644 tests/codegen/emcripten-catch-unwind.rs create mode 100644 tests/codegen/emscripten-catch-unwind-js-eh.rs create mode 100644 tests/codegen/emscripten-catch-unwind-wasm-eh.rs create mode 100644 tests/ui/cfg/disallowed-cli-cfgs.emscripten_wasm_eh_.stderr create mode 100644 tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.rs create mode 100644 tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.stderr (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 628c0b1c29c..b3e1210632f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -109,7 +109,10 @@ unsafe fn configure_llvm(sess: &Session) { add("-wasm-enable-eh", false); } - if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { + if sess.target.os == "emscripten" + && !sess.opts.unstable_opts.emscripten_wasm_eh + && sess.panic_strategy() == PanicStrategy::Unwind + { add("-enable-emscripten-cxx-exceptions", false); } diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 4bc064528f3..0a941a89609 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -2466,10 +2466,12 @@ fn add_order_independent_options( } if sess.target.os == "emscripten" { - cmd.cc_arg("-s").cc_arg(if sess.panic_strategy() == PanicStrategy::Abort { - "DISABLE_EXCEPTION_CATCHING=1" + cmd.cc_arg(if sess.panic_strategy() == PanicStrategy::Abort { + "-sDISABLE_EXCEPTION_CATCHING=1" + } else if sess.opts.unstable_opts.emscripten_wasm_eh { + "-fwasm-exceptions" } else { - "DISABLE_EXCEPTION_CATCHING=0" + "-sDISABLE_EXCEPTION_CATCHING=0" }); } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 77e1fed720d..544578b29f1 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -388,7 +388,8 @@ pub(crate) fn build_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // exceptions. This means that the VM does the unwinding for // us pub fn wants_wasm_eh(sess: &Session) -> bool { - sess.target.is_like_wasm && sess.target.os != "emscripten" + sess.target.is_like_wasm + && (sess.target.os != "emscripten" || sess.opts.unstable_opts.emscripten_wasm_eh) } /// Returns `true` if this session's target will use SEH-based unwinding. diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 4112ae80980..09c8a8a06ec 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -37,6 +37,7 @@ const GATED_CFGS: &[GatedCfg] = &[ (sym::sanitizer_cfi_normalize_integers, sym::cfg_sanitizer_cfi, Features::cfg_sanitizer_cfi), // this is consistent with naming of the compiler flag it's for (sym::fmt_debug, sym::fmt_debug, Features::fmt_debug), + (sym::emscripten_wasm_eh, sym::cfg_emscripten_wasm_eh, Features::cfg_emscripten_wasm_eh), ]; /// Find a gated cfg determined by the `pred`icate which is given the cfg's name. diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index d40823d2ed6..8cc4c18c02a 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -202,6 +202,8 @@ declare_features! ( (internal, allow_internal_unstable, "1.0.0", None), /// Allows using anonymous lifetimes in argument-position impl-trait. (unstable, anonymous_lifetime_in_impl_trait, "1.63.0", None), + /// Allows access to the emscripten_wasm_eh config, used by panic_unwind and unwind + (internal, cfg_emscripten_wasm_eh, "CURRENT_RUSTC_VERSION", None), /// Allows identifying the `compiler_builtins` crate. (internal, compiler_builtins, "1.13.0", None), /// Allows writing custom MIR diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 9ad69039914..b50f3b38f2e 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -782,6 +782,7 @@ fn test_unstable_options_tracking_hash() { tracked!(dwarf_version, Some(5)); tracked!(embed_source, true); tracked!(emit_thin_lto, false); + tracked!(emscripten_wasm_eh, true); tracked!(export_executable_symbols, true); tracked!(fewer_names, Some(true)); tracked!(fixed_x18, true); diff --git a/compiler/rustc_passes/src/weak_lang_items.rs b/compiler/rustc_passes/src/weak_lang_items.rs index 020128f29c5..701f500e4f6 100644 --- a/compiler/rustc_passes/src/weak_lang_items.rs +++ b/compiler/rustc_passes/src/weak_lang_items.rs @@ -26,7 +26,10 @@ pub(crate) fn check_crate( if items.eh_personality().is_none() { items.missing.push(LangItem::EhPersonality); } - if tcx.sess.target.os == "emscripten" && items.eh_catch_typeinfo().is_none() { + if tcx.sess.target.os == "emscripten" + && items.eh_catch_typeinfo().is_none() + && !tcx.sess.opts.unstable_opts.emscripten_wasm_eh + { items.missing.push(LangItem::EhCatchTypeinfo); } diff --git a/compiler/rustc_session/src/config/cfg.rs b/compiler/rustc_session/src/config/cfg.rs index 3426858495b..40016cb76d6 100644 --- a/compiler/rustc_session/src/config/cfg.rs +++ b/compiler/rustc_session/src/config/cfg.rs @@ -143,6 +143,7 @@ pub(crate) fn disallow_cfgs(sess: &Session, user_cfgs: &Cfg) { | (sym::target_has_atomic_load_store, Some(_)) | (sym::target_thread_local, None) => disallow(cfg, "--target"), (sym::fmt_debug, None | Some(_)) => disallow(cfg, "-Z fmt-debug"), + (sym::emscripten_wasm_eh, None | Some(_)) => disallow(cfg, "-Z emscripten_wasm_eh"), _ => {} } } @@ -295,6 +296,10 @@ pub(crate) fn default_configuration(sess: &Session) -> Cfg { ins_none!(sym::ub_checks); } + // Nightly-only implementation detail for the `panic_unwind` and `unwind` crates. + if sess.is_nightly_build() && sess.opts.unstable_opts.emscripten_wasm_eh { + ins_none!(sym::emscripten_wasm_eh); + } ret } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 3772a4a08af..a344ec94a26 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1771,6 +1771,8 @@ options! { "emit a section containing stack size metadata (default: no)"), emit_thin_lto: bool = (true, parse_bool, [TRACKED], "emit the bc module with thin LTO info (default: yes)"), + emscripten_wasm_eh: bool = (false, parse_bool, [TRACKED], + "Use WebAssembly error handling for wasm32-unknown-emscripten"), enforce_type_length_limit: bool = (false, parse_bool, [TRACKED], "enforce the type length limit when monomorphizing instances in codegen"), export_executable_symbols: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 4ecc4201f89..e62107c450e 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -570,6 +570,7 @@ symbols! { cfg_autodiff_fallback, cfg_boolean_literals, cfg_doctest, + cfg_emscripten_wasm_eh, cfg_eval, cfg_fmt_debug, cfg_hide, @@ -823,6 +824,7 @@ symbols! { emit_enum_variant_arg, emit_struct, emit_struct_field, + emscripten_wasm_eh, enable, encode, end, diff --git a/library/panic_unwind/Cargo.toml b/library/panic_unwind/Cargo.toml index 252f118fecf..c2abb79192e 100644 --- a/library/panic_unwind/Cargo.toml +++ b/library/panic_unwind/Cargo.toml @@ -23,4 +23,4 @@ libc = { version = "0.2", default-features = false } [lints.rust.unexpected_cfgs] level = "warn" -check-cfg = [] +check-cfg = ['cfg(emscripten_wasm_eh)'] diff --git a/library/panic_unwind/src/lib.rs b/library/panic_unwind/src/lib.rs index 1981675f409..085c07591d2 100644 --- a/library/panic_unwind/src/lib.rs +++ b/library/panic_unwind/src/lib.rs @@ -25,13 +25,14 @@ // `real_imp` is unused with Miri, so silence warnings. #![cfg_attr(miri, allow(dead_code))] #![allow(internal_features)] +#![cfg_attr(not(bootstrap), feature(cfg_emscripten_wasm_eh))] use alloc::boxed::Box; use core::any::Any; use core::panic::PanicPayload; cfg_if::cfg_if! { - if #[cfg(target_os = "emscripten")] { + if #[cfg(all(target_os = "emscripten", not(emscripten_wasm_eh)))] { #[path = "emcc.rs"] mod imp; } else if #[cfg(target_os = "hermit")] { diff --git a/library/unwind/Cargo.toml b/library/unwind/Cargo.toml index e13c9a06c05..66e8d1a3ffe 100644 --- a/library/unwind/Cargo.toml +++ b/library/unwind/Cargo.toml @@ -37,4 +37,4 @@ system-llvm-libunwind = [] [lints.rust.unexpected_cfgs] level = "warn" -check-cfg = [] +check-cfg = ['cfg(emscripten_wasm_eh)'] diff --git a/library/unwind/src/lib.rs b/library/unwind/src/lib.rs index 40d409310ff..9b93f0b3da0 100644 --- a/library/unwind/src/lib.rs +++ b/library/unwind/src/lib.rs @@ -4,10 +4,11 @@ #![feature(staged_api)] #![cfg_attr(not(target_env = "msvc"), feature(libc))] #![cfg_attr( - all(target_family = "wasm", not(target_os = "emscripten")), + all(target_family = "wasm", any(not(target_os = "emscripten"), emscripten_wasm_eh)), feature(simd_wasm64, wasm_exception_handling_intrinsics) )] #![allow(internal_features)] +#![cfg_attr(not(bootstrap), feature(cfg_emscripten_wasm_eh))] // Force libc to be included even if unused. This is required by many platforms. #[cfg(not(all(windows, target_env = "msvc")))] diff --git a/src/doc/unstable-book/src/compiler-flags/emscripten-wasm-eh.md b/src/doc/unstable-book/src/compiler-flags/emscripten-wasm-eh.md new file mode 100644 index 00000000000..eab29a1744b --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/emscripten-wasm-eh.md @@ -0,0 +1,6 @@ +# `emscripten-wasm-eh` + +Use the WebAssembly exception handling ABI to unwind for the +`wasm32-unknown-emscripten`. If compiling with this setting, the `emcc` linker +should be invoked with `-fwasm-exceptions`. If linking with C/C++ files, the +C/C++ files should also be compiled with `-fwasm-exceptions`. diff --git a/tests/codegen/emcripten-catch-unwind.rs b/tests/codegen/emcripten-catch-unwind.rs deleted file mode 100644 index b15fb40b68f..00000000000 --- a/tests/codegen/emcripten-catch-unwind.rs +++ /dev/null @@ -1,66 +0,0 @@ -//@ compile-flags: -O --target wasm32-unknown-emscripten -//@ needs-llvm-components: webassembly - -// Emscripten has its own unique implementation of catch_unwind (in `codegen_emcc_try`), -// make sure it generates something reasonable. - -#![feature(no_core, lang_items, intrinsics, rustc_attrs)] -#![crate_type = "lib"] -#![no_std] -#![no_core] - -#[lang = "sized"] -trait Sized {} -#[lang = "freeze"] -trait Freeze {} -#[lang = "copy"] -trait Copy {} - -impl Copy for *mut T {} - -#[rustc_intrinsic] -fn size_of() -> usize { - loop {} -} - -extern "rust-intrinsic" { - fn catch_unwind( - try_fn: fn(_: *mut u8), - data: *mut u8, - catch_fn: fn(_: *mut u8, _: *mut u8), - ) -> i32; -} - -// CHECK-LABEL: @ptr_size -#[no_mangle] -pub fn ptr_size() -> usize { - // CHECK: ret [[PTR_SIZE:.*]] - size_of::<*mut u8>() -} - -// CHECK-LABEL: @test_catch_unwind -#[no_mangle] -pub unsafe fn test_catch_unwind( - try_fn: fn(_: *mut u8), - data: *mut u8, - catch_fn: fn(_: *mut u8, _: *mut u8), -) -> i32 { - // CHECK: start: - // CHECK: [[ALLOCA:%.*]] = alloca - - // CHECK: catch.i: - // CHECK: [[LANDINGPAD:%.*]] = landingpad - // CHECK: [[EXCEPTION:%.*]] = extractvalue {{.*}} [[LANDINGPAD]], 0 - // CHECK: [[SELECTOR:%.*]] = extractvalue {{.*}} [[LANDINGPAD]], 1 - - // CHECK: [[IS_RUST_EXN:%.*]] = icmp eq {{.*}}[[SELECTOR]] - // CHECK: [[IS_RUST_EXN_I8:%.*]] = zext i1 [[IS_RUST_EXN]] to i8 - - // CHECK: store ptr [[EXCEPTION]], ptr [[ALLOCA]] - // CHECK: [[IS_RUST_SLOT:%.*]] = getelementptr inbounds{{( nuw)?}} i8, ptr [[ALLOCA]], [[PTR_SIZE]] - // CHECK: store i8 [[IS_RUST_EXN_I8]], ptr [[IS_RUST_SLOT]] - - // CHECK: call void %catch_fn(ptr %data, ptr nonnull [[ALLOCA]]) - - catch_unwind(try_fn, data, catch_fn) -} diff --git a/tests/codegen/emscripten-catch-unwind-js-eh.rs b/tests/codegen/emscripten-catch-unwind-js-eh.rs new file mode 100644 index 00000000000..b15fb40b68f --- /dev/null +++ b/tests/codegen/emscripten-catch-unwind-js-eh.rs @@ -0,0 +1,66 @@ +//@ compile-flags: -O --target wasm32-unknown-emscripten +//@ needs-llvm-components: webassembly + +// Emscripten has its own unique implementation of catch_unwind (in `codegen_emcc_try`), +// make sure it generates something reasonable. + +#![feature(no_core, lang_items, intrinsics, rustc_attrs)] +#![crate_type = "lib"] +#![no_std] +#![no_core] + +#[lang = "sized"] +trait Sized {} +#[lang = "freeze"] +trait Freeze {} +#[lang = "copy"] +trait Copy {} + +impl Copy for *mut T {} + +#[rustc_intrinsic] +fn size_of() -> usize { + loop {} +} + +extern "rust-intrinsic" { + fn catch_unwind( + try_fn: fn(_: *mut u8), + data: *mut u8, + catch_fn: fn(_: *mut u8, _: *mut u8), + ) -> i32; +} + +// CHECK-LABEL: @ptr_size +#[no_mangle] +pub fn ptr_size() -> usize { + // CHECK: ret [[PTR_SIZE:.*]] + size_of::<*mut u8>() +} + +// CHECK-LABEL: @test_catch_unwind +#[no_mangle] +pub unsafe fn test_catch_unwind( + try_fn: fn(_: *mut u8), + data: *mut u8, + catch_fn: fn(_: *mut u8, _: *mut u8), +) -> i32 { + // CHECK: start: + // CHECK: [[ALLOCA:%.*]] = alloca + + // CHECK: catch.i: + // CHECK: [[LANDINGPAD:%.*]] = landingpad + // CHECK: [[EXCEPTION:%.*]] = extractvalue {{.*}} [[LANDINGPAD]], 0 + // CHECK: [[SELECTOR:%.*]] = extractvalue {{.*}} [[LANDINGPAD]], 1 + + // CHECK: [[IS_RUST_EXN:%.*]] = icmp eq {{.*}}[[SELECTOR]] + // CHECK: [[IS_RUST_EXN_I8:%.*]] = zext i1 [[IS_RUST_EXN]] to i8 + + // CHECK: store ptr [[EXCEPTION]], ptr [[ALLOCA]] + // CHECK: [[IS_RUST_SLOT:%.*]] = getelementptr inbounds{{( nuw)?}} i8, ptr [[ALLOCA]], [[PTR_SIZE]] + // CHECK: store i8 [[IS_RUST_EXN_I8]], ptr [[IS_RUST_SLOT]] + + // CHECK: call void %catch_fn(ptr %data, ptr nonnull [[ALLOCA]]) + + catch_unwind(try_fn, data, catch_fn) +} diff --git a/tests/codegen/emscripten-catch-unwind-wasm-eh.rs b/tests/codegen/emscripten-catch-unwind-wasm-eh.rs new file mode 100644 index 00000000000..72395f432d5 --- /dev/null +++ b/tests/codegen/emscripten-catch-unwind-wasm-eh.rs @@ -0,0 +1,65 @@ +//@ compile-flags: -O --target wasm32-unknown-emscripten -Z emscripten-wasm-eh +//@ needs-llvm-components: webassembly + +// Emscripten catch_unwind using wasm exceptions + +#![feature(no_core, lang_items, intrinsics, rustc_attrs)] +#![crate_type = "lib"] +#![no_std] +#![no_core] + +#[lang = "sized"] +trait Sized {} +#[lang = "freeze"] +trait Freeze {} +#[lang = "copy"] +trait Copy {} + +impl Copy for *mut T {} + +#[rustc_intrinsic] +fn size_of() -> usize { + loop {} +} + +extern "rust-intrinsic" { + fn catch_unwind( + try_fn: fn(_: *mut u8), + data: *mut u8, + catch_fn: fn(_: *mut u8, _: *mut u8), + ) -> i32; +} + +// CHECK-LABEL: @ptr_size +#[no_mangle] +pub fn ptr_size() -> usize { + // CHECK: ret [[PTR_SIZE:.*]] + size_of::<*mut u8>() +} + +// CHECK-LABEL: @test_catch_unwind +#[no_mangle] +pub unsafe fn test_catch_unwind( + try_fn: fn(_: *mut u8), + data: *mut u8, + catch_fn: fn(_: *mut u8, _: *mut u8), +) -> i32 { + // CHECK: start: + // CHECK: invoke void %try_fn(ptr %data) + // CHECK: to label %__rust_try.exit unwind label %catchswitch.i + // CHECK: catchswitch.i: ; preds = %start + // CHECK: %catchswitch1.i = catchswitch within none [label %catchpad.i] unwind to caller + + // CHECK: catchpad.i: ; preds = %catchswitch.i + // CHECK: %catchpad2.i = catchpad within %catchswitch1.i [ptr null] + // CHECK: %0 = tail call ptr @llvm.wasm.get.exception(token %catchpad2.i) + // CHECK: %1 = tail call i32 @llvm.wasm.get.ehselector(token %catchpad2.i) + // CHECK: call void %catch_fn(ptr %data, ptr %0) [ "funclet"(token %catchpad2.i) ] + // CHECK: catchret from %catchpad2.i to label %__rust_try.exit + + // CHECK: __rust_try.exit: ; preds = %start, %catchpad.i + // CHECK: %common.ret.op.i = phi i32 [ 0, %start ], [ 1, %catchpad.i ] + // CHECK: ret i32 %common.ret.op.i + + catch_unwind(try_fn, data, catch_fn) +} diff --git a/tests/ui/cfg/disallowed-cli-cfgs.emscripten_wasm_eh_.stderr b/tests/ui/cfg/disallowed-cli-cfgs.emscripten_wasm_eh_.stderr new file mode 100644 index 00000000000..8b2ee0e5c0c --- /dev/null +++ b/tests/ui/cfg/disallowed-cli-cfgs.emscripten_wasm_eh_.stderr @@ -0,0 +1,8 @@ +error: unexpected `--cfg emscripten_wasm_eh` flag + | + = note: config `emscripten_wasm_eh` is only supposed to be controlled by `-Z emscripten_wasm_eh` + = note: manually setting a built-in cfg can and does create incoherent behaviors + = note: `#[deny(explicit_builtin_cfgs_in_flags)]` on by default + +error: aborting due to 1 previous error + diff --git a/tests/ui/cfg/disallowed-cli-cfgs.rs b/tests/ui/cfg/disallowed-cli-cfgs.rs index 3c9ee87f28a..cae9c65cb45 100644 --- a/tests/ui/cfg/disallowed-cli-cfgs.rs +++ b/tests/ui/cfg/disallowed-cli-cfgs.rs @@ -7,6 +7,7 @@ //@ revisions: target_has_atomic_equal_alignment_ target_has_atomic_load_store_ //@ revisions: target_thread_local_ relocation_model_ //@ revisions: fmt_debug_ +//@ revisions: emscripten_wasm_eh_ //@ [overflow_checks_]compile-flags: --cfg overflow_checks //@ [debug_assertions_]compile-flags: --cfg debug_assertions @@ -33,5 +34,6 @@ //@ [target_thread_local_]compile-flags: --cfg target_thread_local //@ [relocation_model_]compile-flags: --cfg relocation_model="a" //@ [fmt_debug_]compile-flags: --cfg fmt_debug="shallow" +//@ [emscripten_wasm_eh_]compile-flags: --cfg emscripten_wasm_eh fn main() {} diff --git a/tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.rs b/tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.rs new file mode 100644 index 00000000000..cff98b43fe7 --- /dev/null +++ b/tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.rs @@ -0,0 +1,4 @@ +//@ compile-flags: --check-cfg=cfg(emscripten_wasm_eh) +#[cfg(not(emscripten_wasm_eh))] +//~^ `cfg(emscripten_wasm_eh)` is experimental +fn main() {} diff --git a/tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.stderr b/tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.stderr new file mode 100644 index 00000000000..67769e3c758 --- /dev/null +++ b/tests/ui/feature-gates/feature-gate-cfg-emscripten-wasm-eh.stderr @@ -0,0 +1,12 @@ +error[E0658]: `cfg(emscripten_wasm_eh)` is experimental and subject to change + --> $DIR/feature-gate-cfg-emscripten-wasm-eh.rs:2:11 + | +LL | #[cfg(not(emscripten_wasm_eh))] + | ^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(cfg_emscripten_wasm_eh)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0658`. -- cgit 1.4.1-3-g733a5 From 7ad45f1d2fc5b237a0f3ce128dc403eb6f618403 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Wed, 8 Jan 2025 15:00:25 +0000 Subject: Change repeat element check into a match --- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 33 +++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 3b62148abb7..8ef25d4e2ff 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -93,23 +93,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - if let OperandValue::Immediate(v) = cg_elem.val { - let start = dest.val.llval; - let size = bx.const_usize(dest.layout.size.bytes()); - - // Use llvm.memset.p0i8.* to initialize all zero arrays - if bx.cx().const_to_opt_u128(v, false) == Some(0) { - let fill = bx.cx().const_u8(0); - bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); - return; - } + match cg_elem.val { + OperandValue::Immediate(v) => { + let start = dest.val.llval; + let size = bx.const_usize(dest.layout.size.bytes()); + + // Use llvm.memset.p0i8.* to initialize all zero arrays + if bx.cx().const_to_opt_u128(v, false) == Some(0) { + let fill = bx.cx().const_u8(0); + bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); + return; + } - // Use llvm.memset.p0i8.* to initialize byte arrays - let v = bx.from_immediate(v); - if bx.cx().val_ty(v) == bx.cx().type_i8() { - bx.memset(start, v, size, dest.val.align, MemFlags::empty()); - return; + // Use llvm.memset.p0i8.* to initialize byte arrays + let v = bx.from_immediate(v); + if bx.cx().val_ty(v) == bx.cx().type_i8() { + bx.memset(start, v, size, dest.val.align, MemFlags::empty()); + return; + } } + _ => (), } let count = self -- cgit 1.4.1-3-g733a5 From 65ea9f3eb40e113284d0142f28e061fd90191c46 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Wed, 8 Jan 2025 16:15:50 +0000 Subject: Pull element init into a reusable closure --- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 35 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 8ef25d4e2ff..e36e0e18d9c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -93,22 +93,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - match cg_elem.val { - OperandValue::Immediate(v) => { - let start = dest.val.llval; - let size = bx.const_usize(dest.layout.size.bytes()); + let try_init_all_same = |bx: &mut Bx, v| { + let start = dest.val.llval; + let size = bx.const_usize(dest.layout.size.bytes()); + + // Use llvm.memset.p0i8.* to initialize all zero arrays + if bx.cx().const_to_opt_u128(v, false) == Some(0) { + let fill = bx.cx().const_u8(0); + bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); + return true; + } - // Use llvm.memset.p0i8.* to initialize all zero arrays - if bx.cx().const_to_opt_u128(v, false) == Some(0) { - let fill = bx.cx().const_u8(0); - bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); - return; - } + // Use llvm.memset.p0i8.* to initialize byte arrays + let v = bx.from_immediate(v); + if bx.cx().val_ty(v) == bx.cx().type_i8() { + bx.memset(start, v, size, dest.val.align, MemFlags::empty()); + return true; + } + false + }; - // Use llvm.memset.p0i8.* to initialize byte arrays - let v = bx.from_immediate(v); - if bx.cx().val_ty(v) == bx.cx().type_i8() { - bx.memset(start, v, size, dest.val.align, MemFlags::empty()); + match cg_elem.val { + OperandValue::Immediate(v) => { + if try_init_all_same(bx, v) { return; } } -- cgit 1.4.1-3-g733a5 From 65b01cb1823926fd4e2f2c94b206aad0220c0e1c Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Wed, 8 Jan 2025 16:29:32 +0000 Subject: Use llvm.memset.p0i8.* to initialize all same-bytes arrays --- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 14 +++++++++----- tests/codegen/slice-init.rs | 8 +++----- 2 files changed, 12 insertions(+), 10 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index e36e0e18d9c..31793641d75 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -97,11 +97,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let start = dest.val.llval; let size = bx.const_usize(dest.layout.size.bytes()); - // Use llvm.memset.p0i8.* to initialize all zero arrays - if bx.cx().const_to_opt_u128(v, false) == Some(0) { - let fill = bx.cx().const_u8(0); - bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); - return true; + // Use llvm.memset.p0i8.* to initialize all same byte arrays + if let Some(int) = bx.cx().const_to_opt_u128(v, false) { + let bytes = &int.to_le_bytes()[..cg_elem.layout.size.bytes_usize()]; + let first = bytes[0]; + if bytes[1..].iter().all(|&b| b == first) { + let fill = bx.cx().const_u8(first); + bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); + return true; + } } // Use llvm.memset.p0i8.* to initialize byte arrays diff --git a/tests/codegen/slice-init.rs b/tests/codegen/slice-init.rs index a01b432add6..1c2dd3e8875 100644 --- a/tests/codegen/slice-init.rs +++ b/tests/codegen/slice-init.rs @@ -65,16 +65,14 @@ pub fn nonzero_integer_array() { const N: usize = 100; -// FIXME: The two bytes of the u16 are the same, so we should -// just use memset, too. // CHECK-LABEL: @u16_init_one_bytes #[no_mangle] pub fn u16_init_one_bytes() -> [u16; N] { // CHECK-NOT: select - // CHECK: br + // CHECK-NOT: br // CHECK-NOT: switch - // CHECK: icmp - // CHECK-NOT: call void @llvm.memset.p0 + // CHECK-NOT: icmp + // CHECK: call void @llvm.memset.p0 [const { u16::from_be_bytes([1, 1]) }; N] } -- cgit 1.4.1-3-g733a5 From f86169a58f212b174a01aa721545df009e96bfda Mon Sep 17 00:00:00 2001 From: David Wood Date: Mon, 23 Sep 2024 18:46:23 +0100 Subject: mir_transform: implement forced inlining Adds `#[rustc_force_inline]` which is similar to always inlining but reports an error if the inlining was not possible, and which always attempts to inline annotated items, regardless of optimisation levels. It can only be applied to free functions to guarantee that the MIR inliner will be able to resolve calls. --- .../rustc_attr_data_structures/src/attributes.rs | 16 + compiler/rustc_codegen_gcc/src/attributes.rs | 2 +- compiler/rustc_codegen_llvm/src/attributes.rs | 4 +- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 63 +- compiler/rustc_feature/src/builtin_attrs.rs | 4 + compiler/rustc_hir_typeck/src/coercion.rs | 19 +- compiler/rustc_middle/src/mir/mono.rs | 7 +- compiler/rustc_middle/src/ty/error.rs | 3 + compiler/rustc_mir_transform/messages.ftl | 11 + .../rustc_mir_transform/src/cross_crate_inline.rs | 7 +- compiler/rustc_mir_transform/src/errors.rs | 28 +- compiler/rustc_mir_transform/src/inline.rs | 1498 ++++++++++++-------- compiler/rustc_mir_transform/src/lib.rs | 8 +- compiler/rustc_mir_transform/src/pass_manager.rs | 10 + compiler/rustc_mir_transform/src/validate.rs | 24 +- compiler/rustc_monomorphize/src/collector.rs | 9 + compiler/rustc_passes/messages.ftl | 8 + compiler/rustc_passes/src/check_attr.rs | 42 +- compiler/rustc_passes/src/errors.rs | 18 + compiler/rustc_span/src/symbol.rs | 1 + compiler/rustc_trait_selection/messages.ftl | 4 +- .../src/error_reporting/infer/mod.rs | 7 +- compiler/rustc_trait_selection/src/errors.rs | 11 +- compiler/rustc_type_ir/src/error.rs | 4 + .../forced.caller.ForceInline.panic-abort.diff | 21 + .../forced.caller.ForceInline.panic-unwind.diff | 21 + tests/mir-opt/inline/forced.rs | 13 + ...orced_async.caller.ForceInline.panic-abort.diff | 12 + ...rced_async.caller.ForceInline.panic-unwind.diff | 12 + tests/mir-opt/inline/forced_async.rs | 14 + ...caller-{closure#0}.ForceInline.panic-abort.diff | 21 + ...aller-{closure#0}.ForceInline.panic-unwind.diff | 21 + tests/mir-opt/inline/forced_closure.rs | 15 + tests/ui/force-inlining/auxiliary/callees.rs | 10 + tests/ui/force-inlining/cast.rs | 25 + tests/ui/force-inlining/cast.stderr | 40 + tests/ui/force-inlining/cross-crate.rs | 13 + tests/ui/force-inlining/deny-async.rs | 25 + tests/ui/force-inlining/deny-async.stderr | 19 + tests/ui/force-inlining/deny-closure.rs | 26 + tests/ui/force-inlining/deny-closure.stderr | 28 + tests/ui/force-inlining/deny.rs | 24 + tests/ui/force-inlining/deny.stderr | 19 + tests/ui/force-inlining/gate.rs | 12 + tests/ui/force-inlining/gate.stderr | 21 + tests/ui/force-inlining/invalid.rs | 164 +++ tests/ui/force-inlining/invalid.stderr | 377 +++++ 47 files changed, 2091 insertions(+), 670 deletions(-) create mode 100644 tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff create mode 100644 tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff create mode 100644 tests/mir-opt/inline/forced.rs create mode 100644 tests/mir-opt/inline/forced_async.caller.ForceInline.panic-abort.diff create mode 100644 tests/mir-opt/inline/forced_async.caller.ForceInline.panic-unwind.diff create mode 100644 tests/mir-opt/inline/forced_async.rs create mode 100644 tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff create mode 100644 tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff create mode 100644 tests/mir-opt/inline/forced_closure.rs create mode 100644 tests/ui/force-inlining/auxiliary/callees.rs create mode 100644 tests/ui/force-inlining/cast.rs create mode 100644 tests/ui/force-inlining/cast.stderr create mode 100644 tests/ui/force-inlining/cross-crate.rs create mode 100644 tests/ui/force-inlining/deny-async.rs create mode 100644 tests/ui/force-inlining/deny-async.stderr create mode 100644 tests/ui/force-inlining/deny-closure.rs create mode 100644 tests/ui/force-inlining/deny-closure.stderr create mode 100644 tests/ui/force-inlining/deny.rs create mode 100644 tests/ui/force-inlining/deny.stderr create mode 100644 tests/ui/force-inlining/gate.rs create mode 100644 tests/ui/force-inlining/gate.stderr create mode 100644 tests/ui/force-inlining/invalid.rs create mode 100644 tests/ui/force-inlining/invalid.stderr (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_attr_data_structures/src/attributes.rs b/compiler/rustc_attr_data_structures/src/attributes.rs index 8986bec57de..def6b16ee8a 100644 --- a/compiler/rustc_attr_data_structures/src/attributes.rs +++ b/compiler/rustc_attr_data_structures/src/attributes.rs @@ -11,6 +11,22 @@ pub enum InlineAttr { Hint, Always, Never, + /// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error + /// if the inlining cannot happen. It is limited to only free functions so that the calls + /// can always be resolved. + Force { + attr_span: Span, + reason: Option, + }, +} + +impl InlineAttr { + pub fn always(&self) -> bool { + match self { + InlineAttr::Always | InlineAttr::Force { .. } => true, + InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false, + } + } } #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)] diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs index 028a5ab5f71..69b04dd5796 100644 --- a/compiler/rustc_codegen_gcc/src/attributes.rs +++ b/compiler/rustc_codegen_gcc/src/attributes.rs @@ -20,7 +20,7 @@ fn inline_attr<'gcc, 'tcx>( ) -> Option> { match inline { InlineAttr::Hint => Some(FnAttribute::Inline), - InlineAttr::Always => Some(FnAttribute::AlwaysInline), + InlineAttr::Always | InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline), InlineAttr::Never => { if cx.sess().target.arch != "amdgpu" { Some(FnAttribute::NoInline) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index f8454fd9960..0250064e55f 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -37,7 +37,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll } match inline { InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), - InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), + InlineAttr::Always | InlineAttr::Force { .. } => { + Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) + } InlineAttr::Never => { if cx.sess().target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index cdb72aba36f..d8f62d543ad 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -18,6 +18,7 @@ use rustc_session::parse::feature_err; use rustc_session::{Session, lint}; use rustc_span::{Ident, Span, sym}; use rustc_target::spec::{SanitizerSet, abi}; +use tracing::debug; use crate::errors; use crate::target_features::{check_target_feature_trait_unsafe, from_target_feature_attr}; @@ -522,26 +523,36 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { mixed_export_name_no_mangle_lint_state.lint_if_mixed(tcx); codegen_fn_attrs.inline = attrs.iter().fold(InlineAttr::None, |ia, attr| { - if !attr.has_name(sym::inline) { - return ia; - } - if attr.is_word() { - InlineAttr::Hint - } else if let Some(ref items) = attr.meta_item_list() { - inline_span = Some(attr.span); - if items.len() != 1 { - struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit(); - InlineAttr::None - } else if list_contains_name(items, sym::always) { - InlineAttr::Always - } else if list_contains_name(items, sym::never) { - InlineAttr::Never - } else { - struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") - .with_help("valid inline arguments are `always` and `never`") - .emit(); + if attr.has_name(sym::inline) { + if attr.is_word() { + InlineAttr::Hint + } else if let Some(ref items) = attr.meta_item_list() { + inline_span = Some(attr.span); + if items.len() != 1 { + struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit(); + InlineAttr::None + } else if list_contains_name(items, sym::always) { + InlineAttr::Always + } else if list_contains_name(items, sym::never) { + InlineAttr::Never + } else { + struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") + .with_help("valid inline arguments are `always` and `never`") + .emit(); - InlineAttr::None + InlineAttr::None + } + } else { + ia + } + } else if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() { + if attr.is_word() { + InlineAttr::Force { attr_span: attr.span, reason: None } + } else if let Some(val) = attr.value_str() { + InlineAttr::Force { attr_span: attr.span, reason: Some(val) } + } else { + debug!("`rustc_force_inline` not checked by attribute validation"); + ia } } else { ia @@ -596,7 +607,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // is probably a poor usage of `#[inline(always)]` and easily avoided by not using the attribute. if tcx.features().target_feature_11() && tcx.is_closure_like(did.to_def_id()) - && codegen_fn_attrs.inline != InlineAttr::Always + && !codegen_fn_attrs.inline.always() { let owner_id = tcx.parent(did.to_def_id()); if tcx.def_kind(owner_id).has_codegen_attrs() { @@ -606,11 +617,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } - // If a function uses #[target_feature] it can't be inlined into general + // If a function uses `#[target_feature]` it can't be inlined into general // purpose functions as they wouldn't have the right target features - // enabled. For that reason we also forbid #[inline(always)] as it can't be + // enabled. For that reason we also forbid `#[inline(always)]` as it can't be // respected. - if !codegen_fn_attrs.target_features.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always + // + // `#[rustc_force_inline]` doesn't need to be prohibited here, that + // is implemented entirely in rustc can attempt to inline and error if it cannot. + if !codegen_fn_attrs.target_features.is_empty() + && matches!(codegen_fn_attrs.inline, InlineAttr::Always) { if let Some(span) = inline_span { tcx.dcx().span_err( @@ -621,7 +636,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } - if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline == InlineAttr::Always { + if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() { if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) { let hir_id = tcx.local_def_id_to_hir_id(did); tcx.node_span_lint( diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 3c3bbc8c31d..c28a4360f6f 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1019,6 +1019,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes, "#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen" ), + rustc_attr!( + rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes, + "#![rustc_force_inline] forces a free function to be inlined" + ), // ========================================================================== // Internal attributes, Testing: diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs index bd26be11279..9ebc7a4657e 100644 --- a/compiler/rustc_hir_typeck/src/coercion.rs +++ b/compiler/rustc_hir_typeck/src/coercion.rs @@ -38,6 +38,7 @@ use std::ops::Deref; use rustc_abi::ExternAbi; +use rustc_attr_parsing::InlineAttr; use rustc_errors::codes::*; use rustc_errors::{Applicability, Diag, struct_span_code_err}; use rustc_hir as hir; @@ -926,8 +927,13 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { return Err(TypeError::IntrinsicCast); } - // Safe `#[target_feature]` functions are not assignable to safe fn pointers (RFC 2396). + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); + if matches!(fn_attrs.inline, InlineAttr::Force { .. }) { + return Err(TypeError::ForceInlineCast); + } + // Safe `#[target_feature]` functions are not assignable to safe fn pointers + // (RFC 2396). if b_hdr.safety.is_safe() && !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty() { @@ -1197,6 +1203,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { return Ok(prev_ty); } + let is_force_inline = |ty: Ty<'tcx>| { + if let ty::FnDef(did, _) = ty.kind() { + matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. }) + } else { + false + } + }; + if is_force_inline(prev_ty) || is_force_inline(new_ty) { + return Err(TypeError::ForceInlineCast); + } + // Special-case that coercion alone cannot handle: // Function items or non-capturing closures of differing IDs or GenericArgs. let (a_sig, b_sig) = { diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 27168b2a9f2..111c3b6956a 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -132,9 +132,10 @@ impl<'tcx> MonoItem<'tcx> { // creating one copy of this `#[inline]` function which may // conflict with upstream crates as it could be an exported // symbol. - match tcx.codegen_fn_attrs(instance.def_id()).inline { - InlineAttr::Always => InstantiationMode::LocalCopy, - _ => InstantiationMode::GloballyShared { may_conflict: true }, + if tcx.codegen_fn_attrs(instance.def_id()).inline.always() { + InstantiationMode::LocalCopy + } else { + InstantiationMode::GloballyShared { may_conflict: true } } } MonoItem::Static(..) | MonoItem::GlobalAsm(..) => { diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 714094db053..35fbaa99569 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -109,6 +109,9 @@ impl<'tcx> TypeError<'tcx> { TypeError::ConstMismatch(ref values) => { format!("expected `{}`, found `{}`", values.expected, values.found).into() } + TypeError::ForceInlineCast => { + "cannot coerce functions which must be inlined to function pointers".into() + } TypeError::IntrinsicCast => "cannot coerce intrinsics to function pointers".into(), TypeError::TargetFeatureCast(_) => { "cannot coerce functions with `#[target_feature]` to safe function pointers".into() diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl index d00bfc66a6a..b0c023cca82 100644 --- a/compiler/rustc_mir_transform/messages.ftl +++ b/compiler/rustc_mir_transform/messages.ftl @@ -19,6 +19,17 @@ mir_transform_ffi_unwind_call = call to {$foreign -> mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer .suggestion = cast `{$ident}` to obtain a function pointer +mir_transform_force_inline = + `{$callee}` could not be inlined into `{$caller}` but is required to be inlined + .call = ...`{$callee}` called here + .attr = inlining due to this annotation + .caller = within `{$caller}`... + .callee = `{$callee}` defined here + .note = could not be inlined due to: {$reason} + +mir_transform_force_inline_justification = + `{$callee}` is required to be inlined to: {$sym} + mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be .label = the value is held across this suspend point .note = {$reason} diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index e1f1dd83f0d..8fce856687c 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // #[inline(never)] to force code generation. match codegen_fn_attrs.inline { InlineAttr::Never => return false, - InlineAttr::Hint | InlineAttr::Always => return true, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true, _ => {} } @@ -69,8 +69,9 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not // enabled. This ensures that we do inference even if someone only passes -Zinline-mir, // which is less confusing than having to also enable -Copt-level=1. - if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline) - { + let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline) + || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id()); + if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run { return false; } diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 2d9eeddea2e..015633d145f 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -4,8 +4,8 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_middle::mir::AssertKind; use rustc_middle::ty::TyCtxt; use rustc_session::lint::{self, Lint}; -use rustc_span::Span; use rustc_span::def_id::DefId; +use rustc_span::{Span, Symbol}; use crate::fluent_generated as fluent; @@ -142,3 +142,29 @@ pub(crate) struct MustNotSuspendReason { #[note(mir_transform_note2)] #[help] pub(crate) struct UndefinedTransmute; + +#[derive(Diagnostic)] +#[diag(mir_transform_force_inline)] +#[note] +pub(crate) struct ForceInlineFailure { + #[label(mir_transform_caller)] + pub caller_span: Span, + #[label(mir_transform_callee)] + pub callee_span: Span, + #[label(mir_transform_attr)] + pub attr_span: Span, + #[primary_span] + #[label(mir_transform_call)] + pub call_span: Span, + pub callee: String, + pub caller: String, + pub reason: &'static str, + #[subdiagnostic] + pub justification: Option, +} + +#[derive(Subdiagnostic)] +#[note(mir_transform_force_inline_justification)] +pub(crate) struct ForceInlineJustification { + pub sym: Symbol, +} diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 339acbad6b9..a5a4b5987f5 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -6,7 +6,7 @@ use std::ops::{Range, RangeFrom}; use rustc_abi::{ExternAbi, FieldIdx}; use rustc_attr_parsing::InlineAttr; use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::Idx; use rustc_index::bit_set::BitSet; use rustc_middle::bug; @@ -29,10 +29,6 @@ pub(crate) mod cycle; const TOP_DOWN_DEPTH_LIMIT: usize = 5; -// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden -// by custom rustc drivers, running all the steps by themselves. See #114628. -pub struct Inline; - #[derive(Clone, Debug)] struct CallSite<'tcx> { callee: Instance<'tcx>, @@ -41,14 +37,12 @@ struct CallSite<'tcx> { source_info: SourceInfo, } +// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden +// by custom rustc drivers, running all the steps by themselves. See #114628. +pub struct Inline; + impl<'tcx> crate::MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined - // MIR correctly when Modified Condition/Decision Coverage is enabled. - if sess.instrument_coverage_mcdc() { - return false; - } - if let Some(enabled) = sess.opts.unstable_opts.inline_mir { return enabled; } @@ -67,7 +61,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id())); let _guard = span.enter(); - if inline(tcx, body) { + if inline::>(tcx, body) { debug!("running simplify cfg on {:?}", body.source); simplify_cfg(body); deref_finder(tcx, body); @@ -75,47 +69,84 @@ impl<'tcx> crate::MirPass<'tcx> for Inline { } } -fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { - let def_id = body.source.def_id().expect_local(); +pub struct ForceInline; - // Only do inlining into fn bodies. - if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { - return false; +impl ForceInline { + pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool { + matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. }) } - if body.source.promoted.is_some() { - return false; +} + +impl<'tcx> crate::MirPass<'tcx> for ForceInline { + fn is_enabled(&self, _: &rustc_session::Session) -> bool { + true } - // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, - // which can create a cycle, even when no attempt is made to inline the function in the other - // direction. - if body.coroutine.is_some() { - return false; + + fn can_be_overridden(&self) -> bool { + false } - let typing_env = body.typing_env(tcx); - let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id())); + let _guard = span.enter(); + if inline::>(tcx, body) { + debug!("running simplify cfg on {:?}", body.source); + simplify_cfg(body); + deref_finder(tcx, body); + } + } +} - let mut this = Inliner { - tcx, - typing_env, - codegen_fn_attrs, - history: Vec::new(), - changed: false, - caller_is_inline_forwarder: matches!( - codegen_fn_attrs.inline, - InlineAttr::Hint | InlineAttr::Always - ) && body_is_forwarder(body), - }; - let blocks = START_BLOCK..body.basic_blocks.next_index(); - this.process_blocks(body, blocks); - this.changed +trait Inliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self; + + fn tcx(&self) -> TyCtxt<'tcx>; + fn typing_env(&self) -> ty::TypingEnv<'tcx>; + fn history(&self) -> &[DefId]; + fn caller_def_id(&self) -> LocalDefId; + + /// Has the caller body been changed? + fn changed(self) -> bool; + + /// Should inlining happen for a given callee? + fn should_inline_for_callee(&self, def_id: DefId) -> bool; + + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool; + + /// Returns inlining decision that is based on the examination of callee MIR body. + /// Assumes that codegen attributes have been checked for compatibility already. + fn check_callee_mir_body( + &self, + callsite: &CallSite<'tcx>, + callee_body: &Body<'tcx>, + callee_attrs: &CodegenFnAttrs, + cross_crate_inlinable: bool, + ) -> Result<(), &'static str>; + + // How many callsites in a body are we allowed to inline? We need to limit this in order + // to prevent super-linear growth in MIR size. + fn inline_limit_for_block(&self) -> Option; + + /// Called when inlining succeeds. + fn on_inline_success( + &mut self, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, + ); + + /// Called when inlining failed or was not performed. + fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str); + + /// Called when the inline limit for a body is reached. + fn on_inline_limit_reached(&self) -> bool; } -struct Inliner<'tcx> { +struct ForceInliner<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, - /// Caller codegen attributes. - codegen_fn_attrs: &'tcx CodegenFnAttrs, + /// `DefId` of caller. + def_id: LocalDefId, /// Stack of inlined instances. /// We only check the `DefId` and not the args because we want to /// avoid inlining cases of polymorphic recursion. @@ -124,366 +155,190 @@ struct Inliner<'tcx> { history: Vec, /// Indicates that the caller body has been modified. changed: bool, - /// Indicates that the caller is #[inline] and just calls another function, - /// and thus we can inline less into it as it'll be inlined itself. - caller_is_inline_forwarder: bool, } -impl<'tcx> Inliner<'tcx> { - fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range) { - // How many callsites in this body are we allowed to inline? We need to limit this in order - // to prevent super-linear growth in MIR size - let inline_limit = match self.history.len() { - 0 => usize::MAX, - 1..=TOP_DOWN_DEPTH_LIMIT => 1, - _ => return, - }; - let mut inlined_count = 0; - for bb in blocks { - let bb_data = &caller_body[bb]; - if bb_data.is_cleanup { - continue; - } - - let Some(callsite) = self.resolve_callsite(caller_body, bb, bb_data) else { - continue; - }; - - let span = trace_span!("process_blocks", %callsite.callee, ?bb); - let _guard = span.enter(); - - match self.try_inlining(caller_body, &callsite) { - Err(reason) => { - debug!("not-inlined {} [{}]", callsite.callee, reason); - } - Ok(new_blocks) => { - debug!("inlined {}", callsite.callee); - self.changed = true; - - self.history.push(callsite.callee.def_id()); - self.process_blocks(caller_body, new_blocks); - self.history.pop(); - - inlined_count += 1; - if inlined_count == inline_limit { - debug!("inline count reached"); - return; - } - } - } - } +impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self { + Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false } } - /// Attempts to inline a callsite into the caller body. When successful returns basic blocks - /// containing the inlined body. Otherwise returns an error describing why inlining didn't take - /// place. - fn try_inlining( - &self, - caller_body: &mut Body<'tcx>, - callsite: &CallSite<'tcx>, - ) -> Result, &'static str> { - self.check_mir_is_available(caller_body, callsite.callee)?; - - let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id()); - let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id()); - self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?; - - // Intrinsic fallback bodies are automatically made cross-crate inlineable, - // but at this stage we don't know whether codegen knows the intrinsic, - // so just conservatively don't inline it. This also ensures that we do not - // accidentally inline the body of an intrinsic that *must* be overridden. - if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { - return Err("Callee is an intrinsic, do not inline fallback bodies"); - } - - let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); - let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; - let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; - for arg in args { - if !arg.node.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.typing_env) - { - // We do not allow inlining functions with unsized params. Inlining these functions - // could create unsized locals, which are unsound and being phased out. - return Err("Call has unsized argument"); - } - } - - let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; - self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; - - let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( - self.tcx, - self.typing_env, - ty::EarlyBinder::bind(callee_body.clone()), - ) else { - return Err("failed to normalize callee body"); - }; - - // Normally, this shouldn't be required, but trait normalization failure can create a - // validation ICE. - if !validate_types(self.tcx, self.typing_env, &callee_body, &caller_body).is_empty() { - return Err("failed to validate callee body"); - } - - // Check call signature compatibility. - // Normally, this shouldn't be required, but trait normalization failure can create a - // validation ICE. - let output_type = callee_body.return_ty(); - if !util::sub_types(self.tcx, self.typing_env, output_type, destination_ty) { - trace!(?output_type, ?destination_ty); - return Err("failed to normalize return type"); - } - if callsite.fn_sig.abi() == ExternAbi::RustCall { - // FIXME: Don't inline user-written `extern "rust-call"` functions, - // since this is generally perf-negative on rustc, and we hope that - // LLVM will inline these functions instead. - if callee_body.spread_arg.is_some() { - return Err("do not inline user-written rust-call functions"); - } + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } - let (self_arg, arg_tuple) = match &args[..] { - [arg_tuple] => (None, arg_tuple), - [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), - _ => bug!("Expected `rust-call` to have 1 or 2 args"), - }; + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env + } - let self_arg_ty = - self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, self.tcx)); + fn history(&self) -> &[DefId] { + &self.history + } - let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, self.tcx); - let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { - bug!("Closure arguments are not passed as a tuple"); - }; + fn caller_def_id(&self) -> LocalDefId { + self.def_id + } - for (arg_ty, input) in - self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) - { - let input_type = callee_body.local_decls[input].ty; - if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { - trace!(?arg_ty, ?input_type); - return Err("failed to normalize tuple argument type"); - } - } - } else { - for (arg, input) in args.iter().zip(callee_body.args_iter()) { - let input_type = callee_body.local_decls[input].ty; - let arg_ty = arg.node.ty(&caller_body.local_decls, self.tcx); - if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { - trace!(?arg_ty, ?input_type); - return Err("failed to normalize argument type"); - } - } - } + fn changed(self) -> bool { + self.changed + } - let old_blocks = caller_body.basic_blocks.next_index(); - self.inline_call(caller_body, callsite, callee_body); - let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); + fn should_inline_for_callee(&self, def_id: DefId) -> bool { + ForceInline::should_run_pass_for_callee(self.tcx(), def_id) + } - Ok(new_blocks) + fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool { + true } - fn check_mir_is_available( + #[instrument(level = "debug", skip(self, callee_body))] + fn check_callee_mir_body( &self, - caller_body: &Body<'tcx>, - callee: Instance<'tcx>, + _: &CallSite<'tcx>, + callee_body: &Body<'tcx>, + _: &CodegenFnAttrs, + _: bool, ) -> Result<(), &'static str> { - let caller_def_id = caller_body.source.def_id(); - let callee_def_id = callee.def_id(); - if callee_def_id == caller_def_id { - return Err("self-recursion"); - } - - match callee.def { - InstanceKind::Item(_) => { - // If there is no MIR available (either because it was not in metadata or - // because it has no MIR because it's an extern function), then the inliner - // won't cause cycles on this. - if !self.tcx.is_mir_available(callee_def_id) { - return Err("item MIR unavailable"); - } - } - // These have no own callable MIR. - InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { - return Err("instance without MIR (intrinsic / virtual)"); - } - - // FIXME(#127030): `ConstParamHasTy` has bad interactions with - // the drop shim builder, which does not evaluate predicates in - // the correct param-env for types being dropped. Stall resolving - // the MIR for this instance until all of its const params are - // substituted. - InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => { - return Err("still needs substitution"); - } - - // This cannot result in an immediate cycle since the callee MIR is a shim, which does - // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we - // do not need to catch this here, we can wait until the inliner decides to continue - // inlining a second time. - InstanceKind::VTableShim(_) - | InstanceKind::ReifyShim(..) - | InstanceKind::FnPtrShim(..) - | InstanceKind::ClosureOnceShim { .. } - | InstanceKind::ConstructCoroutineInClosureShim { .. } - | InstanceKind::DropGlue(..) - | InstanceKind::CloneShim(..) - | InstanceKind::ThreadLocalShim(..) - | InstanceKind::FnPtrAddrShim(..) - | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), - } + if let Some(_) = callee_body.tainted_by_errors { Err("body has errors") } else { Ok(()) } + } - if self.tcx.is_constructor(callee_def_id) { - trace!("constructors always have MIR"); - // Constructor functions cannot cause a query cycle. - return Ok(()); - } + fn inline_limit_for_block(&self) -> Option { + Some(usize::MAX) + } - if callee_def_id.is_local() { - // If we know for sure that the function we're calling will itself try to - // call us, then we avoid inlining that function. - if self.tcx.mir_callgraph_reachable((callee, caller_def_id.expect_local())) { - return Err("caller might be reachable from callee (query cycle avoidance)"); - } + fn on_inline_success( + &mut self, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, + ) { + self.changed = true; - Ok(()) - } else { - // This cannot result in an immediate cycle since the callee MIR is from another crate - // and is already optimized. Any subsequent inlining may cause cycles, but we do - // not need to catch this here, we can wait until the inliner decides to continue - // inlining a second time. - trace!("functions from other crates always have MIR"); - Ok(()) - } + self.history.push(callsite.callee.def_id()); + process_blocks(self, caller_body, new_blocks); + self.history.pop(); } - fn resolve_callsite( - &self, - caller_body: &Body<'tcx>, - bb: BasicBlock, - bb_data: &BasicBlockData<'tcx>, - ) -> Option> { - // Only consider direct calls to functions - let terminator = bb_data.terminator(); - - // FIXME(explicit_tail_calls): figure out if we can inline tail calls - if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { - let func_ty = func.ty(caller_body, self.tcx); - if let ty::FnDef(def_id, args) = *func_ty.kind() { - // To resolve an instance its args have to be fully normalized. - let args = self.tcx.try_normalize_erasing_regions(self.typing_env, args).ok()?; - let callee = Instance::try_resolve(self.tcx, self.typing_env, def_id, args) - .ok() - .flatten()?; - - if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { - return None; - } - - if self.history.contains(&callee.def_id()) { - return None; - } + fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) { + let tcx = self.tcx(); + let InlineAttr::Force { attr_span, reason: justification } = + tcx.codegen_fn_attrs(callsite.callee.def_id()).inline + else { + bug!("called on item without required inlining"); + }; - let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args); + let call_span = callsite.source_info.span; + tcx.dcx().emit_err(crate::errors::ForceInlineFailure { + call_span, + attr_span, + caller_span: tcx.def_span(self.def_id), + caller: tcx.def_path_str(self.def_id), + callee_span: tcx.def_span(callsite.callee.def_id()), + callee: tcx.def_path_str(callsite.callee.def_id()), + reason, + justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }), + }); + } - // Additionally, check that the body that we're inlining actually agrees - // with the ABI of the trait that the item comes from. - if let InstanceKind::Item(instance_def_id) = callee.def - && self.tcx.def_kind(instance_def_id) == DefKind::AssocFn - && let instance_fn_sig = self.tcx.fn_sig(instance_def_id).skip_binder() - && instance_fn_sig.abi() != fn_sig.abi() - { - return None; - } + fn on_inline_limit_reached(&self) -> bool { + false + } +} - let source_info = SourceInfo { span: fn_span, ..terminator.source_info }; +struct NormalInliner<'tcx> { + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + /// `DefId` of caller. + def_id: LocalDefId, + /// Stack of inlined instances. + /// We only check the `DefId` and not the args because we want to + /// avoid inlining cases of polymorphic recursion. + /// The number of `DefId`s is finite, so checking history is enough + /// to ensure that we do not loop endlessly while inlining. + history: Vec, + /// Indicates that the caller body has been modified. + changed: bool, + /// Indicates that the caller is #[inline] and just calls another function, + /// and thus we can inline less into it as it'll be inlined itself. + caller_is_inline_forwarder: bool, +} - return Some(CallSite { callee, fn_sig, block: bb, source_info }); - } +impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { + fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) -> Self { + let typing_env = body.typing_env(tcx); + let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); + + Self { + tcx, + typing_env, + def_id, + history: Vec::new(), + changed: false, + caller_is_inline_forwarder: matches!( + codegen_fn_attrs.inline, + InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } + ) && body_is_forwarder(body), } - - None } - /// Returns an error if inlining is not possible based on codegen attributes alone. A success - /// indicates that inlining decision should be based on other criteria. - fn check_codegen_attributes( - &self, - callsite: &CallSite<'tcx>, - callee_attrs: &CodegenFnAttrs, - cross_crate_inlinable: bool, - ) -> Result<(), &'static str> { - if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) { - return Err("#[rustc_no_mir_inline]"); - } + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } - if let InlineAttr::Never = callee_attrs.inline { - return Err("never inline hint"); - } + fn caller_def_id(&self) -> LocalDefId { + self.def_id + } - // Reachability pass defines which functions are eligible for inlining. Generally inlining - // other functions is incorrect because they could reference symbols that aren't exported. - let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); - if !is_generic && !cross_crate_inlinable { - return Err("not exported"); - } + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env + } - if callsite.fn_sig.c_variadic() { - return Err("C variadic"); - } + fn history(&self) -> &[DefId] { + &self.history + } - if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) { - return Err("cold"); - } + fn changed(self) -> bool { + self.changed + } - if callee_attrs.no_sanitize != self.codegen_fn_attrs.no_sanitize { - return Err("incompatible sanitizer set"); - } + fn should_inline_for_callee(&self, _: DefId) -> bool { + true + } - // Two functions are compatible if the callee has no attribute (meaning - // that it's codegen agnostic), or sets an attribute that is identical - // to this function's attribute. - if callee_attrs.instruction_set.is_some() - && callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set - { - return Err("incompatible instruction set"); + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool { + if body.source.promoted.is_some() { + return false; } - let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name); - let this_feature_names = self.codegen_fn_attrs.target_features.iter().map(|f| f.name); - if callee_feature_names.ne(this_feature_names) { - // In general it is not correct to inline a callee with target features that are a - // subset of the caller. This is because the callee might contain calls, and the ABI of - // those calls depends on the target features of the surrounding function. By moving a - // `Call` terminator from one MIR body to another with more target features, we might - // change the ABI of that call! - return Err("incompatible target features"); + // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, + // which can create a cycle, even when no attempt is made to inline the function in the other + // direction. + if body.coroutine.is_some() { + return false; } - Ok(()) + true } - /// Returns inlining decision that is based on the examination of callee MIR body. - /// Assumes that codegen attributes have been checked for compatibility already. #[instrument(level = "debug", skip(self, callee_body))] - fn check_mir_body( + fn check_callee_mir_body( &self, callsite: &CallSite<'tcx>, callee_body: &Body<'tcx>, callee_attrs: &CodegenFnAttrs, cross_crate_inlinable: bool, ) -> Result<(), &'static str> { - let tcx = self.tcx; + let tcx = self.tcx(); if let Some(_) = callee_body.tainted_by_errors { - return Err("Body is tainted"); + return Err("body has errors"); } let mut threshold = if self.caller_is_inline_forwarder { - self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) + tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) } else if cross_crate_inlinable { - self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) + tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) } else { - self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) + tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) }; // Give a bonus functions with a small number of blocks, @@ -497,7 +352,7 @@ impl<'tcx> Inliner<'tcx> { // FIXME: Give a bonus to functions with only a single caller let mut checker = - CostChecker::new(self.tcx, self.typing_env, Some(callsite.callee), callee_body); + CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body); checker.add_function_level_costs(); @@ -513,20 +368,20 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); + let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id()); if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = callsite.callee.instantiate_mir( - self.tcx, - ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty), - ); - if ty.needs_drop(tcx, self.typing_env) + let ty = callsite + .callee + .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); + if ty.needs_drop(tcx, self.typing_env()) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } - } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set + } else if callee_attrs.instruction_set != caller_attrs.instruction_set && matches!(term.kind, TerminatorKind::InlineAsm { .. }) { // During the attribute checking stage we allow a callee with no @@ -534,7 +389,7 @@ impl<'tcx> Inliner<'tcx> { // assign one. However, during this stage we require an exact match when any // inline-asm is detected. LLVM will still possibly do an inline later on // if the no-attribute function ends up with the same instruction set anyway. - return Err("Cannot move inline-asm across instruction sets"); + return Err("cannot move inline-asm across instruction sets"); } else if let TerminatorKind::TailCall { .. } = term.kind { // FIXME(explicit_tail_calls): figure out how exactly functions containing tail // calls can be inlined (and if they even should) @@ -558,321 +413,710 @@ impl<'tcx> Inliner<'tcx> { } } - fn inline_call( - &self, - caller_body: &mut Body<'tcx>, + fn inline_limit_for_block(&self) -> Option { + match self.history.len() { + 0 => Some(usize::MAX), + 1..=TOP_DOWN_DEPTH_LIMIT => Some(1), + _ => None, + } + } + + fn on_inline_success( + &mut self, callsite: &CallSite<'tcx>, - mut callee_body: Body<'tcx>, + caller_body: &mut Body<'tcx>, + new_blocks: std::ops::Range, ) { - let terminator = caller_body[callsite.block].terminator.take().unwrap(); - let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind - else { - bug!("unexpected terminator kind {:?}", terminator.kind); - }; + self.changed = true; - let return_block = if let Some(block) = target { - // Prepare a new block for code that should execute when call returns. We don't use - // target block directly since it might have other predecessors. - let data = BasicBlockData::new( - Some(Terminator { - source_info: terminator.source_info, - kind: TerminatorKind::Goto { target: block }, - }), - caller_body[block].is_cleanup, - ); - Some(caller_body.basic_blocks_mut().push(data)) - } else { - None + self.history.push(callsite.callee.def_id()); + process_blocks(self, caller_body, new_blocks); + self.history.pop(); + } + + fn on_inline_limit_reached(&self) -> bool { + true + } + + fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {} +} + +fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { + let def_id = body.source.def_id().expect_local(); + + // Only do inlining into fn bodies. + if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() { + return false; + } + + let mut inliner = T::new(tcx, def_id, body); + if !inliner.check_caller_mir_body(body) { + return false; + } + + let blocks = START_BLOCK..body.basic_blocks.next_index(); + process_blocks(&mut inliner, body, blocks); + inliner.changed() +} + +fn process_blocks<'tcx, I: Inliner<'tcx>>( + inliner: &mut I, + caller_body: &mut Body<'tcx>, + blocks: Range, +) { + let Some(inline_limit) = inliner.inline_limit_for_block() else { return }; + let mut inlined_count = 0; + for bb in blocks { + let bb_data = &caller_body[bb]; + if bb_data.is_cleanup { + continue; + } + + let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else { + continue; }; - // If the call is something like `a[*i] = f(i)`, where - // `i : &mut usize`, then just duplicating the `a[*i]` - // Place could result in two different locations if `f` - // writes to `i`. To prevent this we need to create a temporary - // borrow of the place and pass the destination as `*temp` instead. - fn dest_needs_borrow(place: Place<'_>) -> bool { - for elem in place.projection.iter() { - match elem { - ProjectionElem::Deref | ProjectionElem::Index(_) => return true, - _ => {} + let span = trace_span!("process_blocks", %callsite.callee, ?bb); + let _guard = span.enter(); + + if !inliner.should_inline_for_callee(callsite.callee.def_id()) { + debug!("not enabled"); + continue; + } + + match try_inlining(inliner, caller_body, &callsite) { + Err(reason) => { + debug!("not-inlined {} [{}]", callsite.callee, reason); + inliner.on_inline_failure(&callsite, reason); + } + Ok(new_blocks) => { + debug!("inlined {}", callsite.callee); + inliner.on_inline_success(&callsite, caller_body, new_blocks); + + inlined_count += 1; + if inlined_count == inline_limit { + if inliner.on_inline_limit_reached() { + return; + } } } + } + } +} + +fn resolve_callsite<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &Body<'tcx>, + bb: BasicBlock, + bb_data: &BasicBlockData<'tcx>, +) -> Option> { + let tcx = inliner.tcx(); + // Only consider direct calls to functions + let terminator = bb_data.terminator(); + + // FIXME(explicit_tail_calls): figure out if we can inline tail calls + if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { + let func_ty = func.ty(caller_body, tcx); + if let ty::FnDef(def_id, args) = *func_ty.kind() { + // To resolve an instance its args have to be fully normalized. + let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?; + let callee = + Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?; + + if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { + return None; + } - false + if inliner.history().contains(&callee.def_id()) { + return None; + } + + let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args); + + // Additionally, check that the body that we're inlining actually agrees + // with the ABI of the trait that the item comes from. + if let InstanceKind::Item(instance_def_id) = callee.def + && tcx.def_kind(instance_def_id) == DefKind::AssocFn + && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder() + && instance_fn_sig.abi() != fn_sig.abi() + { + return None; + } + + let source_info = SourceInfo { span: fn_span, ..terminator.source_info }; + + return Some(CallSite { callee, fn_sig, block: bb, source_info }); } + } - let dest = if dest_needs_borrow(destination) { - trace!("creating temp for return destination"); - let dest = Rvalue::Ref( - self.tcx.lifetimes.re_erased, - BorrowKind::Mut { kind: MutBorrowKind::Default }, - destination, - ); - let dest_ty = dest.ty(caller_body, self.tcx); - let temp = - Place::from(self.new_call_temp(caller_body, callsite, dest_ty, return_block)); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((temp, dest))), - }); - self.tcx.mk_place_deref(temp) - } else { - destination - }; + None +} - // Always create a local to hold the destination, as `RETURN_PLACE` may appear - // where a full `Place` is not allowed. - let (remap_destination, destination_local) = if let Some(d) = dest.as_local() { - (false, d) - } else { - ( - true, - self.new_call_temp( - caller_body, - callsite, - destination.ty(caller_body, self.tcx).ty, - return_block, - ), - ) - }; +/// Attempts to inline a callsite into the caller body. When successful returns basic blocks +/// containing the inlined body. Otherwise returns an error describing why inlining didn't take +/// place. +fn try_inlining<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, +) -> Result, &'static str> { + let tcx = inliner.tcx(); + check_mir_is_available(inliner, caller_body, callsite.callee)?; + + let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id()); + let cross_crate_inlinable = tcx.cross_crate_inlinable(callsite.callee.def_id()); + check_codegen_attributes(inliner, callsite, callee_attrs, cross_crate_inlinable)?; + + // Intrinsic fallback bodies are automatically made cross-crate inlineable, + // but at this stage we don't know whether codegen knows the intrinsic, + // so just conservatively don't inline it. This also ensures that we do not + // accidentally inline the body of an intrinsic that *must* be overridden. + if tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) { + return Err("callee is an intrinsic"); + } - // Copy the arguments if needed. - let args = self.make_call_args(args, callsite, caller_body, &callee_body, return_block); + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty; + for arg in args { + if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) { + // We do not allow inlining functions with unsized params. Inlining these functions + // could create unsized locals, which are unsound and being phased out. + return Err("call has unsized argument"); + } + } - let mut integrator = Integrator { - args: &args, - new_locals: Local::new(caller_body.local_decls.len()).., - new_scopes: SourceScope::new(caller_body.source_scopes.len()).., - new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., - destination: destination_local, - callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), - callsite, - cleanup_block: unwind, - in_cleanup_block: false, - return_block, - tcx: self.tcx, - always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + let callee_body = try_instance_mir(tcx, callsite.callee.def)?; + inliner.check_callee_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; + + let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( + tcx, + inliner.typing_env(), + ty::EarlyBinder::bind(callee_body.clone()), + ) else { + debug!("failed to normalize callee body"); + return Err("implementation limitation"); + }; + + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() { + debug!("failed to validate callee body"); + return Err("implementation limitation"); + } + + // Check call signature compatibility. + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + let output_type = callee_body.return_ty(); + if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) { + trace!(?output_type, ?destination_ty); + debug!("failed to normalize return type"); + return Err("implementation limitation"); + } + if callsite.fn_sig.abi() == ExternAbi::RustCall { + // FIXME: Don't inline user-written `extern "rust-call"` functions, + // since this is generally perf-negative on rustc, and we hope that + // LLVM will inline these functions instead. + if callee_body.spread_arg.is_some() { + return Err("user-written rust-call functions"); + } + + let (self_arg, arg_tuple) = match &args[..] { + [arg_tuple] => (None, arg_tuple), + [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), + _ => bug!("Expected `rust-call` to have 1 or 2 args"), }; - // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones - // (or existing ones, in a few special cases) in the caller. - integrator.visit_body(&mut callee_body); + let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx)); - // If there are any locals without storage markers, give them storage only for the - // duration of the call. - for local in callee_body.vars_and_temps_iter() { - if integrator.always_live_locals.contains(local) { - let new_local = integrator.map_local(local); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageLive(new_local), - }); + let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx); + let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + for (arg_ty, input) in + self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) + { + let input_type = callee_body.local_decls[input].ty; + if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) { + trace!(?arg_ty, ?input_type); + debug!("failed to normalize tuple argument type"); + return Err("implementation limitation"); } } - if let Some(block) = return_block { - // To avoid repeated O(n) insert, push any new statements to the end and rotate - // the slice once. - let mut n = 0; - if remap_destination { - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new(( - dest, - Rvalue::Use(Operand::Move(destination_local.into())), - ))), - }); - n += 1; + } else { + for (arg, input) in args.iter().zip(callee_body.args_iter()) { + let input_type = callee_body.local_decls[input].ty; + let arg_ty = arg.node.ty(&caller_body.local_decls, tcx); + if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) { + trace!(?arg_ty, ?input_type); + debug!("failed to normalize argument type"); + return Err("implementation limitation"); } - for local in callee_body.vars_and_temps_iter().rev() { - if integrator.always_live_locals.contains(local) { - let new_local = integrator.map_local(local); - caller_body[block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::StorageDead(new_local), - }); - n += 1; - } + } + } + + let old_blocks = caller_body.basic_blocks.next_index(); + inline_call(inliner, caller_body, callsite, callee_body); + let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); + + Ok(new_blocks) +} + +fn check_mir_is_available<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &Body<'tcx>, + callee: Instance<'tcx>, +) -> Result<(), &'static str> { + let caller_def_id = caller_body.source.def_id(); + let callee_def_id = callee.def_id(); + if callee_def_id == caller_def_id { + return Err("self-recursion"); + } + + match callee.def { + InstanceKind::Item(_) => { + // If there is no MIR available (either because it was not in metadata or + // because it has no MIR because it's an extern function), then the inliner + // won't cause cycles on this. + if !inliner.tcx().is_mir_available(callee_def_id) { + debug!("item MIR unavailable"); + return Err("implementation limitation"); } - caller_body[block].statements.rotate_right(n); + } + // These have no own callable MIR. + InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { + debug!("instance without MIR (intrinsic / virtual)"); + return Err("implementation limitation"); } - // Insert all of the (mapped) parts of the callee body into the caller. - caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); - caller_body.source_scopes.append(&mut callee_body.source_scopes); - if self - .tcx - .sess - .opts - .unstable_opts - .inline_mir_preserve_debug - .unwrap_or(self.tcx.sess.opts.debuginfo != DebugInfo::None) - { - // Note that we need to preserve these in the standard library so that - // people working on rust can build with or without debuginfo while - // still getting consistent results from the mir-opt tests. - caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + // FIXME(#127030): `ConstParamHasTy` has bad interactions with + // the drop shim builder, which does not evaluate predicates in + // the correct param-env for types being dropped. Stall resolving + // the MIR for this instance until all of its const params are + // substituted. + InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => { + debug!("still needs substitution"); + return Err("implementation limitation"); } - caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); - caller_body[callsite.block].terminator = Some(Terminator { - source_info: callsite.source_info, - kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, - }); + // This cannot result in an immediate cycle since the callee MIR is a shim, which does + // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we + // do not need to catch this here, we can wait until the inliner decides to continue + // inlining a second time. + InstanceKind::VTableShim(_) + | InstanceKind::ReifyShim(..) + | InstanceKind::FnPtrShim(..) + | InstanceKind::ClosureOnceShim { .. } + | InstanceKind::ConstructCoroutineInClosureShim { .. } + | InstanceKind::DropGlue(..) + | InstanceKind::CloneShim(..) + | InstanceKind::ThreadLocalShim(..) + | InstanceKind::FnPtrAddrShim(..) + | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()), + } - // Copy required constants from the callee_body into the caller_body. Although we are only - // pushing unevaluated consts to `required_consts`, here they may have been evaluated - // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again. - caller_body.required_consts.as_mut().unwrap().extend( - callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()), - ); - // Now that we incorporated the callee's `required_consts`, we can remove the callee from - // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does - // some extra work here to save the monomorphization collector work later. It helps a lot, - // since monomorphization can avoid a lot of work when the "mentioned items" are similar to - // the actually used items. By doing this we can entirely avoid visiting the callee! - // We need to reconstruct the `required_item` for the callee so that we can find and - // remove it. - let callee_item = MentionedItem::Fn(func.ty(caller_body, self.tcx)); - let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap(); - if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) { - // We found the callee, so remove it and add its items instead. - caller_mentioned_items.remove(idx); - caller_mentioned_items.extend(callee_body.mentioned_items()); - } else { - // If we can't find the callee, there's no point in adding its items. Probably it - // already got removed by being inlined elsewhere in the same function, so we already - // took its items. + if inliner.tcx().is_constructor(callee_def_id) { + trace!("constructors always have MIR"); + // Constructor functions cannot cause a query cycle. + return Ok(()); + } + + if callee_def_id.is_local() { + // If we know for sure that the function we're calling will itself try to + // call us, then we avoid inlining that function. + if inliner.tcx().mir_callgraph_reachable((callee, caller_def_id.expect_local())) { + debug!("query cycle avoidance"); + return Err("caller might be reachable from callee"); } + + Ok(()) + } else { + // This cannot result in an immediate cycle since the callee MIR is from another crate + // and is already optimized. Any subsequent inlining may cause cycles, but we do + // not need to catch this here, we can wait until the inliner decides to continue + // inlining a second time. + trace!("functions from other crates always have MIR"); + Ok(()) } +} - fn make_call_args( - &self, - args: Box<[Spanned>]>, - callsite: &CallSite<'tcx>, - caller_body: &mut Body<'tcx>, - callee_body: &Body<'tcx>, - return_block: Option, - ) -> Box<[Local]> { - let tcx = self.tcx; - - // There is a bit of a mismatch between the *caller* of a closure and the *callee*. - // The caller provides the arguments wrapped up in a tuple: - // - // tuple_tmp = (a, b, c) - // Fn::call(closure_ref, tuple_tmp) - // - // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`) - // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has - // the job of unpacking this tuple. But here, we are codegen. =) So we want to create - // a vector like - // - // [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2] - // - // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient - // if we "spill" that into *another* temporary, so that we can map the argument - // variable in the callee MIR directly to an argument variable on our side. - // So we introduce temporaries like: - // - // tmp0 = tuple_tmp.0 - // tmp1 = tuple_tmp.1 - // tmp2 = tuple_tmp.2 - // - // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. - if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() { - // FIXME(edition_2024): switch back to a normal method call. - let mut args = <_>::into_iter(args); - let self_ = self.create_temp_if_necessary( - args.next().unwrap().node, - callsite, - caller_body, - return_block, - ); - let tuple = self.create_temp_if_necessary( - args.next().unwrap().node, - callsite, - caller_body, - return_block, - ); - assert!(args.next().is_none()); - - let tuple = Place::from(tuple); - let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else { - bug!("Closure arguments are not passed as a tuple"); - }; +/// Returns an error if inlining is not possible based on codegen attributes alone. A success +/// indicates that inlining decision should be based on other criteria. +fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>( + inliner: &I, + callsite: &CallSite<'tcx>, + callee_attrs: &CodegenFnAttrs, + cross_crate_inlinable: bool, +) -> Result<(), &'static str> { + let tcx = inliner.tcx(); + if tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) { + return Err("#[rustc_no_mir_inline]"); + } - // The `closure_ref` in our example above. - let closure_ref_arg = iter::once(self_); + if let InlineAttr::Never = callee_attrs.inline { + return Err("never inline attribute"); + } - // The `tmp0`, `tmp1`, and `tmp2` in our example above. - let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { - // This is e.g., `tuple_tmp.0` in our example above. - let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); + // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined + // MIR correctly when Modified Condition/Decision Coverage is enabled. + if tcx.sess.instrument_coverage_mcdc() { + return Err("incompatible with MC/DC coverage"); + } - // Spill to a local to make e.g., `tmp0`. - self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block) - }); + // Reachability pass defines which functions are eligible for inlining. Generally inlining + // other functions is incorrect because they could reference symbols that aren't exported. + let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); + if !is_generic && !cross_crate_inlinable { + return Err("not exported"); + } - closure_ref_arg.chain(tuple_tmp_args).collect() - } else { - // FIXME(edition_2024): switch back to a normal method call. - <_>::into_iter(args) - .map(|a| self.create_temp_if_necessary(a.node, callsite, caller_body, return_block)) - .collect() - } + if callsite.fn_sig.c_variadic() { + return Err("C variadic"); } - /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh - /// temporary `T` and an instruction `T = arg`, and returns `T`. - fn create_temp_if_necessary( - &self, - arg: Operand<'tcx>, - callsite: &CallSite<'tcx>, - caller_body: &mut Body<'tcx>, - return_block: Option, - ) -> Local { - // Reuse the operand if it is a moved temporary. - if let Operand::Move(place) = &arg - && let Some(local) = place.as_local() - && caller_body.local_kind(local) == LocalKind::Temp - { - return local; - } + if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) { + return Err("cold"); + } - // Otherwise, create a temporary for the argument. - trace!("creating temp for argument {:?}", arg); - let arg_ty = arg.ty(caller_body, self.tcx); - let local = self.new_call_temp(caller_body, callsite, arg_ty, return_block); - caller_body[callsite.block].statements.push(Statement { - source_info: callsite.source_info, - kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), - }); - local + let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id()); + if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize { + return Err("incompatible sanitizer set"); } - /// Introduces a new temporary into the caller body that is live for the duration of the call. - fn new_call_temp( - &self, - caller_body: &mut Body<'tcx>, - callsite: &CallSite<'tcx>, - ty: Ty<'tcx>, - return_block: Option, - ) -> Local { - let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); + // Two functions are compatible if the callee has no attribute (meaning + // that it's codegen agnostic), or sets an attribute that is identical + // to this function's attribute. + if callee_attrs.instruction_set.is_some() + && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set + { + return Err("incompatible instruction set"); + } + + let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name); + let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name); + if callee_feature_names.ne(this_feature_names) { + // In general it is not correct to inline a callee with target features that are a + // subset of the caller. This is because the callee might contain calls, and the ABI of + // those calls depends on the target features of the surrounding function. By moving a + // `Call` terminator from one MIR body to another with more target features, we might + // change the ABI of that call! + return Err("incompatible target features"); + } + Ok(()) +} + +fn inline_call<'tcx, I: Inliner<'tcx>>( + inliner: &I, + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, + mut callee_body: Body<'tcx>, +) { + let tcx = inliner.tcx(); + let terminator = caller_body[callsite.block].terminator.take().unwrap(); + let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind + else { + bug!("unexpected terminator kind {:?}", terminator.kind); + }; + + let return_block = if let Some(block) = target { + // Prepare a new block for code that should execute when call returns. We don't use + // target block directly since it might have other predecessors. + let data = BasicBlockData::new( + Some(Terminator { + source_info: terminator.source_info, + kind: TerminatorKind::Goto { target: block }, + }), + caller_body[block].is_cleanup, + ); + Some(caller_body.basic_blocks_mut().push(data)) + } else { + None + }; + + // If the call is something like `a[*i] = f(i)`, where + // `i : &mut usize`, then just duplicating the `a[*i]` + // Place could result in two different locations if `f` + // writes to `i`. To prevent this we need to create a temporary + // borrow of the place and pass the destination as `*temp` instead. + fn dest_needs_borrow(place: Place<'_>) -> bool { + for elem in place.projection.iter() { + match elem { + ProjectionElem::Deref | ProjectionElem::Index(_) => return true, + _ => {} + } + } + + false + } + + let dest = if dest_needs_borrow(destination) { + trace!("creating temp for return destination"); + let dest = Rvalue::Ref( + tcx.lifetimes.re_erased, + BorrowKind::Mut { kind: MutBorrowKind::Default }, + destination, + ); + let dest_ty = dest.ty(caller_body, tcx); + let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block)); caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, - kind: StatementKind::StorageLive(local), + kind: StatementKind::Assign(Box::new((temp, dest))), }); + tcx.mk_place_deref(temp) + } else { + destination + }; + + // Always create a local to hold the destination, as `RETURN_PLACE` may appear + // where a full `Place` is not allowed. + let (remap_destination, destination_local) = if let Some(d) = dest.as_local() { + (false, d) + } else { + ( + true, + new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block), + ) + }; - if let Some(block) = return_block { - caller_body[block].statements.insert(0, Statement { + // Copy the arguments if needed. + let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block); + + let mut integrator = Integrator { + args: &args, + new_locals: Local::new(caller_body.local_decls.len()).., + new_scopes: SourceScope::new(caller_body.source_scopes.len()).., + new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., + destination: destination_local, + callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), + callsite, + cleanup_block: unwind, + in_cleanup_block: false, + return_block, + tcx, + always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + }; + + // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones + // (or existing ones, in a few special cases) in the caller. + integrator.visit_body(&mut callee_body); + + // If there are any locals without storage markers, give them storage only for the + // duration of the call. + for local in callee_body.vars_and_temps_iter() { + if integrator.always_live_locals.contains(local) { + let new_local = integrator.map_local(local); + caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, - kind: StatementKind::StorageDead(local), + kind: StatementKind::StorageLive(new_local), }); } + } + if let Some(block) = return_block { + // To avoid repeated O(n) insert, push any new statements to the end and rotate + // the slice once. + let mut n = 0; + if remap_destination { + caller_body[block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::Assign(Box::new(( + dest, + Rvalue::Use(Operand::Move(destination_local.into())), + ))), + }); + n += 1; + } + for local in callee_body.vars_and_temps_iter().rev() { + if integrator.always_live_locals.contains(local) { + let new_local = integrator.map_local(local); + caller_body[block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(new_local), + }); + n += 1; + } + } + caller_body[block].statements.rotate_right(n); + } + + // Insert all of the (mapped) parts of the callee body into the caller. + caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); + caller_body.source_scopes.append(&mut callee_body.source_scopes); + if tcx + .sess + .opts + .unstable_opts + .inline_mir_preserve_debug + .unwrap_or(tcx.sess.opts.debuginfo != DebugInfo::None) + { + // Note that we need to preserve these in the standard library so that + // people working on rust can build with or without debuginfo while + // still getting consistent results from the mir-opt tests. + caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + } + caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); + + caller_body[callsite.block].terminator = Some(Terminator { + source_info: callsite.source_info, + kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, + }); + + // Copy required constants from the callee_body into the caller_body. Although we are only + // pushing unevaluated consts to `required_consts`, here they may have been evaluated + // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again. + caller_body.required_consts.as_mut().unwrap().extend( + callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()), + ); + // Now that we incorporated the callee's `required_consts`, we can remove the callee from + // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does + // some extra work here to save the monomorphization collector work later. It helps a lot, + // since monomorphization can avoid a lot of work when the "mentioned items" are similar to + // the actually used items. By doing this we can entirely avoid visiting the callee! + // We need to reconstruct the `required_item` for the callee so that we can find and + // remove it. + let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx)); + let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap(); + if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) { + // We found the callee, so remove it and add its items instead. + caller_mentioned_items.remove(idx); + caller_mentioned_items.extend(callee_body.mentioned_items()); + } else { + // If we can't find the callee, there's no point in adding its items. Probably it + // already got removed by being inlined elsewhere in the same function, so we already + // took its items. + } +} + +fn make_call_args<'tcx, I: Inliner<'tcx>>( + inliner: &I, + args: Box<[Spanned>]>, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + callee_body: &Body<'tcx>, + return_block: Option, +) -> Box<[Local]> { + let tcx = inliner.tcx(); + + // There is a bit of a mismatch between the *caller* of a closure and the *callee*. + // The caller provides the arguments wrapped up in a tuple: + // + // tuple_tmp = (a, b, c) + // Fn::call(closure_ref, tuple_tmp) + // + // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`) + // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has + // the job of unpacking this tuple. But here, we are codegen. =) So we want to create + // a vector like + // + // [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2] + // + // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient + // if we "spill" that into *another* temporary, so that we can map the argument + // variable in the callee MIR directly to an argument variable on our side. + // So we introduce temporaries like: + // + // tmp0 = tuple_tmp.0 + // tmp1 = tuple_tmp.1 + // tmp2 = tuple_tmp.2 + // + // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. + if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() { + // FIXME(edition_2024): switch back to a normal method call. + let mut args = <_>::into_iter(args); + let self_ = create_temp_if_necessary( + inliner, + args.next().unwrap().node, + callsite, + caller_body, + return_block, + ); + let tuple = create_temp_if_necessary( + inliner, + args.next().unwrap().node, + callsite, + caller_body, + return_block, + ); + assert!(args.next().is_none()); + + let tuple = Place::from(tuple); + let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + // The `closure_ref` in our example above. + let closure_ref_arg = iter::once(self_); + + // The `tmp0`, `tmp1`, and `tmp2` in our example above. + let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { + // This is e.g., `tuple_tmp.0` in our example above. + let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); + + // Spill to a local to make e.g., `tmp0`. + create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block) + }); + + closure_ref_arg.chain(tuple_tmp_args).collect() + } else { + // FIXME(edition_2024): switch back to a normal method call. + <_>::into_iter(args) + .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block)) + .collect() + } +} - local +/// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh temporary `T` and an +/// instruction `T = arg`, and returns `T`. +fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>( + inliner: &I, + arg: Operand<'tcx>, + callsite: &CallSite<'tcx>, + caller_body: &mut Body<'tcx>, + return_block: Option, +) -> Local { + // Reuse the operand if it is a moved temporary. + if let Operand::Move(place) = &arg + && let Some(local) = place.as_local() + && caller_body.local_kind(local) == LocalKind::Temp + { + return local; + } + + // Otherwise, create a temporary for the argument. + trace!("creating temp for argument {:?}", arg); + let arg_ty = arg.ty(caller_body, inliner.tcx()); + let local = new_call_temp(caller_body, callsite, arg_ty, return_block); + caller_body[callsite.block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))), + }); + local +} + +/// Introduces a new temporary into the caller body that is live for the duration of the call. +fn new_call_temp<'tcx>( + caller_body: &mut Body<'tcx>, + callsite: &CallSite<'tcx>, + ty: Ty<'tcx>, + return_block: Option, +) -> Local { + let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span)); + + caller_body[callsite.block].statements.push(Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageLive(local), + }); + + if let Some(block) = return_block { + caller_body[block].statements.insert(0, Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(local), + }); } + + local } /** diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index e1fba9be5bb..350929ffaa5 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -141,7 +141,7 @@ declare_passes! { mod gvn : GVN; // Made public so that `mir_drops_elaborated_and_const_checked` can be overridden // by custom rustc drivers, running all the steps by themselves. See #114628. - pub mod inline : Inline; + pub mod inline : Inline, ForceInline; mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg }; mod jump_threading : JumpThreading; mod known_panics_lint : KnownPanicsLint; @@ -488,7 +488,9 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { // Do not compute the mir call graph without said call graph actually being used. - if pm::should_run_pass(tcx, &inline::Inline) { + if pm::should_run_pass(tcx, &inline::Inline) + || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id()) + { tcx.ensure_with_value().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id())); } } @@ -664,6 +666,8 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Perform instsimplify before inline to eliminate some trivial calls (like clone // shims). &instsimplify::InstSimplify::BeforeInline, + // Perform inlining of `#[rustc_force_inline]`-annotated callees. + &inline::ForceInline, // Perform inlining, which may add a lot of code. &inline::Inline, // Code from other crates may have storage markers, so this needs to happen after diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 8a45ce0762d..c3f0a989ce1 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -79,6 +79,12 @@ pub(super) trait MirPass<'tcx> { true } + /// Returns `true` if this pass can be overridden by `-Zenable-mir-passes`. This should be + /// true for basically every pass other than those that are necessary for correctness. + fn can_be_overridden(&self) -> bool { + true + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>); fn is_mir_dump_enabled(&self) -> bool { @@ -176,6 +182,10 @@ where { let name = pass.name(); + if !pass.can_be_overridden() { + return pass.is_enabled(tcx.sess); + } + let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes; let overridden = overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| { diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index a670da94fcc..9444c5b61dd 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1,6 +1,7 @@ //! Validates the MIR to ensure that invariants are upheld. use rustc_abi::{ExternAbi, FIRST_VARIANT, Size}; +use rustc_attr_parsing::InlineAttr; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::LangItem; use rustc_index::IndexVec; @@ -79,7 +80,7 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { cfg_checker.fail(location, msg); } - if let MirPhase::Runtime(_) = body.phase { + if let MirPhase::Runtime(phase) = body.phase { if let ty::InstanceKind::Item(_) = body.source.instance { if body.has_free_regions() { cfg_checker.fail( @@ -88,6 +89,27 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { ); } } + + if phase >= RuntimePhase::Optimized + && body + .basic_blocks + .iter() + .filter_map(|bb| match &bb.terminator().kind { + TerminatorKind::Call { func, .. } + | TerminatorKind::TailCall { func, .. } => Some(func), + _ => None, + }) + .filter_map(|func| match func.ty(&body.local_decls, tcx).kind() { + ty::FnDef(did, ..) => Some(did), + _ => None, + }) + .any(|did| matches!(tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })) + { + cfg_checker.fail( + Location::START, + "`#[rustc_force_inline]`-annotated function not inlined", + ); + } } } } diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 00aae03704f..f3232a0e15d 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -207,6 +207,7 @@ use std::path::PathBuf; +use rustc_attr_parsing::InlineAttr; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::{LRef, MTLock, par_for_each_in}; use rustc_data_structures::unord::{UnordMap, UnordSet}; @@ -959,6 +960,14 @@ fn should_codegen_locally<'tcx>(tcx: TyCtxtAt<'tcx>, instance: Instance<'tcx>) - return false; } + if tcx.def_kind(def_id).has_codegen_attrs() + && matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. }) + { + // `#[rustc_force_inline]` items should never be codegened. This should be caught by + // the MIR validator. + return false; + } + if def_id.is_local() { // Local items cannot be referred to locally without monomorphizing them locally. return true; diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl index f39bea2a56f..9df396cbf7a 100644 --- a/compiler/rustc_passes/messages.ftl +++ b/compiler/rustc_passes/messages.ftl @@ -656,6 +656,14 @@ passes_rustc_allow_const_fn_unstable = passes_rustc_dirty_clean = attribute requires -Z query-dep-graph to be enabled +passes_rustc_force_inline = + attribute should be applied to a function + .label = not a function definition + +passes_rustc_force_inline_coro = + attribute cannot be applied to a `async`, `gen` or `async gen` function + .label = `async`, `gen` or `async gen` function + passes_rustc_layout_scalar_valid_range_arg = expected exactly one integer literal argument diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 7656ca86c18..1b2b8ac5dd9 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -247,7 +247,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_coroutine(attr, target); } [sym::linkage, ..] => self.check_linkage(attr, span, target), - [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent( attr.span, span, attrs), + [sym::rustc_pub_transparent, ..] => self.check_rustc_pub_transparent(attr.span, span, attrs), [ // ok sym::allow @@ -332,6 +332,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_repr(attrs, span, target, item, hir_id); self.check_used(attrs, target, span); + self.check_rustc_force_inline(hir_id, attrs, span, target); } fn inline_attr_str_error_with_macro_def(&self, hir_id: HirId, attr: &Attribute, sym: &str) { @@ -2480,6 +2481,45 @@ impl<'tcx> CheckAttrVisitor<'tcx> { } } + fn check_rustc_force_inline( + &self, + hir_id: HirId, + attrs: &[Attribute], + span: Span, + target: Target, + ) { + let force_inline_attr = attrs.iter().find(|attr| attr.has_name(sym::rustc_force_inline)); + match (target, force_inline_attr) { + (Target::Closure, None) => { + let is_coro = matches!( + self.tcx.hir().expect_expr(hir_id).kind, + hir::ExprKind::Closure(hir::Closure { + kind: hir::ClosureKind::Coroutine(..) + | hir::ClosureKind::CoroutineClosure(..), + .. + }) + ); + let parent_did = self.tcx.hir().get_parent_item(hir_id).to_def_id(); + let parent_span = self.tcx.def_span(parent_did); + let parent_force_inline_attr = + self.tcx.get_attr(parent_did, sym::rustc_force_inline); + if let Some(attr) = parent_force_inline_attr + && is_coro + { + self.dcx().emit_err(errors::RustcForceInlineCoro { + attr_span: attr.span, + span: parent_span, + }); + } + } + (Target::Fn, _) => (), + (_, Some(attr)) => { + self.dcx().emit_err(errors::RustcForceInline { attr_span: attr.span, span }); + } + (_, None) => (), + } + } + /// Checks if `#[autodiff]` is applied to an item other than a function item. fn check_autodiff(&self, _hir_id: HirId, _attr: &Attribute, span: Span, target: Target) { debug!("check_autodiff"); diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs index d95fa5db0ce..13da021c614 100644 --- a/compiler/rustc_passes/src/errors.rs +++ b/compiler/rustc_passes/src/errors.rs @@ -688,6 +688,24 @@ pub(crate) struct RustcPubTransparent { pub span: Span, } +#[derive(Diagnostic)] +#[diag(passes_rustc_force_inline)] +pub(crate) struct RustcForceInline { + #[primary_span] + pub attr_span: Span, + #[label] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(passes_rustc_force_inline_coro)] +pub(crate) struct RustcForceInlineCoro { + #[primary_span] + pub attr_span: Span, + #[label] + pub span: Span, +} + #[derive(Diagnostic)] #[diag(passes_link_ordinal)] pub(crate) struct LinkOrdinal { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index ef1e2c20978..a5bf5e06485 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1731,6 +1731,7 @@ symbols! { rustc_error, rustc_evaluate_where_clauses, rustc_expected_cgu_reuse, + rustc_force_inline, rustc_has_incoherent_inherent_impls, rustc_hidden_type_of_opaques, rustc_if_this_changed, diff --git a/compiler/rustc_trait_selection/messages.ftl b/compiler/rustc_trait_selection/messages.ftl index 1ab89ecde7a..b82bb27eb79 100644 --- a/compiler/rustc_trait_selection/messages.ftl +++ b/compiler/rustc_trait_selection/messages.ftl @@ -251,7 +251,9 @@ trait_selection_no_value_in_rustc_on_unimplemented = this attribute must have a trait_selection_nothing = {""} -trait_selection_oc_cant_coerce = cannot coerce intrinsics to function pointers +trait_selection_oc_cant_coerce_force_inline = + cannot coerce functions which must be inlined to function pointers +trait_selection_oc_cant_coerce_intrinsic = cannot coerce intrinsics to function pointers trait_selection_oc_closure_selfref = closure/coroutine type that references itself trait_selection_oc_const_compat = const not compatible with trait trait_selection_oc_fn_lang_correct_type = {$lang_item_name -> diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs index d279590d45a..53300c95fa7 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs @@ -2294,7 +2294,7 @@ impl<'tcx> ObligationCause<'tcx> { { FailureCode::Error0644 } - TypeError::IntrinsicCast => FailureCode::Error0308, + TypeError::IntrinsicCast | TypeError::ForceInlineCast => FailureCode::Error0308, _ => FailureCode::Error0308, }, } @@ -2360,8 +2360,11 @@ impl<'tcx> ObligationCause<'tcx> { { ObligationCauseFailureCode::ClosureSelfref { span } } + TypeError::ForceInlineCast => { + ObligationCauseFailureCode::CantCoerceForceInline { span, subdiags } + } TypeError::IntrinsicCast => { - ObligationCauseFailureCode::CantCoerce { span, subdiags } + ObligationCauseFailureCode::CantCoerceIntrinsic { span, subdiags } } _ => ObligationCauseFailureCode::Generic { span, subdiags }, }, diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs index a8fddff4e4a..53a4e5031c6 100644 --- a/compiler/rustc_trait_selection/src/errors.rs +++ b/compiler/rustc_trait_selection/src/errors.rs @@ -1729,8 +1729,15 @@ pub enum ObligationCauseFailureCode { #[primary_span] span: Span, }, - #[diag(trait_selection_oc_cant_coerce, code = E0308)] - CantCoerce { + #[diag(trait_selection_oc_cant_coerce_force_inline, code = E0308)] + CantCoerceForceInline { + #[primary_span] + span: Span, + #[subdiagnostic] + subdiags: Vec, + }, + #[diag(trait_selection_oc_cant_coerce_intrinsic, code = E0308)] + CantCoerceIntrinsic { #[primary_span] span: Span, #[subdiagnostic] diff --git a/compiler/rustc_type_ir/src/error.rs b/compiler/rustc_type_ir/src/error.rs index 55671b84dbc..68b11489ae7 100644 --- a/compiler/rustc_type_ir/src/error.rs +++ b/compiler/rustc_type_ir/src/error.rs @@ -51,6 +51,9 @@ pub enum TypeError { ConstMismatch(ExpectedFound), IntrinsicCast, + /// `#[rustc_force_inline]` functions must be inlined and must not be codegened independently, + /// so casting to a function pointer must be prohibited. + ForceInlineCast, /// Safe `#[target_feature]` functions are not assignable to safe function pointers. TargetFeatureCast(I::DefId), } @@ -83,6 +86,7 @@ impl TypeError { | ProjectionMismatched(_) | ExistentialMismatch(_) | ConstMismatch(_) + | ForceInlineCast | IntrinsicCast => true, } } diff --git a/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff new file mode 100644 index 00000000000..f894f06e5a0 --- /dev/null +++ b/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff @@ -0,0 +1,21 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> () { + let mut _0: (); + let _1: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_1); +- _1 = callee_forced() -> [return: bb1, unwind unreachable]; +- } +- +- bb1: { + StorageDead(_1); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff new file mode 100644 index 00000000000..7b70fd66566 --- /dev/null +++ b/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff @@ -0,0 +1,21 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> () { + let mut _0: (); + let _1: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_1); +- _1 = callee_forced() -> [return: bb1, unwind continue]; +- } +- +- bb1: { + StorageDead(_1); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced.rs b/tests/mir-opt/inline/forced.rs new file mode 100644 index 00000000000..0447ef2e4d7 --- /dev/null +++ b/tests/mir-opt/inline/forced.rs @@ -0,0 +1,13 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced.caller.ForceInline.diff +pub fn caller() { + callee_forced(); + // CHECK-LABEL: fn caller( + // CHECK: (inlined callee_forced) +} diff --git a/tests/mir-opt/inline/forced_async.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_async.caller.ForceInline.panic-abort.diff new file mode 100644 index 00000000000..42048bceef8 --- /dev/null +++ b/tests/mir-opt/inline/forced_async.caller.ForceInline.panic-abort.diff @@ -0,0 +1,12 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> {async fn body of caller()} { + let mut _0: {async fn body of caller()}; + + bb0: { + _0 = {coroutine@$DIR/forced_async.rs:10:19: 14:2 (#0)}; + return; + } + } + diff --git a/tests/mir-opt/inline/forced_async.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_async.caller.ForceInline.panic-unwind.diff new file mode 100644 index 00000000000..42048bceef8 --- /dev/null +++ b/tests/mir-opt/inline/forced_async.caller.ForceInline.panic-unwind.diff @@ -0,0 +1,12 @@ +- // MIR for `caller` before ForceInline ++ // MIR for `caller` after ForceInline + + fn caller() -> {async fn body of caller()} { + let mut _0: {async fn body of caller()}; + + bb0: { + _0 = {coroutine@$DIR/forced_async.rs:10:19: 14:2 (#0)}; + return; + } + } + diff --git a/tests/mir-opt/inline/forced_async.rs b/tests/mir-opt/inline/forced_async.rs new file mode 100644 index 00000000000..456f8fdc5ce --- /dev/null +++ b/tests/mir-opt/inline/forced_async.rs @@ -0,0 +1,14 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +//@ edition: 2021 +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced_async.caller.ForceInline.diff +async fn caller() { + callee_forced(); + // CHECK-LABEL: fn caller( + // CHECK: (inlined callee_forced) +} diff --git a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff new file mode 100644 index 00000000000..def2375efd0 --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff @@ -0,0 +1,21 @@ +- // MIR for `caller::{closure#0}` before ForceInline ++ // MIR for `caller::{closure#0}` after ForceInline + + fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure.rs:10:6: 10:8}) -> () { + let mut _0: (); + let _2: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_2); +- _2 = callee_forced() -> [return: bb1, unwind unreachable]; +- } +- +- bb1: { + StorageDead(_2); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff new file mode 100644 index 00000000000..8f50c20587d --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff @@ -0,0 +1,21 @@ +- // MIR for `caller::{closure#0}` before ForceInline ++ // MIR for `caller::{closure#0}` after ForceInline + + fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure.rs:10:6: 10:8}) -> () { + let mut _0: (); + let _2: (); ++ scope 1 (inlined callee_forced) { ++ } + + bb0: { + StorageLive(_2); +- _2 = callee_forced() -> [return: bb1, unwind continue]; +- } +- +- bb1: { + StorageDead(_2); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/inline/forced_closure.rs b/tests/mir-opt/inline/forced_closure.rs new file mode 100644 index 00000000000..a5590d6c7f0 --- /dev/null +++ b/tests/mir-opt/inline/forced_closure.rs @@ -0,0 +1,15 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -Copt-level=0 --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee_forced() {} + +// EMIT_MIR forced_closure.caller-{closure#0}.ForceInline.diff +pub fn caller() { + (|| { + callee_forced(); + // CHECK-LABEL: fn caller::{closure#0}( + // CHECK: (inlined callee_forced) + })(); +} diff --git a/tests/ui/force-inlining/auxiliary/callees.rs b/tests/ui/force-inlining/auxiliary/callees.rs new file mode 100644 index 00000000000..7468b8f1098 --- /dev/null +++ b/tests/ui/force-inlining/auxiliary/callees.rs @@ -0,0 +1,10 @@ +//@ compile-flags: --crate-type=lib +#![feature(rustc_attrs)] + +#[rustc_force_inline = "the test requires it"] +pub fn forced_with_reason() { +} + +#[rustc_force_inline] +pub fn forced() { +} diff --git a/tests/ui/force-inlining/cast.rs b/tests/ui/force-inlining/cast.rs new file mode 100644 index 00000000000..8f51ec83977 --- /dev/null +++ b/tests/ui/force-inlining/cast.rs @@ -0,0 +1,25 @@ +//@ check-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +#[rustc_force_inline] +pub fn callee(x: isize) -> usize { unimplemented!() } + +fn a() { + let _: fn(isize) -> usize = callee; +//~^ ERROR cannot coerce functions which must be inlined to function pointers +} + +fn b() { + let _ = callee as fn(isize) -> usize; +//~^ ERROR non-primitive cast +} + +fn c() { + let _: [fn(isize) -> usize; 2] = [ + callee, +//~^ ERROR cannot coerce functions which must be inlined to function pointers + callee, + ]; +} diff --git a/tests/ui/force-inlining/cast.stderr b/tests/ui/force-inlining/cast.stderr new file mode 100644 index 00000000000..116919e5fe7 --- /dev/null +++ b/tests/ui/force-inlining/cast.stderr @@ -0,0 +1,40 @@ +error[E0308]: cannot coerce functions which must be inlined to function pointers + --> $DIR/cast.rs:10:33 + | +LL | let _: fn(isize) -> usize = callee; + | ------------------ ^^^^^^ cannot coerce functions which must be inlined to function pointers + | | + | expected due to this + | + = note: expected fn pointer `fn(_) -> _` + found fn item `fn(_) -> _ {callee}` + = note: fn items are distinct from fn pointers +help: consider casting to a fn pointer + | +LL | let _: fn(isize) -> usize = callee as fn(isize) -> usize; + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error[E0605]: non-primitive cast: `fn(isize) -> usize {callee}` as `fn(isize) -> usize` + --> $DIR/cast.rs:15:13 + | +LL | let _ = callee as fn(isize) -> usize; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid cast + +error[E0308]: cannot coerce functions which must be inlined to function pointers + --> $DIR/cast.rs:21:9 + | +LL | callee, + | ^^^^^^ cannot coerce functions which must be inlined to function pointers + | + = note: expected fn pointer `fn(_) -> _` + found fn item `fn(_) -> _ {callee}` + = note: fn items are distinct from fn pointers +help: consider casting to a fn pointer + | +LL | callee as fn(isize) -> usize, + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error: aborting due to 3 previous errors + +Some errors have detailed explanations: E0308, E0605. +For more information about an error, try `rustc --explain E0308`. diff --git a/tests/ui/force-inlining/cross-crate.rs b/tests/ui/force-inlining/cross-crate.rs new file mode 100644 index 00000000000..a3f0c2c873e --- /dev/null +++ b/tests/ui/force-inlining/cross-crate.rs @@ -0,0 +1,13 @@ +//@ aux-build:callees.rs +//@ build-pass +//@ compile-flags: --crate-type=lib + +extern crate callees; + +// Test that forced inlining across crates works as expected. + +pub fn caller() { + callees::forced(); + + callees::forced_with_reason(); +} diff --git a/tests/ui/force-inlining/deny-async.rs b/tests/ui/force-inlining/deny-async.rs new file mode 100644 index 00000000000..d5b6dc000f7 --- /dev/null +++ b/tests/ui/force-inlining/deny-async.rs @@ -0,0 +1,25 @@ +//@ check-fail +//@ compile-flags: --crate-type=lib +//@ edition: 2021 +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining into async functions w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +async fn async_caller() { + callee(); +//~^ ERROR `callee` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `async_caller::{closure#0}` but is required to be inlined +} diff --git a/tests/ui/force-inlining/deny-async.stderr b/tests/ui/force-inlining/deny-async.stderr new file mode 100644 index 00000000000..7b077174ee8 --- /dev/null +++ b/tests/ui/force-inlining/deny-async.stderr @@ -0,0 +1,19 @@ +error: `callee` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + --> $DIR/deny-async.rs:20:5 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `async_caller::{closure#0}` but is required to be inlined + --> $DIR/deny-async.rs:23:5 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/deny-closure.rs b/tests/ui/force-inlining/deny-closure.rs new file mode 100644 index 00000000000..0057f1e7854 --- /dev/null +++ b/tests/ui/force-inlining/deny-closure.rs @@ -0,0 +1,26 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining into closures w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +pub fn caller() { + (|| { + callee(); +//~^ ERROR `callee` could not be inlined into `caller::{closure#0}` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `caller::{closure#0}` but is required to be inlined + })(); +} diff --git a/tests/ui/force-inlining/deny-closure.stderr b/tests/ui/force-inlining/deny-closure.stderr new file mode 100644 index 00000000000..7d42145f1c8 --- /dev/null +++ b/tests/ui/force-inlining/deny-closure.stderr @@ -0,0 +1,28 @@ +error: `callee` could not be inlined into `caller::{closure#0}` but is required to be inlined + --> $DIR/deny-closure.rs:20:9 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `caller::{closure#0}` but is required to be inlined + --> $DIR/deny-closure.rs:23:9 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +note: the above error was encountered while instantiating `fn caller::{closure#0}` + --> $DIR/deny-closure.rs:19:5 + | +LL | / (|| { +LL | | callee(); +... | +LL | | })(); + | |________^ + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/deny.rs b/tests/ui/force-inlining/deny.rs new file mode 100644 index 00000000000..a19e820cff2 --- /dev/null +++ b/tests/ui/force-inlining/deny.rs @@ -0,0 +1,24 @@ +//@ build-fail +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] +#![feature(rustc_attrs)] + +// Test that forced inlining w/ errors works as expected. + +#[rustc_no_mir_inline] +#[rustc_force_inline] +pub fn callee() { +} + +#[rustc_no_mir_inline] +#[rustc_force_inline = "the test requires it"] +pub fn callee_justified() { +} + +pub fn caller() { + callee(); +//~^ ERROR `callee` could not be inlined into `caller` but is required to be inlined + + callee_justified(); +//~^ ERROR `callee_justified` could not be inlined into `caller` but is required to be inlined +} diff --git a/tests/ui/force-inlining/deny.stderr b/tests/ui/force-inlining/deny.stderr new file mode 100644 index 00000000000..3983bead182 --- /dev/null +++ b/tests/ui/force-inlining/deny.stderr @@ -0,0 +1,19 @@ +error: `callee` could not be inlined into `caller` but is required to be inlined + --> $DIR/deny.rs:19:5 + | +LL | callee(); + | ^^^^^^^^ ...`callee` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + +error: `callee_justified` could not be inlined into `caller` but is required to be inlined + --> $DIR/deny.rs:22:5 + | +LL | callee_justified(); + | ^^^^^^^^^^^^^^^^^^ ...`callee_justified` called here + | + = note: could not be inlined due to: #[rustc_no_mir_inline] + = note: `callee_justified` is required to be inlined to: the test requires it + +error: aborting due to 2 previous errors + diff --git a/tests/ui/force-inlining/gate.rs b/tests/ui/force-inlining/gate.rs new file mode 100644 index 00000000000..d6a01a74a44 --- /dev/null +++ b/tests/ui/force-inlining/gate.rs @@ -0,0 +1,12 @@ +//@ compile-flags: --crate-type=lib +#![allow(internal_features)] + +#[rustc_force_inline] +//~^ ERROR #![rustc_force_inline] forces a free function to be inlined +pub fn bare() { +} + +#[rustc_force_inline = "the test requires it"] +//~^ ERROR #![rustc_force_inline] forces a free function to be inlined +pub fn justified() { +} diff --git a/tests/ui/force-inlining/gate.stderr b/tests/ui/force-inlining/gate.stderr new file mode 100644 index 00000000000..e3973a08c23 --- /dev/null +++ b/tests/ui/force-inlining/gate.stderr @@ -0,0 +1,21 @@ +error[E0658]: #![rustc_force_inline] forces a free function to be inlined + --> $DIR/gate.rs:4:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error[E0658]: #![rustc_force_inline] forces a free function to be inlined + --> $DIR/gate.rs:9:1 + | +LL | #[rustc_force_inline = "the test requires it"] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: add `#![feature(rustc_attrs)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0658`. diff --git a/tests/ui/force-inlining/invalid.rs b/tests/ui/force-inlining/invalid.rs new file mode 100644 index 00000000000..7574078b245 --- /dev/null +++ b/tests/ui/force-inlining/invalid.rs @@ -0,0 +1,164 @@ +//@ edition: 2024 +#![allow(internal_features, unused_imports, unused_macros)] +#![feature(extern_types)] +#![feature(gen_blocks)] +#![feature(rustc_attrs)] +#![feature(stmt_expr_attributes)] +#![feature(trait_alias)] + +// Test that invalid force inlining attributes error as expected. + +#[rustc_force_inline("foo")] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced1() { +} + +#[rustc_force_inline(bar, baz)] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced2() { +} + +#[rustc_force_inline(2)] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced3() { +} + +#[rustc_force_inline = 2] +//~^ ERROR malformed `rustc_force_inline` attribute input +pub fn forced4() { +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +extern crate std as other_std; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +use std::collections::HashMap; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +static _FOO: &'static str = "FOO"; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +const _BAR: u32 = 3; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +mod foo { } + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +unsafe extern "C" { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + static X: &'static u32; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + type Y; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo(); +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +type Foo = u32; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +enum Bar<#[rustc_force_inline] T> { +//~^ ERROR attribute should be applied to a function + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + Baz(std::marker::PhantomData), +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +struct Qux { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + field: u32, +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +union FooBar { + x: u32, + y: u32, +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +trait FooBaz { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + type Foo; + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + const Bar: i32; + + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo() {} +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +trait FooQux = FooBaz; + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +impl Bar { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + fn foo() {} +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +impl FooBaz for Bar { + type Foo = u32; + const Bar: i32 = 3; +} + +#[rustc_force_inline] +//~^ ERROR attribute should be applied to a function +macro_rules! barqux { ($foo:tt) => { $foo }; } + +fn barqux(#[rustc_force_inline] _x: u32) {} +//~^ ERROR allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters +//~^^ ERROR attribute should be applied to a function + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +async fn async_foo() {} + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +gen fn gen_foo() {} + +#[rustc_force_inline] +//~^ ERROR attribute cannot be applied to a `async`, `gen` or `async gen` function +async gen fn async_gen_foo() {} + +fn main() { + let _x = #[rustc_force_inline] || { }; +//~^ ERROR attribute should be applied to a function + let _y = #[rustc_force_inline] 3 + 4; +//~^ ERROR attribute should be applied to a function + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + let _z = 3; + + match _z { + #[rustc_force_inline] +//~^ ERROR attribute should be applied to a function + 1 => (), + _ => (), + } +} diff --git a/tests/ui/force-inlining/invalid.stderr b/tests/ui/force-inlining/invalid.stderr new file mode 100644 index 00000000000..5d280124129 --- /dev/null +++ b/tests/ui/force-inlining/invalid.stderr @@ -0,0 +1,377 @@ +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:11:1 + | +LL | #[rustc_force_inline("foo")] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:16:1 + | +LL | #[rustc_force_inline(bar, baz)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:21:1 + | +LL | #[rustc_force_inline(2)] + | ^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: malformed `rustc_force_inline` attribute input + --> $DIR/invalid.rs:26:1 + | +LL | #[rustc_force_inline = 2] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | +help: the following are the possible correct uses + | +LL | #[rustc_force_inline = "reason"] + | +LL | #[rustc_force_inline] + | + +error: allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters + --> $DIR/invalid.rs:133:11 + | +LL | fn barqux(#[rustc_force_inline] _x: u32) {} + | ^^^^^^^^^^^^^^^^^^^^^ + +error: attribute should be applied to a function + --> $DIR/invalid.rs:31:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | extern crate std as other_std; + | ------------------------------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:35:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | use std::collections::HashMap; + | ------------------------------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:39:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | static _FOO: &'static str = "FOO"; + | ---------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:43:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | const _BAR: u32 = 3; + | -------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:47:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | mod foo { } + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:51:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / unsafe extern "C" { +LL | | #[rustc_force_inline] +LL | | +LL | | static X: &'static u32; +... | +LL | | fn foo(); +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:67:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Foo = u32; + | --------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:71:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / enum Bar<#[rustc_force_inline] T> { +LL | | +LL | | #[rustc_force_inline] +... | +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:73:10 + | +LL | enum Bar<#[rustc_force_inline] T> { + | ^^^^^^^^^^^^^^^^^^^^^ - not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:75:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | Baz(std::marker::PhantomData), + | -------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:80:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / struct Qux { +LL | | #[rustc_force_inline] +LL | | +LL | | field: u32, +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:83:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | field: u32, + | ---------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:88:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / union FooBar { +LL | | x: u32, +LL | | y: u32, +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:95:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / trait FooBaz { +LL | | #[rustc_force_inline] +LL | | +LL | | type Foo; +... | +LL | | fn foo() {} +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:110:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | trait FooQux = FooBaz; + | ---------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:114:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / impl Bar { +LL | | #[rustc_force_inline] +LL | | +LL | | fn foo() {} +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:122:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | / impl FooBaz for Bar { +LL | | type Foo = u32; +LL | | const Bar: i32 = 3; +LL | | } + | |_- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:129:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | macro_rules! barqux { ($foo:tt) => { $foo }; } + | ---------------------------------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:133:11 + | +LL | fn barqux(#[rustc_force_inline] _x: u32) {} + | ^^^^^^^^^^^^^^^^^^^^^-------- + | | + | not a function definition + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:137:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | async fn async_foo() {} + | -------------------- `async`, `gen` or `async gen` function + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:141:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | gen fn gen_foo() {} + | ---------------- `async`, `gen` or `async gen` function + +error: attribute cannot be applied to a `async`, `gen` or `async gen` function + --> $DIR/invalid.rs:145:1 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | async gen fn async_gen_foo() {} + | ---------------------------- `async`, `gen` or `async gen` function + +error: attribute should be applied to a function + --> $DIR/invalid.rs:150:14 + | +LL | let _x = #[rustc_force_inline] || { }; + | ^^^^^^^^^^^^^^^^^^^^^ ------ not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:152:14 + | +LL | let _y = #[rustc_force_inline] 3 + 4; + | ^^^^^^^^^^^^^^^^^^^^^ - not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:154:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | let _z = 3; + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:159:9 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | 1 => (), + | ------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:98:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Foo; + | --------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:101:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | const Bar: i32; + | --------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:105:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo() {} + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:117:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo() {} + | ----------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:54:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | static X: &'static u32; + | ----------------------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:58:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | type Y; + | ------- not a function definition + +error: attribute should be applied to a function + --> $DIR/invalid.rs:62:5 + | +LL | #[rustc_force_inline] + | ^^^^^^^^^^^^^^^^^^^^^ +LL | +LL | fn foo(); + | --------- not a function definition + +error: aborting due to 38 previous errors + -- cgit 1.4.1-3-g733a5 From 02d423cd24944b17783e9fa4062e6a4e8e4d351f Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 10 Dec 2024 12:05:24 +0000 Subject: codegen_attrs: force inlining takes precedence --- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 61 +++++++++++++------------ 1 file changed, 33 insertions(+), 28 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index d8f62d543ad..a78826a4629 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -523,41 +523,46 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { mixed_export_name_no_mangle_lint_state.lint_if_mixed(tcx); codegen_fn_attrs.inline = attrs.iter().fold(InlineAttr::None, |ia, attr| { - if attr.has_name(sym::inline) { - if attr.is_word() { - InlineAttr::Hint - } else if let Some(ref items) = attr.meta_item_list() { - inline_span = Some(attr.span); - if items.len() != 1 { - struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit(); - InlineAttr::None - } else if list_contains_name(items, sym::always) { - InlineAttr::Always - } else if list_contains_name(items, sym::never) { - InlineAttr::Never - } else { - struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") - .with_help("valid inline arguments are `always` and `never`") - .emit(); + if !attr.has_name(sym::inline) { + return ia; + } - InlineAttr::None - } - } else { - ia - } - } else if attr.has_name(sym::rustc_force_inline) && tcx.features().rustc_attrs() { - if attr.is_word() { - InlineAttr::Force { attr_span: attr.span, reason: None } - } else if let Some(val) = attr.value_str() { - InlineAttr::Force { attr_span: attr.span, reason: Some(val) } + if attr.is_word() { + InlineAttr::Hint + } else if let Some(ref items) = attr.meta_item_list() { + inline_span = Some(attr.span); + if items.len() != 1 { + struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit(); + InlineAttr::None + } else if list_contains_name(items, sym::always) { + InlineAttr::Always + } else if list_contains_name(items, sym::never) { + InlineAttr::Never } else { - debug!("`rustc_force_inline` not checked by attribute validation"); - ia + struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument") + .with_help("valid inline arguments are `always` and `never`") + .emit(); + + InlineAttr::None } } else { ia } }); + codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| { + if !attr.has_name(sym::rustc_force_inline) || !tcx.features().rustc_attrs() { + return ia; + } + + if attr.is_word() { + InlineAttr::Force { attr_span: attr.span, reason: None } + } else if let Some(val) = attr.value_str() { + InlineAttr::Force { attr_span: attr.span, reason: Some(val) } + } else { + debug!("`rustc_force_inline` not checked by attribute validation"); + ia + } + }); // naked function MUST NOT be inlined! This attribute is required for the rust compiler itself, // but not for the code generation backend because at that point the naked function will just be -- cgit 1.4.1-3-g733a5 From ce602acfc22724c30b6154f650225d2e1d1550ed Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Dec 2024 13:54:41 +0000 Subject: clarify `target_feature` + forced inlining --- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index a78826a4629..37b53bb5bea 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -627,17 +627,19 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { // enabled. For that reason we also forbid `#[inline(always)]` as it can't be // respected. // - // `#[rustc_force_inline]` doesn't need to be prohibited here, that - // is implemented entirely in rustc can attempt to inline and error if it cannot. + // `#[rustc_force_inline]` doesn't need to be prohibited here, only + // `#[inline(always)]`, as forced inlining is implemented entirely within + // rustc (and so the MIR inliner can do any necessary checks for compatible target + // features). + // + // This sidesteps the LLVM blockers in enabling `target_features` + + // `inline(always)` to be used together (see rust-lang/rust#116573 and + // llvm/llvm-project#70563). if !codegen_fn_attrs.target_features.is_empty() && matches!(codegen_fn_attrs.inline, InlineAttr::Always) { if let Some(span) = inline_span { - tcx.dcx().span_err( - span, - "cannot use `#[inline(always)]` with \ - `#[target_feature]`", - ); + tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`"); } } -- cgit 1.4.1-3-g733a5 From 47573bf61ec0807cee3cf2c5c1ca88bb7d5b89cd Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 18 Dec 2024 22:03:07 +0100 Subject: add `-Zmin-function-alignment` --- compiler/rustc_codegen_llvm/src/attributes.rs | 6 ++- compiler/rustc_codegen_ssa/src/mir/naked_asm.rs | 8 +++- compiler/rustc_interface/src/tests.rs | 2 + compiler/rustc_session/src/config.rs | 2 + compiler/rustc_session/src/options.rs | 19 ++++++++++ .../src/compiler-flags/min-function-alignment.md | 24 ++++++++++++ tests/codegen/min-function-alignment.rs | 43 +++++++++++++++++++++ tests/codegen/naked-fn/min-function-alignment.rs | 44 ++++++++++++++++++++++ 8 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 src/doc/unstable-book/src/compiler-flags/min-function-alignment.md create mode 100644 tests/codegen/min-function-alignment.rs create mode 100644 tests/codegen/naked-fn/min-function-alignment.rs (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index f8454fd9960..f84cafb0c4f 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -472,7 +472,11 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); } - if let Some(align) = codegen_fn_attrs.alignment { + // function alignment can be set globally with the `-Zmin-function-alignment=` flag; + // the alignment from a `#[repr(align())]` is used if it specifies a higher alignment. + if let Some(align) = + Ord::max(cx.tcx.sess.opts.unstable_opts.min_function_alignment, codegen_fn_attrs.alignment) + { llvm::set_alignment(llfn, align); } if let Some(backchain) = backchain_attr(cx) { diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index cac3cc587cb..8df270abc81 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -132,9 +132,13 @@ fn prefix_and_suffix<'tcx>( let attrs = tcx.codegen_fn_attrs(instance.def_id()); let link_section = attrs.link_section.map(|symbol| symbol.as_str().to_string()); - let align = attrs.alignment.map(|a| a.bytes()).unwrap_or(4); - // See https://sourceware.org/binutils/docs/as/ARM-Directives.html for info on these directives. + // function alignment can be set globally with the `-Zmin-function-alignment=` flag; + // the alignment from a `#[repr(align())]` is used if it specifies a higher alignment. + // if no alignment is specified, an alignment of 4 bytes is used. + let min_function_alignment = tcx.sess.opts.unstable_opts.min_function_alignment; + let align = Ord::max(min_function_alignment, attrs.alignment).map(|a| a.bytes()).unwrap_or(4); + // In particular, `.arm` can also be written `.code 32` and `.thumb` as `.code 16`. let (arch_prefix, arch_suffix) = if is_arm { ( diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index d103f7f45e2..5648b9a2311 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -24,6 +24,7 @@ use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, build_session, filesearc use rustc_span::edition::{DEFAULT_EDITION, Edition}; use rustc_span::source_map::{RealFileLoader, SourceMapInputs}; use rustc_span::{FileName, SourceFileHashAlgorithm, sym}; +use rustc_target::abi::Align; use rustc_target::spec::{ CodeModel, FramePointer, LinkerFlavorCli, MergeFunctions, OnBrokenPipe, PanicStrategy, RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel, WasmCAbi, @@ -802,6 +803,7 @@ fn test_unstable_options_tracking_hash() { tracked!(location_detail, LocationDetail { file: true, line: false, column: false }); tracked!(maximal_hir_to_mir_coverage, true); tracked!(merge_functions, Some(MergeFunctions::Disabled)); + tracked!(min_function_alignment, Some(Align::EIGHT)); tracked!(mir_emit_retag, true); tracked!(mir_enable_passes, vec![("DestProp".to_string(), false)]); tracked!(mir_keep_place_mention, true); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 4784a4d1953..ea6bb18d5a5 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2891,6 +2891,7 @@ pub(crate) mod dep_tracking { use std::num::NonZero; use std::path::PathBuf; + use rustc_abi::Align; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::stable_hasher::Hash64; use rustc_errors::LanguageIdentifier; @@ -3011,6 +3012,7 @@ pub(crate) mod dep_tracking { InliningThreshold, FunctionReturn, WasmCAbi, + Align, ); impl DepTrackingHash for (T1, T2) diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 3873a203bd3..69cee65c5cd 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -4,6 +4,7 @@ use std::num::{IntErrorKind, NonZero}; use std::path::PathBuf; use std::str; +use rustc_abi::Align; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::profiling::TimePassesFormat; use rustc_data_structures::stable_hasher::Hash64; @@ -455,6 +456,7 @@ mod desc { pub(crate) const parse_wasm_c_abi: &str = "`legacy` or `spec`"; pub(crate) const parse_mir_include_spans: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or `nll` (default: `nll`)"; + pub(crate) const parse_align: &str = "a number that is a power of 2 between 1 and 2^29"; } pub mod parse { @@ -1533,6 +1535,21 @@ pub mod parse { true } + + pub(crate) fn parse_align(slot: &mut Option, v: Option<&str>) -> bool { + let mut bytes = 0u64; + if !parse_number(&mut bytes, v) { + return false; + } + + let Ok(align) = Align::from_bytes(bytes) else { + return false; + }; + + *slot = Some(align); + + true + } } options! { @@ -1888,6 +1905,8 @@ options! { "gather metadata statistics (default: no)"), metrics_dir: Option = (None, parse_opt_pathbuf, [UNTRACKED], "the directory metrics emitted by rustc are dumped into (implicitly enables default set of metrics)"), + min_function_alignment: Option = (None, parse_align, [TRACKED], + "align all functions to at least this many bytes. Must be a power of 2"), mir_emit_retag: bool = (false, parse_bool, [TRACKED], "emit Retagging MIR statements, interpreted e.g., by miri; implies -Zmir-opt-level=0 \ (default: no)"), diff --git a/src/doc/unstable-book/src/compiler-flags/min-function-alignment.md b/src/doc/unstable-book/src/compiler-flags/min-function-alignment.md new file mode 100644 index 00000000000..b7a3aa71fc4 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/min-function-alignment.md @@ -0,0 +1,24 @@ +# `min-function-alignment` + +The tracking issue for this feature is: https://github.com/rust-lang/rust/issues/82232. + +------------------------ + +The `-Zmin-function-alignment=` flag specifies the minimum alignment of functions for which code is generated. +The `align` value must be a power of 2, other values are rejected. + +Note that `-Zbuild-std` (or similar) is required to apply this minimum alignment to standard library functions. +By default, these functions come precompiled and their alignments won't respect the `min-function-alignment` flag. + +This flag is equivalent to: + +- `-fmin-function-alignment` for [GCC](https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html#index-fmin-function-alignment_003dn) +- `-falign-functions` for [Clang](https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang1-falign-functions) + +The specified alignment is a minimum. A higher alignment can be specified for specific functions by using the [`repr(align(...))`](https://github.com/rust-lang/rust/issues/82232) feature and annotating the function with a `#[repr(align())]` attribute. The attribute's value is ignored when it is lower than the value passed to `min-function-alignment`. + +There are two additional edge cases for this flag: + +- targets have a minimum alignment for functions (e.g. on x86_64 the lowest that LLVM generates is 16 bytes). + A `min-function-alignment` value lower than the target's minimum has no effect. +- the maximum alignment supported by rust (and LLVM) is `2^29`. Trying to set a higher value results in an error. diff --git a/tests/codegen/min-function-alignment.rs b/tests/codegen/min-function-alignment.rs new file mode 100644 index 00000000000..7c0ad12402a --- /dev/null +++ b/tests/codegen/min-function-alignment.rs @@ -0,0 +1,43 @@ +//@ revisions: align16 align1024 +//@ compile-flags: -C no-prepopulate-passes -Z mir-opt-level=0 +//@ [align16] compile-flags: -Zmin-function-alignment=16 +//@ [align1024] compile-flags: -Zmin-function-alignment=1024 + +#![crate_type = "lib"] +#![feature(fn_align)] + +// functions without explicit alignment use the global minimum +// +// CHECK-LABEL: @no_explicit_align +// align16: align 16 +// align1024: align 1024 +#[no_mangle] +pub fn no_explicit_align() {} + +// CHECK-LABEL: @lower_align +// align16: align 16 +// align1024: align 1024 +#[no_mangle] +#[repr(align(8))] +pub fn lower_align() {} + +// the higher value of min-function-alignment and repr(align) wins out +// +// CHECK-LABEL: @higher_align +// align16: align 32 +// align1024: align 1024 +#[no_mangle] +#[repr(align(32))] +pub fn higher_align() {} + +// cold functions follow the same rules as other functions +// +// in GCC, the `-falign-functions` does not apply to cold functions, but +// `-Zmin-function-alignment` applies to all functions. +// +// CHECK-LABEL: @no_explicit_align_cold +// align16: align 16 +// align1024: align 1024 +#[no_mangle] +#[cold] +pub fn no_explicit_align_cold() {} diff --git a/tests/codegen/naked-fn/min-function-alignment.rs b/tests/codegen/naked-fn/min-function-alignment.rs new file mode 100644 index 00000000000..1330d796d39 --- /dev/null +++ b/tests/codegen/naked-fn/min-function-alignment.rs @@ -0,0 +1,44 @@ +//@ compile-flags: -C no-prepopulate-passes -Copt-level=0 -Zmin-function-alignment=16 +//@ needs-asm-support +//@ ignore-arm no "ret" mnemonic + +#![feature(naked_functions, fn_align)] +#![crate_type = "lib"] + +// functions without explicit alignment use the global minimum +// +// CHECK: .balign 16 +#[no_mangle] +#[naked] +pub unsafe extern "C" fn naked_no_explicit_align() { + core::arch::naked_asm!("ret") +} + +// CHECK: .balign 16 +#[no_mangle] +#[repr(align(8))] +#[naked] +pub unsafe extern "C" fn naked_lower_align() { + core::arch::naked_asm!("ret") +} + +// CHECK: .balign 32 +#[no_mangle] +#[repr(align(32))] +#[naked] +pub unsafe extern "C" fn naked_higher_align() { + core::arch::naked_asm!("ret") +} + +// cold functions follow the same rules as other functions +// +// in GCC, the `-falign-functions` does not apply to cold functions, but +// `-Zmin-function-alignment` applies to all functions. +// +// CHECK: .balign 16 +#[no_mangle] +#[cold] +#[naked] +pub unsafe extern "C" fn no_explicit_align_cold() { + core::arch::naked_asm!("ret") +} -- cgit 1.4.1-3-g733a5 From a13354bea05968799a5be5521691322274fa6a9e Mon Sep 17 00:00:00 2001 From: Rémy Rakic Date: Tue, 7 Jan 2025 15:19:05 +0000 Subject: rename `BitSet` to `DenseBitSet` This should make it clearer that this bitset is dense, with the advantages and disadvantages that it entails. --- compiler/rustc_borrowck/src/borrow_set.rs | 9 ++- compiler/rustc_borrowck/src/dataflow.rs | 14 ++-- compiler/rustc_borrowck/src/lib.rs | 6 +- .../src/type_check/liveness/trace.rs | 6 +- .../src/debuginfo/create_scope_map.rs | 10 +-- compiler/rustc_codegen_ssa/src/mir/analyze.rs | 6 +- compiler/rustc_codegen_ssa/src/mir/mod.rs | 6 +- .../rustc_const_eval/src/check_consts/check.rs | 6 +- .../src/graph/implementation/mod.rs | 8 +- .../rustc_data_structures/src/graph/iterate/mod.rs | 14 ++-- .../rustc_data_structures/src/stable_hasher.rs | 4 +- .../src/stable_hasher/tests.rs | 6 +- compiler/rustc_data_structures/src/work_queue.rs | 6 +- compiler/rustc_hir_analysis/src/check/check.rs | 2 +- compiler/rustc_hir_analysis/src/check/mod.rs | 2 +- compiler/rustc_index/src/bit_set.rs | 94 +++++++++++----------- compiler/rustc_index/src/bit_set/tests.rs | 46 +++++------ compiler/rustc_metadata/src/rmeta/mod.rs | 4 +- compiler/rustc_middle/src/mir/coverage.rs | 6 +- compiler/rustc_middle/src/mir/mod.rs | 2 +- compiler/rustc_middle/src/mir/traversal.rs | 26 +++--- compiler/rustc_middle/src/query/mod.rs | 4 +- compiler/rustc_middle/src/ty/context.rs | 2 +- compiler/rustc_middle/src/ty/parameterized.rs | 2 +- compiler/rustc_mir_dataflow/src/debuginfo.rs | 8 +- .../rustc_mir_dataflow/src/framework/cursor.rs | 4 +- compiler/rustc_mir_dataflow/src/framework/fmt.rs | 4 +- .../rustc_mir_dataflow/src/framework/graphviz.rs | 4 +- .../rustc_mir_dataflow/src/framework/lattice.rs | 10 +-- compiler/rustc_mir_dataflow/src/framework/mod.rs | 6 +- compiler/rustc_mir_dataflow/src/framework/tests.rs | 10 +-- .../src/impls/borrowed_locals.rs | 12 +-- .../rustc_mir_dataflow/src/impls/initialized.rs | 8 +- compiler/rustc_mir_dataflow/src/impls/liveness.rs | 20 ++--- .../src/impls/storage_liveness.rs | 28 +++---- compiler/rustc_mir_dataflow/src/points.rs | 6 +- compiler/rustc_mir_dataflow/src/value_analysis.rs | 10 +-- compiler/rustc_mir_transform/src/copy_prop.rs | 14 ++-- compiler/rustc_mir_transform/src/coroutine.rs | 40 ++++----- .../rustc_mir_transform/src/coverage/counters.rs | 11 ++- compiler/rustc_mir_transform/src/coverage/graph.rs | 6 +- .../rustc_mir_transform/src/coverage/mappings.rs | 10 +-- compiler/rustc_mir_transform/src/coverage/query.rs | 28 +++---- .../src/dead_store_elimination.rs | 4 +- .../rustc_mir_transform/src/deduce_param_attrs.rs | 6 +- compiler/rustc_mir_transform/src/dest_prop.rs | 13 +-- .../rustc_mir_transform/src/elaborate_drops.rs | 6 +- compiler/rustc_mir_transform/src/gvn.rs | 8 +- compiler/rustc_mir_transform/src/inline.rs | 8 +- compiler/rustc_mir_transform/src/jump_threading.rs | 8 +- .../rustc_mir_transform/src/known_panics_lint.rs | 10 +-- compiler/rustc_mir_transform/src/lint.rs | 4 +- .../src/multiple_return_terminators.rs | 4 +- compiler/rustc_mir_transform/src/nrvo.rs | 4 +- compiler/rustc_mir_transform/src/prettify.rs | 10 ++- compiler/rustc_mir_transform/src/ref_prop.rs | 10 +-- .../src/remove_noop_landing_pads.rs | 6 +- .../rustc_mir_transform/src/single_use_consts.rs | 10 +-- compiler/rustc_mir_transform/src/sroa.rs | 18 ++--- compiler/rustc_mir_transform/src/ssa.rs | 14 ++-- compiler/rustc_mir_transform/src/validate.rs | 4 +- compiler/rustc_pattern_analysis/src/constructor.rs | 4 +- compiler/rustc_pattern_analysis/src/usefulness.rs | 14 ++-- compiler/rustc_ty_utils/src/layout.rs | 8 +- compiler/rustc_ty_utils/src/representability.rs | 8 +- compiler/rustc_ty_utils/src/ty.rs | 8 +- compiler/rustc_type_ir/src/interner.rs | 4 +- 67 files changed, 367 insertions(+), 356 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_borrowck/src/borrow_set.rs b/compiler/rustc_borrowck/src/borrow_set.rs index a29833464fb..303fa469332 100644 --- a/compiler/rustc_borrowck/src/borrow_set.rs +++ b/compiler/rustc_borrowck/src/borrow_set.rs @@ -2,7 +2,7 @@ use std::fmt; use std::ops::Index; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{MutatingUseContext, NonUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, Body, Local, Location, traversal}; use rustc_middle::span_bug; @@ -131,7 +131,7 @@ impl<'tcx> fmt::Display for BorrowData<'tcx> { pub enum LocalsStateAtExit { AllAreInvalidated, - SomeAreInvalidated { has_storage_dead_or_moved: BitSet }, + SomeAreInvalidated { has_storage_dead_or_moved: DenseBitSet }, } impl LocalsStateAtExit { @@ -140,7 +140,7 @@ impl LocalsStateAtExit { body: &Body<'tcx>, move_data: &MoveData<'tcx>, ) -> Self { - struct HasStorageDead(BitSet); + struct HasStorageDead(DenseBitSet); impl<'tcx> Visitor<'tcx> for HasStorageDead { fn visit_local(&mut self, local: Local, ctx: PlaceContext, _: Location) { @@ -153,7 +153,8 @@ impl LocalsStateAtExit { if locals_are_invalidated_at_exit { LocalsStateAtExit::AllAreInvalidated } else { - let mut has_storage_dead = HasStorageDead(BitSet::new_empty(body.local_decls.len())); + let mut has_storage_dead = + HasStorageDead(DenseBitSet::new_empty(body.local_decls.len())); has_storage_dead.visit_body(body); let mut has_storage_dead_or_moved = has_storage_dead.0; for move_out in &move_data.moves { diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs index abe4d4f20ec..a7a6f2da509 100644 --- a/compiler/rustc_borrowck/src/dataflow.rs +++ b/compiler/rustc_borrowck/src/dataflow.rs @@ -2,7 +2,7 @@ use std::fmt; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::graph; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{ self, BasicBlock, Body, CallReturnPlaces, Location, Place, TerminatorEdges, }; @@ -180,7 +180,7 @@ pub struct Borrows<'a, 'tcx> { } struct OutOfScopePrecomputer<'a, 'tcx> { - visited: BitSet, + visited: DenseBitSet, visit_stack: Vec, body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>, @@ -190,7 +190,7 @@ struct OutOfScopePrecomputer<'a, 'tcx> { impl<'a, 'tcx> OutOfScopePrecomputer<'a, 'tcx> { fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self { OutOfScopePrecomputer { - visited: BitSet::new_empty(body.basic_blocks.len()), + visited: DenseBitSet::new_empty(body.basic_blocks.len()), visit_stack: vec![], body, regioncx, @@ -292,7 +292,7 @@ pub fn calculate_borrows_out_of_scope_at_location<'tcx>( } struct PoloniusOutOfScopePrecomputer<'a, 'tcx> { - visited: BitSet, + visited: DenseBitSet, visit_stack: Vec, body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>, @@ -303,7 +303,7 @@ struct PoloniusOutOfScopePrecomputer<'a, 'tcx> { impl<'a, 'tcx> PoloniusOutOfScopePrecomputer<'a, 'tcx> { fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self { Self { - visited: BitSet::new_empty(body.basic_blocks.len()), + visited: DenseBitSet::new_empty(body.basic_blocks.len()), visit_stack: vec![], body, regioncx, @@ -559,7 +559,7 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> { } } -type BorrowsDomain = BitSet; +type BorrowsDomain = DenseBitSet; /// Forward dataflow computation of the set of borrows that are in scope at a particular location. /// - we gen the introduced loans @@ -575,7 +575,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { fn bottom_value(&self, _: &mir::Body<'tcx>) -> Self::Domain { // bottom = nothing is reserved or activated yet; - BitSet::new_empty(self.borrow_set.len()) + DenseBitSet::new_empty(self.borrow_set.len()) } fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) { diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 4f46a2cef7c..45764f9c6b8 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -28,7 +28,7 @@ use rustc_errors::LintDiagnostic; use rustc_hir as hir; use rustc_hir::CRATE_HIR_ID; use rustc_hir::def_id::LocalDefId; -use rustc_index::bit_set::{BitSet, MixedBitSet}; +use rustc_index::bit_set::{DenseBitSet, MixedBitSet}; use rustc_index::{IndexSlice, IndexVec}; use rustc_infer::infer::{ InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt, @@ -1017,11 +1017,11 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { &self, location: Location, state: &'s BorrowckDomain, - ) -> Cow<'s, BitSet> { + ) -> Cow<'s, DenseBitSet> { if let Some(polonius) = &self.polonius_output { // Use polonius output if it has been enabled. let location = self.location_table.start_index(location); - let mut polonius_output = BitSet::new_empty(self.borrow_set.len()); + let mut polonius_output = DenseBitSet::new_empty(self.borrow_set.len()); for &idx in polonius.errors_at(location) { polonius_output.insert(idx); } diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 7fce49fd16b..4c0d3138f2d 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -1,5 +1,5 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::IntervalSet; use rustc_infer::infer::canonical::QueryRegionConstraints; use rustc_infer::infer::outlives::for_liveness; @@ -129,7 +129,7 @@ struct LivenessResults<'a, 'typeck, 'b, 'tcx> { cx: LivenessContext<'a, 'typeck, 'b, 'tcx>, /// Set of points that define the current local. - defs: BitSet, + defs: DenseBitSet, /// Points where the current variable is "use live" -- meaning /// that there is a future "full use" that may use its value. @@ -152,7 +152,7 @@ impl<'a, 'typeck, 'b, 'tcx> LivenessResults<'a, 'typeck, 'b, 'tcx> { let num_points = cx.location_map.num_points(); LivenessResults { cx, - defs: BitSet::new_empty(num_points), + defs: DenseBitSet::new_empty(num_points), use_live_at: IntervalSet::new(num_points), drop_live_at: IntervalSet::new(num_points), drop_locations: vec![], diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 07bd0f4d1c1..e545ce386ed 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -4,7 +4,7 @@ use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; use rustc_index::Idx; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{Body, SourceScope}; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv}; use rustc_middle::ty::{self, Instance}; @@ -27,7 +27,7 @@ pub(crate) fn compute_mir_scopes<'ll, 'tcx>( ) { // Find all scopes with variables defined in them. let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { - let mut vars = BitSet::new_empty(mir.source_scopes.len()); + let mut vars = DenseBitSet::new_empty(mir.source_scopes.len()); // FIXME(eddyb) take into account that arguments always have debuginfo, // irrespective of their name (assuming full debuginfo is enabled). // NOTE(eddyb) actually, on second thought, those are always in the @@ -40,7 +40,7 @@ pub(crate) fn compute_mir_scopes<'ll, 'tcx>( // Nothing to emit, of course. None }; - let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); + let mut instantiated = DenseBitSet::new_empty(mir.source_scopes.len()); let mut discriminators = FxHashMap::default(); // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { @@ -63,9 +63,9 @@ fn make_mir_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - variables: &Option>, + variables: &Option>, debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, - instantiated: &mut BitSet, + instantiated: &mut DenseBitSet, discriminators: &mut FxHashMap, scope: SourceScope, ) { diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 43b8230c679..23baab3124e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -2,7 +2,7 @@ //! which do not. use rustc_data_structures::graph::dominators::Dominators; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind, traversal}; @@ -16,7 +16,7 @@ use crate::traits::*; pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( fx: &FunctionCx<'a, 'tcx, Bx>, traversal_order: &[mir::BasicBlock], -) -> BitSet { +) -> DenseBitSet { let mir = fx.mir; let dominators = mir.basic_blocks.dominators(); let locals = mir @@ -44,7 +44,7 @@ pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( analyzer.visit_basic_block_data(bb, data); } - let mut non_ssa_locals = BitSet::new_empty(analyzer.locals.len()); + let mut non_ssa_locals = DenseBitSet::new_empty(analyzer.locals.len()); for (local, kind) in analyzer.locals.iter_enumerated() { if matches!(kind, LocalKind::Memory) { non_ssa_locals.insert(local); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 62f69af3f2f..3a896071bc6 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -1,7 +1,7 @@ use std::iter; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::{UnwindTerminateReason, traversal}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout}; @@ -293,7 +293,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // So drop the builder of `start_llbb` to avoid having two at the same time. drop(start_bx); - let mut unreached_blocks = BitSet::new_filled(mir.basic_blocks.len()); + let mut unreached_blocks = DenseBitSet::new_filled(mir.basic_blocks.len()); // Codegen the body of each reachable block using our reverse postorder list. for bb in traversal_order { fx.codegen_block(bb); @@ -316,7 +316,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, fx: &mut FunctionCx<'a, 'tcx, Bx>, - memory_locals: &BitSet, + memory_locals: &DenseBitSet, ) -> Vec> { let mir = fx.mir; let mut idx = 0; diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index 015e1671682..7a5f6c17268 100644 --- a/compiler/rustc_const_eval/src/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -10,7 +10,7 @@ use rustc_attr_parsing::{ConstStability, StabilityLevel}; use rustc_errors::{Diag, ErrorGuaranteed}; use rustc_hir::def_id::DefId; use rustc_hir::{self as hir, LangItem}; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_infer::infer::TyCtxtInferExt; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; @@ -172,7 +172,7 @@ pub struct Checker<'mir, 'tcx> { /// A set that stores for each local whether it is "transient", i.e. guaranteed to be dead /// when this MIR body returns. - transient_locals: Option>, + transient_locals: Option>, error_emitted: Option, secondary_errors: Vec>, @@ -242,7 +242,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { // And then check all `Return` in the MIR, and if a local is "maybe live" at a // `Return` then it is definitely not transient. - let mut transient = BitSet::new_filled(ccx.body.local_decls.len()); + let mut transient = DenseBitSet::new_filled(ccx.body.local_decls.len()); // Make sure to only visit reachable blocks, the dataflow engine can ICE otherwise. for (bb, data) in traversal::reachable(&ccx.body) { if matches!(data.terminator().kind, TerminatorKind::Return) { diff --git a/compiler/rustc_data_structures/src/graph/implementation/mod.rs b/compiler/rustc_data_structures/src/graph/implementation/mod.rs index 43fdfe6ee0d..7724e9347d8 100644 --- a/compiler/rustc_data_structures/src/graph/implementation/mod.rs +++ b/compiler/rustc_data_structures/src/graph/implementation/mod.rs @@ -22,7 +22,7 @@ use std::fmt::Debug; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use tracing::debug; #[cfg(test)] @@ -214,7 +214,7 @@ impl Graph { direction: Direction, entry_node: NodeIndex, ) -> Vec { - let mut visited = BitSet::new_empty(self.len_nodes()); + let mut visited = DenseBitSet::new_empty(self.len_nodes()); let mut stack = vec![]; let mut result = Vec::with_capacity(self.len_nodes()); let mut push_node = |stack: &mut Vec<_>, node: NodeIndex| { @@ -287,7 +287,7 @@ impl<'g, N: Debug, E: Debug> Iterator for AdjacentEdges<'g, N, E> { pub struct DepthFirstTraversal<'g, N, E> { graph: &'g Graph, stack: Vec, - visited: BitSet, + visited: DenseBitSet, direction: Direction, } @@ -297,7 +297,7 @@ impl<'g, N: Debug, E: Debug> DepthFirstTraversal<'g, N, E> { start_node: NodeIndex, direction: Direction, ) -> Self { - let mut visited = BitSet::new_empty(graph.len_nodes()); + let mut visited = DenseBitSet::new_empty(graph.len_nodes()); visited.insert(start_node.node_id()); DepthFirstTraversal { graph, stack: vec![start_node], visited, direction } } diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs index cbc6664d853..ecda7d3fba8 100644 --- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs +++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs @@ -1,6 +1,6 @@ use std::ops::ControlFlow; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; use super::{DirectedGraph, StartNode, Successors}; @@ -78,7 +78,7 @@ where { graph: G, stack: Vec, - visited: BitSet, + visited: DenseBitSet, } impl DepthFirstSearch @@ -86,7 +86,7 @@ where G: DirectedGraph + Successors, { pub fn new(graph: G) -> Self { - Self { stack: vec![], visited: BitSet::new_empty(graph.num_nodes()), graph } + Self { stack: vec![], visited: DenseBitSet::new_empty(graph.num_nodes()), graph } } /// Version of `push_start_node` that is convenient for chained @@ -207,8 +207,8 @@ where { graph: &'graph G, stack: Vec>, - visited: BitSet, - settled: BitSet, + visited: DenseBitSet, + settled: DenseBitSet, } impl<'graph, G> TriColorDepthFirstSearch<'graph, G> @@ -219,8 +219,8 @@ where TriColorDepthFirstSearch { graph, stack: vec![], - visited: BitSet::new_empty(graph.num_nodes()), - settled: BitSet::new_empty(graph.num_nodes()), + visited: DenseBitSet::new_empty(graph.num_nodes()), + settled: DenseBitSet::new_empty(graph.num_nodes()), } } diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs index 0872bd2c9ac..9cd0cc499ca 100644 --- a/compiler/rustc_data_structures/src/stable_hasher.rs +++ b/compiler/rustc_data_structures/src/stable_hasher.rs @@ -3,7 +3,7 @@ use std::marker::PhantomData; use std::mem; use std::num::NonZero; -use rustc_index::bit_set::{self, BitSet}; +use rustc_index::bit_set::{self, DenseBitSet}; use rustc_index::{Idx, IndexSlice, IndexVec}; use smallvec::SmallVec; @@ -544,7 +544,7 @@ where } } -impl HashStable for BitSet { +impl HashStable for DenseBitSet { fn hash_stable(&self, _ctx: &mut CTX, hasher: &mut StableHasher) { ::std::hash::Hash::hash(self, hasher); } diff --git a/compiler/rustc_data_structures/src/stable_hasher/tests.rs b/compiler/rustc_data_structures/src/stable_hasher/tests.rs index aab50a13af0..635f241847c 100644 --- a/compiler/rustc_data_structures/src/stable_hasher/tests.rs +++ b/compiler/rustc_data_structures/src/stable_hasher/tests.rs @@ -17,9 +17,9 @@ fn hash>(t: &T) -> Hash128 { // Check that bit set hash includes the domain size. #[test] fn test_hash_bit_set() { - use rustc_index::bit_set::BitSet; - let a: BitSet = BitSet::new_empty(1); - let b: BitSet = BitSet::new_empty(2); + use rustc_index::bit_set::DenseBitSet; + let a: DenseBitSet = DenseBitSet::new_empty(1); + let b: DenseBitSet = DenseBitSet::new_empty(2); assert_ne!(a, b); assert_ne!(hash(&a), hash(&b)); } diff --git a/compiler/rustc_data_structures/src/work_queue.rs b/compiler/rustc_data_structures/src/work_queue.rs index ca052e2eac6..815756edfeb 100644 --- a/compiler/rustc_data_structures/src/work_queue.rs +++ b/compiler/rustc_data_structures/src/work_queue.rs @@ -1,7 +1,7 @@ use std::collections::VecDeque; use rustc_index::Idx; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; /// A work queue is a handy data structure for tracking work left to /// do. (For example, basic blocks left to process.) It is basically a @@ -11,14 +11,14 @@ use rustc_index::bit_set::BitSet; /// and also use a bit set to track occupancy. pub struct WorkQueue { deque: VecDeque, - set: BitSet, + set: DenseBitSet, } impl WorkQueue { /// Creates a new work queue that starts empty, where elements range from (0..len). #[inline] pub fn with_none(len: usize) -> Self { - WorkQueue { deque: VecDeque::with_capacity(len), set: BitSet::new_empty(len) } + WorkQueue { deque: VecDeque::with_capacity(len), set: DenseBitSet::new_empty(len) } } /// Attempt to enqueue `element` in the work queue. Returns false if it was already present. diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index 8c6059d49a8..821829455d9 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -1666,7 +1666,7 @@ fn check_type_alias_type_params_are_used<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalD .collect::>() }); - let mut params_used = BitSet::new_empty(generics.own_params.len()); + let mut params_used = DenseBitSet::new_empty(generics.own_params.len()); for leaf in ty.walk() { if let GenericArgKind::Type(leaf_ty) = leaf.unpack() && let ty::Param(param) = leaf_ty.kind() diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs index 0b0c92a726d..92b18c80fd8 100644 --- a/compiler/rustc_hir_analysis/src/check/mod.rs +++ b/compiler/rustc_hir_analysis/src/check/mod.rs @@ -79,7 +79,7 @@ use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_errors::{Diag, ErrorGuaranteed, pluralize, struct_span_code_err}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::Visitor; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_infer::infer::outlives::env::OutlivesEnvironment; use rustc_infer::infer::{self, TyCtxtInferExt as _}; use rustc_infer::traits::ObligationCause; diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 38e2dbbde7d..73dd040a69c 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -108,33 +108,33 @@ macro_rules! bit_relations_inherent_impls { /// #[cfg_attr(feature = "nightly", derive(Decodable_Generic, Encodable_Generic))] #[derive(Eq, PartialEq, Hash)] -pub struct BitSet { +pub struct DenseBitSet { domain_size: usize, words: SmallVec<[Word; 2]>, marker: PhantomData, } -impl BitSet { +impl DenseBitSet { /// Gets the domain size. pub fn domain_size(&self) -> usize { self.domain_size } } -impl BitSet { +impl DenseBitSet { /// Creates a new, empty bitset with a given `domain_size`. #[inline] - pub fn new_empty(domain_size: usize) -> BitSet { + pub fn new_empty(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); - BitSet { domain_size, words: smallvec![0; num_words], marker: PhantomData } + DenseBitSet { domain_size, words: smallvec![0; num_words], marker: PhantomData } } /// Creates a new, filled bitset with a given `domain_size`. #[inline] - pub fn new_filled(domain_size: usize) -> BitSet { + pub fn new_filled(domain_size: usize) -> DenseBitSet { let num_words = num_words(domain_size); let mut result = - BitSet { domain_size, words: smallvec![!0; num_words], marker: PhantomData }; + DenseBitSet { domain_size, words: smallvec![!0; num_words], marker: PhantomData }; result.clear_excess_bits(); result } @@ -165,7 +165,7 @@ impl BitSet { /// Is `self` is a (non-strict) superset of `other`? #[inline] - pub fn superset(&self, other: &BitSet) -> bool { + pub fn superset(&self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); self.words.iter().zip(&other.words).all(|(a, b)| (a & b) == *b) } @@ -278,32 +278,36 @@ impl BitSet { } // dense REL dense -impl BitRelations> for BitSet { - fn union(&mut self, other: &BitSet) -> bool { +impl BitRelations> for DenseBitSet { + fn union(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); bitwise(&mut self.words, &other.words, |a, b| a | b) } - fn subtract(&mut self, other: &BitSet) -> bool { + fn subtract(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); bitwise(&mut self.words, &other.words, |a, b| a & !b) } - fn intersect(&mut self, other: &BitSet) -> bool { + fn intersect(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); bitwise(&mut self.words, &other.words, |a, b| a & b) } } -impl From> for BitSet { +impl From> for DenseBitSet { fn from(bit_set: GrowableBitSet) -> Self { bit_set.bit_set } } -impl Clone for BitSet { +impl Clone for DenseBitSet { fn clone(&self) -> Self { - BitSet { domain_size: self.domain_size, words: self.words.clone(), marker: PhantomData } + DenseBitSet { + domain_size: self.domain_size, + words: self.words.clone(), + marker: PhantomData, + } } fn clone_from(&mut self, from: &Self) { @@ -312,13 +316,13 @@ impl Clone for BitSet { } } -impl fmt::Debug for BitSet { +impl fmt::Debug for DenseBitSet { fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { w.debug_list().entries(self.iter()).finish() } } -impl ToString for BitSet { +impl ToString for DenseBitSet { fn to_string(&self) -> String { let mut result = String::new(); let mut sep = '['; @@ -902,7 +906,7 @@ impl BitRelations> for ChunkedBitSet { } } -impl BitRelations> for BitSet { +impl BitRelations> for DenseBitSet { fn union(&mut self, other: &ChunkedBitSet) -> bool { sequential_update(|elem| self.insert(elem), other.iter()) } @@ -1114,10 +1118,10 @@ where false } -/// A bitset with a mixed representation, using `BitSet` for small and medium -/// bitsets, and `ChunkedBitSet` for large bitsets, i.e. those with enough bits -/// for at least two chunks. This is a good choice for many bitsets that can -/// have large domain sizes (e.g. 5000+). +/// A bitset with a mixed representation, using `DenseBitSet` for small and +/// medium bitsets, and `ChunkedBitSet` for large bitsets, i.e. those with +/// enough bits for at least two chunks. This is a good choice for many bitsets +/// that can have large domain sizes (e.g. 5000+). /// /// `T` is an index type, typically a newtyped `usize` wrapper, but it can also /// just be `usize`. @@ -1127,7 +1131,7 @@ where /// will panic if the bitsets have differing domain sizes. #[derive(PartialEq, Eq)] pub enum MixedBitSet { - Small(BitSet), + Small(DenseBitSet), Large(ChunkedBitSet), } @@ -1144,7 +1148,7 @@ impl MixedBitSet { #[inline] pub fn new_empty(domain_size: usize) -> MixedBitSet { if domain_size <= CHUNK_BITS { - MixedBitSet::Small(BitSet::new_empty(domain_size)) + MixedBitSet::Small(DenseBitSet::new_empty(domain_size)) } else { MixedBitSet::Large(ChunkedBitSet::new_empty(domain_size)) } @@ -1283,7 +1287,7 @@ impl<'a, T: Idx> Iterator for MixedBitIter<'a, T> { /// to or greater than the domain size. #[derive(Clone, Debug, PartialEq)] pub struct GrowableBitSet { - bit_set: BitSet, + bit_set: DenseBitSet, } impl Default for GrowableBitSet { @@ -1306,11 +1310,11 @@ impl GrowableBitSet { } pub fn new_empty() -> GrowableBitSet { - GrowableBitSet { bit_set: BitSet::new_empty(0) } + GrowableBitSet { bit_set: DenseBitSet::new_empty(0) } } pub fn with_capacity(capacity: usize) -> GrowableBitSet { - GrowableBitSet { bit_set: BitSet::new_empty(capacity) } + GrowableBitSet { bit_set: DenseBitSet::new_empty(capacity) } } /// Returns `true` if the set has changed. @@ -1349,8 +1353,8 @@ impl GrowableBitSet { } } -impl From> for GrowableBitSet { - fn from(bit_set: BitSet) -> Self { +impl From> for GrowableBitSet { + fn from(bit_set: DenseBitSet) -> Self { Self { bit_set } } } @@ -1386,7 +1390,7 @@ impl BitMatrix { } /// Creates a new matrix, with `row` used as the value for every row. - pub fn from_row_n(row: &BitSet, num_rows: usize) -> BitMatrix { + pub fn from_row_n(row: &DenseBitSet, num_rows: usize) -> BitMatrix { let num_columns = row.domain_size(); let words_per_row = num_words(num_columns); assert_eq!(words_per_row, row.words.len()); @@ -1484,7 +1488,7 @@ impl BitMatrix { /// Adds the bits from `with` to the bits from row `write`, and /// returns `true` if anything changed. - pub fn union_row_with(&mut self, with: &BitSet, write: R) -> bool { + pub fn union_row_with(&mut self, with: &DenseBitSet, write: R) -> bool { assert!(write.index() < self.num_rows); assert_eq!(with.domain_size(), self.num_columns); let (write_start, write_end) = self.range(write); @@ -1541,8 +1545,8 @@ impl fmt::Debug for BitMatrix { /// A fixed-column-size, variable-row-size 2D bit matrix with a moderately /// sparse representation. /// -/// Initially, every row has no explicit representation. If any bit within a -/// row is set, the entire row is instantiated as `Some()`. +/// Initially, every row has no explicit representation. If any bit within a row +/// is set, the entire row is instantiated as `Some()`. /// Furthermore, any previously uninstantiated rows prior to it will be /// instantiated as `None`. Those prior rows may themselves become fully /// instantiated later on if any of their bits are set. @@ -1556,7 +1560,7 @@ where C: Idx, { num_columns: usize, - rows: IndexVec>>, + rows: IndexVec>>, } impl SparseBitMatrix { @@ -1565,10 +1569,10 @@ impl SparseBitMatrix { Self { num_columns, rows: IndexVec::new() } } - fn ensure_row(&mut self, row: R) -> &mut BitSet { - // Instantiate any missing rows up to and including row `row` with an empty `BitSet`. - // Then replace row `row` with a full `BitSet` if necessary. - self.rows.get_or_insert_with(row, || BitSet::new_empty(self.num_columns)) + fn ensure_row(&mut self, row: R) -> &mut DenseBitSet { + // Instantiate any missing rows up to and including row `row` with an empty `DenseBitSet`. + // Then replace row `row` with a full `DenseBitSet` if necessary. + self.rows.get_or_insert_with(row, || DenseBitSet::new_empty(self.num_columns)) } /// Sets the cell at `(row, column)` to true. Put another way, insert @@ -1642,17 +1646,17 @@ impl SparseBitMatrix { self.row(row).into_iter().flat_map(|r| r.iter()) } - pub fn row(&self, row: R) -> Option<&BitSet> { + pub fn row(&self, row: R) -> Option<&DenseBitSet> { self.rows.get(row)?.as_ref() } - /// Intersects `row` with `set`. `set` can be either `BitSet` or + /// Intersects `row` with `set`. `set` can be either `DenseBitSet` or /// `ChunkedBitSet`. Has no effect if `row` does not exist. /// /// Returns true if the row was changed. pub fn intersect_row(&mut self, row: R, set: &Set) -> bool where - BitSet: BitRelations, + DenseBitSet: BitRelations, { match self.rows.get_mut(row) { Some(Some(row)) => row.intersect(set), @@ -1660,13 +1664,13 @@ impl SparseBitMatrix { } } - /// Subtracts `set` from `row`. `set` can be either `BitSet` or + /// Subtracts `set` from `row`. `set` can be either `DenseBitSet` or /// `ChunkedBitSet`. Has no effect if `row` does not exist. /// /// Returns true if the row was changed. pub fn subtract_row(&mut self, row: R, set: &Set) -> bool where - BitSet: BitRelations, + DenseBitSet: BitRelations, { match self.rows.get_mut(row) { Some(Some(row)) => row.subtract(set), @@ -1674,13 +1678,13 @@ impl SparseBitMatrix { } } - /// Unions `row` with `set`. `set` can be either `BitSet` or + /// Unions `row` with `set`. `set` can be either `DenseBitSet` or /// `ChunkedBitSet`. /// /// Returns true if the row was changed. pub fn union_row(&mut self, row: R, set: &Set) -> bool where - BitSet: BitRelations, + DenseBitSet: BitRelations, { self.ensure_row(row).union(set) } diff --git a/compiler/rustc_index/src/bit_set/tests.rs b/compiler/rustc_index/src/bit_set/tests.rs index f6142323979..0350740aa81 100644 --- a/compiler/rustc_index/src/bit_set/tests.rs +++ b/compiler/rustc_index/src/bit_set/tests.rs @@ -8,7 +8,7 @@ use test::Bencher; #[test] fn test_new_filled() { for i in 0..128 { - let idx_buf = BitSet::new_filled(i); + let idx_buf = DenseBitSet::new_filled(i); let elems: Vec = idx_buf.iter().collect(); let expected: Vec = (0..i).collect(); assert_eq!(elems, expected); @@ -17,7 +17,7 @@ fn test_new_filled() { #[test] fn bitset_iter_works() { - let mut bitset: BitSet = BitSet::new_empty(100); + let mut bitset: DenseBitSet = DenseBitSet::new_empty(100); bitset.insert(1); bitset.insert(10); bitset.insert(19); @@ -32,7 +32,7 @@ fn bitset_iter_works() { #[test] fn bitset_iter_works_2() { - let mut bitset: BitSet = BitSet::new_empty(320); + let mut bitset: DenseBitSet = DenseBitSet::new_empty(320); bitset.insert(0); bitset.insert(127); bitset.insert(191); @@ -43,25 +43,25 @@ fn bitset_iter_works_2() { #[test] fn bitset_clone_from() { - let mut a: BitSet = BitSet::new_empty(10); + let mut a: DenseBitSet = DenseBitSet::new_empty(10); a.insert(4); a.insert(7); a.insert(9); - let mut b = BitSet::new_empty(2); + let mut b = DenseBitSet::new_empty(2); b.clone_from(&a); assert_eq!(b.domain_size(), 10); assert_eq!(b.iter().collect::>(), [4, 7, 9]); - b.clone_from(&BitSet::new_empty(40)); + b.clone_from(&DenseBitSet::new_empty(40)); assert_eq!(b.domain_size(), 40); assert_eq!(b.iter().collect::>(), []); } #[test] fn union_two_sets() { - let mut set1: BitSet = BitSet::new_empty(65); - let mut set2: BitSet = BitSet::new_empty(65); + let mut set1: DenseBitSet = DenseBitSet::new_empty(65); + let mut set2: DenseBitSet = DenseBitSet::new_empty(65); assert!(set1.insert(3)); assert!(!set1.insert(3)); assert!(set2.insert(5)); @@ -268,8 +268,8 @@ fn with_elements_chunked(elements: &[usize], domain_size: usize) -> ChunkedBitSe s } -fn with_elements_standard(elements: &[usize], domain_size: usize) -> BitSet { - let mut s = BitSet::new_empty(domain_size); +fn with_elements_standard(elements: &[usize], domain_size: usize) -> DenseBitSet { + let mut s = DenseBitSet::new_empty(domain_size); for &e in elements { assert!(s.insert(e)); } @@ -503,15 +503,15 @@ fn sparse_matrix_operations() { matrix.insert(2, 99); matrix.insert(4, 0); - let mut disjoint: BitSet = BitSet::new_empty(100); + let mut disjoint: DenseBitSet = DenseBitSet::new_empty(100); disjoint.insert(33); - let mut superset = BitSet::new_empty(100); + let mut superset = DenseBitSet::new_empty(100); superset.insert(22); superset.insert(75); superset.insert(33); - let mut subset = BitSet::new_empty(100); + let mut subset = DenseBitSet::new_empty(100); subset.insert(22); // SparseBitMatrix::remove @@ -568,7 +568,7 @@ fn dense_insert_range() { where R: RangeBounds + Clone + IntoIterator + std::fmt::Debug, { - let mut set = BitSet::new_empty(domain); + let mut set = DenseBitSet::new_empty(domain); set.insert_range(range.clone()); for i in set.iter() { assert!(range.contains(&i)); @@ -609,7 +609,7 @@ fn dense_insert_range() { #[test] fn dense_last_set_before() { - fn easy(set: &BitSet, needle: impl RangeBounds) -> Option { + fn easy(set: &DenseBitSet, needle: impl RangeBounds) -> Option { let mut last_leq = None; for e in set.iter() { if needle.contains(&e) { @@ -620,7 +620,7 @@ fn dense_last_set_before() { } #[track_caller] - fn cmp(set: &BitSet, needle: impl RangeBounds + Clone + std::fmt::Debug) { + fn cmp(set: &DenseBitSet, needle: impl RangeBounds + Clone + std::fmt::Debug) { assert_eq!( set.last_set_in(needle.clone()), easy(set, needle.clone()), @@ -629,7 +629,7 @@ fn dense_last_set_before() { set ); } - let mut set = BitSet::new_empty(300); + let mut set = DenseBitSet::new_empty(300); cmp(&set, 50..=50); set.insert(WORD_BITS); cmp(&set, WORD_BITS..=WORD_BITS); @@ -645,7 +645,7 @@ fn dense_last_set_before() { for i in 0..=WORD_BITS * 2 { for j in i..=WORD_BITS * 2 { for k in 0..WORD_BITS * 2 { - let mut set = BitSet::new_empty(300); + let mut set = DenseBitSet::new_empty(300); cmp(&set, i..j); cmp(&set, i..=j); set.insert(k); @@ -658,7 +658,7 @@ fn dense_last_set_before() { #[bench] fn bench_insert(b: &mut Bencher) { - let mut bs = BitSet::new_filled(99999usize); + let mut bs = DenseBitSet::new_filled(99999usize); b.iter(|| { black_box(bs.insert(black_box(100u32))); }); @@ -666,7 +666,7 @@ fn bench_insert(b: &mut Bencher) { #[bench] fn bench_remove(b: &mut Bencher) { - let mut bs = BitSet::new_filled(99999usize); + let mut bs = DenseBitSet::new_filled(99999usize); b.iter(|| { black_box(bs.remove(black_box(100u32))); }); @@ -674,7 +674,7 @@ fn bench_remove(b: &mut Bencher) { #[bench] fn bench_iter(b: &mut Bencher) { - let bs = BitSet::new_filled(99999usize); + let bs = DenseBitSet::new_filled(99999usize); b.iter(|| { bs.iter().map(|b: usize| black_box(b)).for_each(drop); }); @@ -682,8 +682,8 @@ fn bench_iter(b: &mut Bencher) { #[bench] fn bench_intersect(b: &mut Bencher) { - let mut ba: BitSet = BitSet::new_filled(99999usize); - let bb = BitSet::new_filled(99999usize); + let mut ba: DenseBitSet = DenseBitSet::new_filled(99999usize); + let bb = DenseBitSet::new_filled(99999usize); b.iter(|| { ba.intersect(black_box(&bb)); }); diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 4fe8f73efd6..4f9cdc9a474 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -15,7 +15,7 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIndex, DefPathHash, Stable use rustc_hir::definitions::DefKey; use rustc_hir::lang_items::LangItem; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_macros::{ Decodable, Encodable, MetadataDecodable, MetadataEncodable, TyDecodable, TyEncodable, }; @@ -450,7 +450,7 @@ define_tables! { trait_item_def_id: Table, expn_that_defined: Table>, default_fields: Table>, - params_in_repr: Table>>, + params_in_repr: Table>>, repr_options: Table>, // `def_keys` and `def_path_hashes` represent a lazy version of a // `DefPathTable`. This allows us to avoid deserializing an entire diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index 29f26180c97..cd6b2d65bf1 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -3,7 +3,7 @@ use std::fmt::{self, Debug, Formatter}; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; use rustc_span::Span; @@ -303,8 +303,8 @@ pub struct MCDCDecisionSpan { /// Used by the `coverage_ids_info` query. #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable)] pub struct CoverageIdsInfo { - pub counters_seen: BitSet, - pub zero_expressions: BitSet, + pub counters_seen: DenseBitSet, + pub zero_expressions: DenseBitSet, } impl CoverageIdsInfo { diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 98ef7d58a50..bbb8bdce4a0 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -21,7 +21,7 @@ use rustc_hir::def_id::{CRATE_DEF_ID, DefId}; use rustc_hir::{ self as hir, BindingMode, ByRef, CoroutineDesugaring, CoroutineKind, HirId, ImplicitSelfKind, }; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; use rustc_serialize::{Decodable, Encodable}; diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index b8b74da401c..0e7dcc24daf 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -21,7 +21,7 @@ use super::*; #[derive(Clone)] pub struct Preorder<'a, 'tcx> { body: &'a Body<'tcx>, - visited: BitSet, + visited: DenseBitSet, worklist: Vec, root_is_start_block: bool, } @@ -32,7 +32,7 @@ impl<'a, 'tcx> Preorder<'a, 'tcx> { Preorder { body, - visited: BitSet::new_empty(body.basic_blocks.len()), + visited: DenseBitSet::new_empty(body.basic_blocks.len()), worklist, root_is_start_block: root == START_BLOCK, } @@ -106,7 +106,7 @@ impl<'a, 'tcx> Iterator for Preorder<'a, 'tcx> { /// A Postorder traversal of this graph is `D B C A` or `D C B A` pub struct Postorder<'a, 'tcx, C> { basic_blocks: &'a IndexSlice>, - visited: BitSet, + visited: DenseBitSet, visit_stack: Vec<(BasicBlock, Successors<'a>)>, root_is_start_block: bool, extra: C, @@ -123,7 +123,7 @@ where ) -> Postorder<'a, 'tcx, C> { let mut po = Postorder { basic_blocks, - visited: BitSet::new_empty(basic_blocks.len()), + visited: DenseBitSet::new_empty(basic_blocks.len()), visit_stack: Vec::new(), root_is_start_block: root == START_BLOCK, extra, @@ -285,8 +285,8 @@ pub fn reachable<'a, 'tcx>( preorder(body) } -/// Returns a `BitSet` containing all basic blocks reachable from the `START_BLOCK`. -pub fn reachable_as_bitset(body: &Body<'_>) -> BitSet { +/// Returns a `DenseBitSet` containing all basic blocks reachable from the `START_BLOCK`. +pub fn reachable_as_bitset(body: &Body<'_>) -> DenseBitSet { let mut iter = preorder(body); while let Some(_) = iter.next() {} iter.visited @@ -340,13 +340,13 @@ pub fn mono_reachable<'a, 'tcx>( MonoReachable::new(body, tcx, instance) } -/// [`MonoReachable`] internally accumulates a [`BitSet`] of visited blocks. This is just a +/// [`MonoReachable`] internally accumulates a [`DenseBitSet`] of visited blocks. This is just a /// convenience function to run that traversal then extract its set of reached blocks. pub fn mono_reachable_as_bitset<'a, 'tcx>( body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, -) -> BitSet { +) -> DenseBitSet { let mut iter = mono_reachable(body, tcx, instance); while let Some(_) = iter.next() {} iter.visited @@ -356,11 +356,11 @@ pub struct MonoReachable<'a, 'tcx> { body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, - visited: BitSet, + visited: DenseBitSet, // Other traversers track their worklist in a Vec. But we don't care about order, so we can - // store ours in a BitSet and thus save allocations because BitSet has a small size + // store ours in a DenseBitSet and thus save allocations because DenseBitSet has a small size // optimization. - worklist: BitSet, + worklist: DenseBitSet, } impl<'a, 'tcx> MonoReachable<'a, 'tcx> { @@ -369,13 +369,13 @@ impl<'a, 'tcx> MonoReachable<'a, 'tcx> { tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ) -> MonoReachable<'a, 'tcx> { - let mut worklist = BitSet::new_empty(body.basic_blocks.len()); + let mut worklist = DenseBitSet::new_empty(body.basic_blocks.len()); worklist.insert(START_BLOCK); MonoReachable { body, tcx, instance, - visited: BitSet::new_empty(body.basic_blocks.len()), + visited: DenseBitSet::new_empty(body.basic_blocks.len()), worklist, } } diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 283675573d4..7d334847d44 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -297,7 +297,7 @@ rustc_queries! { separate_provide_extern } - query unsizing_params_for_adt(key: DefId) -> &'tcx rustc_index::bit_set::BitSet + query unsizing_params_for_adt(key: DefId) -> &'tcx rustc_index::bit_set::DenseBitSet { arena_cache desc { |tcx| @@ -494,7 +494,7 @@ rustc_queries! { } /// Set of param indexes for type params that are in the type's representation - query params_in_repr(key: DefId) -> &'tcx rustc_index::bit_set::BitSet { + query params_in_repr(key: DefId) -> &'tcx rustc_index::bit_set::DenseBitSet { desc { "finding type parameters in the representation" } arena_cache no_hash diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index d26c007d227..24f10b4fbe7 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -613,7 +613,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { self.coroutine_is_async_gen(coroutine_def_id) } - type UnsizingParams = &'tcx rustc_index::bit_set::BitSet; + type UnsizingParams = &'tcx rustc_index::bit_set::DenseBitSet; fn unsizing_params_for_adt(self, adt_def_id: DefId) -> Self::UnsizingParams { self.unsizing_params_for_adt(adt_def_id) } diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs index 86a95827e84..6b6c6f3c72f 100644 --- a/compiler/rustc_middle/src/ty/parameterized.rs +++ b/compiler/rustc_middle/src/ty/parameterized.rs @@ -96,7 +96,7 @@ trivially_parameterized_over_tcx! { rustc_hir::def_id::DefIndex, rustc_hir::definitions::DefKey, rustc_hir::OpaqueTyOrigin, - rustc_index::bit_set::BitSet, + rustc_index::bit_set::DenseBitSet, rustc_index::bit_set::FiniteBitSet, rustc_session::cstore::ForeignModule, rustc_session::cstore::LinkagePreference, diff --git a/compiler/rustc_mir_dataflow/src/debuginfo.rs b/compiler/rustc_mir_dataflow/src/debuginfo.rs index fd5e8cf2955..0d25ce91c9a 100644 --- a/compiler/rustc_mir_dataflow/src/debuginfo.rs +++ b/compiler/rustc_mir_dataflow/src/debuginfo.rs @@ -1,17 +1,17 @@ -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; /// Return the set of locals that appear in debuginfo. -pub fn debuginfo_locals(body: &Body<'_>) -> BitSet { - let mut visitor = DebuginfoLocals(BitSet::new_empty(body.local_decls.len())); +pub fn debuginfo_locals(body: &Body<'_>) -> DenseBitSet { + let mut visitor = DebuginfoLocals(DenseBitSet::new_empty(body.local_decls.len())); for debuginfo in body.var_debug_info.iter() { visitor.visit_var_debug_info(debuginfo); } visitor.0 } -struct DebuginfoLocals(BitSet); +struct DebuginfoLocals(DenseBitSet); impl Visitor<'_> for DebuginfoLocals { fn visit_local(&mut self, local: Local, _: PlaceContext, _: Location) { diff --git a/compiler/rustc_mir_dataflow/src/framework/cursor.rs b/compiler/rustc_mir_dataflow/src/framework/cursor.rs index 89ff93d9943..c46ae9775cf 100644 --- a/compiler/rustc_mir_dataflow/src/framework/cursor.rs +++ b/compiler/rustc_mir_dataflow/src/framework/cursor.rs @@ -4,7 +4,7 @@ use std::cmp::Ordering; use std::ops::{Deref, DerefMut}; #[cfg(debug_assertions)] -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{self, BasicBlock, Location}; use super::{Analysis, Direction, Effect, EffectIndex, Results}; @@ -71,7 +71,7 @@ where state_needs_reset: bool, #[cfg(debug_assertions)] - reachable_blocks: BitSet, + reachable_blocks: DenseBitSet, } impl<'mir, 'tcx, A> ResultsCursor<'mir, 'tcx, A> diff --git a/compiler/rustc_mir_dataflow/src/framework/fmt.rs b/compiler/rustc_mir_dataflow/src/framework/fmt.rs index faf2c411dde..38599cd0949 100644 --- a/compiler/rustc_mir_dataflow/src/framework/fmt.rs +++ b/compiler/rustc_mir_dataflow/src/framework/fmt.rs @@ -4,7 +4,7 @@ use std::fmt; use rustc_index::Idx; -use rustc_index::bit_set::{BitSet, ChunkedBitSet, MixedBitSet}; +use rustc_index::bit_set::{ChunkedBitSet, DenseBitSet, MixedBitSet}; use super::lattice::MaybeReachable; @@ -73,7 +73,7 @@ where // Impls -impl DebugWithContext for BitSet +impl DebugWithContext for DenseBitSet where T: Idx + DebugWithContext, { diff --git a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs index 60c97710c7f..e457b514936 100644 --- a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs +++ b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs @@ -9,7 +9,7 @@ use std::{io, ops, str}; use regex::Regex; use rustc_hir::def_id::DefId; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{ self, BasicBlock, Body, Location, create_dump_file, dump_enabled, graphviz_safe_def_name, traversal, @@ -205,7 +205,7 @@ where // the operations that involve the mutation, i.e. within the `borrow_mut`. cursor: RefCell>, style: OutputStyle, - reachable: BitSet, + reachable: DenseBitSet, } impl<'mir, 'tcx, A> Formatter<'mir, 'tcx, A> diff --git a/compiler/rustc_mir_dataflow/src/framework/lattice.rs b/compiler/rustc_mir_dataflow/src/framework/lattice.rs index cb8159ce37b..919e346bda7 100644 --- a/compiler/rustc_mir_dataflow/src/framework/lattice.rs +++ b/compiler/rustc_mir_dataflow/src/framework/lattice.rs @@ -39,7 +39,7 @@ //! [poset]: https://en.wikipedia.org/wiki/Partially_ordered_set use rustc_index::Idx; -use rustc_index::bit_set::{BitSet, MixedBitSet}; +use rustc_index::bit_set::{DenseBitSet, MixedBitSet}; use crate::framework::BitSetExt; @@ -68,10 +68,10 @@ pub trait HasTop { const TOP: Self; } -/// A `BitSet` represents the lattice formed by the powerset of all possible values of -/// the index type `T` ordered by inclusion. Equivalently, it is a tuple of "two-point" lattices, -/// one for each possible value of `T`. -impl JoinSemiLattice for BitSet { +/// A `DenseBitSet` represents the lattice formed by the powerset of all possible values of the +/// index type `T` ordered by inclusion. Equivalently, it is a tuple of "two-point" lattices, one +/// for each possible value of `T`. +impl JoinSemiLattice for DenseBitSet { fn join(&mut self, other: &Self) -> bool { self.union(other) } diff --git a/compiler/rustc_mir_dataflow/src/framework/mod.rs b/compiler/rustc_mir_dataflow/src/framework/mod.rs index 3de2c6e3f47..60c5cb0cae8 100644 --- a/compiler/rustc_mir_dataflow/src/framework/mod.rs +++ b/compiler/rustc_mir_dataflow/src/framework/mod.rs @@ -35,7 +35,7 @@ use std::cmp::Ordering; use rustc_data_structures::work_queue::WorkQueue; -use rustc_index::bit_set::{BitSet, MixedBitSet}; +use rustc_index::bit_set::{DenseBitSet, MixedBitSet}; use rustc_index::{Idx, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{self, BasicBlock, CallReturnPlaces, Location, TerminatorEdges, traversal}; @@ -65,7 +65,7 @@ pub trait BitSetExt { fn contains(&self, elem: T) -> bool; } -impl BitSetExt for BitSet { +impl BitSetExt for DenseBitSet { fn contains(&self, elem: T) -> bool { self.contains(elem) } @@ -334,7 +334,7 @@ pub trait GenKill { } } -impl GenKill for BitSet { +impl GenKill for DenseBitSet { fn gen_(&mut self, elem: T) { self.insert(elem); } diff --git a/compiler/rustc_mir_dataflow/src/framework/tests.rs b/compiler/rustc_mir_dataflow/src/framework/tests.rs index 8e7d4ab0fa3..5b3a9ccba69 100644 --- a/compiler/rustc_mir_dataflow/src/framework/tests.rs +++ b/compiler/rustc_mir_dataflow/src/framework/tests.rs @@ -84,13 +84,13 @@ impl MockAnalysis<'_, D> { /// The entry set for each `BasicBlock` is the ID of that block offset by a fixed amount to /// avoid colliding with the statement/terminator effects. - fn mock_entry_set(&self, bb: BasicBlock) -> BitSet { + fn mock_entry_set(&self, bb: BasicBlock) -> DenseBitSet { let mut ret = self.bottom_value(self.body); ret.insert(Self::BASIC_BLOCK_OFFSET + bb.index()); ret } - fn mock_entry_states(&self) -> IndexVec> { + fn mock_entry_states(&self) -> IndexVec> { let empty = self.bottom_value(self.body); let mut ret = IndexVec::from_elem(empty, &self.body.basic_blocks); @@ -121,7 +121,7 @@ impl MockAnalysis<'_, D> { /// For example, the expected state when calling /// `seek_before_primary_effect(Location { block: 2, statement_index: 2 })` /// would be `[102, 0, 1, 2, 3, 4]`. - fn expected_state_at_target(&self, target: SeekTarget) -> BitSet { + fn expected_state_at_target(&self, target: SeekTarget) -> DenseBitSet { let block = target.block(); let mut ret = self.bottom_value(self.body); ret.insert(Self::BASIC_BLOCK_OFFSET + block.index()); @@ -155,13 +155,13 @@ impl MockAnalysis<'_, D> { } impl<'tcx, D: Direction> Analysis<'tcx> for MockAnalysis<'tcx, D> { - type Domain = BitSet; + type Domain = DenseBitSet; type Direction = D; const NAME: &'static str = "mock"; fn bottom_value(&self, body: &mir::Body<'tcx>) -> Self::Domain { - BitSet::new_empty(Self::BASIC_BLOCK_OFFSET + body.basic_blocks.len()) + DenseBitSet::new_empty(Self::BASIC_BLOCK_OFFSET + body.basic_blocks.len()) } fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) { diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs index 217594b3238..f2ef5018c49 100644 --- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs +++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs @@ -1,4 +1,4 @@ -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; @@ -21,12 +21,12 @@ impl MaybeBorrowedLocals { } impl<'tcx> Analysis<'tcx> for MaybeBorrowedLocals { - type Domain = BitSet; + type Domain = DenseBitSet; const NAME: &'static str = "maybe_borrowed_locals"; fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain { // bottom = unborrowed - BitSet::new_empty(body.local_decls().len()) + DenseBitSet::new_empty(body.local_decls().len()) } fn initialize_start_block(&self, _: &Body<'tcx>, _: &mut Self::Domain) { @@ -137,8 +137,8 @@ where } /// The set of locals that are borrowed at some point in the MIR body. -pub fn borrowed_locals(body: &Body<'_>) -> BitSet { - struct Borrowed(BitSet); +pub fn borrowed_locals(body: &Body<'_>) -> DenseBitSet { + struct Borrowed(DenseBitSet); impl GenKill for Borrowed { #[inline] @@ -151,7 +151,7 @@ pub fn borrowed_locals(body: &Body<'_>) -> BitSet { } } - let mut borrowed = Borrowed(BitSet::new_empty(body.local_decls.len())); + let mut borrowed = Borrowed(DenseBitSet::new_empty(body.local_decls.len())); TransferFunction { trans: &mut borrowed }.visit_body(body); borrowed.0 } diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized.rs b/compiler/rustc_mir_dataflow/src/impls/initialized.rs index 769f9c7cfc3..760f94af52d 100644 --- a/compiler/rustc_mir_dataflow/src/impls/initialized.rs +++ b/compiler/rustc_mir_dataflow/src/impls/initialized.rs @@ -2,7 +2,7 @@ use std::assert_matches::assert_matches; use rustc_abi::VariantIdx; use rustc_index::Idx; -use rustc_index::bit_set::{BitSet, MixedBitSet}; +use rustc_index::bit_set::{DenseBitSet, MixedBitSet}; use rustc_middle::bug; use rustc_middle::mir::{self, Body, CallReturnPlaces, Location, TerminatorEdges}; use rustc_middle::ty::util::Discr; @@ -207,7 +207,7 @@ pub struct MaybeUninitializedPlaces<'a, 'tcx> { move_data: &'a MoveData<'tcx>, mark_inactive_variants_as_uninit: bool, - skip_unreachable_unwind: BitSet, + skip_unreachable_unwind: DenseBitSet, } impl<'a, 'tcx> MaybeUninitializedPlaces<'a, 'tcx> { @@ -217,7 +217,7 @@ impl<'a, 'tcx> MaybeUninitializedPlaces<'a, 'tcx> { body, move_data, mark_inactive_variants_as_uninit: false, - skip_unreachable_unwind: BitSet::new_empty(body.basic_blocks.len()), + skip_unreachable_unwind: DenseBitSet::new_empty(body.basic_blocks.len()), } } @@ -233,7 +233,7 @@ impl<'a, 'tcx> MaybeUninitializedPlaces<'a, 'tcx> { pub fn skipping_unreachable_unwind( mut self, - unreachable_unwind: BitSet, + unreachable_unwind: DenseBitSet, ) -> Self { self.skip_unreachable_unwind = unreachable_unwind; self diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs index b2050a6adf9..6ec1b03a34e 100644 --- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs @@ -1,4 +1,4 @@ -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{ self, CallReturnPlaces, Local, Location, Place, StatementKind, TerminatorEdges, @@ -26,14 +26,14 @@ use crate::{Analysis, Backward, GenKill}; pub struct MaybeLiveLocals; impl<'tcx> Analysis<'tcx> for MaybeLiveLocals { - type Domain = BitSet; + type Domain = DenseBitSet; type Direction = Backward; const NAME: &'static str = "liveness"; fn bottom_value(&self, body: &mir::Body<'tcx>) -> Self::Domain { // bottom = not live - BitSet::new_empty(body.local_decls.len()) + DenseBitSet::new_empty(body.local_decls.len()) } fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) { @@ -81,7 +81,7 @@ impl<'tcx> Analysis<'tcx> for MaybeLiveLocals { } } -pub struct TransferFunction<'a>(pub &'a mut BitSet); +pub struct TransferFunction<'a>(pub &'a mut DenseBitSet); impl<'tcx> Visitor<'tcx> for TransferFunction<'_> { fn visit_place(&mut self, place: &mir::Place<'tcx>, context: PlaceContext, location: Location) { @@ -117,7 +117,7 @@ impl<'tcx> Visitor<'tcx> for TransferFunction<'_> { } } -struct YieldResumeEffect<'a>(&'a mut BitSet); +struct YieldResumeEffect<'a>(&'a mut DenseBitSet); impl<'tcx> Visitor<'tcx> for YieldResumeEffect<'_> { fn visit_place(&mut self, place: &mir::Place<'tcx>, context: PlaceContext, location: Location) { @@ -137,7 +137,7 @@ enum DefUse { } impl DefUse { - fn apply(state: &mut BitSet, place: Place<'_>, context: PlaceContext) { + fn apply(state: &mut DenseBitSet, place: Place<'_>, context: PlaceContext) { match DefUse::for_place(place, context) { Some(DefUse::Def) => state.kill(place.local), Some(DefUse::Use) => state.gen_(place.local), @@ -204,7 +204,7 @@ impl DefUse { /// /// All of the caveats of `MaybeLiveLocals` apply. pub struct MaybeTransitiveLiveLocals<'a> { - always_live: &'a BitSet, + always_live: &'a DenseBitSet, } impl<'a> MaybeTransitiveLiveLocals<'a> { @@ -212,20 +212,20 @@ impl<'a> MaybeTransitiveLiveLocals<'a> { /// considered live. /// /// This should include at least all locals that are ever borrowed. - pub fn new(always_live: &'a BitSet) -> Self { + pub fn new(always_live: &'a DenseBitSet) -> Self { MaybeTransitiveLiveLocals { always_live } } } impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> { - type Domain = BitSet; + type Domain = DenseBitSet; type Direction = Backward; const NAME: &'static str = "transitive liveness"; fn bottom_value(&self, body: &mir::Body<'tcx>) -> Self::Domain { // bottom = not live - BitSet::new_empty(body.local_decls.len()) + DenseBitSet::new_empty(body.local_decls.len()) } fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) { diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs index 65b480d3a5e..e3aa8f5a620 100644 --- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs @@ -1,6 +1,6 @@ use std::borrow::Cow; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -10,8 +10,8 @@ use crate::{Analysis, GenKill, ResultsCursor}; /// The set of locals in a MIR body that do not have `StorageLive`/`StorageDead` annotations. /// /// These locals have fixed storage for the duration of the body. -pub fn always_storage_live_locals(body: &Body<'_>) -> BitSet { - let mut always_live_locals = BitSet::new_filled(body.local_decls.len()); +pub fn always_storage_live_locals(body: &Body<'_>) -> DenseBitSet { + let mut always_live_locals = DenseBitSet::new_filled(body.local_decls.len()); for block in &*body.basic_blocks { for statement in &block.statements { @@ -25,23 +25,23 @@ pub fn always_storage_live_locals(body: &Body<'_>) -> BitSet { } pub struct MaybeStorageLive<'a> { - always_live_locals: Cow<'a, BitSet>, + always_live_locals: Cow<'a, DenseBitSet>, } impl<'a> MaybeStorageLive<'a> { - pub fn new(always_live_locals: Cow<'a, BitSet>) -> Self { + pub fn new(always_live_locals: Cow<'a, DenseBitSet>) -> Self { MaybeStorageLive { always_live_locals } } } impl<'a, 'tcx> Analysis<'tcx> for MaybeStorageLive<'a> { - type Domain = BitSet; + type Domain = DenseBitSet; const NAME: &'static str = "maybe_storage_live"; fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain { // bottom = dead - BitSet::new_empty(body.local_decls.len()) + DenseBitSet::new_empty(body.local_decls.len()) } fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) { @@ -67,23 +67,23 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeStorageLive<'a> { } pub struct MaybeStorageDead<'a> { - always_live_locals: Cow<'a, BitSet>, + always_live_locals: Cow<'a, DenseBitSet>, } impl<'a> MaybeStorageDead<'a> { - pub fn new(always_live_locals: Cow<'a, BitSet>) -> Self { + pub fn new(always_live_locals: Cow<'a, DenseBitSet>) -> Self { MaybeStorageDead { always_live_locals } } } impl<'a, 'tcx> Analysis<'tcx> for MaybeStorageDead<'a> { - type Domain = BitSet; + type Domain = DenseBitSet; const NAME: &'static str = "maybe_storage_dead"; fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain { // bottom = live - BitSet::new_empty(body.local_decls.len()) + DenseBitSet::new_empty(body.local_decls.len()) } fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) { @@ -125,13 +125,13 @@ impl<'mir, 'tcx> MaybeRequiresStorage<'mir, 'tcx> { } impl<'tcx> Analysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> { - type Domain = BitSet; + type Domain = DenseBitSet; const NAME: &'static str = "requires_storage"; fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain { // bottom = dead - BitSet::new_empty(body.local_decls.len()) + DenseBitSet::new_empty(body.local_decls.len()) } fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) { @@ -304,7 +304,7 @@ impl<'tcx> MaybeRequiresStorage<'_, 'tcx> { struct MoveVisitor<'a, 'mir, 'tcx> { borrowed_locals: &'a mut BorrowedLocalsResults<'mir, 'tcx>, - state: &'a mut BitSet, + state: &'a mut DenseBitSet, } impl<'tcx> Visitor<'tcx> for MoveVisitor<'_, '_, 'tcx> { diff --git a/compiler/rustc_mir_dataflow/src/points.rs b/compiler/rustc_mir_dataflow/src/points.rs index 74209da876a..5d2a78acbf5 100644 --- a/compiler/rustc_mir_dataflow/src/points.rs +++ b/compiler/rustc_mir_dataflow/src/points.rs @@ -1,4 +1,4 @@ -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::SparseIntervalMatrix; use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::{self, BasicBlock, Body, Location}; @@ -102,7 +102,7 @@ pub fn save_as_intervals<'tcx, N, A>( ) -> SparseIntervalMatrix where N: Idx, - A: Analysis<'tcx, Domain = BitSet>, + A: Analysis<'tcx, Domain = DenseBitSet>, { let values = SparseIntervalMatrix::new(elements.num_points()); let mut visitor = Visitor { elements, values }; @@ -122,7 +122,7 @@ struct Visitor<'a, N: Idx> { impl<'mir, 'tcx, A, N> ResultsVisitor<'mir, 'tcx, A> for Visitor<'_, N> where - A: Analysis<'tcx, Domain = BitSet>, + A: Analysis<'tcx, Domain = DenseBitSet>, N: Idx, { fn visit_after_primary_statement_effect( diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 9328870c7ae..a51af8c40fd 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -6,7 +6,7 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::{FxHashMap, FxIndexSet, StdEntry}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::tcx::PlaceTy; use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -399,7 +399,7 @@ impl<'tcx> Map<'tcx> { &mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - exclude: BitSet, + exclude: DenseBitSet, value_limit: Option, ) { // Start by constructing the places for each bare local. @@ -912,9 +912,9 @@ pub fn iter_fields<'tcx>( } /// Returns all locals with projections that have their reference or address taken. -pub fn excluded_locals(body: &Body<'_>) -> BitSet { +pub fn excluded_locals(body: &Body<'_>) -> DenseBitSet { struct Collector { - result: BitSet, + result: DenseBitSet, } impl<'tcx> Visitor<'tcx> for Collector { @@ -932,7 +932,7 @@ pub fn excluded_locals(body: &Body<'_>) -> BitSet { } } - let mut collector = Collector { result: BitSet::new_empty(body.local_decls.len()) }; + let mut collector = Collector { result: DenseBitSet::new_empty(body.local_decls.len()) }; collector.visit_body(body); collector.result } diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index 9b3443d3209..f149bf97cde 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -1,5 +1,5 @@ use rustc_index::IndexSlice; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -34,7 +34,7 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { let fully_moved = fully_moved_locals(&ssa, body); debug!(?fully_moved); - let mut storage_to_remove = BitSet::new_empty(fully_moved.domain_size()); + let mut storage_to_remove = DenseBitSet::new_empty(fully_moved.domain_size()); for (local, &head) in ssa.copy_classes().iter_enumerated() { if local != head { storage_to_remove.insert(head); @@ -68,8 +68,8 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { /// This means that replacing it by a copy of `_a` if ok, since this copy happens before `_c` is /// moved, and therefore that `_d` is moved. #[instrument(level = "trace", skip(ssa, body))] -fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet { - let mut fully_moved = BitSet::new_filled(body.local_decls.len()); +fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> DenseBitSet { + let mut fully_moved = DenseBitSet::new_filled(body.local_decls.len()); for (_, rvalue, _) in ssa.assignments(body) { let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) @@ -96,9 +96,9 @@ fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet { /// Utility to help performing substitution of `*pattern` by `target`. struct Replacer<'a, 'tcx> { tcx: TyCtxt<'tcx>, - fully_moved: BitSet, - storage_to_remove: BitSet, - borrowed_locals: &'a BitSet, + fully_moved: DenseBitSet, + storage_to_remove: DenseBitSet, + borrowed_locals: &'a DenseBitSet, copy_classes: &'a IndexSlice, } diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index f6536d78761..a3715b5d485 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -60,7 +60,7 @@ use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_hir::{CoroutineDesugaring, CoroutineKind}; -use rustc_index::bit_set::{BitMatrix, BitSet, GrowableBitSet}; +use rustc_index::bit_set::{BitMatrix, DenseBitSet, GrowableBitSet}; use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -185,13 +185,13 @@ struct TransformVisitor<'tcx> { remap: IndexVec, VariantIdx, FieldIdx)>>, // A map from a suspension point in a block to the locals which have live storage at that point - storage_liveness: IndexVec>>, + storage_liveness: IndexVec>>, // A list of suspension points, generated during the transform suspension_points: Vec>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. - always_live_locals: BitSet, + always_live_locals: DenseBitSet, // The original RETURN_PLACE local old_ret_local: Local, @@ -633,7 +633,7 @@ struct LivenessInfo { saved_locals: CoroutineSavedLocals, /// The set of saved locals live at each suspension point. - live_locals_at_suspension_points: Vec>, + live_locals_at_suspension_points: Vec>, /// Parallel vec to the above with SourceInfo for each yield terminator. source_info_at_suspension_points: Vec, @@ -645,7 +645,7 @@ struct LivenessInfo { /// For every suspending block, the locals which are storage-live across /// that suspension point. - storage_liveness: IndexVec>>, + storage_liveness: IndexVec>>, } /// Computes which locals have to be stored in the state-machine for the @@ -659,7 +659,7 @@ struct LivenessInfo { fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &BitSet, + always_live_locals: &DenseBitSet, movable: bool, ) -> LivenessInfo { // Calculate when MIR locals have live storage. This gives us an upper bound of their @@ -688,7 +688,7 @@ fn locals_live_across_suspend_points<'tcx>( let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); let mut live_locals_at_suspension_points = Vec::new(); let mut source_info_at_suspension_points = Vec::new(); - let mut live_locals_at_any_suspension_point = BitSet::new_empty(body.local_decls.len()); + let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); for (block, data) in body.basic_blocks.iter_enumerated() { if let TerminatorKind::Yield { .. } = data.terminator().kind { @@ -768,7 +768,7 @@ fn locals_live_across_suspend_points<'tcx>( /// `CoroutineSavedLocal` is indexed in terms of the elements in this set; /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local /// included in this set. -struct CoroutineSavedLocals(BitSet); +struct CoroutineSavedLocals(DenseBitSet); impl CoroutineSavedLocals { /// Returns an iterator over each `CoroutineSavedLocal` along with the `Local` it corresponds @@ -777,11 +777,11 @@ impl CoroutineSavedLocals { self.iter().enumerate().map(|(i, l)| (CoroutineSavedLocal::from(i), l)) } - /// Transforms a `BitSet` that contains only locals saved across yield points to the - /// equivalent `BitSet`. - fn renumber_bitset(&self, input: &BitSet) -> BitSet { + /// Transforms a `DenseBitSet` that contains only locals saved across yield points to the + /// equivalent `DenseBitSet`. + fn renumber_bitset(&self, input: &DenseBitSet) -> DenseBitSet { assert!(self.superset(input), "{:?} not a superset of {:?}", self.0, input); - let mut out = BitSet::new_empty(self.count()); + let mut out = DenseBitSet::new_empty(self.count()); for (saved_local, local) in self.iter_enumerated() { if input.contains(local) { out.insert(saved_local); @@ -801,7 +801,7 @@ impl CoroutineSavedLocals { } impl ops::Deref for CoroutineSavedLocals { - type Target = BitSet; + type Target = DenseBitSet; fn deref(&self) -> &Self::Target { &self.0 @@ -815,7 +815,7 @@ impl ops::Deref for CoroutineSavedLocals { fn compute_storage_conflicts<'mir, 'tcx>( body: &'mir Body<'tcx>, saved_locals: &'mir CoroutineSavedLocals, - always_live_locals: BitSet, + always_live_locals: DenseBitSet, mut requires_storage: Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>, ) -> BitMatrix { assert_eq!(body.local_decls.len(), saved_locals.domain_size()); @@ -833,7 +833,7 @@ fn compute_storage_conflicts<'mir, 'tcx>( body, saved_locals, local_conflicts: BitMatrix::from_row_n(&ineligible_locals, body.local_decls.len()), - eligible_storage_live: BitSet::new_empty(body.local_decls.len()), + eligible_storage_live: DenseBitSet::new_empty(body.local_decls.len()), }; requires_storage.visit_reachable_with(body, &mut visitor); @@ -871,7 +871,7 @@ struct StorageConflictVisitor<'a, 'tcx> { // benchmarks for coroutines. local_conflicts: BitMatrix, // We keep this bitset as a buffer to avoid reallocating memory. - eligible_storage_live: BitSet, + eligible_storage_live: DenseBitSet, } impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>> @@ -880,7 +880,7 @@ impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>> fn visit_after_early_statement_effect( &mut self, _results: &mut Results<'tcx, MaybeRequiresStorage<'a, 'tcx>>, - state: &BitSet, + state: &DenseBitSet, _statement: &'a Statement<'tcx>, loc: Location, ) { @@ -890,7 +890,7 @@ impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>> fn visit_after_early_terminator_effect( &mut self, _results: &mut Results<'tcx, MaybeRequiresStorage<'a, 'tcx>>, - state: &BitSet, + state: &DenseBitSet, _terminator: &'a Terminator<'tcx>, loc: Location, ) { @@ -899,7 +899,7 @@ impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>> } impl StorageConflictVisitor<'_, '_> { - fn apply_state(&mut self, state: &BitSet, loc: Location) { + fn apply_state(&mut self, state: &DenseBitSet, loc: Location) { // Ignore unreachable blocks. if let TerminatorKind::Unreachable = self.body.basic_blocks[loc.block].terminator().kind { return; @@ -924,7 +924,7 @@ fn compute_layout<'tcx>( ) -> ( IndexVec, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, - IndexVec>>, + IndexVec>>, ) { let LivenessInfo { saved_locals, diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index 9e80f1f1c4a..a9111a5ef9b 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -5,7 +5,7 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::DirectedGraph; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op}; use tracing::{debug, debug_span, instrument}; @@ -77,7 +77,7 @@ impl CoverageCounters { /// counters or counter expressions for nodes and edges as required. pub(super) fn make_bcb_counters( graph: &CoverageGraph, - bcb_needs_counter: &BitSet, + bcb_needs_counter: &DenseBitSet, ) -> Self { let mut builder = CountersBuilder::new(graph, bcb_needs_counter); builder.make_bcb_counters(); @@ -220,13 +220,16 @@ fn sibling_out_edge_targets( /// the set of nodes that need counters. struct CountersBuilder<'a> { graph: &'a CoverageGraph, - bcb_needs_counter: &'a BitSet, + bcb_needs_counter: &'a DenseBitSet, site_counters: FxHashMap, } impl<'a> CountersBuilder<'a> { - fn new(graph: &'a CoverageGraph, bcb_needs_counter: &'a BitSet) -> Self { + fn new( + graph: &'a CoverageGraph, + bcb_needs_counter: &'a DenseBitSet, + ) -> Self { assert_eq!(graph.num_nodes(), bcb_needs_counter.domain_size()); Self { graph, bcb_needs_counter, site_counters: FxHashMap::default() } } diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index ad6774fccd6..3fa8b063fa7 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -8,7 +8,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::graph::dominators::Dominators; use rustc_data_structures::graph::{self, DirectedGraph, StartNode}; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{self, BasicBlock, Terminator, TerminatorKind}; use tracing::debug; @@ -27,7 +27,7 @@ pub(crate) struct CoverageGraph { /// their relative order is consistent but arbitrary. dominator_order_rank: IndexVec, /// A loop header is a node that dominates one or more of its predecessors. - is_loop_header: BitSet, + is_loop_header: DenseBitSet, /// For each node, the loop header node of its nearest enclosing loop. /// This forms a linked list that can be traversed to find all enclosing loops. enclosing_loop_header: IndexVec>, @@ -72,7 +72,7 @@ impl CoverageGraph { predecessors, dominators: None, dominator_order_rank: IndexVec::from_elem_n(0, num_nodes), - is_loop_header: BitSet::new_empty(num_nodes), + is_loop_header: DenseBitSet::new_empty(num_nodes), enclosing_loop_header: IndexVec::from_elem_n(None, num_nodes), }; assert_eq!(num_nodes, this.num_nodes()); diff --git a/compiler/rustc_mir_transform/src/coverage/mappings.rs b/compiler/rustc_mir_transform/src/coverage/mappings.rs index 5bd20e00eb6..8d0d92dc367 100644 --- a/compiler/rustc_mir_transform/src/coverage/mappings.rs +++ b/compiler/rustc_mir_transform/src/coverage/mappings.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::graph::DirectedGraph; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::coverage::{ BlockMarkerId, BranchSpan, ConditionId, ConditionInfo, CoverageInfoHi, CoverageKind, }; @@ -128,7 +128,7 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>( } impl ExtractedMappings { - pub(super) fn all_bcbs_with_counter_mappings(&self) -> BitSet { + pub(super) fn all_bcbs_with_counter_mappings(&self) -> DenseBitSet { // Fully destructure self to make sure we don't miss any fields that have mappings. let Self { num_bcbs, @@ -140,7 +140,7 @@ impl ExtractedMappings { } = self; // Identify which BCBs have one or more mappings. - let mut bcbs_with_counter_mappings = BitSet::new_empty(*num_bcbs); + let mut bcbs_with_counter_mappings = DenseBitSet::new_empty(*num_bcbs); let mut insert = |bcb| { bcbs_with_counter_mappings.insert(bcb); }; @@ -172,8 +172,8 @@ impl ExtractedMappings { } /// Returns the set of BCBs that have one or more `Code` mappings. - pub(super) fn bcbs_with_ordinary_code_mappings(&self) -> BitSet { - let mut bcbs = BitSet::new_empty(self.num_bcbs); + pub(super) fn bcbs_with_ordinary_code_mappings(&self) -> DenseBitSet { + let mut bcbs = DenseBitSet::new_empty(self.num_bcbs); for &CodeMapping { span: _, bcb } in &self.code_mappings { bcbs.insert(bcb); } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index edaec3c7965..3e7cf8541c2 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -1,5 +1,5 @@ use rustc_data_structures::captures::Captures; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::{ CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, ExpressionId, @@ -92,13 +92,13 @@ fn coverage_ids_info<'tcx>( let Some(fn_cov_info) = mir_body.function_coverage_info.as_deref() else { return CoverageIdsInfo { - counters_seen: BitSet::new_empty(0), - zero_expressions: BitSet::new_empty(0), + counters_seen: DenseBitSet::new_empty(0), + zero_expressions: DenseBitSet::new_empty(0), }; }; - let mut counters_seen = BitSet::new_empty(fn_cov_info.num_counters); - let mut expressions_seen = BitSet::new_filled(fn_cov_info.expressions.len()); + let mut counters_seen = DenseBitSet::new_empty(fn_cov_info.num_counters); + let mut expressions_seen = DenseBitSet::new_filled(fn_cov_info.expressions.len()); // For each expression ID that is directly used by one or more mappings, // mark it as not-yet-seen. This indicates that we expect to see a @@ -148,23 +148,23 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool { scope_data.inlined.is_some() || scope_data.inlined_parent_scope.is_some() } -/// Identify expressions that will always have a value of zero, and note -/// their IDs in a `BitSet`. Mappings that refer to a zero expression -/// can instead become mappings to a constant zero value. +/// Identify expressions that will always have a value of zero, and note their +/// IDs in a `DenseBitSet`. Mappings that refer to a zero expression can instead +/// become mappings to a constant zero value. /// /// This function mainly exists to preserve the simplifications that were /// already being performed by the Rust-side expression renumbering, so that /// the resulting coverage mappings don't get worse. fn identify_zero_expressions( fn_cov_info: &FunctionCoverageInfo, - counters_seen: &BitSet, - expressions_seen: &BitSet, -) -> BitSet { + counters_seen: &DenseBitSet, + expressions_seen: &DenseBitSet, +) -> DenseBitSet { // The set of expressions that either were optimized out entirely, or // have zero as both of their operands, and will therefore always have // a value of zero. Other expressions that refer to these as operands // can have those operands replaced with `CovTerm::Zero`. - let mut zero_expressions = BitSet::new_empty(fn_cov_info.expressions.len()); + let mut zero_expressions = DenseBitSet::new_empty(fn_cov_info.expressions.len()); // Simplify a copy of each expression based on lower-numbered expressions, // and then update the set of always-zero expressions if necessary. @@ -228,8 +228,8 @@ fn identify_zero_expressions( /// into account knowledge of which counters are unused and which expressions /// are always zero. fn is_zero_term( - counters_seen: &BitSet, - zero_expressions: &BitSet, + counters_seen: &DenseBitSet, + zero_expressions: &DenseBitSet, term: CovTerm, ) -> bool { match term { diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index 0c75cdadc92..434e921d439 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -26,8 +26,8 @@ use crate::util::is_within_packed; /// Performs the optimization on the body /// -/// The `borrowed` set must be a `BitSet` of all the locals that are ever borrowed in this body. It -/// can be generated via the [`borrowed_locals`] function. +/// The `borrowed` set must be a `DenseBitSet` of all the locals that are ever borrowed in this +/// body. It can be generated via the [`borrowed_locals`] function. fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let borrowed_locals = borrowed_locals(body); diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index 67b215c7c9d..049f13ce96d 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -6,7 +6,7 @@ //! dependent crates can use them. use rustc_hir::def_id::LocalDefId; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{Body, Location, Operand, Place, RETURN_PLACE, Terminator, TerminatorKind}; use rustc_middle::ty::{self, DeducedParamAttrs, Ty, TyCtxt}; @@ -18,13 +18,13 @@ struct DeduceReadOnly { /// Each bit is indexed by argument number, starting at zero (so 0 corresponds to local decl /// 1). The bit is true if the argument may have been mutated or false if we know it hasn't /// been up to the point we're at. - mutable_args: BitSet, + mutable_args: DenseBitSet, } impl DeduceReadOnly { /// Returns a new DeduceReadOnly instance. fn new(arg_count: usize) -> Self { - Self { mutable_args: BitSet::new_empty(arg_count) } + Self { mutable_args: DenseBitSet::new_empty(arg_count) } } } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index e99bee6a01f..1ac4d835946 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -132,7 +132,7 @@ //! [attempt 3]: https://github.com/rust-lang/rust/pull/72632 use rustc_data_structures::fx::{FxIndexMap, IndexEntry, IndexOccupiedEntry}; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::SparseIntervalMatrix; use rustc_middle::bug; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; @@ -204,7 +204,8 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { // Because we only filter once per round, it is unsound to use a local for more than // one merge operation within a single round of optimizations. We store here which ones // we have already used. - let mut merged_locals: BitSet = BitSet::new_empty(body.local_decls.len()); + let mut merged_locals: DenseBitSet = + DenseBitSet::new_empty(body.local_decls.len()); // This is the set of merges we will apply this round. It is a subset of the candidates. let mut merges = FxIndexMap::default(); @@ -274,7 +275,7 @@ fn apply_merges<'tcx>( body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>, merges: FxIndexMap, - merged_locals: BitSet, + merged_locals: DenseBitSet, ) { let mut merger = Merger { tcx, merges, merged_locals }; merger.visit_body_preserves_cfg(body); @@ -283,7 +284,7 @@ fn apply_merges<'tcx>( struct Merger<'tcx> { tcx: TyCtxt<'tcx>, merges: FxIndexMap, - merged_locals: BitSet, + merged_locals: DenseBitSet, } impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> { @@ -351,7 +352,7 @@ impl Candidates { /// Collects the candidates for merging. /// /// This is responsible for enforcing the first and third bullet point. - fn reset_and_find<'tcx>(&mut self, body: &Body<'tcx>, borrowed: &BitSet) { + fn reset_and_find<'tcx>(&mut self, body: &Body<'tcx>, borrowed: &DenseBitSet) { self.c.clear(); self.reverse.clear(); let mut visitor = FindAssignments { body, candidates: &mut self.c, borrowed }; @@ -735,7 +736,7 @@ fn places_to_candidate_pair<'tcx>( struct FindAssignments<'a, 'tcx> { body: &'a Body<'tcx>, candidates: &'a mut FxIndexMap>, - borrowed: &'a BitSet, + borrowed: &'a DenseBitSet, } impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 3ebc9113725..988f1a25561 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -2,7 +2,7 @@ use std::fmt; use rustc_abi::{FieldIdx, VariantIdx}; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; @@ -96,10 +96,10 @@ impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops { fn compute_dead_unwinds<'a, 'tcx>( body: &'a Body<'tcx>, flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>, -) -> BitSet { +) -> DenseBitSet { // We only need to do this pass once, because unwind edges can only // reach cleanup blocks, which can't have unwind edges themselves. - let mut dead_unwinds = BitSet::new_empty(body.basic_blocks.len()); + let mut dead_unwinds = DenseBitSet::new_empty(body.basic_blocks.len()); for (bb, bb_data) in body.basic_blocks.iter_enumerated() { let TerminatorKind::Drop { place, unwind: UnwindAction::Cleanup(_), .. } = bb_data.terminator().kind diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 71bec38c405..affc4cf0afc 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -94,7 +94,7 @@ use rustc_const_eval::interpret::{ use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::graph::dominators::Dominators; use rustc_hir::def::DefKind; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexVec, newtype_index}; use rustc_middle::bug; use rustc_middle::mir::interpret::GlobalAlloc; @@ -256,7 +256,7 @@ struct VnState<'body, 'tcx> { feature_unsized_locals: bool, ssa: &'body SsaLocals, dominators: Dominators, - reused_locals: BitSet, + reused_locals: DenseBitSet, } impl<'body, 'tcx> VnState<'body, 'tcx> { @@ -287,7 +287,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { feature_unsized_locals: tcx.features().unsized_locals(), ssa, dominators, - reused_locals: BitSet::new_empty(local_decls.len()), + reused_locals: DenseBitSet::new_empty(local_decls.len()), } } @@ -1714,7 +1714,7 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> { struct StorageRemover<'tcx> { tcx: TyCtxt<'tcx>, - reused_locals: BitSet, + reused_locals: DenseBitSet, } impl<'tcx> MutVisitor<'tcx> for StorageRemover<'tcx> { diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index e4daa2b9757..470393c9ae1 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -8,7 +8,7 @@ use rustc_attr_parsing::InlineAttr; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_index::Idx; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::mir::visit::*; @@ -369,7 +369,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; - let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); + let mut visited = DenseBitSet::new_empty(callee_body.basic_blocks.len()); while let Some(bb) = work_list.pop() { if !visited.insert(bb.index()) { continue; @@ -885,7 +885,7 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( in_cleanup_block: false, return_block, tcx, - always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), + always_live_locals: DenseBitSet::new_filled(callee_body.local_decls.len()), }; // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones @@ -1127,7 +1127,7 @@ struct Integrator<'a, 'tcx> { in_cleanup_block: bool, return_block: Option, tcx: TyCtxt<'tcx>, - always_live_locals: BitSet, + always_live_locals: DenseBitSet, } impl Integrator<'_, '_> { diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 8feb90ff7a0..c73a03489c5 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -40,7 +40,7 @@ use rustc_const_eval::const_eval::DummyMachine; use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::Visitor; @@ -121,7 +121,7 @@ struct TOFinder<'a, 'tcx> { ecx: InterpCx<'tcx, DummyMachine>, body: &'a Body<'tcx>, map: Map<'tcx>, - loop_headers: BitSet, + loop_headers: DenseBitSet, /// We use an arena to avoid cloning the slices when cloning `state`. arena: &'a DroplessArena, opportunities: Vec, @@ -832,8 +832,8 @@ enum Update { /// at least a predecessor which it dominates. This definition is only correct for reducible CFGs. /// But if the CFG is already irreducible, there is no point in trying much harder. /// is already irreducible. -fn loop_headers(body: &Body<'_>) -> BitSet { - let mut loop_headers = BitSet::new_empty(body.basic_blocks.len()); +fn loop_headers(body: &Body<'_>) -> DenseBitSet { + let mut loop_headers = DenseBitSet::new_empty(body.basic_blocks.len()); let dominators = body.basic_blocks.dominators(); // Only visit reachable blocks. for (bb, bbdata) in traversal::preorder(body) { diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index f1705d0c831..9b3a0e67295 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -13,7 +13,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_hir::HirId; use rustc_hir::def::DefKind; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -67,7 +67,7 @@ struct ConstPropagator<'mir, 'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, worklist: Vec, - visited_blocks: BitSet, + visited_blocks: DenseBitSet, locals: IndexVec>, body: &'mir Body<'tcx>, written_only_inside_own_block_locals: FxHashSet, @@ -190,7 +190,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { tcx, typing_env, worklist: vec![START_BLOCK], - visited_blocks: BitSet::new_empty(body.basic_blocks.len()), + visited_blocks: DenseBitSet::new_empty(body.basic_blocks.len()), locals: IndexVec::from_elem_n(Value::Uninit, body.local_decls.len()), body, can_const_prop, @@ -852,7 +852,7 @@ enum ConstPropMode { struct CanConstProp { can_const_prop: IndexVec, // False at the beginning. Once set, no more assignments are allowed to that local. - found_assignment: BitSet, + found_assignment: DenseBitSet, } impl CanConstProp { @@ -864,7 +864,7 @@ impl CanConstProp { ) -> IndexVec { let mut cpv = CanConstProp { can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls), - found_assignment: BitSet::new_empty(body.local_decls.len()), + found_assignment: DenseBitSet::new_empty(body.local_decls.len()), }; for (local, val) in cpv.can_const_prop.iter_enumerated_mut() { let ty = body.local_decls[local].ty; diff --git a/compiler/rustc_mir_transform/src/lint.rs b/compiler/rustc_mir_transform/src/lint.rs index 29e762af8de..f472c7cb493 100644 --- a/compiler/rustc_mir_transform/src/lint.rs +++ b/compiler/rustc_mir_transform/src/lint.rs @@ -5,7 +5,7 @@ use std::borrow::Cow; use rustc_data_structures::fx::FxHashSet; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -43,7 +43,7 @@ struct Lint<'a, 'tcx> { when: String, body: &'a Body<'tcx>, is_fn_like: bool, - always_live_locals: &'a BitSet, + always_live_locals: &'a DenseBitSet, maybe_storage_live: ResultsCursor<'a, 'tcx, MaybeStorageLive<'a>>, maybe_storage_dead: ResultsCursor<'a, 'tcx, MaybeStorageDead<'a>>, places: FxHashSet>, diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index a9227524ce5..6dfa14d6b52 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -1,7 +1,7 @@ //! This pass removes jumps to basic blocks containing only a return, and replaces them with a //! return instead. -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -16,7 +16,7 @@ impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators { fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // find basic blocks with no statement and a return terminator - let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks.len()); + let mut bbs_simple_returns = DenseBitSet::new_empty(body.basic_blocks.len()); let bbs = body.basic_blocks_mut(); for idx in bbs.indices() { if bbs[idx].statements.is_empty() diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index cd026ed6806..35872de3852 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -1,7 +1,7 @@ //! See the docs for [`RenameReturnPlace`]. use rustc_hir::Mutability; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, BasicBlock, Local, Location}; @@ -116,7 +116,7 @@ fn local_eligible_for_nrvo(body: &mir::Body<'_>) -> Option { fn find_local_assigned_to_return_place(start: BasicBlock, body: &mir::Body<'_>) -> Option { let mut block = start; - let mut seen = BitSet::new_empty(body.basic_blocks.len()); + let mut seen = DenseBitSet::new_empty(body.basic_blocks.len()); // Iterate as long as `block` has exactly one predecessor that we have not yet visited. while seen.insert(block) { diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs index 937c207776b..51abd4da86e 100644 --- a/compiler/rustc_mir_transform/src/prettify.rs +++ b/compiler/rustc_mir_transform/src/prettify.rs @@ -4,7 +4,7 @@ //! (`-Zmir-enable-passes=+ReorderBasicBlocks,+ReorderLocals`) //! to make the MIR easier to read for humans. -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -51,8 +51,10 @@ impl<'tcx> crate::MirPass<'tcx> for ReorderLocals { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let mut finder = - LocalFinder { map: IndexVec::new(), seen: BitSet::new_empty(body.local_decls.len()) }; + let mut finder = LocalFinder { + map: IndexVec::new(), + seen: DenseBitSet::new_empty(body.local_decls.len()), + }; // We can't reorder the return place or the arguments for local in (0..=body.arg_count).map(Local::from_usize) { @@ -113,7 +115,7 @@ impl<'tcx> MutVisitor<'tcx> for BasicBlockUpdater<'tcx> { struct LocalFinder { map: IndexVec, - seen: BitSet, + seen: DenseBitSet, } impl LocalFinder { diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index 96bcdfa6fac..95b05f94270 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; @@ -132,7 +132,7 @@ fn compute_replacement<'tcx>( let mut targets = IndexVec::from_elem(Value::Unknown, &body.local_decls); // Set of locals for which we will remove their storage statement. This is useful for // reborrowed references. - let mut storage_to_remove = BitSet::new_empty(body.local_decls.len()); + let mut storage_to_remove = DenseBitSet::new_empty(body.local_decls.len()); let fully_replacable_locals = fully_replacable_locals(ssa); @@ -324,8 +324,8 @@ fn compute_replacement<'tcx>( /// /// We consider a local to be replacable iff it's only used in a `Deref` projection `*_local` or /// non-use position (like storage statements and debuginfo). -fn fully_replacable_locals(ssa: &SsaLocals) -> BitSet { - let mut replacable = BitSet::new_empty(ssa.num_locals()); +fn fully_replacable_locals(ssa: &SsaLocals) -> DenseBitSet { + let mut replacable = DenseBitSet::new_empty(ssa.num_locals()); // First pass: for each local, whether its uses can be fully replaced. for local in ssa.locals() { @@ -344,7 +344,7 @@ fn fully_replacable_locals(ssa: &SsaLocals) -> BitSet { struct Replacer<'tcx> { tcx: TyCtxt<'tcx>, targets: IndexVec>, - storage_to_remove: BitSet, + storage_to_remove: DenseBitSet, allowed_replacements: FxHashSet<(Local, Location)>, any_replacement: bool, } diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index fd49e956f43..76a3edfe0be 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -1,4 +1,4 @@ -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -40,7 +40,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveNoopLandingPads { let mut jumps_folded = 0; let mut landing_pads_removed = 0; - let mut nop_landing_pads = BitSet::new_empty(body.basic_blocks.len()); + let mut nop_landing_pads = DenseBitSet::new_empty(body.basic_blocks.len()); // This is a post-order traversal, so that if A post-dominates B // then A will be visited before B. @@ -81,7 +81,7 @@ impl RemoveNoopLandingPads { &self, bb: BasicBlock, body: &Body<'_>, - nop_landing_pads: &BitSet, + nop_landing_pads: &DenseBitSet, ) -> bool { for stmt in &body[bb].statements { match &stmt.kind { diff --git a/compiler/rustc_mir_transform/src/single_use_consts.rs b/compiler/rustc_mir_transform/src/single_use_consts.rs index 277a33c0311..10b3c0ae94f 100644 --- a/compiler/rustc_mir_transform/src/single_use_consts.rs +++ b/compiler/rustc_mir_transform/src/single_use_consts.rs @@ -1,5 +1,5 @@ use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -28,9 +28,9 @@ impl<'tcx> crate::MirPass<'tcx> for SingleUseConsts { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let mut finder = SingleUseConstsFinder { - ineligible_locals: BitSet::new_empty(body.local_decls.len()), + ineligible_locals: DenseBitSet::new_empty(body.local_decls.len()), locations: IndexVec::from_elem(LocationPair::new(), &body.local_decls), - locals_in_debug_info: BitSet::new_empty(body.local_decls.len()), + locals_in_debug_info: DenseBitSet::new_empty(body.local_decls.len()), }; finder.ineligible_locals.insert_range(..=Local::from_usize(body.arg_count)); @@ -96,9 +96,9 @@ impl LocationPair { } struct SingleUseConstsFinder { - ineligible_locals: BitSet, + ineligible_locals: DenseBitSet, locations: IndexVec, - locals_in_debug_info: BitSet, + locals_in_debug_info: DenseBitSet, } impl<'tcx> Visitor<'tcx> for SingleUseConstsFinder { diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 52b9ec1e0a3..d54ea3feab6 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -2,7 +2,7 @@ use rustc_abi::{FIRST_VARIANT, FieldIdx}; use rustc_data_structures::flat_map_in_place::FlatMapInPlace; use rustc_hir::LangItem; use rustc_index::IndexVec; -use rustc_index::bit_set::{BitSet, GrowableBitSet}; +use rustc_index::bit_set::{DenseBitSet, GrowableBitSet}; use rustc_middle::bug; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::visit::*; @@ -60,9 +60,9 @@ impl<'tcx> crate::MirPass<'tcx> for ScalarReplacementOfAggregates { fn escaping_locals<'tcx>( tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, - excluded: &BitSet, + excluded: &DenseBitSet, body: &Body<'tcx>, -) -> BitSet { +) -> DenseBitSet { let is_excluded_ty = |ty: Ty<'tcx>| { if ty.is_union() || ty.is_enum() { return true; @@ -97,7 +97,7 @@ fn escaping_locals<'tcx>( false }; - let mut set = BitSet::new_empty(body.local_decls.len()); + let mut set = DenseBitSet::new_empty(body.local_decls.len()); set.insert_range(RETURN_PLACE..=Local::from_usize(body.arg_count)); for (local, decl) in body.local_decls().iter_enumerated() { if excluded.contains(local) || is_excluded_ty(decl.ty) { @@ -109,7 +109,7 @@ fn escaping_locals<'tcx>( return visitor.set; struct EscapeVisitor { - set: BitSet, + set: DenseBitSet, } impl<'tcx> Visitor<'tcx> for EscapeVisitor { @@ -198,7 +198,7 @@ fn compute_flattening<'tcx>( tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, body: &mut Body<'tcx>, - escaping: BitSet, + escaping: DenseBitSet, ) -> ReplacementMap<'tcx> { let mut fragments = IndexVec::from_elem(None, &body.local_decls); @@ -226,8 +226,8 @@ fn replace_flattened_locals<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, replacements: ReplacementMap<'tcx>, -) -> BitSet { - let mut all_dead_locals = BitSet::new_empty(replacements.fragments.len()); +) -> DenseBitSet { + let mut all_dead_locals = DenseBitSet::new_empty(replacements.fragments.len()); for (local, replacements) in replacements.fragments.iter_enumerated() { if replacements.is_some() { all_dead_locals.insert(local); @@ -267,7 +267,7 @@ struct ReplacementVisitor<'tcx, 'll> { /// Work to do. replacements: &'ll ReplacementMap<'tcx>, /// This is used to check that we are not leaving references to replaced locals behind. - all_dead_locals: BitSet, + all_dead_locals: DenseBitSet, patch: MirPatch<'tcx>, } diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 5653aef0aae..a24b3b2e80f 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -7,7 +7,7 @@ //! of a `Freeze` local. Those can still be considered to be SSA. use rustc_data_structures::graph::dominators::Dominators; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::bug; use rustc_middle::middle::resolve_bound_vars::Set1; @@ -29,7 +29,7 @@ pub(super) struct SsaLocals { /// We ignore non-uses (Storage statements, debuginfo). direct_uses: IndexVec, /// Set of SSA locals that are immutably borrowed. - borrowed_locals: BitSet, + borrowed_locals: DenseBitSet, } pub(super) enum AssignedValue<'a, 'tcx> { @@ -50,7 +50,7 @@ impl SsaLocals { let dominators = body.basic_blocks.dominators(); let direct_uses = IndexVec::from_elem(0, &body.local_decls); - let borrowed_locals = BitSet::new_empty(body.local_decls.len()); + let borrowed_locals = DenseBitSet::new_empty(body.local_decls.len()); let mut visitor = SsaVisitor { body, assignments, @@ -202,12 +202,12 @@ impl SsaLocals { } /// Set of SSA locals that are immutably borrowed. - pub(super) fn borrowed_locals(&self) -> &BitSet { + pub(super) fn borrowed_locals(&self) -> &DenseBitSet { &self.borrowed_locals } /// Make a property uniform on a copy equivalence class by removing elements. - pub(super) fn meet_copy_equivalence(&self, property: &mut BitSet) { + pub(super) fn meet_copy_equivalence(&self, property: &mut DenseBitSet) { // Consolidate to have a local iff all its copies are. // // `copy_classes` defines equivalence classes between locals. The `local`s that recursively @@ -241,7 +241,7 @@ struct SsaVisitor<'a, 'tcx> { assignment_order: Vec, direct_uses: IndexVec, // Track locals that are immutably borrowed, so we can check their type is `Freeze` later. - borrowed_locals: BitSet, + borrowed_locals: DenseBitSet, } impl SsaVisitor<'_, '_> { @@ -396,7 +396,7 @@ pub(crate) struct StorageLiveLocals { impl StorageLiveLocals { pub(crate) fn new( body: &Body<'_>, - always_storage_live_locals: &BitSet, + always_storage_live_locals: &DenseBitSet, ) -> StorageLiveLocals { let mut storage_live = IndexVec::from_elem(Set1::Empty, &body.local_decls); for local in always_storage_live_locals.iter() { diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 035670d4903..414477d9004 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -5,7 +5,7 @@ use rustc_attr_parsing::InlineAttr; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::LangItem; use rustc_index::IndexVec; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::traits::{Obligation, ObligationCause}; use rustc_middle::mir::coverage::CoverageKind; @@ -98,7 +98,7 @@ struct CfgChecker<'a, 'tcx> { body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, unwind_edge_count: usize, - reachable_blocks: BitSet, + reachable_blocks: DenseBitSet, value_cache: FxHashSet, // If `false`, then the MIR must not contain `UnwindAction::Continue` or // `TerminatorKind::Resume`. diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs index 8fce4266345..4ce868f014f 100644 --- a/compiler/rustc_pattern_analysis/src/constructor.rs +++ b/compiler/rustc_pattern_analysis/src/constructor.rs @@ -182,7 +182,7 @@ use std::iter::once; use rustc_apfloat::ieee::{DoubleS, HalfS, IeeeFloat, QuadS, SingleS}; use rustc_index::IndexVec; -use rustc_index::bit_set::{BitSet, GrowableBitSet}; +use rustc_index::bit_set::{DenseBitSet, GrowableBitSet}; use smallvec::SmallVec; use self::Constructor::*; @@ -1072,7 +1072,7 @@ impl ConstructorSet { } } ConstructorSet::Variants { variants, non_exhaustive } => { - let mut seen_set = BitSet::new_empty(variants.len()); + let mut seen_set = DenseBitSet::new_empty(variants.len()); for idx in seen.iter().filter_map(|c| c.as_variant()) { seen_set.insert(idx); } diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs index 99261eaa95c..cc09cd491af 100644 --- a/compiler/rustc_pattern_analysis/src/usefulness.rs +++ b/compiler/rustc_pattern_analysis/src/usefulness.rs @@ -712,7 +712,7 @@ use std::fmt; #[cfg(feature = "rustc")] use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hash::{FxHashMap, FxHashSet}; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use smallvec::{SmallVec, smallvec}; use tracing::{debug, instrument}; @@ -1129,7 +1129,7 @@ struct MatrixRow<'p, Cx: PatCx> { /// ``` /// Here the `(true, true)` case is irrelevant. Since we skip it, we will not detect that row 0 /// intersects rows 1 and 2. - intersects_at_least: BitSet, + intersects_at_least: DenseBitSet, /// Whether the head pattern is a branch (see definition of "branch pattern" at /// [`BranchPatUsefulness`]) head_is_branch: bool, @@ -1142,7 +1142,7 @@ impl<'p, Cx: PatCx> MatrixRow<'p, Cx> { parent_row: arm_id, is_under_guard: arm.has_guard, useful: false, - intersects_at_least: BitSet::new_empty(0), // Initialized in `Matrix::push`. + intersects_at_least: DenseBitSet::new_empty(0), // Initialized in `Matrix::push`. // This pattern is a branch because it comes from a match arm. head_is_branch: true, } @@ -1171,7 +1171,7 @@ impl<'p, Cx: PatCx> MatrixRow<'p, Cx> { parent_row, is_under_guard: self.is_under_guard, useful: false, - intersects_at_least: BitSet::new_empty(0), // Initialized in `Matrix::push`. + intersects_at_least: DenseBitSet::new_empty(0), // Initialized in `Matrix::push`. head_is_branch: is_or_pat, }) } @@ -1191,7 +1191,7 @@ impl<'p, Cx: PatCx> MatrixRow<'p, Cx> { parent_row, is_under_guard: self.is_under_guard, useful: false, - intersects_at_least: BitSet::new_empty(0), // Initialized in `Matrix::push`. + intersects_at_least: DenseBitSet::new_empty(0), // Initialized in `Matrix::push`. head_is_branch: false, }) } @@ -1230,7 +1230,7 @@ struct Matrix<'p, Cx: PatCx> { impl<'p, Cx: PatCx> Matrix<'p, Cx> { /// Pushes a new row to the matrix. Internal method, prefer [`Matrix::new`]. fn push(&mut self, mut row: MatrixRow<'p, Cx>) { - row.intersects_at_least = BitSet::new_empty(self.rows.len()); + row.intersects_at_least = DenseBitSet::new_empty(self.rows.len()); self.rows.push(row); } @@ -1824,7 +1824,7 @@ pub struct UsefulnessReport<'p, Cx: PatCx> { pub non_exhaustiveness_witnesses: Vec>, /// For each arm, a set of indices of arms above it that have non-empty intersection, i.e. there /// is a value matched by both arms. This may miss real intersections. - pub arm_intersections: Vec>, + pub arm_intersections: Vec>, } /// Computes whether a match is exhaustive and which of its arms are useful. diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index fae787f9a40..ab606478c51 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -9,7 +9,7 @@ use rustc_abi::{ HasDataLayout, Layout, LayoutCalculatorError, LayoutData, Niche, ReprOptions, Scalar, Size, StructKind, TagEncoding, VariantIdx, Variants, WrappingRange, }; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{CoroutineLayout, CoroutineSavedLocal}; @@ -724,7 +724,7 @@ enum SavedLocalEligibility { /// Compute the eligibility and assignment of each local. fn coroutine_saved_local_eligibility( info: &CoroutineLayout<'_>, -) -> (BitSet, IndexVec) { +) -> (DenseBitSet, IndexVec) { use SavedLocalEligibility::*; let mut assignments: IndexVec = @@ -732,7 +732,7 @@ fn coroutine_saved_local_eligibility( // The saved locals not eligible for overlap. These will get // "promoted" to the prefix of our coroutine. - let mut ineligible_locals = BitSet::new_empty(info.field_tys.len()); + let mut ineligible_locals = DenseBitSet::new_empty(info.field_tys.len()); // Figure out which of our saved locals are fields in only // one variant. The rest are deemed ineligible for overlap. @@ -792,7 +792,7 @@ fn coroutine_saved_local_eligibility( // lay them out with the other locals in the prefix and eliminate // unnecessary padding bytes. { - let mut used_variants = BitSet::new_empty(info.variant_fields.len()); + let mut used_variants = DenseBitSet::new_empty(info.variant_fields.len()); for assignment in &assignments { if let Assigned(idx) = assignment { used_variants.insert(*idx); diff --git a/compiler/rustc_ty_utils/src/representability.rs b/compiler/rustc_ty_utils/src/representability.rs index 0ffb7f62496..98b1550e1a3 100644 --- a/compiler/rustc_ty_utils/src/representability.rs +++ b/compiler/rustc_ty_utils/src/representability.rs @@ -1,5 +1,5 @@ use rustc_hir::def::DefKind; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::query::Providers; use rustc_middle::ty::{self, Representability, Ty, TyCtxt}; @@ -83,10 +83,10 @@ fn representability_adt_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Representab Representability::Representable } -fn params_in_repr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> BitSet { +fn params_in_repr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> DenseBitSet { let adt_def = tcx.adt_def(def_id); let generics = tcx.generics_of(def_id); - let mut params_in_repr = BitSet::new_empty(generics.own_params.len()); + let mut params_in_repr = DenseBitSet::new_empty(generics.own_params.len()); for variant in adt_def.variants() { for field in variant.fields.iter() { params_in_repr_ty( @@ -99,7 +99,7 @@ fn params_in_repr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> BitSet { params_in_repr } -fn params_in_repr_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, params_in_repr: &mut BitSet) { +fn params_in_repr_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, params_in_repr: &mut DenseBitSet) { match *ty.kind() { ty::Adt(adt, args) => { let inner_params_in_repr = tcx.params_in_repr(adt.did()); diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs index 7eed32e3a33..8ed45b4e541 100644 --- a/compiler/rustc_ty_utils/src/ty.rs +++ b/compiler/rustc_ty_utils/src/ty.rs @@ -2,7 +2,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; use rustc_hir::LangItem; use rustc_hir::def::DefKind; -use rustc_index::bit_set::BitSet; +use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; use rustc_middle::query::Providers; use rustc_middle::ty::fold::fold_regions; @@ -317,7 +317,7 @@ fn asyncness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Asyncness { }) } -fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> BitSet { +fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> DenseBitSet { let def = tcx.adt_def(def_id); let num_params = tcx.generics_of(def_id).count(); @@ -338,10 +338,10 @@ fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> BitSet bool; fn coroutine_is_async_gen(self, coroutine_def_id: Self::DefId) -> bool; - type UnsizingParams: Deref>; + type UnsizingParams: Deref>; fn unsizing_params_for_adt(self, adt_def_id: Self::DefId) -> Self::UnsizingParams; fn find_const_ty_from_env( -- cgit 1.4.1-3-g733a5