diff options
Diffstat (limited to 'compiler')
123 files changed, 2855 insertions, 1519 deletions
diff --git a/compiler/rustc_abi/src/extern_abi/mod.rs b/compiler/rustc_abi/src/extern_abi/mod.rs index 28860afb37a..130834d560f 100644 --- a/compiler/rustc_abi/src/extern_abi/mod.rs +++ b/compiler/rustc_abi/src/extern_abi/mod.rs @@ -45,6 +45,9 @@ pub enum ExternAbi { PtxKernel, Msp430Interrupt, X86Interrupt, + /// An entry-point function called by the GPU's host + // FIXME: should not be callable from Rust on GPU targets, is for host's use only + GpuKernel, EfiApi, AvrInterrupt, AvrNonBlockingInterrupt, @@ -122,6 +125,7 @@ const AbiDatas: &[AbiData] = &[ AbiData { abi: Abi::PtxKernel, name: "ptx-kernel" }, AbiData { abi: Abi::Msp430Interrupt, name: "msp430-interrupt" }, AbiData { abi: Abi::X86Interrupt, name: "x86-interrupt" }, + AbiData { abi: Abi::GpuKernel, name: "gpu-kernel" }, AbiData { abi: Abi::EfiApi, name: "efiapi" }, AbiData { abi: Abi::AvrInterrupt, name: "avr-interrupt" }, AbiData { abi: Abi::AvrNonBlockingInterrupt, name: "avr-non-blocking-interrupt" }, @@ -239,6 +243,10 @@ pub fn is_stable(name: &str) -> Result<(), AbiDisabled> { feature: sym::abi_x86_interrupt, explain: "x86-interrupt ABI is experimental and subject to change", }), + "gpu-kernel" => Err(AbiDisabled::Unstable { + feature: sym::abi_gpu_kernel, + explain: "gpu-kernel ABI is experimental and subject to change", + }), "avr-interrupt" | "avr-non-blocking-interrupt" => Err(AbiDisabled::Unstable { feature: sym::abi_avr_interrupt, explain: "avr-interrupt and avr-non-blocking-interrupt ABIs are experimental and subject to change", @@ -293,20 +301,21 @@ impl Abi { PtxKernel => 19, Msp430Interrupt => 20, X86Interrupt => 21, - EfiApi => 22, - AvrInterrupt => 23, - AvrNonBlockingInterrupt => 24, - CCmseNonSecureCall => 25, - CCmseNonSecureEntry => 26, + GpuKernel => 22, + EfiApi => 23, + AvrInterrupt => 24, + AvrNonBlockingInterrupt => 25, + CCmseNonSecureCall => 26, + CCmseNonSecureEntry => 27, // Cross-platform ABIs - System { unwind: false } => 27, - System { unwind: true } => 28, - RustIntrinsic => 29, - RustCall => 30, - Unadjusted => 31, - RustCold => 32, - RiscvInterruptM => 33, - RiscvInterruptS => 34, + System { unwind: false } => 28, + System { unwind: true } => 29, + RustIntrinsic => 30, + RustCall => 31, + Unadjusted => 32, + RustCold => 33, + RiscvInterruptM => 34, + RiscvInterruptS => 35, }; debug_assert!( AbiDatas diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 97385b2eaab..51f18580013 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -723,6 +723,8 @@ impl MetaItemLit { pub trait AttributeExt: Debug { fn id(&self) -> AttrId; + /// For a single-segment attribute (i.e., `#[attr]` and not `#[path::atrr]`), + /// return the name of the attribute, else return the empty identifier. fn name_or_empty(&self) -> Symbol { self.ident().unwrap_or_else(Ident::empty).name } diff --git a/compiler/rustc_ast_lowering/src/delegation.rs b/compiler/rustc_ast_lowering/src/delegation.rs index 758f1dc1c35..266e77c0e02 100644 --- a/compiler/rustc_ast_lowering/src/delegation.rs +++ b/compiler/rustc_ast_lowering/src/delegation.rs @@ -188,7 +188,14 @@ impl<'hir> LoweringContext<'_, 'hir> { ) -> hir::FnSig<'hir> { let header = if let Some(local_sig_id) = sig_id.as_local() { match self.resolver.delegation_fn_sigs.get(&local_sig_id) { - Some(sig) => self.lower_fn_header(sig.header, hir::Safety::Safe), + Some(sig) => self.lower_fn_header( + sig.header, + // HACK: we override the default safety instead of generating attributes from the ether. + // We are not forwarding the attributes, as the delegation fn sigs are collected on the ast, + // and here we need the hir attributes. + if sig.target_feature { hir::Safety::Unsafe } else { hir::Safety::Safe }, + &[], + ), None => self.generate_header_error(), } } else { @@ -198,7 +205,11 @@ impl<'hir> LoweringContext<'_, 'hir> { Asyncness::No => hir::IsAsync::NotAsync, }; hir::FnHeader { - safety: sig.safety, + safety: if self.tcx.codegen_fn_attrs(sig_id).safe_target_features { + hir::HeaderSafety::SafeTargetFeatures + } else { + hir::HeaderSafety::Normal(sig.safety) + }, constness: self.tcx.constness(sig_id), asyncness, abi: sig.abi, @@ -384,7 +395,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn generate_header_error(&self) -> hir::FnHeader { hir::FnHeader { - safety: hir::Safety::Safe, + safety: hir::Safety::Safe.into(), constness: hir::Constness::NotConst, asyncness: hir::IsAsync::NotAsync, abi: abi::Abi::Rust, diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index 6fce9116938..61d7da429f8 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -231,7 +231,7 @@ impl<'hir> LoweringContext<'_, 'hir> { }); let sig = hir::FnSig { decl, - header: this.lower_fn_header(*header, hir::Safety::Safe), + header: this.lower_fn_header(*header, hir::Safety::Safe, attrs), span: this.lower_span(*fn_sig_span), }; hir::ItemKind::Fn { sig, generics, body: body_id, has_body: body.is_some() } @@ -610,7 +610,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_foreign_item(&mut self, i: &ForeignItem) -> &'hir hir::ForeignItem<'hir> { let hir_id = hir::HirId::make_owner(self.current_hir_id_owner.def_id); let owner_id = hir_id.expect_owner(); - self.lower_attrs(hir_id, &i.attrs); + let attrs = self.lower_attrs(hir_id, &i.attrs); let item = hir::ForeignItem { owner_id, ident: self.lower_ident(i.ident), @@ -634,7 +634,7 @@ impl<'hir> LoweringContext<'_, 'hir> { }); // Unmarked safety in unsafe block defaults to unsafe. - let header = self.lower_fn_header(sig.header, hir::Safety::Unsafe); + let header = self.lower_fn_header(sig.header, hir::Safety::Unsafe, attrs); hir::ForeignItemKind::Fn( hir::FnSig { header, decl, span: self.lower_span(sig.span) }, @@ -776,6 +776,7 @@ impl<'hir> LoweringContext<'_, 'hir> { i.id, FnDeclKind::Trait, sig.header.coroutine_kind, + attrs, ); (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)), false) } @@ -795,6 +796,7 @@ impl<'hir> LoweringContext<'_, 'hir> { i.id, FnDeclKind::Trait, sig.header.coroutine_kind, + attrs, ); (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)), true) } @@ -911,6 +913,7 @@ impl<'hir> LoweringContext<'_, 'hir> { i.id, if self.is_in_trait_impl { FnDeclKind::Impl } else { FnDeclKind::Inherent }, sig.header.coroutine_kind, + attrs, ); (generics, hir::ImplItemKind::Fn(sig, body_id)) @@ -1339,8 +1342,9 @@ impl<'hir> LoweringContext<'_, 'hir> { id: NodeId, kind: FnDeclKind, coroutine_kind: Option<CoroutineKind>, + attrs: &[hir::Attribute], ) -> (&'hir hir::Generics<'hir>, hir::FnSig<'hir>) { - let header = self.lower_fn_header(sig.header, hir::Safety::Safe); + let header = self.lower_fn_header(sig.header, hir::Safety::Safe, attrs); let itctx = ImplTraitContext::Universal; let (generics, decl) = self.lower_generics(generics, id, itctx, |this| { this.lower_fn_decl(&sig.decl, id, sig.span, kind, coroutine_kind) @@ -1352,14 +1356,28 @@ impl<'hir> LoweringContext<'_, 'hir> { &mut self, h: FnHeader, default_safety: hir::Safety, + attrs: &[hir::Attribute], ) -> hir::FnHeader { let asyncness = if let Some(CoroutineKind::Async { span, .. }) = h.coroutine_kind { hir::IsAsync::Async(span) } else { hir::IsAsync::NotAsync }; + + let safety = self.lower_safety(h.safety, default_safety); + + // Treat safe `#[target_feature]` functions as unsafe, but also remember that we did so. + let safety = if attrs.iter().any(|attr| attr.has_name(sym::target_feature)) + && safety.is_safe() + && !self.tcx.sess.target.is_like_wasm + { + hir::HeaderSafety::SafeTargetFeatures + } else { + safety.into() + }; + hir::FnHeader { - safety: self.lower_safety(h.safety, default_safety), + safety, asyncness, constness: self.lower_constness(h.constness), abi: self.lower_extern(h.ext), diff --git a/compiler/rustc_attr_data_structures/src/stability.rs b/compiler/rustc_attr_data_structures/src/stability.rs index 3c77d4c766c..dfda04387ec 100644 --- a/compiler/rustc_attr_data_structures/src/stability.rs +++ b/compiler/rustc_attr_data_structures/src/stability.rs @@ -101,6 +101,16 @@ impl PartialConstStability { } } +#[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)] +#[derive(HashStable_Generic)] +pub enum AllowedThroughUnstableModules { + /// This does not get a deprecation warning. We still generally would prefer people to use the + /// fully stable path, and a warning will likely be emitted in the future. + WithoutDeprecation, + /// Emit the given deprecation warning. + WithDeprecation(Symbol), +} + /// The available stability levels. #[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)] #[derive(HashStable_Generic)] @@ -137,9 +147,8 @@ pub enum StabilityLevel { Stable { /// Rust release which stabilized this feature. since: StableSince, - /// Is this item allowed to be referred to on stable, despite being contained in unstable - /// modules? - allowed_through_unstable_modules: bool, + /// This is `Some` if this item allowed to be referred to on stable via unstable modules. + allowed_through_unstable_modules: Option<AllowedThroughUnstableModules>, }, } diff --git a/compiler/rustc_attr_parsing/src/attributes/stability.rs b/compiler/rustc_attr_parsing/src/attributes/stability.rs index 89937e1c593..bfbe51b27d8 100644 --- a/compiler/rustc_attr_parsing/src/attributes/stability.rs +++ b/compiler/rustc_attr_parsing/src/attributes/stability.rs @@ -6,8 +6,8 @@ use rustc_ast::MetaItem; use rustc_ast::attr::AttributeExt; use rustc_ast_pretty::pprust; use rustc_attr_data_structures::{ - ConstStability, DefaultBodyStability, Stability, StabilityLevel, StableSince, UnstableReason, - VERSION_PLACEHOLDER, + AllowedThroughUnstableModules, ConstStability, DefaultBodyStability, Stability, StabilityLevel, + StableSince, UnstableReason, VERSION_PLACEHOLDER, }; use rustc_errors::ErrorGuaranteed; use rustc_session::Session; @@ -24,11 +24,16 @@ pub fn find_stability( item_sp: Span, ) -> Option<(Stability, Span)> { let mut stab: Option<(Stability, Span)> = None; - let mut allowed_through_unstable_modules = false; + let mut allowed_through_unstable_modules = None; for attr in attrs { match attr.name_or_empty() { - sym::rustc_allowed_through_unstable_modules => allowed_through_unstable_modules = true, + sym::rustc_allowed_through_unstable_modules => { + allowed_through_unstable_modules = Some(match attr.value_str() { + Some(msg) => AllowedThroughUnstableModules::WithDeprecation(msg), + None => AllowedThroughUnstableModules::WithoutDeprecation, + }) + } sym::unstable => { if stab.is_some() { sess.dcx().emit_err(session_diagnostics::MultipleStabilityLevels { @@ -56,15 +61,15 @@ pub fn find_stability( } } - if allowed_through_unstable_modules { + if let Some(allowed_through_unstable_modules) = allowed_through_unstable_modules { match &mut stab { Some(( Stability { - level: StabilityLevel::Stable { allowed_through_unstable_modules, .. }, + level: StabilityLevel::Stable { allowed_through_unstable_modules: in_stab, .. }, .. }, _, - )) => *allowed_through_unstable_modules = true, + )) => *in_stab = Some(allowed_through_unstable_modules), _ => { sess.dcx() .emit_err(session_diagnostics::RustcAllowedUnstablePairing { span: item_sp }); @@ -283,7 +288,7 @@ fn parse_stability(sess: &Session, attr: &impl AttributeExt) -> Option<(Symbol, match feature { Ok(feature) => { - let level = StabilityLevel::Stable { since, allowed_through_unstable_modules: false }; + let level = StabilityLevel::Stable { since, allowed_through_unstable_modules: None }; Some((feature, level)) } Err(ErrorGuaranteed { .. }) => None, diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs index a7a6f2da509..7511a55b03a 100644 --- a/compiler/rustc_borrowck/src/dataflow.rs +++ b/compiler/rustc_borrowck/src/dataflow.rs @@ -187,19 +187,28 @@ struct OutOfScopePrecomputer<'a, 'tcx> { borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>, } -impl<'a, 'tcx> OutOfScopePrecomputer<'a, 'tcx> { - fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self { - OutOfScopePrecomputer { +impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> { + fn compute( + body: &Body<'tcx>, + regioncx: &RegionInferenceContext<'tcx>, + borrow_set: &BorrowSet<'tcx>, + ) -> FxIndexMap<Location, Vec<BorrowIndex>> { + let mut prec = OutOfScopePrecomputer { visited: DenseBitSet::new_empty(body.basic_blocks.len()), visit_stack: vec![], body, regioncx, borrows_out_of_scope_at_location: FxIndexMap::default(), + }; + for (borrow_index, borrow_data) in borrow_set.iter_enumerated() { + let borrow_region = borrow_data.region; + let location = borrow_data.reserve_location; + prec.precompute_borrows_out_of_scope(borrow_index, borrow_region, location); } + + prec.borrows_out_of_scope_at_location } -} -impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> { fn precompute_borrows_out_of_scope( &mut self, borrow_index: BorrowIndex, @@ -280,15 +289,7 @@ pub fn calculate_borrows_out_of_scope_at_location<'tcx>( regioncx: &RegionInferenceContext<'tcx>, borrow_set: &BorrowSet<'tcx>, ) -> FxIndexMap<Location, Vec<BorrowIndex>> { - let mut prec = OutOfScopePrecomputer::new(body, regioncx); - for (borrow_index, borrow_data) in borrow_set.iter_enumerated() { - let borrow_region = borrow_data.region; - let location = borrow_data.reserve_location; - - prec.precompute_borrows_out_of_scope(borrow_index, borrow_region, location); - } - - prec.borrows_out_of_scope_at_location + OutOfScopePrecomputer::compute(body, regioncx, borrow_set) } struct PoloniusOutOfScopePrecomputer<'a, 'tcx> { @@ -300,19 +301,30 @@ struct PoloniusOutOfScopePrecomputer<'a, 'tcx> { loans_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>, } -impl<'a, 'tcx> PoloniusOutOfScopePrecomputer<'a, 'tcx> { - fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self { - Self { +impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { + fn compute( + body: &Body<'tcx>, + regioncx: &RegionInferenceContext<'tcx>, + borrow_set: &BorrowSet<'tcx>, + ) -> FxIndexMap<Location, Vec<BorrowIndex>> { + // The in-tree polonius analysis computes loans going out of scope using the + // set-of-loans model. + let mut prec = PoloniusOutOfScopePrecomputer { visited: DenseBitSet::new_empty(body.basic_blocks.len()), visit_stack: vec![], body, regioncx, loans_out_of_scope_at_location: FxIndexMap::default(), + }; + for (loan_idx, loan_data) in borrow_set.iter_enumerated() { + let issuing_region = loan_data.region; + let loan_issued_at = loan_data.reserve_location; + prec.precompute_loans_out_of_scope(loan_idx, issuing_region, loan_issued_at); } + + prec.loans_out_of_scope_at_location } -} -impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { /// Loans are in scope while they are live: whether they are contained within any live region. /// In the location-insensitive analysis, a loan will be contained in a region if the issuing /// region can reach it in the subset graph. So this is a reachability problem. @@ -325,10 +337,17 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { let sccs = self.regioncx.constraint_sccs(); let universal_regions = self.regioncx.universal_regions(); + // The loop below was useful for the location-insensitive analysis but shouldn't be + // impactful in the location-sensitive case. It seems that it does, however, as without it a + // handful of tests fail. That likely means some liveness or outlives data related to choice + // regions is missing + // FIXME: investigate the impact of loans traversing applied member constraints and why some + // tests fail otherwise. + // // We first handle the cases where the loan doesn't go out of scope, depending on the // issuing region's successors. for successor in graph::depth_first_search(&self.regioncx.region_graph(), issuing_region) { - // 1. Via applied member constraints + // Via applied member constraints // // The issuing region can flow into the choice regions, and they are either: // - placeholders or free regions themselves, @@ -346,14 +365,6 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { return; } } - - // 2. Via regions that are live at all points: placeholders and free regions. - // - // If the issuing region outlives such a region, its loan escapes the function and - // cannot go out of scope. We can early return. - if self.regioncx.is_region_live_at_all_points(successor) { - return; - } } let first_block = loan_issued_at.block; @@ -461,34 +472,12 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> { regioncx: &RegionInferenceContext<'tcx>, borrow_set: &'a BorrowSet<'tcx>, ) -> Self { - let mut borrows_out_of_scope_at_location = - calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set); - - // The in-tree polonius analysis computes loans going out of scope using the set-of-loans - // model, and makes sure they're identical to the existing computation of the set-of-points - // model. - if tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { - let mut polonius_prec = PoloniusOutOfScopePrecomputer::new(body, regioncx); - for (loan_idx, loan_data) in borrow_set.iter_enumerated() { - let issuing_region = loan_data.region; - let loan_issued_at = loan_data.reserve_location; - - polonius_prec.precompute_loans_out_of_scope( - loan_idx, - issuing_region, - loan_issued_at, - ); - } - - assert_eq!( - borrows_out_of_scope_at_location, polonius_prec.loans_out_of_scope_at_location, - "polonius loan scopes differ from NLL borrow scopes, for body {:?}", - body.span, - ); - - borrows_out_of_scope_at_location = polonius_prec.loans_out_of_scope_at_location; - } - + let borrows_out_of_scope_at_location = + if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { + calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set) + } else { + PoloniusOutOfScopePrecomputer::compute(body, regioncx, borrow_set) + }; Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location } } diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 45764f9c6b8..c6e6d962ce5 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -182,9 +182,6 @@ fn do_mir_borrowck<'tcx>( let location_table = PoloniusLocationTable::new(body); let move_data = MoveData::gather_moves(body, tcx, |_| true); - let promoted_move_data = promoted - .iter_enumerated() - .map(|(idx, body)| (idx, MoveData::gather_moves(body, tcx, |_| true))); let flow_inits = MaybeInitializedPlaces::new(tcx, body, &move_data) .iterate_to_fixpoint(tcx, body, Some("borrowck")) @@ -242,10 +239,14 @@ fn do_mir_borrowck<'tcx>( false }; - for (idx, move_data) in promoted_move_data { + // While promoteds should mostly be correct by construction, we need to check them for + // invalid moves to detect moving out of arrays:`struct S; fn main() { &([S][0]); }`. + for promoted_body in &promoted { use rustc_middle::mir::visit::Visitor; - - let promoted_body = &promoted[idx]; + // This assumes that we won't use some of the fields of the `promoted_mbcx` + // when detecting and reporting move errors. While it would be nice to move + // this check out of `MirBorrowckCtxt`, actually doing so is far from trivial. + let move_data = MoveData::gather_moves(promoted_body, tcx, |_| true); let mut promoted_mbcx = MirBorrowckCtxt { infcx: &infcx, body: promoted_body, @@ -270,9 +271,6 @@ fn do_mir_borrowck<'tcx>( move_errors: Vec::new(), diags_buffer, }; - MoveVisitor { ctxt: &mut promoted_mbcx }.visit_body(promoted_body); - promoted_mbcx.report_move_errors(); - struct MoveVisitor<'a, 'b, 'infcx, 'tcx> { ctxt: &'a mut MirBorrowckCtxt<'b, 'infcx, 'tcx>, } @@ -284,6 +282,8 @@ fn do_mir_borrowck<'tcx>( } } } + MoveVisitor { ctxt: &mut promoted_mbcx }.visit_body(promoted_body); + promoted_mbcx.report_move_errors(); } let mut mbcx = MirBorrowckCtxt { diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index aa0bfd72147..35264bd1a70 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -103,7 +103,7 @@ pub(crate) fn compute_regions<'a, 'tcx>( constraints, universal_region_relations, opaque_type_values, - mut polonius_context, + polonius_context, } = type_check::type_check( infcx, body, @@ -142,10 +142,10 @@ pub(crate) fn compute_regions<'a, 'tcx>( location_map, ); - // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives - // constraints. - let localized_outlives_constraints = polonius_context.as_mut().map(|polonius_context| { - polonius_context.create_localized_constraints(infcx.tcx, ®ioncx, body) + // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints + // and use them to compute loan liveness. + let localized_outlives_constraints = polonius_context.as_ref().map(|polonius_context| { + polonius_context.compute_loan_liveness(infcx.tcx, &mut regioncx, body, borrow_set) }); // If requested: dump NLL facts, and run legacy polonius analysis. diff --git a/compiler/rustc_borrowck/src/polonius/loan_liveness.rs b/compiler/rustc_borrowck/src/polonius/loan_liveness.rs new file mode 100644 index 00000000000..768c12a97a6 --- /dev/null +++ b/compiler/rustc_borrowck/src/polonius/loan_liveness.rs @@ -0,0 +1,307 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; +use rustc_middle::mir::visit::Visitor; +use rustc_middle::mir::{ + Body, Local, Location, Place, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, +}; +use rustc_middle::ty::{RegionVid, TyCtxt}; +use rustc_mir_dataflow::points::PointIndex; + +use super::{LiveLoans, LocalizedOutlivesConstraintSet}; +use crate::constraints::OutlivesConstraint; +use crate::dataflow::BorrowIndex; +use crate::region_infer::values::LivenessValues; +use crate::type_check::Locations; +use crate::{BorrowSet, PlaceConflictBias, places_conflict}; + +/// Compute loan reachability, stop at kills, and trace loan liveness throughout the CFG, by +/// traversing the full graph of constraints that combines: +/// - the localized constraints (the physical edges), +/// - with the constraints that hold at all points (the logical edges). +pub(super) fn compute_loan_liveness<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + liveness: &LivenessValues, + outlives_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>, + borrow_set: &BorrowSet<'tcx>, + localized_outlives_constraints: &LocalizedOutlivesConstraintSet, +) -> LiveLoans { + let mut live_loans = LiveLoans::new(borrow_set.len()); + + // FIXME: it may be preferable for kills to be encoded in the edges themselves, to simplify and + // likely make traversal (and constraint generation) more efficient. We also display kills on + // edges when visualizing the constraint graph anyways. + let kills = collect_kills(body, tcx, borrow_set); + + // Create the full graph with the physical edges we've localized earlier, and the logical edges + // of constraints that hold at all points. + let logical_constraints = + outlives_constraints.filter(|c| matches!(c.locations, Locations::All(_))); + let graph = LocalizedConstraintGraph::new(&localized_outlives_constraints, logical_constraints); + let mut visited = FxHashSet::default(); + let mut stack = Vec::new(); + + // Compute reachability per loan by traversing each loan's subgraph starting from where it is + // introduced. + for (loan_idx, loan) in borrow_set.iter_enumerated() { + visited.clear(); + stack.clear(); + + let start_node = LocalizedNode { + region: loan.region, + point: liveness.point_from_location(loan.reserve_location), + }; + stack.push(start_node); + + while let Some(node) = stack.pop() { + if !visited.insert(node) { + continue; + } + + // Record the loan as being live on entry to this point. + live_loans.insert(node.point, loan_idx); + + // Here, we have a conundrum. There's currently a weakness in our theory, in that + // we're using a single notion of reachability to represent what used to be _two_ + // different transitive closures. It didn't seem impactful when coming up with the + // single-graph and reachability through space (regions) + time (CFG) concepts, but in + // practice the combination of time-traveling with kills is more impactful than + // initially anticipated. + // + // Kills should prevent a loan from reaching its successor points in the CFG, but not + // while time-traveling: we're not actually at that CFG point, but looking for + // predecessor regions that contain the loan. One of the two TCs we had pushed the + // transitive subset edges to each point instead of having backward edges, and the + // problem didn't exist before. In the abstract, naive reachability is not enough to + // model this, we'd need a slightly different solution. For example, maybe with a + // two-step traversal: + // - at each point we first traverse the subgraph (and possibly time-travel) looking for + // exit nodes while ignoring kills, + // - and then when we're back at the current point, we continue normally. + // + // Another (less annoying) subtlety is that kills and the loan use-map are + // flow-insensitive. Kills can actually appear in places before a loan is introduced, or + // at a location that is actually unreachable in the CFG from the introduction point, + // and these can also be encountered during time-traveling. + // + // The simplest change that made sense to "fix" the issues above is taking into + // account kills that are: + // - reachable from the introduction point + // - encountered during forward traversal. Note that this is not transitive like the + // two-step traversal described above: only kills encountered on exit via a backward + // edge are ignored. + // + // In our test suite, there are a couple of cases where kills are encountered while + // time-traveling, however as far as we can tell, always in cases where they would be + // unreachable. We have reason to believe that this is a property of the single-graph + // approach (but haven't proved it yet): + // - reachable kills while time-traveling would also be encountered via regular + // traversal + // - it makes _some_ sense to ignore unreachable kills, but subtleties around dead code + // in general need to be better thought through (like they were for NLLs). + // - ignoring kills is a conservative approximation: the loan is still live and could + // cause false positive errors at another place access. Soundness issues in this + // domain should look more like the absence of reachability instead. + // + // This is enough in practice to pass tests, and therefore is what we have implemented + // for now. + // + // FIXME: all of the above. Analyze potential unsoundness, possibly in concert with a + // borrowck implementation in a-mir-formality, fuzzing, or manually crafting + // counter-examples. + + // Continuing traversal will depend on whether the loan is killed at this point, and + // whether we're time-traveling. + let current_location = liveness.location_from_point(node.point); + let is_loan_killed = + kills.get(¤t_location).is_some_and(|kills| kills.contains(&loan_idx)); + + for succ in graph.outgoing_edges(node) { + // If the loan is killed at this point, it is killed _on exit_. But only during + // forward traversal. + if is_loan_killed { + let destination = liveness.location_from_point(succ.point); + if current_location.is_predecessor_of(destination, body) { + continue; + } + } + stack.push(succ); + } + } + } + + live_loans +} + +/// The localized constraint graph indexes the physical and logical edges to compute a given node's +/// successors during traversal. +struct LocalizedConstraintGraph { + /// The actual, physical, edges we have recorded for a given node. + edges: FxHashMap<LocalizedNode, FxIndexSet<LocalizedNode>>, + + /// The logical edges representing the outlives constraints that hold at all points in the CFG, + /// which we don't localize to avoid creating a lot of unnecessary edges in the graph. Some CFGs + /// can be big, and we don't need to create such a physical edge for every point in the CFG. + logical_edges: FxHashMap<RegionVid, FxIndexSet<RegionVid>>, +} + +/// A node in the graph to be traversed, one of the two vertices of a localized outlives constraint. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +struct LocalizedNode { + region: RegionVid, + point: PointIndex, +} + +impl LocalizedConstraintGraph { + /// Traverses the constraints and returns the indexed graph of edges per node. + fn new<'tcx>( + constraints: &LocalizedOutlivesConstraintSet, + logical_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>, + ) -> Self { + let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); + for constraint in &constraints.outlives { + let source = LocalizedNode { region: constraint.source, point: constraint.from }; + let target = LocalizedNode { region: constraint.target, point: constraint.to }; + edges.entry(source).or_default().insert(target); + } + + let mut logical_edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); + for constraint in logical_constraints { + logical_edges.entry(constraint.sup).or_default().insert(constraint.sub); + } + + LocalizedConstraintGraph { edges, logical_edges } + } + + /// Returns the outgoing edges of a given node, not its transitive closure. + fn outgoing_edges(&self, node: LocalizedNode) -> impl Iterator<Item = LocalizedNode> + use<'_> { + // The outgoing edges are: + // - the physical edges present at this node, + // - the materialized logical edges that exist virtually at all points for this node's + // region, localized at this point. + let physical_edges = + self.edges.get(&node).into_iter().flat_map(|targets| targets.iter().copied()); + let materialized_edges = + self.logical_edges.get(&node.region).into_iter().flat_map(move |targets| { + targets + .iter() + .copied() + .map(move |target| LocalizedNode { point: node.point, region: target }) + }); + physical_edges.chain(materialized_edges) + } +} + +/// Traverses the MIR and collects kills. +fn collect_kills<'tcx>( + body: &Body<'tcx>, + tcx: TyCtxt<'tcx>, + borrow_set: &BorrowSet<'tcx>, +) -> BTreeMap<Location, BTreeSet<BorrowIndex>> { + let mut collector = KillsCollector { borrow_set, tcx, body, kills: BTreeMap::default() }; + for (block, data) in body.basic_blocks.iter_enumerated() { + collector.visit_basic_block_data(block, data); + } + collector.kills +} + +struct KillsCollector<'a, 'tcx> { + body: &'a Body<'tcx>, + tcx: TyCtxt<'tcx>, + borrow_set: &'a BorrowSet<'tcx>, + + /// The set of loans killed at each location. + kills: BTreeMap<Location, BTreeSet<BorrowIndex>>, +} + +// This visitor has a similar structure to the `Borrows` dataflow computation with respect to kills, +// and the datalog polonius fact generation for the `loan_killed_at` relation. +impl<'tcx> KillsCollector<'_, 'tcx> { + /// Records the borrows on the specified place as `killed`. For example, when assigning to a + /// local, or on a call's return destination. + fn record_killed_borrows_for_place(&mut self, place: Place<'tcx>, location: Location) { + // For the reasons described in graph traversal, we also filter out kills + // unreachable from the loan's introduction point, as they would stop traversal when + // e.g. checking for reachability in the subset graph through invariance constraints + // higher up. + let filter_unreachable_kills = |loan| { + let introduction = self.borrow_set[loan].reserve_location; + let reachable = introduction.is_predecessor_of(location, self.body); + reachable + }; + + let other_borrows_of_local = self + .borrow_set + .local_map + .get(&place.local) + .into_iter() + .flat_map(|bs| bs.iter()) + .copied(); + + // If the borrowed place is a local with no projections, all other borrows of this + // local must conflict. This is purely an optimization so we don't have to call + // `places_conflict` for every borrow. + if place.projection.is_empty() { + if !self.body.local_decls[place.local].is_ref_to_static() { + self.kills + .entry(location) + .or_default() + .extend(other_borrows_of_local.filter(|&loan| filter_unreachable_kills(loan))); + } + return; + } + + // By passing `PlaceConflictBias::NoOverlap`, we conservatively assume that any given + // pair of array indices are not equal, so that when `places_conflict` returns true, we + // will be assured that two places being compared definitely denotes the same sets of + // locations. + let definitely_conflicting_borrows = other_borrows_of_local + .filter(|&i| { + places_conflict( + self.tcx, + self.body, + self.borrow_set[i].borrowed_place, + place, + PlaceConflictBias::NoOverlap, + ) + }) + .filter(|&loan| filter_unreachable_kills(loan)); + + self.kills.entry(location).or_default().extend(definitely_conflicting_borrows); + } + + /// Records the borrows on the specified local as `killed`. + fn record_killed_borrows_for_local(&mut self, local: Local, location: Location) { + if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) { + self.kills.entry(location).or_default().extend(borrow_indices.iter()); + } + } +} + +impl<'tcx> Visitor<'tcx> for KillsCollector<'_, 'tcx> { + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + // Make sure there are no remaining borrows for locals that have gone out of scope. + if let StatementKind::StorageDead(local) = statement.kind { + self.record_killed_borrows_for_local(local, location); + } + + self.super_statement(statement, location); + } + + fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + // When we see `X = ...`, then kill borrows of `(*X).foo` and so forth. + self.record_killed_borrows_for_place(*place, location); + self.super_assign(place, rvalue, location); + } + + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { + // A `Call` terminator's return value can be a local which has borrows, so we need to record + // those as killed as well. + if let TerminatorKind::Call { destination, .. } = terminator.kind { + self.record_killed_borrows_for_place(destination, location); + } + + self.super_terminator(terminator, location); + } +} diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index 7d0f9397021..502c868194a 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -37,6 +37,7 @@ mod constraints; mod dump; pub(crate) mod legacy; mod liveness_constraints; +mod loan_liveness; mod typeck_constraints; use std::collections::BTreeMap; @@ -49,8 +50,12 @@ use rustc_mir_dataflow::points::PointIndex; pub(crate) use self::constraints::*; pub(crate) use self::dump::dump_polonius_mir; use self::liveness_constraints::create_liveness_constraints; +use self::loan_liveness::compute_loan_liveness; use self::typeck_constraints::convert_typeck_constraints; -use crate::RegionInferenceContext; +use crate::dataflow::BorrowIndex; +use crate::{BorrowSet, RegionInferenceContext}; + +pub(crate) type LiveLoans = SparseBitMatrix<PointIndex, BorrowIndex>; /// This struct holds the data needed to create the Polonius localized constraints. pub(crate) struct PoloniusContext { @@ -82,14 +87,20 @@ impl PoloniusContext { Self { live_region_variances: BTreeMap::new(), live_regions: None } } - /// Creates a constraint set for `-Zpolonius=next` by: + /// Computes live loans using the set of loans model for `-Zpolonius=next`. + /// + /// First, creates a constraint graph combining regions and CFG points, by: /// - converting NLL typeck constraints to be localized /// - encoding liveness constraints - pub(crate) fn create_localized_constraints<'tcx>( + /// + /// Then, this graph is traversed, and combined with kills, reachability is recorded as loan + /// liveness, to be used by the loan scope and active loans computations. + pub(crate) fn compute_loan_liveness<'tcx>( &self, tcx: TyCtxt<'tcx>, - regioncx: &RegionInferenceContext<'tcx>, + regioncx: &mut RegionInferenceContext<'tcx>, body: &Body<'tcx>, + borrow_set: &BorrowSet<'tcx>, ) -> LocalizedOutlivesConstraintSet { let mut localized_outlives_constraints = LocalizedOutlivesConstraintSet::default(); convert_typeck_constraints( @@ -113,8 +124,17 @@ impl PoloniusContext { &mut localized_outlives_constraints, ); - // FIXME: here, we can trace loan reachability in the constraint graph and record this as loan - // liveness for the next step in the chain, the NLL loan scope and active loans computations. + // Now that we have a complete graph, we can compute reachability to trace the liveness of + // loans for the next step in the chain, the NLL loan scope and active loans computations. + let live_loans = compute_loan_liveness( + tcx, + body, + regioncx.liveness_constraints(), + regioncx.outlives_constraints(), + borrow_set, + &localized_outlives_constraints, + ); + regioncx.record_live_loans(live_loans); localized_outlives_constraints } diff --git a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs index 8235b844886..1289b1899eb 100644 --- a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs @@ -22,23 +22,11 @@ pub(super) fn convert_typeck_constraints<'tcx>( for outlives_constraint in outlives_constraints { match outlives_constraint.locations { Locations::All(_) => { - // For now, turn logical constraints holding at all points into physical edges at - // every point in the graph. - // FIXME: encode this into *traversal* instead. - for (block, bb) in body.basic_blocks.iter_enumerated() { - let statement_count = bb.statements.len(); - for statement_index in 0..=statement_count { - let current_location = Location { block, statement_index }; - let current_point = liveness.point_from_location(current_location); - - localized_outlives_constraints.push(LocalizedOutlivesConstraint { - source: outlives_constraint.sup, - from: current_point, - target: outlives_constraint.sub, - to: current_point, - }); - } - } + // We don't turn constraints holding at all points into physical edges at every + // point in the graph. They are encoded into *traversal* instead: a given node's + // successors will combine these logical edges with the regular, physical, localized + // edges. + continue; } Locations::Single(location) => { diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs index c177538ee17..d2268c4779d 100644 --- a/compiler/rustc_borrowck/src/region_infer/mod.rs +++ b/compiler/rustc_borrowck/src/region_infer/mod.rs @@ -31,6 +31,7 @@ use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstra use crate::dataflow::BorrowIndex; use crate::diagnostics::{RegionErrorKind, RegionErrors, UniverseInfo}; use crate::member_constraints::{MemberConstraintSet, NllMemberConstraintIndex}; +use crate::polonius::LiveLoans; use crate::polonius::legacy::PoloniusOutput; use crate::region_infer::reverse_sccs::ReverseSccGraph; use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues, ToElementIndex}; @@ -2171,28 +2172,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { self.constraint_graph.region_graph(&self.constraints, self.universal_regions().fr_static) } - /// Returns whether the given region is considered live at all points: whether it is a - /// placeholder or a free region. - pub(crate) fn is_region_live_at_all_points(&self, region: RegionVid) -> bool { - // FIXME: there must be a cleaner way to find this information. At least, when - // higher-ranked subtyping is abstracted away from the borrowck main path, we'll only - // need to check whether this is a universal region. - let origin = self.region_definition(region).origin; - let live_at_all_points = matches!( - origin, - NllRegionVariableOrigin::Placeholder(_) | NllRegionVariableOrigin::FreeRegion - ); - live_at_all_points - } - - /// Returns whether the `loan_idx` is live at the given `location`: whether its issuing - /// region is contained within the type of a variable that is live at this point. - /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`. - pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool { - let point = self.liveness_constraints.point_from_location(location); - self.liveness_constraints.is_loan_live_at(loan_idx, point) - } - /// Returns the representative `RegionVid` for a given SCC. /// See `RegionTracker` for how a region variable ID is chosen. /// @@ -2208,6 +2187,20 @@ impl<'tcx> RegionInferenceContext<'tcx> { pub(crate) fn liveness_constraints(&self) -> &LivenessValues { &self.liveness_constraints } + + /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active + /// loans dataflow computations. + pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) { + self.liveness_constraints.record_live_loans(live_loans); + } + + /// Returns whether the `loan_idx` is live at the given `location`: whether its issuing + /// region is contained within the type of a variable that is live at this point. + /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`. + pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool { + let point = self.liveness_constraints.point_from_location(location); + self.liveness_constraints.is_loan_live_at(loan_idx, point) + } } impl<'tcx> RegionDefinition<'tcx> { diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs index 11fb125ca22..f1bcb353dc6 100644 --- a/compiler/rustc_borrowck/src/region_infer/values.rs +++ b/compiler/rustc_borrowck/src/region_infer/values.rs @@ -11,6 +11,7 @@ use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; use tracing::debug; use crate::BorrowIndex; +use crate::polonius::LiveLoans; rustc_index::newtype_index! { /// A single integer representing a `ty::Placeholder`. @@ -50,29 +51,8 @@ pub(crate) struct LivenessValues { /// region is live, only that it is. points: Option<SparseIntervalMatrix<RegionVid, PointIndex>>, - /// When using `-Zpolonius=next`, for each point: the loans flowing into the live regions at - /// that point. - pub(crate) loans: Option<LiveLoans>, -} - -/// Data used to compute the loans that are live at a given point in the CFG, when using -/// `-Zpolonius=next`. -pub(crate) struct LiveLoans { - /// The set of loans that flow into a given region. When individual regions are marked as live - /// in the CFG, these inflowing loans are recorded as live. - pub(crate) inflowing_loans: SparseBitMatrix<RegionVid, BorrowIndex>, - - /// The set of loans that are live at a given point in the CFG. - pub(crate) live_loans: SparseBitMatrix<PointIndex, BorrowIndex>, -} - -impl LiveLoans { - pub(crate) fn new(num_loans: usize) -> Self { - LiveLoans { - live_loans: SparseBitMatrix::new(num_loans), - inflowing_loans: SparseBitMatrix::new(num_loans), - } - } + /// When using `-Zpolonius=next`, the set of loans that are live at a given point in the CFG. + live_loans: Option<LiveLoans>, } impl LivenessValues { @@ -82,7 +62,7 @@ impl LivenessValues { live_regions: None, points: Some(SparseIntervalMatrix::new(location_map.num_points())), location_map, - loans: None, + live_loans: None, } } @@ -95,7 +75,7 @@ impl LivenessValues { live_regions: Some(Default::default()), points: None, location_map, - loans: None, + live_loans: None, } } @@ -129,13 +109,6 @@ impl LivenessValues { } else if self.location_map.point_in_range(point) { self.live_regions.as_mut().unwrap().insert(region); } - - // When available, record the loans flowing into this region as live at the given point. - if let Some(loans) = self.loans.as_mut() { - if let Some(inflowing) = loans.inflowing_loans.row(region) { - loans.live_loans.union_row(point, inflowing); - } - } } /// Records `region` as being live at all the given `points`. @@ -146,17 +119,6 @@ impl LivenessValues { } else if points.iter().any(|point| self.location_map.point_in_range(point)) { self.live_regions.as_mut().unwrap().insert(region); } - - // When available, record the loans flowing into this region as live at the given points. - if let Some(loans) = self.loans.as_mut() { - if let Some(inflowing) = loans.inflowing_loans.row(region) { - if !inflowing.is_empty() { - for point in points.iter() { - loans.live_loans.union_row(point, inflowing); - } - } - } - } } /// Records `region` as being live at all the control-flow points. @@ -213,12 +175,17 @@ impl LivenessValues { self.location_map.to_location(point) } + /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active + /// loans dataflow computations. + pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) { + self.live_loans = Some(live_loans); + } + /// When using `-Zpolonius=next`, returns whether the `loan_idx` is live at the given `point`. pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, point: PointIndex) -> bool { - self.loans + self.live_loans .as_ref() .expect("Accessing live loans requires `-Zpolonius=next`") - .live_loans .contains(point, loan_idx) } } diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs index d83d6ade203..e355d2b415b 100644 --- a/compiler/rustc_borrowck/src/renumber.rs +++ b/compiler/rustc_borrowck/src/renumber.rs @@ -34,7 +34,6 @@ pub(crate) enum RegionCtxt { Location(Location), TyContext(TyContext), Free(Symbol), - Bound(Symbol), LateBound(Symbol), Existential(Option<Symbol>), Placeholder(Symbol), diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs index ea965eb6545..edf612f4e97 100644 --- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs +++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs @@ -5,13 +5,14 @@ use rustc_infer::infer::canonical::QueryRegionConstraints; use rustc_infer::infer::outlives::env::RegionBoundPairs; use rustc_infer::infer::region_constraints::GenericKind; use rustc_infer::infer::{InferCtxt, outlives}; +use rustc_infer::traits::ScrubbedTraitError; use rustc_middle::mir::ConstraintCategory; use rustc_middle::traits::ObligationCause; use rustc_middle::traits::query::OutlivesBound; use rustc_middle::ty::{self, RegionVid, Ty, TypeVisitableExt}; use rustc_span::{ErrorGuaranteed, Span}; -use rustc_trait_selection::error_reporting::InferCtxtErrorExt; -use rustc_trait_selection::solve::deeply_normalize; +use rustc_trait_selection::solve::NoSolution; +use rustc_trait_selection::traits::query::type_op::custom::CustomTypeOp; use rustc_trait_selection::traits::query::type_op::{self, TypeOp}; use tracing::{debug, instrument}; use type_op::TypeOpOutput; @@ -229,24 +230,14 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { let mut constraints = vec![]; let mut known_type_outlives_obligations = vec![]; for bound in param_env.caller_bounds() { - let Some(mut outlives) = bound.as_type_outlives_clause() else { continue }; - - // In the new solver, normalize the type-outlives obligation assumptions. - if self.infcx.next_trait_solver() { - match deeply_normalize( - self.infcx.at(&ObligationCause::misc(span, defining_ty_def_id), param_env), + if let Some(outlives) = bound.as_type_outlives_clause() { + self.normalize_and_push_type_outlives_obligation( outlives, - ) { - Ok(normalized_outlives) => { - outlives = normalized_outlives; - } - Err(e) => { - self.infcx.err_ctxt().report_fulfillment_errors(e); - } - } - } - - known_type_outlives_obligations.push(outlives); + span, + &mut known_type_outlives_obligations, + &mut constraints, + ); + }; } let unnormalized_input_output_tys = self @@ -356,6 +347,44 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { } } + fn normalize_and_push_type_outlives_obligation( + &self, + mut outlives: ty::PolyTypeOutlivesPredicate<'tcx>, + span: Span, + known_type_outlives_obligations: &mut Vec<ty::PolyTypeOutlivesPredicate<'tcx>>, + constraints: &mut Vec<&QueryRegionConstraints<'tcx>>, + ) { + // In the new solver, normalize the type-outlives obligation assumptions. + if self.infcx.next_trait_solver() { + let Ok(TypeOpOutput { + output: normalized_outlives, + constraints: constraints_normalize, + error_info: _, + }) = CustomTypeOp::new( + |ocx| { + ocx.deeply_normalize( + &ObligationCause::dummy_with_span(span), + self.param_env, + outlives, + ) + .map_err(|_: Vec<ScrubbedTraitError<'tcx>>| NoSolution) + }, + "normalize type outlives obligation", + ) + .fully_perform(self.infcx, span) + else { + self.infcx.dcx().delayed_bug(format!("could not normalize {outlives:?}")); + return; + }; + outlives = normalized_outlives; + if let Some(c) = constraints_normalize { + constraints.push(c); + } + } + + known_type_outlives_obligations.push(outlives); + } + /// Update the type of a single local, which should represent /// either the return type of the MIR or one of its arguments. At /// the same time, compute and add any implied bounds that come diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs index f23602d0358..4e0b2a4e296 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs @@ -38,11 +38,19 @@ pub(super) fn generate<'a, 'tcx>( ) { debug!("liveness::generate"); - let free_regions = regions_that_outlive_free_regions( - typeck.infcx.num_region_vars(), - &typeck.universal_regions, - &typeck.constraints.outlives_constraints, - ); + // NLLs can avoid computing some liveness data here because its constraints are + // location-insensitive, but that doesn't work in polonius: locals whose type contains a region + // that outlives a free region are not necessarily live everywhere in a flow-sensitive setting, + // unlike NLLs. + let free_regions = if !typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() { + regions_that_outlive_free_regions( + typeck.infcx.num_region_vars(), + &typeck.universal_regions, + &typeck.constraints.outlives_constraints, + ) + } else { + typeck.universal_regions.universal_regions_iter().collect() + }; let (relevant_live_locals, boring_locals) = compute_relevant_live_locals(typeck.tcx(), &free_regions, body); diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 4c0d3138f2d..c564d85616e 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -16,7 +16,7 @@ use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOp, Type use tracing::debug; use crate::polonius; -use crate::region_infer::values::{self, LiveLoans}; +use crate::region_infer::values; use crate::type_check::liveness::local_use_map::LocalUseMap; use crate::type_check::{NormalizeLocation, TypeChecker}; @@ -44,37 +44,6 @@ pub(super) fn trace<'a, 'tcx>( boring_locals: Vec<Local>, ) { let local_use_map = &LocalUseMap::build(&relevant_live_locals, location_map, body); - - // When using `-Zpolonius=next`, compute the set of loans that can reach a given region. - if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() { - let borrow_set = &typeck.borrow_set; - let mut live_loans = LiveLoans::new(borrow_set.len()); - let outlives_constraints = &typeck.constraints.outlives_constraints; - let graph = outlives_constraints.graph(typeck.infcx.num_region_vars()); - let region_graph = - graph.region_graph(outlives_constraints, typeck.universal_regions.fr_static); - - // Traverse each issuing region's constraints, and record the loan as flowing into the - // outlived region. - for (loan, issuing_region_data) in borrow_set.iter_enumerated() { - for succ in rustc_data_structures::graph::depth_first_search( - ®ion_graph, - issuing_region_data.region, - ) { - // We don't need to mention that a loan flows into its issuing region. - if succ == issuing_region_data.region { - continue; - } - - live_loans.inflowing_loans.insert(succ, loan); - } - } - - // Store the inflowing loans in the liveness constraints: they will be used to compute live - // loans when liveness data is recorded there. - typeck.constraints.liveness_constraints.loans = Some(live_loans); - }; - let cx = LivenessContext { typeck, body, diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index a1979c8b8ab..eca8a688ff4 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1654,7 +1654,20 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { match *cast_kind { CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer, coercion_source) => { let is_implicit_coercion = coercion_source == CoercionSource::Implicit; - let src_sig = op.ty(body, tcx).fn_sig(tcx); + let src_ty = op.ty(body, tcx); + let mut src_sig = src_ty.fn_sig(tcx); + if let ty::FnDef(def_id, _) = src_ty.kind() + && let ty::FnPtr(_, target_hdr) = *ty.kind() + && tcx.codegen_fn_attrs(def_id).safe_target_features + && target_hdr.safety.is_safe() + && let Some(safe_sig) = tcx.adjust_target_feature_sig( + *def_id, + src_sig, + body.source.def_id(), + ) + { + src_sig = safe_sig; + } // HACK: This shouldn't be necessary... We can remove this when we actually // get binders with where clauses, then elaborate implied bounds into that diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs index 3dc4569c57b..26af86c0cdd 100644 --- a/compiler/rustc_borrowck/src/universal_regions.rs +++ b/compiler/rustc_borrowck/src/universal_regions.rs @@ -467,15 +467,13 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { self.infcx.tcx.local_parent(self.mir_def), |r| { debug!(?r); - if !indices.indices.contains_key(&r) { - let region_vid = { - let name = r.get_name_or_anon(); - self.infcx.next_nll_region_var(FR, || RegionCtxt::LateBound(name)) - }; - - debug!(?region_vid); - indices.insert_late_bound_region(r, region_vid.as_var()); - } + let region_vid = { + let name = r.get_name_or_anon(); + self.infcx.next_nll_region_var(FR, || RegionCtxt::LateBound(name)) + }; + + debug!(?region_vid); + indices.insert_late_bound_region(r, region_vid.as_var()); }, ); @@ -484,21 +482,17 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { self.infcx.num_region_vars() }; - // "Liberate" the late-bound regions. These correspond to - // "local" free regions. + // Converse of above, if this is a function/closure then the late-bound regions declared + // on its signature are local. + // + // We manually loop over `bound_inputs_and_output` instead of using + // `for_each_late_bound_region_in_item` as we may need to add the otherwise + // implicit `ClosureEnv` region. let bound_inputs_and_output = self.compute_inputs_and_output(&indices, defining_ty); - - let inputs_and_output = self.infcx.replace_bound_regions_with_nll_infer_vars( - FR, - self.mir_def, - bound_inputs_and_output, - &mut indices, - ); - // Converse of above, if this is a function/closure then the late-bound regions declared on its - // signature are local. - for_each_late_bound_region_in_item(self.infcx.tcx, self.mir_def, |r| { - debug!(?r); - if !indices.indices.contains_key(&r) { + for (idx, bound_var) in bound_inputs_and_output.bound_vars().iter().enumerate() { + if let ty::BoundVariableKind::Region(kind) = bound_var { + let kind = ty::LateParamRegionKind::from_bound(ty::BoundVar::from_usize(idx), kind); + let r = ty::Region::new_late_param(self.infcx.tcx, self.mir_def.to_def_id(), kind); let region_vid = { let name = r.get_name_or_anon(); self.infcx.next_nll_region_var(FR, || RegionCtxt::LateBound(name)) @@ -507,7 +501,12 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { debug!(?region_vid); indices.insert_late_bound_region(r, region_vid.as_var()); } - }); + } + let inputs_and_output = self.infcx.replace_bound_regions_with_nll_infer_vars( + self.mir_def, + bound_inputs_and_output, + &indices, + ); let (unnormalized_output_ty, mut unnormalized_input_tys) = inputs_and_output.split_last().unwrap(); @@ -832,10 +831,9 @@ impl<'tcx> BorrowckInferCtxt<'tcx> { #[instrument(level = "debug", skip(self, indices))] fn replace_bound_regions_with_nll_infer_vars<T>( &self, - origin: NllRegionVariableOrigin, all_outlive_scope: LocalDefId, value: ty::Binder<'tcx, T>, - indices: &mut UniversalRegionIndices<'tcx>, + indices: &UniversalRegionIndices<'tcx>, ) -> T where T: TypeFoldable<TyCtxt<'tcx>>, @@ -845,18 +843,7 @@ impl<'tcx> BorrowckInferCtxt<'tcx> { let kind = ty::LateParamRegionKind::from_bound(br.var, br.kind); let liberated_region = ty::Region::new_late_param(self.tcx, all_outlive_scope.to_def_id(), kind); - let region_vid = { - let name = match br.kind.get_name() { - Some(name) => name, - _ => sym::anon, - }; - - self.next_nll_region_var(origin, || RegionCtxt::Bound(name)) - }; - - indices.insert_late_bound_region(liberated_region, region_vid.as_var()); - debug!(?liberated_region, ?region_vid); - region_vid + ty::Region::new_var(self.tcx, indices.to_region_vid(liberated_region)) }); value } @@ -870,7 +857,7 @@ impl<'tcx> UniversalRegionIndices<'tcx> { /// well. These are used for error reporting. fn insert_late_bound_region(&mut self, r: ty::Region<'tcx>, vid: ty::RegionVid) { debug!("insert_late_bound_region({:?}, {:?})", r, vid); - self.indices.insert(r, vid); + assert_eq!(self.indices.insert(r, vid), None); } /// Converts `r` into a local inference variable: `r` can either diff --git a/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch b/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch index bf07e455a75..bf58e485158 100644 --- a/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch +++ b/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch @@ -16,8 +16,8 @@ index 7165c3e48af..968552ad435 100644 [dependencies] core = { path = "../core" } --compiler_builtins = { version = "=0.1.140", features = ['rustc-dep-of-std'] } -+compiler_builtins = { version = "=0.1.140", features = ['rustc-dep-of-std', 'no-f16-f128'] } +-compiler_builtins = { version = "=0.1.143", features = ['rustc-dep-of-std'] } ++compiler_builtins = { version = "=0.1.143", features = ['rustc-dep-of-std', 'no-f16-f128'] } [dev-dependencies] rand = { version = "0.8.5", default-features = false, features = ["alloc"] } diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 2466bfe60c7..2c99597922e 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -65,7 +65,11 @@ pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: Call sess.dcx().fatal("C-cmse-nonsecure-entry call conv is not yet implemented"); } - Conv::Msp430Intr | Conv::PtxKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => { + Conv::Msp430Intr + | Conv::PtxKernel + | Conv::GpuKernel + | Conv::AvrInterrupt + | Conv::AvrNonBlockingInterrupt => { unreachable!("tried to use {c:?} call conv which only exists on an unsupported target"); } } diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs index 025667e66b2..dcfd7ddabbc 100644 --- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs +++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs @@ -76,20 +76,22 @@ pub(crate) fn maybe_codegen_mul_checked<'tcx>( } let is_signed = type_sign(lhs.layout().ty); - - let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]); - let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); + let oflow_out_place = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32)); let param_types = vec![ - AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn), AbiParam::new(types::I128), AbiParam::new(types::I128), + AbiParam::special(fx.pointer_type, ArgumentPurpose::Normal), ]; - let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)]; - fx.lib_call( + let args = [lhs.load_scalar(fx), rhs.load_scalar(fx), oflow_out_place.to_ptr().get_addr(fx)]; + let ret = fx.lib_call( if is_signed { "__rust_i128_mulo" } else { "__rust_u128_mulo" }, param_types, - vec![], + vec![AbiParam::new(types::I128)], &args, ); - Some(out_place.to_cvalue(fx)) + let mul = ret[0]; + let oflow = oflow_out_place.to_cvalue(fx).load_scalar(fx); + let oflow = clif_intcast(fx, oflow, types::I8, false); + let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool])); + Some(CValue::by_val_pair(mul, oflow, layout)) } diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs index f8e3a034421..2484c10848e 100644 --- a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs +++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs @@ -43,7 +43,7 @@ builtin_functions! { fn __divti3(n: i128, d: i128) -> i128; fn __umodti3(n: u128, d: u128) -> u128; fn __modti3(n: i128, d: i128) -> i128; - fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool); + fn __rust_u128_mulo(a: u128, b: u128, oflow: &mut i32) -> u128; // floats fn __floattisf(i: i128) -> f32; diff --git a/compiler/rustc_codegen_gcc/example/mini_core.rs b/compiler/rustc_codegen_gcc/example/mini_core.rs index bd7a4612a92..5a4ee0a198c 100644 --- a/compiler/rustc_codegen_gcc/example/mini_core.rs +++ b/compiler/rustc_codegen_gcc/example/mini_core.rs @@ -689,7 +689,7 @@ impl<T> Index<usize> for [T] { } } -extern { +extern "C" { type VaListImpl; } diff --git a/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs b/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs index dcfa34cb729..1d51e0a1856 100644 --- a/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs +++ b/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs @@ -258,13 +258,13 @@ fn main() { assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42); - extern { + extern "C" { #[linkage = "weak"] static ABC: *const u8; } { - extern { + extern "C" { #[linkage = "weak"] static ABC: *const u8; } diff --git a/compiler/rustc_codegen_gcc/example/mod_bench.rs b/compiler/rustc_codegen_gcc/example/mod_bench.rs index cae911c1073..e8a9cade747 100644 --- a/compiler/rustc_codegen_gcc/example/mod_bench.rs +++ b/compiler/rustc_codegen_gcc/example/mod_bench.rs @@ -3,7 +3,7 @@ #![allow(internal_features)] #[link(name = "c")] -extern {} +extern "C" {} #[panic_handler] fn panic_handler(_: &core::panic::PanicInfo<'_>) -> ! { diff --git a/compiler/rustc_codegen_gcc/example/std_example.rs b/compiler/rustc_codegen_gcc/example/std_example.rs index 9e43b4635f0..5fa1e0afb06 100644 --- a/compiler/rustc_codegen_gcc/example/std_example.rs +++ b/compiler/rustc_codegen_gcc/example/std_example.rs @@ -7,7 +7,7 @@ use std::arch::x86_64::*; use std::io::Write; use std::ops::Coroutine; -extern { +extern "C" { pub fn printf(format: *const i8, ...) -> i32; } diff --git a/compiler/rustc_codegen_gcc/src/int.rs b/compiler/rustc_codegen_gcc/src/int.rs index 7b9d1feb8d7..fe6a65bed03 100644 --- a/compiler/rustc_codegen_gcc/src/int.rs +++ b/compiler/rustc_codegen_gcc/src/int.rs @@ -322,36 +322,26 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { }, } } else { - match new_kind { - Int(I128) | Uint(U128) => { - let func_name = match oop { - OverflowOp::Add => match new_kind { - Int(I128) => "__rust_i128_addo", - Uint(U128) => "__rust_u128_addo", - _ => unreachable!(), - }, - OverflowOp::Sub => match new_kind { - Int(I128) => "__rust_i128_subo", - Uint(U128) => "__rust_u128_subo", - _ => unreachable!(), - }, - OverflowOp::Mul => match new_kind { - Int(I128) => "__rust_i128_mulo", // TODO(antoyo): use __muloti4d instead? - Uint(U128) => "__rust_u128_mulo", - _ => unreachable!(), - }, - }; - return self.operation_with_overflow(func_name, lhs, rhs); - } - _ => match oop { - OverflowOp::Mul => match new_kind { - Int(I32) => "__mulosi4", - Int(I64) => "__mulodi4", - _ => unreachable!(), - }, - _ => unimplemented!("overflow operation for {:?}", new_kind), + let (func_name, width) = match oop { + OverflowOp::Add => match new_kind { + Int(I128) => ("__rust_i128_addo", 128), + Uint(U128) => ("__rust_u128_addo", 128), + _ => unreachable!(), }, - } + OverflowOp::Sub => match new_kind { + Int(I128) => ("__rust_i128_subo", 128), + Uint(U128) => ("__rust_u128_subo", 128), + _ => unreachable!(), + }, + OverflowOp::Mul => match new_kind { + Int(I32) => ("__mulosi4", 32), + Int(I64) => ("__mulodi4", 64), + Int(I128) => ("__rust_i128_mulo", 128), // TODO(antoyo): use __muloti4d instead? + Uint(U128) => ("__rust_u128_mulo", 128), + _ => unreachable!(), + }, + }; + return self.operation_with_overflow(func_name, lhs, rhs, width); }; let intrinsic = self.context.get_builtin_function(name); @@ -364,80 +354,87 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { (res.dereference(self.location).to_rvalue(), overflow) } + /// Non-`__builtin_*` overflow operations with a `fn(T, T, &mut i32) -> T` signature. pub fn operation_with_overflow( &self, func_name: &str, lhs: RValue<'gcc>, rhs: RValue<'gcc>, + width: u64, ) -> (RValue<'gcc>, RValue<'gcc>) { let a_type = lhs.get_type(); let b_type = rhs.get_type(); debug_assert!(a_type.dyncast_array().is_some()); debug_assert!(b_type.dyncast_array().is_some()); + let overflow_type = self.i32_type; + let overflow_param_type = overflow_type.make_pointer(); + let res_type = a_type; + + let overflow_value = + self.current_func().new_local(self.location, overflow_type, "overflow"); + let overflow_addr = overflow_value.get_address(self.location); + let param_a = self.context.new_parameter(self.location, a_type, "a"); let param_b = self.context.new_parameter(self.location, b_type, "b"); - let result_field = self.context.new_field(self.location, a_type, "result"); - let overflow_field = self.context.new_field(self.location, self.bool_type, "overflow"); - - let ret_ty = Ty::new_tup(self.tcx, &[self.tcx.types.i128, self.tcx.types.bool]); + let param_overflow = + self.context.new_parameter(self.location, overflow_param_type, "overflow"); + + let a_elem_type = a_type.dyncast_array().expect("non-array a value"); + debug_assert!(a_elem_type.is_integral()); + let res_ty = match width { + 32 => self.tcx.types.i32, + 64 => self.tcx.types.i64, + 128 => self.tcx.types.i128, + _ => unreachable!("unexpected integer size"), + }; let layout = self .tcx - .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ret_ty)) + .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(res_ty)) .unwrap(); let arg_abi = ArgAbi { layout, mode: PassMode::Direct(ArgAttributes::new()) }; let mut fn_abi = FnAbi { - args: vec![arg_abi.clone(), arg_abi.clone()].into_boxed_slice(), + args: vec![arg_abi.clone(), arg_abi.clone(), arg_abi.clone()].into_boxed_slice(), ret: arg_abi, c_variadic: false, - fixed_count: 2, + fixed_count: 3, conv: Conv::C, can_unwind: false, }; fn_abi.adjust_for_foreign_abi(self.cx, spec::abi::Abi::C { unwind: false }).unwrap(); - let indirect = matches!(fn_abi.ret.mode, PassMode::Indirect { .. }); - - let return_type = self - .context - .new_struct_type(self.location, "result_overflow", &[result_field, overflow_field]); - let result = if indirect { - let return_value = - self.current_func().new_local(self.location, return_type.as_type(), "return_value"); - let return_param_type = return_type.as_type().make_pointer(); - let return_param = - self.context.new_parameter(self.location, return_param_type, "return_value"); + let ret_indirect = matches!(fn_abi.ret.mode, PassMode::Indirect { .. }); + + let result = if ret_indirect { + let res_value = self.current_func().new_local(self.location, res_type, "result_value"); + let res_addr = res_value.get_address(self.location); + let res_param_type = res_type.make_pointer(); + let param_res = self.context.new_parameter(self.location, res_param_type, "result"); + let func = self.context.new_function( self.location, FunctionType::Extern, self.type_void(), - &[return_param, param_a, param_b], + &[param_res, param_a, param_b, param_overflow], func_name, false, ); - self.llbb().add_eval( - self.location, - self.context.new_call(self.location, func, &[ - return_value.get_address(self.location), - lhs, - rhs, - ]), - ); - return_value.to_rvalue() + let _void = + self.context.new_call(self.location, func, &[res_addr, lhs, rhs, overflow_addr]); + res_value.to_rvalue() } else { let func = self.context.new_function( self.location, FunctionType::Extern, - return_type.as_type(), - &[param_a, param_b], + res_type, + &[param_a, param_b, param_overflow], func_name, false, ); - self.context.new_call(self.location, func, &[lhs, rhs]) + self.context.new_call(self.location, func, &[lhs, rhs, overflow_addr]) }; - let overflow = result.access_field(self.location, overflow_field); - let int_result = result.access_field(self.location, result_field); - (int_result, overflow) + + (result, self.context.new_cast(self.location, overflow_value, self.bool_type).to_rvalue()) } pub fn gcc_icmp( diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 42d189d1b7d..48606f5f91c 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -1001,7 +1001,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { 128 => "__rust_i128_addo", _ => unreachable!(), }; - let (int_result, overflow) = self.operation_with_overflow(func_name, lhs, rhs); + let (int_result, overflow) = + self.operation_with_overflow(func_name, lhs, rhs, width); self.llbb().add_assignment(self.location, res, int_result); overflow }; @@ -1071,7 +1072,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { 128 => "__rust_i128_subo", _ => unreachable!(), }; - let (int_result, overflow) = self.operation_with_overflow(func_name, lhs, rhs); + let (int_result, overflow) = + self.operation_with_overflow(func_name, lhs, rhs, width); self.llbb().add_assignment(self.location, res, int_result); overflow }; diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 1d35138b013..31ee0eeca11 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,3 +1,4 @@ +use std::borrow::Borrow; use std::cmp; use libc::c_uint; @@ -312,7 +313,7 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; - fn llvm_cconv(&self) -> llvm::CallConv; + fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv; /// Apply attributes to a function declaration/definition. fn apply_attrs_llfn( @@ -404,8 +405,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { cx.type_ptr_ext(cx.data_layout().instruction_address_space) } - fn llvm_cconv(&self) -> llvm::CallConv { - self.conv.into() + fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv { + llvm::CallConv::from_conv(self.conv, cx.tcx.sess.target.arch.borrow()) } fn apply_attrs_llfn( @@ -617,7 +618,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } - let cconv = self.llvm_cconv(); + let cconv = self.llvm_cconv(&bx.cx); if cconv != llvm::CCallConv { llvm::SetInstructionCallConv(callsite, cconv); } @@ -655,8 +656,8 @@ impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } -impl From<Conv> for llvm::CallConv { - fn from(conv: Conv) -> Self { +impl llvm::CallConv { + pub fn from_conv(conv: Conv, arch: &str) -> Self { match conv { Conv::C | Conv::Rust @@ -666,6 +667,15 @@ impl From<Conv> for llvm::CallConv { Conv::Cold => llvm::ColdCallConv, Conv::PreserveMost => llvm::PreserveMost, Conv::PreserveAll => llvm::PreserveAll, + Conv::GpuKernel => { + if arch == "amdgpu" { + llvm::AmdgpuKernel + } else if arch == "nvptx64" { + llvm::PtxKernel + } else { + panic!("Architecture {arch} does not support GpuKernel calling convention"); + } + } Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, Conv::ArmAapcs => llvm::ArmAapcsCallConv, diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d8fbe51b975..65345751842 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -741,7 +741,10 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { if self.get_declared_value(entry_name).is_none() { Some(self.declare_entry_fn( entry_name, - self.sess().target.entry_abi.into(), + llvm::CallConv::from_conv( + self.sess().target.entry_abi, + self.sess().target.arch.borrow(), + ), llvm::UnnamedAddr::Global, fn_type, )) diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 3ec386f6b07..c72b5b5611f 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -125,7 +125,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let llfn = declare_raw_fn( self, name, - fn_abi.llvm_cconv(), + fn_abi.llvm_cconv(self), llvm::UnnamedAddr::Global, llvm::Visibility::Default, fn_abi.llvm_type(self), diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index cb4a8c9a5f2..ec6c84f6f25 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -120,6 +120,7 @@ pub enum CallConv { X86_Intr = 83, AvrNonBlockingInterrupt = 84, AvrInterrupt = 85, + AmdgpuKernel = 91, } /// Must match the layout of `LLVMLinkage`. diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 37b53bb5bea..1daa17fbaf3 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -250,10 +250,14 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { } } sym::target_feature => { - if !tcx.is_closure_like(did.to_def_id()) - && let Some(fn_sig) = fn_sig() - && fn_sig.skip_binder().safety().is_safe() - { + let Some(sig) = tcx.hir_node_by_def_id(did).fn_sig() else { + tcx.dcx().span_delayed_bug(attr.span, "target_feature applied to non-fn"); + continue; + }; + let safe_target_features = + matches!(sig.header.safety, hir::HeaderSafety::SafeTargetFeatures); + codegen_fn_attrs.safe_target_features = safe_target_features; + if safe_target_features { if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { // The `#[target_feature]` attribute is allowed on // WebAssembly targets on all functions, including safe diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index 7a5f6c17268..6c940124193 100644 --- a/compiler/rustc_const_eval/src/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -35,6 +35,12 @@ use crate::errors; type QualifResults<'mir, 'tcx, Q> = rustc_mir_dataflow::ResultsCursor<'mir, 'tcx, FlowSensitiveAnalysis<'mir, 'mir, 'tcx, Q>>; +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +enum ConstConditionsHold { + Yes, + No, +} + #[derive(Default)] pub(crate) struct Qualifs<'mir, 'tcx> { has_mut_interior: Option<QualifResults<'mir, 'tcx, HasMutInterior>>, @@ -376,15 +382,15 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { callee: DefId, callee_args: ty::GenericArgsRef<'tcx>, call_span: Span, - ) -> bool { + ) -> Option<ConstConditionsHold> { let tcx = self.tcx; if !tcx.is_conditionally_const(callee) { - return false; + return None; } let const_conditions = tcx.const_conditions(callee).instantiate(tcx, callee_args); if const_conditions.is_empty() { - return false; + return None; } let (infcx, param_env) = tcx.infer_ctxt().build_with_typing_env(self.body.typing_env(tcx)); @@ -413,12 +419,13 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { })); let errors = ocx.select_all_or_error(); - if !errors.is_empty() { + if errors.is_empty() { + Some(ConstConditionsHold::Yes) + } else { tcx.dcx() .span_delayed_bug(call_span, "this should have reported a ~const error in HIR"); + Some(ConstConditionsHold::No) } - - true } pub fn check_drop_terminator( @@ -706,7 +713,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { trace!("attempting to call a trait method"); let trait_is_const = tcx.is_const_trait(trait_did); - if trait_is_const { + // Only consider a trait to be const if the const conditions hold. + // Otherwise, it's really misleading to call something "conditionally" + // const when it's very obviously not conditionally const. + if trait_is_const && has_const_conditions == Some(ConstConditionsHold::Yes) { // Trait calls are always conditionally-const. self.check_op(ops::ConditionallyConstCall { callee, @@ -730,7 +740,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { } // Even if we know the callee, ensure we can use conditionally-const calls. - if has_const_conditions { + if has_const_conditions.is_some() { self.check_op(ops::ConditionallyConstCall { callee, args: fn_args, diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs index 8af17d01b0a..35c3e3ed315 100644 --- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs @@ -4,14 +4,18 @@ use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; -fn parent_impl_constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness { +fn parent_impl_or_trait_constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness { let parent_id = tcx.local_parent(def_id); - if matches!(tcx.def_kind(parent_id), DefKind::Impl { .. }) - && let Some(header) = tcx.impl_trait_header(parent_id) - { - header.constness - } else { - hir::Constness::NotConst + match tcx.def_kind(parent_id) { + DefKind::Impl { of_trait: true } => tcx.impl_trait_header(parent_id).unwrap().constness, + DefKind::Trait => { + if tcx.is_const_trait(parent_id.into()) { + hir::Constness::Const + } else { + hir::Constness::NotConst + } + } + _ => hir::Constness::NotConst, } } @@ -34,7 +38,7 @@ fn constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness { // If the function itself is not annotated with `const`, it may still be a `const fn` // if it resides in a const trait impl. - parent_impl_constness(tcx, def_id) + parent_impl_or_trait_constness(tcx, def_id) } else { tcx.dcx().span_bug( tcx.def_span(def_id), diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index ba7fbb254c6..cfdfbdb7880 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -360,10 +360,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { // sensitive check here. But we can at least rule out functions that are not const at // all. That said, we have to allow calling functions inside a trait marked with // #[const_trait]. These *are* const-checked! - // FIXME(const_trait_impl): why does `is_const_fn` not classify them as const? - if (!ecx.tcx.is_const_fn(def) && !ecx.tcx.is_const_default_method(def)) - || ecx.tcx.has_attr(def, sym::rustc_do_not_const_check) - { + if !ecx.tcx.is_const_fn(def) || ecx.tcx.has_attr(def, sym::rustc_do_not_const_check) { // We certainly do *not* want to actually call the fn // though, so be sure we return here. throw_unsup_format!("calling non-const function `{}`", instance) diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 0790db984e3..2772c94d52b 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -1481,22 +1481,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// Test if this value might be null. /// If the machine does not support ptr-to-int casts, this is conservative. pub fn scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool> { - interp_ok(match scalar.try_to_scalar_int() { - Ok(int) => int.is_null(), + match scalar.try_to_scalar_int() { + Ok(int) => interp_ok(int.is_null()), Err(_) => { - // Can only happen during CTFE. + // We can't cast this pointer to an integer. Can only happen during CTFE. let ptr = scalar.to_pointer(self)?; match self.ptr_try_get_alloc_id(ptr, 0) { Ok((alloc_id, offset, _)) => { - let size = self.get_alloc_info(alloc_id).size; - // If the pointer is out-of-bounds, it may be null. - // Note that one-past-the-end (offset == size) is still inbounds, and never null. - offset > size + let info = self.get_alloc_info(alloc_id); + // If the pointer is in-bounds (including "at the end"), it is definitely not null. + if offset <= info.size { + return interp_ok(false); + } + // If the allocation is N-aligned, and the offset is not divisible by N, + // then `base + offset` has a non-zero remainder after division by `N`, + // which means `base + offset` cannot be null. + if offset.bytes() % info.align.bytes() != 0 { + return interp_ok(false); + } + // We don't know enough, this might be null. + interp_ok(true) } Err(_offset) => bug!("a non-int scalar is always a pointer"), } } - }) + } } /// Turning a "maybe pointer" into a proper pointer (and some information diff --git a/compiler/rustc_data_structures/src/flock.rs b/compiler/rustc_data_structures/src/flock.rs index e03962a54ec..292a33d5646 100644 --- a/compiler/rustc_data_structures/src/flock.rs +++ b/compiler/rustc_data_structures/src/flock.rs @@ -4,6 +4,7 @@ //! green/native threading. This is just a bare-bones enough solution for //! librustdoc, it is not production quality at all. +#[cfg(bootstrap)] cfg_match! { cfg(target_os = "linux") => { mod linux; @@ -27,4 +28,28 @@ cfg_match! { } } +#[cfg(not(bootstrap))] +cfg_match! { + target_os = "linux" => { + mod linux; + use linux as imp; + } + target_os = "redox" => { + mod linux; + use linux as imp; + } + unix => { + mod unix; + use unix as imp; + } + windows => { + mod windows; + use self::windows as imp; + } + _ => { + mod unsupported; + use unsupported as imp; + } +} + pub use imp::Lock; diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs index ecda7d3fba8..7b4573d7a84 100644 --- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs +++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs @@ -125,6 +125,16 @@ where pub fn visited(&self, node: G::Node) -> bool { self.visited.contains(node) } + + /// Returns a reference to the set of nodes that have been visited, with + /// the same caveats as [`Self::visited`]. + /// + /// When incorporating the visited nodes into another bitset, using bulk + /// operations like `union` or `intersect` can be more efficient than + /// processing each node individually. + pub fn visited_set(&self) -> &DenseBitSet<G::Node> { + &self.visited + } } impl<G> std::fmt::Debug for DepthFirstSearch<G> diff --git a/compiler/rustc_data_structures/src/graph/mod.rs b/compiler/rustc_data_structures/src/graph/mod.rs index 103ddd917bf..92035e8bc48 100644 --- a/compiler/rustc_data_structures/src/graph/mod.rs +++ b/compiler/rustc_data_structures/src/graph/mod.rs @@ -4,6 +4,7 @@ pub mod dominators; pub mod implementation; pub mod iterate; mod reference; +pub mod reversed; pub mod scc; pub mod vec_graph; diff --git a/compiler/rustc_data_structures/src/graph/reversed.rs b/compiler/rustc_data_structures/src/graph/reversed.rs new file mode 100644 index 00000000000..9b726deaa15 --- /dev/null +++ b/compiler/rustc_data_structures/src/graph/reversed.rs @@ -0,0 +1,42 @@ +use crate::graph::{DirectedGraph, Predecessors, Successors}; + +/// View that reverses the direction of edges in its underlying graph, so that +/// successors become predecessors and vice-versa. +/// +/// Because of `impl<G: Graph> Graph for &G`, the underlying graph can be +/// wrapped by-reference instead of by-value if desired. +#[derive(Clone, Copy, Debug)] +pub struct ReversedGraph<G> { + pub inner: G, +} + +impl<G> ReversedGraph<G> { + pub fn new(inner: G) -> Self { + Self { inner } + } +} + +impl<G: DirectedGraph> DirectedGraph for ReversedGraph<G> { + type Node = G::Node; + + fn num_nodes(&self) -> usize { + self.inner.num_nodes() + } +} + +// Implementing `StartNode` is not possible in general, because the start node +// of an underlying graph is instead an _end_ node in the reversed graph. +// But would be possible to define another wrapper type that adds an explicit +// start node to its underlying graph, if desired. + +impl<G: Predecessors> Successors for ReversedGraph<G> { + fn successors(&self, node: Self::Node) -> impl Iterator<Item = Self::Node> { + self.inner.predecessors(node) + } +} + +impl<G: Successors> Predecessors for ReversedGraph<G> { + fn predecessors(&self, node: Self::Node) -> impl Iterator<Item = Self::Node> { + self.inner.successors(node) + } +} diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs index 19050746c2f..18e98e6c39f 100644 --- a/compiler/rustc_data_structures/src/profiling.rs +++ b/compiler/rustc_data_structures/src/profiling.rs @@ -860,6 +860,7 @@ fn get_thread_id() -> u32 { } // Memory reporting +#[cfg(bootstrap)] cfg_match! { cfg(windows) => { pub fn get_resident_set_size() -> Option<usize> { @@ -921,5 +922,67 @@ cfg_match! { } } +#[cfg(not(bootstrap))] +cfg_match! { + windows => { + pub fn get_resident_set_size() -> Option<usize> { + use std::mem; + + use windows::{ + Win32::System::ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, + Win32::System::Threading::GetCurrentProcess, + }; + + let mut pmc = PROCESS_MEMORY_COUNTERS::default(); + let pmc_size = mem::size_of_val(&pmc); + unsafe { + K32GetProcessMemoryInfo( + GetCurrentProcess(), + &mut pmc, + pmc_size as u32, + ) + } + .ok() + .ok()?; + + Some(pmc.WorkingSetSize) + } + } + target_os = "macos" => { + pub fn get_resident_set_size() -> Option<usize> { + use libc::{c_int, c_void, getpid, proc_pidinfo, proc_taskinfo, PROC_PIDTASKINFO}; + use std::mem; + const PROC_TASKINFO_SIZE: c_int = mem::size_of::<proc_taskinfo>() as c_int; + + unsafe { + let mut info: proc_taskinfo = mem::zeroed(); + let info_ptr = &mut info as *mut proc_taskinfo as *mut c_void; + let pid = getpid() as c_int; + let ret = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, info_ptr, PROC_TASKINFO_SIZE); + if ret == PROC_TASKINFO_SIZE { + Some(info.pti_resident_size as usize) + } else { + None + } + } + } + } + unix => { + pub fn get_resident_set_size() -> Option<usize> { + let field = 1; + let contents = fs::read("/proc/self/statm").ok()?; + let contents = String::from_utf8(contents).ok()?; + let s = contents.split_whitespace().nth(field)?; + let npages = s.parse::<usize>().ok()?; + Some(npages * 4096) + } + } + _ => { + pub fn get_resident_set_size() -> Option<usize> { + None + } + } +} + #[cfg(test)] mod tests; diff --git a/compiler/rustc_error_codes/src/error_codes/E0207.md b/compiler/rustc_error_codes/src/error_codes/E0207.md index 95e7c9fc76c..5b35748f472 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0207.md +++ b/compiler/rustc_error_codes/src/error_codes/E0207.md @@ -195,6 +195,30 @@ impl<'a> Contains for Foo { Please note that unconstrained lifetime parameters are not supported if they are being used by an associated type. +In cases where the associated type's lifetime is meant to be tied to the the +self type, and none of the methods on the trait need ownership or different +mutability, then an option is to implement the trait on a borrowed type: + +```rust +struct Foo(i32); + +trait Contents { + type Item; + + fn get(&self) -> Self::Item; +} + +// Note the lifetime `'a` is used both for the self type... +impl<'a> Contents for &'a Foo { + // ...and the associated type. + type Item = &'a i32; + + fn get(&self) -> Self::Item { + &self.0 + } +} +``` + ### Additional information For more information, please see [RFC 447]. diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index c28a4360f6f..5510e7e09e5 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -623,7 +623,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ EncodeCrossCrate::No, "allow_internal_unsafe side-steps the unsafe_code lint", ), rustc_attr!( - rustc_allowed_through_unstable_modules, Normal, template!(Word), + rustc_allowed_through_unstable_modules, Normal, template!(Word, NameValueStr: "deprecation message"), WarnFollowing, EncodeCrossCrate::No, "rustc_allowed_through_unstable_modules special cases accidental stabilizations of stable items \ through unstable paths" diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index 6b88651715e..1dcde453331 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -361,6 +361,8 @@ declare_features! ( (unstable, abi_avr_interrupt, "1.45.0", Some(69664)), /// Allows `extern "C-cmse-nonsecure-call" fn()`. (unstable, abi_c_cmse_nonsecure_call, "1.51.0", Some(81391)), + /// Allows `extern "gpu-kernel" fn()`. + (unstable, abi_gpu_kernel, "CURRENT_RUSTC_VERSION", Some(135467)), /// Allows `extern "msp430-interrupt" fn()`. (unstable, abi_msp430_interrupt, "1.16.0", Some(38487)), /// Allows `extern "ptx-*" fn()`. diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index dd96b30fefc..5339feb5d27 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -34,6 +34,7 @@ use crate::intravisit::FnKind; #[derive(Debug, Copy, Clone, HashStable_Generic)] pub struct Lifetime { + #[stable_hasher(ignore)] pub hir_id: HirId, /// Either "`'a`", referring to a named lifetime definition, @@ -214,6 +215,7 @@ impl Path<'_> { pub struct PathSegment<'hir> { /// The identifier portion of this path segment. pub ident: Ident, + #[stable_hasher(ignore)] pub hir_id: HirId, pub res: Res, @@ -304,6 +306,7 @@ pub enum ConstArgKind<'hir> { #[derive(Clone, Copy, Debug, HashStable_Generic)] pub struct InferArg { + #[stable_hasher(ignore)] pub hir_id: HirId, pub span: Span, } @@ -592,6 +595,7 @@ pub enum GenericParamKind<'hir> { #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct GenericParam<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub def_id: LocalDefId, pub name: ParamName, @@ -850,6 +854,7 @@ impl<'hir> Generics<'hir> { /// A single predicate in a where-clause. #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct WherePredicate<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub span: Span, pub kind: &'hir WherePredicateKind<'hir>, @@ -1521,6 +1526,7 @@ impl fmt::Debug for DotDotPos { #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct PatExpr<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub span: Span, pub kind: PatExprKind<'hir>, @@ -1610,6 +1616,7 @@ pub enum PatKind<'hir> { /// A statement. #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct Stmt<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub kind: StmtKind<'hir>, pub span: Span, @@ -1641,6 +1648,7 @@ pub struct LetStmt<'hir> { pub init: Option<&'hir Expr<'hir>>, /// Else block for a `let...else` binding. pub els: Option<&'hir Block<'hir>>, + #[stable_hasher(ignore)] pub hir_id: HirId, pub span: Span, /// Can be `ForLoopDesugar` if the `let` statement is part of a `for` loop @@ -1937,6 +1945,7 @@ pub type Lit = Spanned<LitKind>; /// `const N: usize = { ... }` with `tcx.hir().opt_const_param_default_param_def_id(..)` #[derive(Copy, Clone, Debug, HashStable_Generic)] pub struct AnonConst { + #[stable_hasher(ignore)] pub hir_id: HirId, pub def_id: LocalDefId, pub body: BodyId, @@ -1946,6 +1955,7 @@ pub struct AnonConst { /// An inline constant expression `const { something }`. #[derive(Copy, Clone, Debug, HashStable_Generic)] pub struct ConstBlock { + #[stable_hasher(ignore)] pub hir_id: HirId, pub def_id: LocalDefId, pub body: BodyId, @@ -1961,6 +1971,7 @@ pub struct ConstBlock { /// [rust lang reference]: https://doc.rust-lang.org/reference/expressions.html #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct Expr<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub kind: ExprKind<'hir>, pub span: Span, @@ -2839,6 +2850,7 @@ pub enum ImplItemKind<'hir> { /// * the `f(..): Bound` in `Trait<f(..): Bound>` (feature `return_type_notation`) #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct AssocItemConstraint<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub ident: Ident, pub gen_args: &'hir GenericArgs<'hir>, @@ -2907,6 +2919,7 @@ impl<'hir> AssocItemConstraintKind<'hir> { #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct Ty<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub kind: TyKind<'hir>, pub span: Span, @@ -3102,6 +3115,7 @@ pub struct UnsafeBinderTy<'hir> { #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct OpaqueTy<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub def_id: LocalDefId, pub bounds: GenericBounds<'hir>, @@ -3138,6 +3152,7 @@ impl PreciseCapturingArg<'_> { /// since resolve_bound_vars operates on `Lifetime`s. #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct PreciseCapturingNonLifetimeArg { + #[stable_hasher(ignore)] pub hir_id: HirId, pub ident: Ident, pub res: Res, @@ -3311,6 +3326,7 @@ impl InlineAsm<'_> { /// Represents a parameter in a function header. #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct Param<'hir> { + #[stable_hasher(ignore)] pub hir_id: HirId, pub pat: &'hir Pat<'hir>, pub ty_span: Span, @@ -3468,6 +3484,7 @@ pub struct Variant<'hir> { /// Name of the variant. pub ident: Ident, /// Id of the variant (not the constructor, see `VariantData::ctor_hir_id()`). + #[stable_hasher(ignore)] pub hir_id: HirId, pub def_id: LocalDefId, /// Fields and constructor id of the variant. @@ -3540,6 +3557,7 @@ pub struct FieldDef<'hir> { pub span: Span, pub vis_span: Span, pub ident: Ident, + #[stable_hasher(ignore)] pub hir_id: HirId, pub def_id: LocalDefId, pub ty: &'hir Ty<'hir>, @@ -3564,11 +3582,11 @@ pub enum VariantData<'hir> { /// A tuple variant. /// /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`. - Tuple(&'hir [FieldDef<'hir>], HirId, LocalDefId), + Tuple(&'hir [FieldDef<'hir>], #[stable_hasher(ignore)] HirId, LocalDefId), /// A unit variant. /// /// E.g., `Bar = ..` as in `enum Foo { Bar = .. }`. - Unit(HirId, LocalDefId), + Unit(#[stable_hasher(ignore)] HirId, LocalDefId), } impl<'hir> VariantData<'hir> { @@ -3762,9 +3780,30 @@ impl fmt::Display for Constness { } } +/// The actualy safety specified in syntax. We may treat +/// its safety different within the type system to create a +/// "sound by default" system that needs checking this enum +/// explicitly to allow unsafe operations. +#[derive(Copy, Clone, Debug, HashStable_Generic, PartialEq, Eq)] +pub enum HeaderSafety { + /// A safe function annotated with `#[target_features]`. + /// The type system treats this function as an unsafe function, + /// but safety checking will check this enum to treat it as safe + /// and allowing calling other safe target feature functions with + /// the same features without requiring an additional unsafe block. + SafeTargetFeatures, + Normal(Safety), +} + +impl From<Safety> for HeaderSafety { + fn from(v: Safety) -> Self { + Self::Normal(v) + } +} + #[derive(Copy, Clone, Debug, HashStable_Generic)] pub struct FnHeader { - pub safety: Safety, + pub safety: HeaderSafety, pub constness: Constness, pub asyncness: IsAsync, pub abi: ExternAbi, @@ -3780,7 +3819,18 @@ impl FnHeader { } pub fn is_unsafe(&self) -> bool { - self.safety.is_unsafe() + self.safety().is_unsafe() + } + + pub fn is_safe(&self) -> bool { + self.safety().is_safe() + } + + pub fn safety(&self) -> Safety { + match self.safety { + HeaderSafety::SafeTargetFeatures => Safety::Unsafe, + HeaderSafety::Normal(safety) => safety, + } } } diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl index 0c3ed9b5c60..d7ab6eca84b 100644 --- a/compiler/rustc_hir_analysis/messages.ftl +++ b/compiler/rustc_hir_analysis/messages.ftl @@ -135,7 +135,7 @@ hir_analysis_dispatch_from_dyn_multi = implementing the `DispatchFromDyn` trait hir_analysis_dispatch_from_dyn_repr = structs implementing `DispatchFromDyn` may not have `#[repr(packed)]` or `#[repr(C)]` -hir_analysis_dispatch_from_dyn_zst = the trait `DispatchFromDyn` may only be implemented for structs containing the field being coerced, ZST fields with 1 byte alignment, and nothing else +hir_analysis_dispatch_from_dyn_zst = the trait `DispatchFromDyn` may only be implemented for structs containing the field being coerced, ZST fields with 1 byte alignment that don't mention type/const generics, and nothing else .note = extra field `{$name}` of type `{$ty}` is not allowed hir_analysis_drop_impl_negative = negative `Drop` impls are not supported diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index ec82644ea5b..b0a6922ff72 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -1637,7 +1637,6 @@ fn check_type_alias_type_params_are_used<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalD let ty = tcx.type_of(def_id).instantiate_identity(); if ty.references_error() { // If there is already another error, do not emit an error for not using a type parameter. - assert!(tcx.dcx().has_errors().is_some()); return; } diff --git a/compiler/rustc_hir_analysis/src/coherence/builtin.rs b/compiler/rustc_hir_analysis/src/coherence/builtin.rs index 3b98f358b1e..b43a808ccdc 100644 --- a/compiler/rustc_hir_analysis/src/coherence/builtin.rs +++ b/compiler/rustc_hir_analysis/src/coherence/builtin.rs @@ -259,19 +259,37 @@ fn visit_implementation_of_dispatch_from_dyn(checker: &Checker<'_>) -> Result<() let coerced_fields = fields .iter() .filter(|field| { + // Ignore PhantomData fields + let unnormalized_ty = tcx.type_of(field.did).instantiate_identity(); + if tcx + .try_normalize_erasing_regions( + ty::TypingEnv::non_body_analysis(tcx, def_a.did()), + unnormalized_ty, + ) + .unwrap_or(unnormalized_ty) + .is_phantom_data() + { + return false; + } + let ty_a = field.ty(tcx, args_a); let ty_b = field.ty(tcx, args_b); - if let Ok(layout) = - tcx.layout_of(infcx.typing_env(param_env).as_query_input(ty_a)) - { - if layout.is_1zst() { + // FIXME: We could do normalization here, but is it really worth it? + if ty_a == ty_b { + // Allow 1-ZSTs that don't mention type params. + // + // Allowing type params here would allow us to possibly transmute + // between ZSTs, which may be used to create library unsoundness. + if let Ok(layout) = + tcx.layout_of(infcx.typing_env(param_env).as_query_input(ty_a)) + && layout.is_1zst() + && !ty_a.has_non_region_param() + { // ignore 1-ZST fields return false; } - } - if ty_a == ty_b { res = Err(tcx.dcx().emit_err(errors::DispatchFromDynZST { span, name: field.name, @@ -460,8 +478,16 @@ pub(crate) fn coerce_unsized_info<'tcx>( .filter_map(|(i, f)| { let (a, b) = (f.ty(tcx, args_a), f.ty(tcx, args_b)); - if tcx.type_of(f.did).instantiate_identity().is_phantom_data() { - // Ignore PhantomData fields + // Ignore PhantomData fields + let unnormalized_ty = tcx.type_of(f.did).instantiate_identity(); + if tcx + .try_normalize_erasing_regions( + ty::TypingEnv::non_body_analysis(tcx, def_a.did()), + unnormalized_ty, + ) + .unwrap_or(unnormalized_ty) + .is_phantom_data() + { return None; } diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs index d41b03640b6..86c6532c97d 100644 --- a/compiler/rustc_hir_analysis/src/collect.rs +++ b/compiler/rustc_hir_analysis/src/collect.rs @@ -1336,7 +1336,7 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_, ty::PolyFn { icx.lowerer().lower_fn_ty( hir_id, - sig.header.safety, + sig.header.safety(), sig.header.abi, sig.decl, Some(generics), @@ -1351,13 +1351,18 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_, ty::PolyFn kind: TraitItemKind::Fn(FnSig { header, decl, span: _ }, _), generics, .. - }) => { - icx.lowerer().lower_fn_ty(hir_id, header.safety, header.abi, decl, Some(generics), None) - } + }) => icx.lowerer().lower_fn_ty( + hir_id, + header.safety(), + header.abi, + decl, + Some(generics), + None, + ), ForeignItem(&hir::ForeignItem { kind: ForeignItemKind::Fn(sig, _, _), .. }) => { let abi = tcx.hir().get_foreign_abi(hir_id); - compute_sig_of_foreign_fn_decl(tcx, def_id, sig.decl, abi, sig.header.safety) + compute_sig_of_foreign_fn_decl(tcx, def_id, sig.decl, abi, sig.header.safety()) } Ctor(data) | Variant(hir::Variant { data, .. }) if data.ctor().is_some() => { @@ -1405,7 +1410,7 @@ fn lower_fn_sig_recovering_infer_ret_ty<'tcx>( icx.lowerer().lower_fn_ty( icx.tcx().local_def_id_to_hir_id(def_id), - sig.header.safety, + sig.header.safety(), sig.header.abi, sig.decl, Some(generics), diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs index 1569c0963c8..8a975786a92 100644 --- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs @@ -653,7 +653,7 @@ pub(super) fn implied_predicates_with_filter<'tcx>( } } } - PredicateFilter::SelfAndAssociatedTypeBounds => { + PredicateFilter::All | PredicateFilter::SelfAndAssociatedTypeBounds => { for &(pred, span) in implied_bounds { debug!("superbound: {:?}", pred); if let ty::ClauseKind::Trait(bound) = pred.kind().skip_binder() diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index 39b6823cf0e..3ff6acd79fc 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -2407,7 +2407,7 @@ impl<'a> State<'a> { self.print_fn( decl, hir::FnHeader { - safety, + safety: safety.into(), abi, constness: hir::Constness::NotConst, asyncness: hir::IsAsync::NotAsync, @@ -2423,12 +2423,20 @@ impl<'a> State<'a> { fn print_fn_header_info(&mut self, header: hir::FnHeader) { self.print_constness(header.constness); + let safety = match header.safety { + hir::HeaderSafety::SafeTargetFeatures => { + self.word_nbsp("#[target_feature]"); + hir::Safety::Safe + } + hir::HeaderSafety::Normal(safety) => safety, + }; + match header.asyncness { hir::IsAsync::NotAsync => {} hir::IsAsync::Async(_) => self.word_nbsp("async"), } - self.print_safety(header.safety); + self.print_safety(safety); if header.abi != ExternAbi::Rust { self.word_nbsp("extern"); diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs index 9ebc7a4657e..6945dbc3216 100644 --- a/compiler/rustc_hir_typeck/src/coercion.rs +++ b/compiler/rustc_hir_typeck/src/coercion.rs @@ -920,7 +920,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { match b.kind() { ty::FnPtr(_, b_hdr) => { - let a_sig = a.fn_sig(self.tcx); + let mut a_sig = a.fn_sig(self.tcx); if let ty::FnDef(def_id, _) = *a.kind() { // Intrinsics are not coercible to function pointers if self.tcx.intrinsic(def_id).is_some() { @@ -932,12 +932,20 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { return Err(TypeError::ForceInlineCast); } - // Safe `#[target_feature]` functions are not assignable to safe fn pointers - // (RFC 2396). if b_hdr.safety.is_safe() - && !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty() + && self.tcx.codegen_fn_attrs(def_id).safe_target_features { - return Err(TypeError::TargetFeatureCast(def_id)); + // Allow the coercion if the current function has all the features that would be + // needed to call the coercee safely. + if let Some(safe_sig) = self.tcx.adjust_target_feature_sig( + def_id, + a_sig, + self.fcx.body_id.into(), + ) { + a_sig = safe_sig; + } else { + return Err(TypeError::TargetFeatureCast(def_id)); + } } } diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs index 39511ca30e0..53e055fdeef 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs @@ -2682,6 +2682,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { if let hir::ExprKind::Unary(hir::UnOp::Deref, inner) = expr.kind && let Some(1) = self.deref_steps_for_suggestion(expected, checked_ty) + && self.typeck_results.borrow().expr_ty(inner).is_ref() { // We have `*&T`, check if what was expected was `&T`. // If so, we may want to suggest removing a `*`. diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs index a406ec9a8fb..cb21961f36b 100644 --- a/compiler/rustc_hir_typeck/src/lib.rs +++ b/compiler/rustc_hir_typeck/src/lib.rs @@ -139,7 +139,7 @@ fn typeck_with_fallback<'tcx>( // type that has an infer in it, lower the type directly so that it'll // be correctly filled with infer. We'll use this inference to provide // a suggestion later on. - fcx.lowerer().lower_fn_ty(id, header.safety, header.abi, decl, None, None) + fcx.lowerer().lower_fn_ty(id, header.safety(), header.abi, decl, None, None) } else { tcx.fn_sig(def_id).instantiate_identity() }; diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index d93707b745d..f12df831cb5 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -281,6 +281,24 @@ impl<T: Idx> DenseBitSet<T> { } bit_relations_inherent_impls! {} + + /// Sets `self = self | !other`. + /// + /// FIXME: Incorporate this into [`BitRelations`] and fill out + /// implementations for other bitset types, if needed. + pub fn union_not(&mut self, other: &DenseBitSet<T>) { + assert_eq!(self.domain_size, other.domain_size); + + // FIXME(Zalathar): If we were to forcibly _set_ all excess bits before + // the bitwise update, and then clear them again afterwards, we could + // quickly and accurately detect whether the update changed anything. + // But that's only worth doing if there's an actual use-case. + + bitwise(&mut self.words, &other.words, |a, b| a | !b); + // The bitwise update `a | !b` can result in the last word containing + // out-of-domain bits, so we need to clear them. + self.clear_excess_bits(); + } } // dense REL dense @@ -1087,6 +1105,18 @@ impl<T: Idx> fmt::Debug for ChunkedBitSet<T> { } } +/// Sets `out_vec[i] = op(out_vec[i], in_vec[i])` for each index `i` in both +/// slices. The slices must have the same length. +/// +/// Returns true if at least one bit in `out_vec` was changed. +/// +/// ## Warning +/// Some bitwise operations (e.g. union-not, xor) can set output bits that were +/// unset in in both inputs. If this happens in the last word/chunk of a bitset, +/// it can cause the bitset to contain out-of-domain values, which need to +/// be cleared with `clear_excess_bits_in_final_word`. This also makes the +/// "changed" return value unreliable, because the change might have only +/// affected excess bits. #[inline] fn bitwise<Op>(out_vec: &mut [Word], in_vec: &[Word], op: Op) -> bool where diff --git a/compiler/rustc_index/src/bit_set/tests.rs b/compiler/rustc_index/src/bit_set/tests.rs index 0350740aa81..eaa4aafe721 100644 --- a/compiler/rustc_index/src/bit_set/tests.rs +++ b/compiler/rustc_index/src/bit_set/tests.rs @@ -76,6 +76,32 @@ fn union_two_sets() { } #[test] +fn union_not() { + let mut a = DenseBitSet::<usize>::new_empty(100); + let mut b = DenseBitSet::<usize>::new_empty(100); + + a.insert(3); + a.insert(5); + a.insert(80); + a.insert(81); + + b.insert(5); // Already in `a`. + b.insert(7); + b.insert(63); + b.insert(81); // Already in `a`. + b.insert(90); + + a.union_not(&b); + + // After union-not, `a` should contain all values in the domain, except for + // the ones that are in `b` and were _not_ already in `a`. + assert_eq!( + a.iter().collect::<Vec<_>>(), + (0usize..100).filter(|&x| !matches!(x, 7 | 63 | 90)).collect::<Vec<_>>(), + ); +} + +#[test] fn chunked_bitset() { let mut b0 = ChunkedBitSet::<usize>::new_empty(0); let b0b = b0.clone(); diff --git a/compiler/rustc_infer/src/traits/util.rs b/compiler/rustc_infer/src/traits/util.rs index ab8ada1596c..66ed49fe326 100644 --- a/compiler/rustc_infer/src/traits/util.rs +++ b/compiler/rustc_infer/src/traits/util.rs @@ -1,5 +1,5 @@ use rustc_data_structures::fx::FxHashSet; -use rustc_middle::ty::{self, ToPolyTraitRef, TyCtxt}; +use rustc_middle::ty::{self, TyCtxt}; use rustc_span::{Ident, Span}; pub use rustc_type_ir::elaborate::*; @@ -125,8 +125,8 @@ pub fn transitive_bounds_that_define_assoc_item<'tcx>( .iter_identity_copied() .map(|(clause, _)| clause.instantiate_supertrait(tcx, trait_ref)) .filter_map(|clause| clause.as_trait_clause()) - // FIXME: Negative supertraits are elaborated here lol - .map(|trait_pred| trait_pred.to_poly_trait_ref()), + .filter(|clause| clause.polarity() == ty::PredicatePolarity::Positive) + .map(|clause| clause.map_bound(|clause| clause.trait_ref)), ); return Some(trait_ref); diff --git a/compiler/rustc_lint/src/types/literal.rs b/compiler/rustc_lint/src/types/literal.rs index 83942918e3b..4b5163522f8 100644 --- a/compiler/rustc_lint/src/types/literal.rs +++ b/compiler/rustc_lint/src/types/literal.rs @@ -204,20 +204,35 @@ fn get_type_suggestion(t: Ty<'_>, val: u128, negative: bool) -> Option<&'static match t.kind() { ty::Uint(ty::UintTy::Usize) | ty::Int(ty::IntTy::Isize) => None, ty::Uint(_) => Some(Integer::fit_unsigned(val).uint_ty_str()), - ty::Int(_) if negative => Some(Integer::fit_signed(-(val as i128)).int_ty_str()), - ty::Int(int) => { - let signed = Integer::fit_signed(val as i128); - let unsigned = Integer::fit_unsigned(val); - Some(if Some(unsigned.size().bits()) == int.bit_width() { - unsigned.uint_ty_str() + ty::Int(_) => { + let signed = literal_to_i128(val, negative).map(Integer::fit_signed); + if negative { + signed.map(Integer::int_ty_str) } else { - signed.int_ty_str() - }) + let unsigned = Integer::fit_unsigned(val); + Some(if let Some(signed) = signed { + if unsigned.size() < signed.size() { + unsigned.uint_ty_str() + } else { + signed.int_ty_str() + } + } else { + unsigned.uint_ty_str() + }) + } } _ => None, } } +fn literal_to_i128(val: u128, negative: bool) -> Option<i128> { + if negative { + (val <= i128::MAX as u128 + 1).then(|| val.wrapping_neg() as i128) + } else { + val.try_into().ok() + } +} + fn lint_int_literal<'tcx>( cx: &LateContext<'tcx>, type_limits: &TypeLimits, diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 0c6147f4a46..9fc527a6a3a 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -3597,7 +3597,7 @@ declare_lint! { /// /// [Other ABIs]: https://doc.rust-lang.org/reference/items/external-blocks.html#abi pub MISSING_ABI, - Allow, + Warn, "No declared ABI for extern declaration" } diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs index 16d868300db..e05f42af6fd 100644 --- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs +++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs @@ -30,6 +30,8 @@ pub struct CodegenFnAttrs { /// features (only enabled features are supported right now). /// Implied target features have already been applied. pub target_features: Vec<TargetFeature>, + /// Whether the function was declared safe, but has target features + pub safe_target_features: bool, /// The `#[linkage = "..."]` attribute on Rust-defined items and the value we found. pub linkage: Option<Linkage>, /// The `#[linkage = "..."]` attribute on foreign items and the value we found. @@ -150,6 +152,7 @@ impl CodegenFnAttrs { link_name: None, link_ordinal: None, target_features: vec![], + safe_target_features: false, linkage: None, import_linkage: None, link_section: None, diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs index 84c3c2eb49e..77a7da2c74b 100644 --- a/compiler/rustc_middle/src/middle/stability.rs +++ b/compiler/rustc_middle/src/middle/stability.rs @@ -30,6 +30,14 @@ pub enum StabilityLevel { Stable, } +#[derive(Copy, Clone)] +pub enum UnstableKind { + /// Enforcing regular stability of an item + Regular, + /// Enforcing const stability of an item + Const(Span), +} + /// An entry in the `depr_map`. #[derive(Copy, Clone, HashStable, Debug, Encodable, Decodable)] pub struct DeprecationEntry { @@ -108,10 +116,16 @@ pub fn report_unstable( is_soft: bool, span: Span, soft_handler: impl FnOnce(&'static Lint, Span, String), + kind: UnstableKind, ) { + let qual = match kind { + UnstableKind::Regular => "", + UnstableKind::Const(_) => " const", + }; + let msg = match reason { - Some(r) => format!("use of unstable library feature `{feature}`: {r}"), - None => format!("use of unstable library feature `{feature}`"), + Some(r) => format!("use of unstable{qual} library feature `{feature}`: {r}"), + None => format!("use of unstable{qual} library feature `{feature}`"), }; if is_soft { @@ -121,6 +135,9 @@ pub fn report_unstable( if let Some((inner_types, msg, sugg, applicability)) = suggestion { err.span_suggestion(inner_types, msg, sugg, applicability); } + if let UnstableKind::Const(kw) = kind { + err.span_label(kw, "trait is not stable as const yet"); + } err.emit(); } } @@ -232,9 +249,18 @@ fn late_report_deprecation( return; } + let is_in_effect = depr.is_in_effect(); + let lint = deprecation_lint(is_in_effect); + + // Calculating message for lint involves calling `self.def_path_str`, + // which will by default invoke the expensive `visible_parent_map` query. + // Skip all that work if the lint is allowed anyway. + if tcx.lint_level_at_node(lint, hir_id).0 == Level::Allow { + return; + } + let def_path = with_no_trimmed_paths!(tcx.def_path_str(def_id)); let def_kind = tcx.def_descr(def_id); - let is_in_effect = depr.is_in_effect(); let method_span = method_span.unwrap_or(span); let suggestion = @@ -250,7 +276,7 @@ fn late_report_deprecation( note: depr.note, since_kind: deprecated_since_kind(is_in_effect, depr.since), }; - tcx.emit_node_span_lint(deprecation_lint(is_in_effect), hir_id, method_span, diag); + tcx.emit_node_span_lint(lint, hir_id, method_span, diag); } /// Result of `TyCtxt::eval_stability`. @@ -360,13 +386,7 @@ impl<'tcx> TyCtxt<'tcx> { // hierarchy. let depr_attr = &depr_entry.attr; if !skip || depr_attr.is_since_rustc_version() { - // Calculating message for lint involves calling `self.def_path_str`. - // Which by default to calculate visible path will invoke expensive `visible_parent_map` query. - // So we skip message calculation altogether, if lint is allowed. - let lint = deprecation_lint(depr_attr.is_in_effect()); - if self.lint_level_at_node(lint, id).0 != Level::Allow { - late_report_deprecation(self, depr_attr, span, method_span, id, def_id); - } + late_report_deprecation(self, depr_attr, span, method_span, id, def_id); } }; } @@ -587,6 +607,7 @@ impl<'tcx> TyCtxt<'tcx> { is_soft, span, soft_handler, + UnstableKind::Regular, ), EvalResult::Unmarked => unmarked(span, def_id), } @@ -594,6 +615,73 @@ impl<'tcx> TyCtxt<'tcx> { is_allowed } + /// This function is analogous to `check_optional_stability` but with the logic in + /// `eval_stability_allow_unstable` inlined, and which operating on const stability + /// instead of regular stability. + /// + /// This enforces *syntactical* const stability of const traits. In other words, + /// it enforces the ability to name `~const`/`const` traits in trait bounds in various + /// syntax positions in HIR (including in the trait of an impl header). + pub fn check_const_stability(self, def_id: DefId, span: Span, const_kw_span: Span) { + let is_staged_api = self.lookup_stability(def_id.krate.as_def_id()).is_some(); + if !is_staged_api { + return; + } + + // Only the cross-crate scenario matters when checking unstable APIs + let cross_crate = !def_id.is_local(); + if !cross_crate { + return; + } + + let stability = self.lookup_const_stability(def_id); + debug!( + "stability: \ + inspecting def_id={:?} span={:?} of stability={:?}", + def_id, span, stability + ); + + match stability { + Some(ConstStability { + level: attr::StabilityLevel::Unstable { reason, issue, is_soft, implied_by, .. }, + feature, + .. + }) => { + assert!(!is_soft); + + if span.allows_unstable(feature) { + debug!("body stability: skipping span={:?} since it is internal", span); + return; + } + if self.features().enabled(feature) { + return; + } + + // If this item was previously part of a now-stabilized feature which is still + // enabled (i.e. the user hasn't removed the attribute for the stabilized feature + // yet) then allow use of this item. + if let Some(implied_by) = implied_by + && self.features().enabled(implied_by) + { + return; + } + + report_unstable( + self.sess, + feature, + reason.to_opt_reason(), + issue, + None, + false, + span, + |_, _, _| {}, + UnstableKind::Const(const_kw_span), + ); + } + Some(_) | None => {} + } + } + pub fn lookup_deprecation(self, id: DefId) -> Option<Deprecation> { self.lookup_deprecation_entry(id).map(|depr| depr.attr) } diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index cd6b2d65bf1..65f51ae9d39 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -71,11 +71,7 @@ impl ConditionId { /// Enum that can hold a constant zero value, the ID of an physical coverage /// counter, or the ID of a coverage-counter expression. -/// -/// This was originally only used for expression operands (and named `Operand`), -/// but the zero/counter/expression distinction is also useful for representing -/// the value of code/gap mappings, and the true/false arms of branch mappings. -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] #[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] pub enum CovTerm { Zero, @@ -171,7 +167,7 @@ impl Op { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] #[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] pub struct Expression { pub lhs: CovTerm, diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index fab0047babf..7035e641f39 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -60,7 +60,7 @@ use crate::dep_graph::{DepGraph, DepKindStruct}; use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarInfo, CanonicalVarInfos}; use crate::lint::lint_level; use crate::metadata::ModChild; -use crate::middle::codegen_fn_attrs::CodegenFnAttrs; +use crate::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature}; use crate::middle::{resolve_bound_vars, stability}; use crate::mir::interpret::{self, Allocation, ConstAllocation}; use crate::mir::{Body, Local, Place, PlaceElem, ProjectionKind, Promoted}; @@ -1776,6 +1776,37 @@ impl<'tcx> TyCtxt<'tcx> { pub fn dcx(self) -> DiagCtxtHandle<'tcx> { self.sess.dcx() } + + pub fn is_target_feature_call_safe( + self, + callee_features: &[TargetFeature], + body_features: &[TargetFeature], + ) -> bool { + // If the called function has target features the calling function hasn't, + // the call requires `unsafe`. Don't check this on wasm + // targets, though. For more information on wasm see the + // is_like_wasm check in hir_analysis/src/collect.rs + self.sess.target.options.is_like_wasm + || callee_features + .iter() + .all(|feature| body_features.iter().any(|f| f.name == feature.name)) + } + + /// Returns the safe version of the signature of the given function, if calling it + /// would be safe in the context of the given caller. + pub fn adjust_target_feature_sig( + self, + fun_def: DefId, + fun_sig: ty::Binder<'tcx, ty::FnSig<'tcx>>, + caller: DefId, + ) -> Option<ty::Binder<'tcx, ty::FnSig<'tcx>>> { + let fun_features = &self.codegen_fn_attrs(fun_def).target_features; + let callee_features = &self.codegen_fn_attrs(caller).target_features; + if self.is_target_feature_call_safe(&fun_features, &callee_features) { + return Some(fun_sig.map_bound(|sig| ty::FnSig { safety: hir::Safety::Safe, ..sig })); + } + None + } } impl<'tcx> TyCtxtAt<'tcx> { diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 8d4d127607d..1e67cdfc32a 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -1241,6 +1241,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi) PtxKernel | Msp430Interrupt | X86Interrupt + | GpuKernel | EfiApi | AvrInterrupt | AvrNonBlockingInterrupt diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index bf0ccdc0f10..ca70ae794c5 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -79,8 +79,7 @@ pub use self::predicate::{ PolyExistentialPredicate, PolyExistentialProjection, PolyExistentialTraitRef, PolyProjectionPredicate, PolyRegionOutlivesPredicate, PolySubtypePredicate, PolyTraitPredicate, PolyTraitRef, PolyTypeOutlivesPredicate, Predicate, PredicateKind, ProjectionPredicate, - RegionOutlivesPredicate, SubtypePredicate, ToPolyTraitRef, TraitPredicate, TraitRef, - TypeOutlivesPredicate, + RegionOutlivesPredicate, SubtypePredicate, TraitPredicate, TraitRef, TypeOutlivesPredicate, }; pub use self::region::{ BoundRegion, BoundRegionKind, EarlyParamRegion, LateParamRegion, LateParamRegionKind, Region, @@ -222,6 +221,7 @@ pub struct DelegationFnSig { pub param_count: usize, pub has_self: bool, pub c_variadic: bool, + pub target_feature: bool, } #[derive(Clone, Copy, Debug)] diff --git a/compiler/rustc_middle/src/ty/predicate.rs b/compiler/rustc_middle/src/ty/predicate.rs index 32d6455e825..584cac22ae8 100644 --- a/compiler/rustc_middle/src/ty/predicate.rs +++ b/compiler/rustc_middle/src/ty/predicate.rs @@ -476,16 +476,6 @@ impl<'tcx> Clause<'tcx> { } } -pub trait ToPolyTraitRef<'tcx> { - fn to_poly_trait_ref(&self) -> PolyTraitRef<'tcx>; -} - -impl<'tcx> ToPolyTraitRef<'tcx> for PolyTraitPredicate<'tcx> { - fn to_poly_trait_ref(&self) -> PolyTraitRef<'tcx> { - self.map_bound_ref(|trait_pred| trait_pred.trait_ref) - } -} - impl<'tcx> UpcastFrom<TyCtxt<'tcx>, PredicateKind<'tcx>> for Predicate<'tcx> { fn upcast_from(from: PredicateKind<'tcx>, tcx: TyCtxt<'tcx>) -> Self { ty::Binder::dummy(from).upcast(tcx) diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index 9fe1caa4b58..ac900edefe1 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -690,7 +690,14 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write { if with_reduced_queries() { p!(print_def_path(def_id, args)); } else { - let sig = self.tcx().fn_sig(def_id).instantiate(self.tcx(), args); + let mut sig = self.tcx().fn_sig(def_id).instantiate(self.tcx(), args); + if self.tcx().codegen_fn_attrs(def_id).safe_target_features { + p!("#[target_features] "); + sig = sig.map_bound(|mut sig| { + sig.safety = hir::Safety::Safe; + sig + }); + } p!(print(sig), " {{", print_value_path(def_id, args), "}}"); } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 2980964898c..bf37ae05c82 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -401,7 +401,7 @@ impl<'tcx> Ty<'tcx> { /// The more specific methods will often optimize their creation. #[allow(rustc::usage_of_ty_tykind)] #[inline] - pub fn new(tcx: TyCtxt<'tcx>, st: TyKind<'tcx>) -> Ty<'tcx> { + fn new(tcx: TyCtxt<'tcx>, st: TyKind<'tcx>) -> Ty<'tcx> { tcx.mk_ty_from_kind(st) } @@ -613,6 +613,41 @@ impl<'tcx> Ty<'tcx> { #[inline] pub fn new_adt(tcx: TyCtxt<'tcx>, def: AdtDef<'tcx>, args: GenericArgsRef<'tcx>) -> Ty<'tcx> { tcx.debug_assert_args_compatible(def.did(), args); + if cfg!(debug_assertions) { + match tcx.def_kind(def.did()) { + DefKind::Struct | DefKind::Union | DefKind::Enum => {} + DefKind::Mod + | DefKind::Variant + | DefKind::Trait + | DefKind::TyAlias + | DefKind::ForeignTy + | DefKind::TraitAlias + | DefKind::AssocTy + | DefKind::TyParam + | DefKind::Fn + | DefKind::Const + | DefKind::ConstParam + | DefKind::Static { .. } + | DefKind::Ctor(..) + | DefKind::AssocFn + | DefKind::AssocConst + | DefKind::Macro(..) + | DefKind::ExternCrate + | DefKind::Use + | DefKind::ForeignMod + | DefKind::AnonConst + | DefKind::InlineConst + | DefKind::OpaqueTy + | DefKind::Field + | DefKind::LifetimeParam + | DefKind::GlobalAsm + | DefKind::Impl { .. } + | DefKind::Closure + | DefKind::SyntheticCoroutineBody => { + bug!("not an adt: {def:?} ({:?})", tcx.def_kind(def.did())) + } + } + } Ty::new(tcx, Adt(def, args)) } @@ -772,7 +807,7 @@ impl<'tcx> Ty<'tcx> { } } }); - Ty::new(tcx, Adt(adt_def, args)) + Ty::new_adt(tcx, adt_def, args) } #[inline] diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs index f7071eb139f..5eed9ef798d 100644 --- a/compiler/rustc_mir_build/src/check_unsafety.rs +++ b/compiler/rustc_mir_build/src/check_unsafety.rs @@ -478,23 +478,26 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> { return; // don't visit the whole expression } ExprKind::Call { fun, ty: _, args: _, from_hir_call: _, fn_span: _ } => { - if self.thir[fun].ty.fn_sig(self.tcx).safety().is_unsafe() { - let func_id = if let ty::FnDef(func_id, _) = self.thir[fun].ty.kind() { + let fn_ty = self.thir[fun].ty; + let sig = fn_ty.fn_sig(self.tcx); + let (callee_features, safe_target_features): (&[_], _) = match fn_ty.kind() { + ty::FnDef(func_id, ..) => { + let cg_attrs = self.tcx.codegen_fn_attrs(func_id); + (&cg_attrs.target_features, cg_attrs.safe_target_features) + } + _ => (&[], false), + }; + if sig.safety().is_unsafe() && !safe_target_features { + let func_id = if let ty::FnDef(func_id, _) = fn_ty.kind() { Some(*func_id) } else { None }; self.requires_unsafe(expr.span, CallToUnsafeFunction(func_id)); - } else if let &ty::FnDef(func_did, _) = self.thir[fun].ty.kind() { - // If the called function has target features the calling function hasn't, - // the call requires `unsafe`. Don't check this on wasm - // targets, though. For more information on wasm see the - // is_like_wasm check in hir_analysis/src/collect.rs - let callee_features = &self.tcx.codegen_fn_attrs(func_did).target_features; - if !self.tcx.sess.target.options.is_like_wasm - && !callee_features.iter().all(|feature| { - self.body_target_features.iter().any(|f| f.name == feature.name) - }) + } else if let &ty::FnDef(func_did, _) = fn_ty.kind() { + if !self + .tcx + .is_target_feature_call_safe(callee_features, self.body_target_features) { let missing: Vec<_> = callee_features .iter() @@ -739,7 +742,10 @@ impl UnsafeOpKind { ) { let parent_id = tcx.hir().get_parent_item(hir_id); let parent_owner = tcx.hir_owner_node(parent_id); - let should_suggest = parent_owner.fn_sig().is_some_and(|sig| sig.header.is_unsafe()); + let should_suggest = parent_owner.fn_sig().is_some_and(|sig| { + // Do not suggest for safe target_feature functions + matches!(sig.header.safety, hir::HeaderSafety::Normal(hir::Safety::Unsafe)) + }); let unsafe_not_inherited_note = if should_suggest { suggest_unsafe_block.then(|| { let body_span = tcx.hir().body(parent_owner.body_id().unwrap()).value.span; @@ -902,7 +908,7 @@ impl UnsafeOpKind { { true } else if let Some(sig) = tcx.hir().fn_sig_by_hir_id(*id) - && sig.header.is_unsafe() + && matches!(sig.header.safety, hir::HeaderSafety::Normal(hir::Safety::Unsafe)) { true } else { @@ -1111,7 +1117,16 @@ pub(crate) fn check_unsafety(tcx: TyCtxt<'_>, def: LocalDefId) { let hir_id = tcx.local_def_id_to_hir_id(def); let safety_context = tcx.hir().fn_sig_by_hir_id(hir_id).map_or(SafetyContext::Safe, |fn_sig| { - if fn_sig.header.safety.is_unsafe() { SafetyContext::UnsafeFn } else { SafetyContext::Safe } + match fn_sig.header.safety { + // We typeck the body as safe, but otherwise treat it as unsafe everywhere else. + // Call sites to other SafeTargetFeatures functions are checked explicitly and don't need + // to care about safety of the body. + hir::HeaderSafety::SafeTargetFeatures => SafetyContext::Safe, + hir::HeaderSafety::Normal(safety) => match safety { + hir::Safety::Unsafe => SafetyContext::UnsafeFn, + hir::Safety::Safe => SafetyContext::Safe, + }, + } }); let body_target_features = &tcx.body_codegen_attrs(def.to_def_id()).target_features; let mut warnings = Vec::new(); diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index a9111a5ef9b..8d397f63cc7 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -1,117 +1,181 @@ use std::cmp::Ordering; -use std::fmt::{self, Debug}; +use either::Either; +use itertools::Itertools; use rustc_data_structures::captures::Captures; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_data_structures::graph::DirectedGraph; use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op}; -use tracing::{debug, debug_span, instrument}; -use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, ReadyFirstTraversal}; +use crate::coverage::counters::balanced_flow::BalancedFlowGraph; +use crate::coverage::counters::iter_nodes::IterNodes; +use crate::coverage::counters::node_flow::{CounterTerm, MergedNodeFlowGraph, NodeCounters}; +use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph}; -#[cfg(test)] -mod tests; +mod balanced_flow; +mod iter_nodes; +mod node_flow; +mod union_find; -/// The coverage counter or counter expression associated with a particular -/// BCB node or BCB edge. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -enum BcbCounter { - Counter { id: CounterId }, - Expression { id: ExpressionId }, +/// Ensures that each BCB node needing a counter has one, by creating physical +/// counters or counter expressions for nodes as required. +pub(super) fn make_bcb_counters( + graph: &CoverageGraph, + bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>, +) -> CoverageCounters { + // Create the derived graphs that are necessary for subsequent steps. + let balanced_graph = BalancedFlowGraph::for_graph(graph, |n| !graph[n].is_out_summable); + let merged_graph = MergedNodeFlowGraph::for_balanced_graph(&balanced_graph); + + // Use those graphs to determine which nodes get physical counters, and how + // to compute the execution counts of other nodes from those counters. + let nodes = make_node_counter_priority_list(graph, balanced_graph); + let node_counters = merged_graph.make_node_counters(&nodes); + + // Convert the counters into a form suitable for embedding into MIR. + transcribe_counters(&node_counters, bcb_needs_counter) } -impl BcbCounter { - fn as_term(&self) -> CovTerm { - match *self { - BcbCounter::Counter { id, .. } => CovTerm::Counter(id), - BcbCounter::Expression { id, .. } => CovTerm::Expression(id), - } - } +/// Arranges the nodes in `balanced_graph` into a list, such that earlier nodes +/// take priority in being given a counter expression instead of a physical counter. +fn make_node_counter_priority_list( + graph: &CoverageGraph, + balanced_graph: BalancedFlowGraph<&CoverageGraph>, +) -> Vec<BasicCoverageBlock> { + // A "reloop" node has exactly one out-edge, which jumps back to the top + // of an enclosing loop. Reloop nodes are typically visited more times + // than loop-exit nodes, so try to avoid giving them physical counters. + let is_reloop_node = IndexVec::from_fn_n( + |node| match graph.successors[node].as_slice() { + &[succ] => graph.dominates(succ, node), + _ => false, + }, + graph.num_nodes(), + ); + + let mut nodes = balanced_graph.iter_nodes().rev().collect::<Vec<_>>(); + // The first node is the sink, which must not get a physical counter. + assert_eq!(nodes[0], balanced_graph.sink); + // Sort the real nodes, such that earlier (lesser) nodes take priority + // in being given a counter expression instead of a physical counter. + nodes[1..].sort_by(|&a, &b| { + // Start with a dummy `Equal` to make the actual tests line up nicely. + Ordering::Equal + // Prefer a physical counter for return/yield nodes. + .then_with(|| Ord::cmp(&graph[a].is_out_summable, &graph[b].is_out_summable)) + // Prefer an expression for reloop nodes (see definition above). + .then_with(|| Ord::cmp(&is_reloop_node[a], &is_reloop_node[b]).reverse()) + // Otherwise, prefer a physical counter for dominating nodes. + .then_with(|| graph.cmp_in_dominator_order(a, b).reverse()) + }); + nodes } -impl Debug for BcbCounter { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Counter { id, .. } => write!(fmt, "Counter({:?})", id.index()), - Self::Expression { id } => write!(fmt, "Expression({:?})", id.index()), - } - } -} +// Converts node counters into a form suitable for embedding into MIR. +fn transcribe_counters( + old: &NodeCounters<BasicCoverageBlock>, + bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>, +) -> CoverageCounters { + let mut new = CoverageCounters::with_num_bcbs(bcb_needs_counter.domain_size()); + + for bcb in bcb_needs_counter.iter() { + // Our counter-creation algorithm doesn't guarantee that a counter + // expression starts or ends with a positive term, so partition the + // counters into "positive" and "negative" lists for easier handling. + let (mut pos, mut neg): (Vec<_>, Vec<_>) = + old.counter_expr(bcb).iter().partition_map(|&CounterTerm { node, op }| match op { + Op::Add => Either::Left(node), + Op::Subtract => Either::Right(node), + }); + + if pos.is_empty() { + // If we somehow end up with no positive terms, fall back to + // creating a physical counter. There's no known way for this + // to happen, but we can avoid an ICE if it does. + debug_assert!(false, "{bcb:?} has no positive counter terms"); + pos = vec![bcb]; + neg = vec![]; + } + + // These intermediate sorts are not strictly necessary, but were helpful + // in reducing churn when switching to the current counter-creation scheme. + // They also help to slightly decrease the overall size of the expression + // table, due to more subexpressions being shared. + pos.sort(); + neg.sort(); + + let mut new_counters_for_sites = |sites: Vec<BasicCoverageBlock>| { + sites.into_iter().map(|node| new.ensure_phys_counter(node)).collect::<Vec<_>>() + }; + let mut pos = new_counters_for_sites(pos); + let mut neg = new_counters_for_sites(neg); -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -struct BcbExpression { - lhs: BcbCounter, - op: Op, - rhs: BcbCounter, -} + // These sorts are also not strictly necessary; see above. + pos.sort(); + neg.sort(); + + let pos_counter = new.make_sum(&pos).expect("`pos` should not be empty"); + let new_counter = new.make_subtracted_sum(pos_counter, &neg); + new.set_node_counter(bcb, new_counter); + } -/// Enum representing either a node or an edge in the coverage graph. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(super) enum Site { - Node { bcb: BasicCoverageBlock }, - Edge { from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock }, + new } /// Generates and stores coverage counter and coverage expression information -/// associated with nodes/edges in the BCB graph. +/// associated with nodes in the coverage graph. pub(super) struct CoverageCounters { /// List of places where a counter-increment statement should be injected /// into MIR, each with its corresponding counter ID. - counter_increment_sites: IndexVec<CounterId, Site>, + phys_counter_for_node: FxIndexMap<BasicCoverageBlock, CounterId>, + next_counter_id: CounterId, /// Coverage counters/expressions that are associated with individual BCBs. - node_counters: IndexVec<BasicCoverageBlock, Option<BcbCounter>>, + node_counters: IndexVec<BasicCoverageBlock, Option<CovTerm>>, /// Table of expression data, associating each expression ID with its /// corresponding operator (+ or -) and its LHS/RHS operands. - expressions: IndexVec<ExpressionId, BcbExpression>, + expressions: IndexVec<ExpressionId, Expression>, /// Remember expressions that have already been created (or simplified), /// so that we don't create unnecessary duplicates. - expressions_memo: FxHashMap<BcbExpression, BcbCounter>, + expressions_memo: FxHashMap<Expression, CovTerm>, } impl CoverageCounters { - /// Ensures that each BCB node needing a counter has one, by creating physical - /// counters or counter expressions for nodes and edges as required. - pub(super) fn make_bcb_counters( - graph: &CoverageGraph, - bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>, - ) -> Self { - let mut builder = CountersBuilder::new(graph, bcb_needs_counter); - builder.make_bcb_counters(); - - builder.into_coverage_counters() - } - fn with_num_bcbs(num_bcbs: usize) -> Self { Self { - counter_increment_sites: IndexVec::new(), + phys_counter_for_node: FxIndexMap::default(), + next_counter_id: CounterId::ZERO, node_counters: IndexVec::from_elem_n(None, num_bcbs), expressions: IndexVec::new(), expressions_memo: FxHashMap::default(), } } - /// Creates a new physical counter for a BCB node or edge. - fn make_phys_counter(&mut self, site: Site) -> BcbCounter { - let id = self.counter_increment_sites.push(site); - BcbCounter::Counter { id } + /// Returns the physical counter for the given node, creating it if necessary. + fn ensure_phys_counter(&mut self, bcb: BasicCoverageBlock) -> CovTerm { + let id = *self.phys_counter_for_node.entry(bcb).or_insert_with(|| { + let id = self.next_counter_id; + self.next_counter_id = id + 1; + id + }); + CovTerm::Counter(id) } - fn make_expression(&mut self, lhs: BcbCounter, op: Op, rhs: BcbCounter) -> BcbCounter { - let new_expr = BcbExpression { lhs, op, rhs }; - *self.expressions_memo.entry(new_expr).or_insert_with(|| { + fn make_expression(&mut self, lhs: CovTerm, op: Op, rhs: CovTerm) -> CovTerm { + let new_expr = Expression { lhs, op, rhs }; + *self.expressions_memo.entry(new_expr.clone()).or_insert_with(|| { let id = self.expressions.push(new_expr); - BcbCounter::Expression { id } + CovTerm::Expression(id) }) } /// Creates a counter that is the sum of the given counters. /// /// Returns `None` if the given list of counters was empty. - fn make_sum(&mut self, counters: &[BcbCounter]) -> Option<BcbCounter> { + fn make_sum(&mut self, counters: &[CovTerm]) -> Option<CovTerm> { counters .iter() .copied() @@ -119,16 +183,18 @@ impl CoverageCounters { } /// Creates a counter whose value is `lhs - SUM(rhs)`. - fn make_subtracted_sum(&mut self, lhs: BcbCounter, rhs: &[BcbCounter]) -> BcbCounter { + fn make_subtracted_sum(&mut self, lhs: CovTerm, rhs: &[CovTerm]) -> CovTerm { let Some(rhs_sum) = self.make_sum(rhs) else { return lhs }; self.make_expression(lhs, Op::Subtract, rhs_sum) } pub(super) fn num_counters(&self) -> usize { - self.counter_increment_sites.len() + let num_counters = self.phys_counter_for_node.len(); + assert_eq!(num_counters, self.next_counter_id.as_usize()); + num_counters } - fn set_node_counter(&mut self, bcb: BasicCoverageBlock, counter: BcbCounter) -> BcbCounter { + fn set_node_counter(&mut self, bcb: BasicCoverageBlock, counter: CovTerm) -> CovTerm { let existing = self.node_counters[bcb].replace(counter); assert!( existing.is_none(), @@ -138,16 +204,16 @@ impl CoverageCounters { } pub(super) fn term_for_bcb(&self, bcb: BasicCoverageBlock) -> Option<CovTerm> { - self.node_counters[bcb].map(|counter| counter.as_term()) + self.node_counters[bcb] } - /// Returns an iterator over all the nodes/edges in the coverage graph that + /// Returns an iterator over all the nodes in the coverage graph that /// should have a counter-increment statement injected into MIR, along with /// each site's corresponding counter ID. pub(super) fn counter_increment_sites( &self, - ) -> impl Iterator<Item = (CounterId, Site)> + Captures<'_> { - self.counter_increment_sites.iter_enumerated().map(|(id, &site)| (id, site)) + ) -> impl Iterator<Item = (CounterId, BasicCoverageBlock)> + Captures<'_> { + self.phys_counter_for_node.iter().map(|(&site, &id)| (id, site)) } /// Returns an iterator over the subset of BCB nodes that have been associated @@ -157,435 +223,13 @@ impl CoverageCounters { ) -> impl Iterator<Item = (BasicCoverageBlock, ExpressionId)> + Captures<'_> { self.node_counters.iter_enumerated().filter_map(|(bcb, &counter)| match counter { // Yield the BCB along with its associated expression ID. - Some(BcbCounter::Expression { id }) => Some((bcb, id)), + Some(CovTerm::Expression(id)) => Some((bcb, id)), // This BCB is associated with a counter or nothing, so skip it. - Some(BcbCounter::Counter { .. }) | None => None, + Some(CovTerm::Counter { .. } | CovTerm::Zero) | None => None, }) } pub(super) fn into_expressions(self) -> IndexVec<ExpressionId, Expression> { - let old_len = self.expressions.len(); - let expressions = self - .expressions - .into_iter() - .map(|BcbExpression { lhs, op, rhs }| Expression { - lhs: lhs.as_term(), - op, - rhs: rhs.as_term(), - }) - .collect::<IndexVec<ExpressionId, _>>(); - - // Expression IDs are indexes into this vector, so make sure we didn't - // accidentally invalidate them by changing its length. - assert_eq!(old_len, expressions.len()); - expressions - } -} - -/// Symbolic representation of the coverage counter to be used for a particular -/// node or edge in the coverage graph. The same site counter can be used for -/// multiple sites, if they have been determined to have the same count. -#[derive(Clone, Copy, Debug)] -enum SiteCounter { - /// A physical counter at some node/edge. - Phys { site: Site }, - /// A counter expression for a node that takes the sum of all its in-edge - /// counters. - NodeSumExpr { bcb: BasicCoverageBlock }, - /// A counter expression for an edge that takes the counter of its source - /// node, and subtracts the counters of all its sibling out-edges. - EdgeDiffExpr { from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock }, -} - -/// Yields the graph successors of `from_bcb` that aren't `to_bcb`. This is -/// used when creating a counter expression for [`SiteCounter::EdgeDiffExpr`]. -/// -/// For example, in this diagram the sibling out-edge targets of edge `AC` are -/// the nodes `B` and `D`. -/// -/// ```text -/// A -/// / | \ -/// B C D -/// ``` -fn sibling_out_edge_targets( - graph: &CoverageGraph, - from_bcb: BasicCoverageBlock, - to_bcb: BasicCoverageBlock, -) -> impl Iterator<Item = BasicCoverageBlock> + Captures<'_> { - graph.successors[from_bcb].iter().copied().filter(move |&t| t != to_bcb) -} - -/// Helper struct that allows counter creation to inspect the BCB graph, and -/// the set of nodes that need counters. -struct CountersBuilder<'a> { - graph: &'a CoverageGraph, - bcb_needs_counter: &'a DenseBitSet<BasicCoverageBlock>, - - site_counters: FxHashMap<Site, SiteCounter>, -} - -impl<'a> CountersBuilder<'a> { - fn new( - graph: &'a CoverageGraph, - bcb_needs_counter: &'a DenseBitSet<BasicCoverageBlock>, - ) -> Self { - assert_eq!(graph.num_nodes(), bcb_needs_counter.domain_size()); - Self { graph, bcb_needs_counter, site_counters: FxHashMap::default() } - } - - fn make_bcb_counters(&mut self) { - debug!("make_bcb_counters(): adding a counter or expression to each BasicCoverageBlock"); - - // Traverse the coverage graph, ensuring that every node that needs a - // coverage counter has one. - for bcb in ReadyFirstTraversal::new(self.graph) { - let _span = debug_span!("traversal", ?bcb).entered(); - if self.bcb_needs_counter.contains(bcb) { - self.make_node_counter_and_out_edge_counters(bcb); - } - } + self.expressions } - - /// Make sure the given node has a node counter, and then make sure each of - /// its out-edges has an edge counter (if appropriate). - #[instrument(level = "debug", skip(self))] - fn make_node_counter_and_out_edge_counters(&mut self, from_bcb: BasicCoverageBlock) { - // First, ensure that this node has a counter of some kind. - // We might also use that counter to compute one of the out-edge counters. - self.get_or_make_node_counter(from_bcb); - - // If this node's out-edges won't sum to the node's counter, - // then there's no reason to create edge counters here. - if !self.graph[from_bcb].is_out_summable { - return; - } - - // When choosing which out-edge should be given a counter expression, ignore edges that - // already have counters, or could use the existing counter of their target node. - let out_edge_has_counter = |to_bcb| { - if self.site_counters.contains_key(&Site::Edge { from_bcb, to_bcb }) { - return true; - } - self.graph.sole_predecessor(to_bcb) == Some(from_bcb) - && self.site_counters.contains_key(&Site::Node { bcb: to_bcb }) - }; - - // Determine the set of out-edges that could benefit from being given an expression. - let candidate_successors = self.graph.successors[from_bcb] - .iter() - .copied() - .filter(|&to_bcb| !out_edge_has_counter(to_bcb)) - .collect::<Vec<_>>(); - debug!(?candidate_successors); - - // If there are out-edges without counters, choose one to be given an expression - // (computed from this node and the other out-edges) instead of a physical counter. - let Some(to_bcb) = self.choose_out_edge_for_expression(from_bcb, &candidate_successors) - else { - return; - }; - - // For each out-edge other than the one that was chosen to get an expression, - // ensure that it has a counter (existing counter/expression or a new counter). - for target in sibling_out_edge_targets(self.graph, from_bcb, to_bcb) { - self.get_or_make_edge_counter(from_bcb, target); - } - - // Now create an expression for the chosen edge, by taking the counter - // for its source node and subtracting the sum of its sibling out-edges. - let counter = SiteCounter::EdgeDiffExpr { from_bcb, to_bcb }; - self.site_counters.insert(Site::Edge { from_bcb, to_bcb }, counter); - } - - #[instrument(level = "debug", skip(self))] - fn get_or_make_node_counter(&mut self, bcb: BasicCoverageBlock) -> SiteCounter { - // If the BCB already has a counter, return it. - if let Some(&counter) = self.site_counters.get(&Site::Node { bcb }) { - debug!("{bcb:?} already has a counter: {counter:?}"); - return counter; - } - - let counter = self.make_node_counter_inner(bcb); - self.site_counters.insert(Site::Node { bcb }, counter); - counter - } - - fn make_node_counter_inner(&mut self, bcb: BasicCoverageBlock) -> SiteCounter { - // If the node's sole in-edge already has a counter, use that. - if let Some(sole_pred) = self.graph.sole_predecessor(bcb) - && let Some(&edge_counter) = - self.site_counters.get(&Site::Edge { from_bcb: sole_pred, to_bcb: bcb }) - { - return edge_counter; - } - - let predecessors = self.graph.predecessors[bcb].as_slice(); - - // Handle cases where we can't compute a node's count from its in-edges: - // - START_BCB has no in-edges, so taking the sum would panic (or be wrong). - // - For nodes with one in-edge, or that directly loop to themselves, - // trying to get the in-edge counts would require this node's counter, - // leading to infinite recursion. - if predecessors.len() <= 1 || predecessors.contains(&bcb) { - debug!(?bcb, ?predecessors, "node has <=1 predecessors or is its own predecessor"); - let counter = SiteCounter::Phys { site: Site::Node { bcb } }; - debug!(?bcb, ?counter, "node gets a physical counter"); - return counter; - } - - // A BCB with multiple incoming edges can compute its count by ensuring that counters - // exist for each of those edges, and then adding them up to get a total count. - for &from_bcb in predecessors { - self.get_or_make_edge_counter(from_bcb, bcb); - } - let sum_of_in_edges = SiteCounter::NodeSumExpr { bcb }; - - debug!("{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}"); - sum_of_in_edges - } - - #[instrument(level = "debug", skip(self))] - fn get_or_make_edge_counter( - &mut self, - from_bcb: BasicCoverageBlock, - to_bcb: BasicCoverageBlock, - ) -> SiteCounter { - // If the edge already has a counter, return it. - if let Some(&counter) = self.site_counters.get(&Site::Edge { from_bcb, to_bcb }) { - debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter:?}"); - return counter; - } - - let counter = self.make_edge_counter_inner(from_bcb, to_bcb); - self.site_counters.insert(Site::Edge { from_bcb, to_bcb }, counter); - counter - } - - fn make_edge_counter_inner( - &mut self, - from_bcb: BasicCoverageBlock, - to_bcb: BasicCoverageBlock, - ) -> SiteCounter { - // If the target node has exactly one in-edge (i.e. this one), then just - // use the node's counter, since it will have the same value. - if let Some(sole_pred) = self.graph.sole_predecessor(to_bcb) { - assert_eq!(sole_pred, from_bcb); - // This call must take care not to invoke `get_or_make_edge` for - // this edge, since that would result in infinite recursion! - return self.get_or_make_node_counter(to_bcb); - } - - // If the source node has exactly one out-edge (i.e. this one) and would have - // the same execution count as that edge, then just use the node's counter. - if let Some(simple_succ) = self.graph.simple_successor(from_bcb) { - assert_eq!(simple_succ, to_bcb); - return self.get_or_make_node_counter(from_bcb); - } - - // Make a new counter to count this edge. - let counter = SiteCounter::Phys { site: Site::Edge { from_bcb, to_bcb } }; - debug!(?from_bcb, ?to_bcb, ?counter, "edge gets a physical counter"); - counter - } - - /// Given a set of candidate out-edges (represented by their successor node), - /// choose one to be given a counter expression instead of a physical counter. - fn choose_out_edge_for_expression( - &self, - from_bcb: BasicCoverageBlock, - candidate_successors: &[BasicCoverageBlock], - ) -> Option<BasicCoverageBlock> { - // Try to find a candidate that leads back to the top of a loop, - // because reloop edges tend to be executed more times than loop-exit edges. - if let Some(reloop_target) = self.find_good_reloop_edge(from_bcb, &candidate_successors) { - debug!("Selecting reloop target {reloop_target:?} to get an expression"); - return Some(reloop_target); - } - - // We couldn't identify a "good" edge, so just choose an arbitrary one. - let arbitrary_target = candidate_successors.first().copied()?; - debug!(?arbitrary_target, "selecting arbitrary out-edge to get an expression"); - Some(arbitrary_target) - } - - /// Given a set of candidate out-edges (represented by their successor node), - /// tries to find one that leads back to the top of a loop. - /// - /// Reloop edges are good candidates for counter expressions, because they - /// will tend to be executed more times than a loop-exit edge, so it's nice - /// for them to be able to avoid a physical counter increment. - fn find_good_reloop_edge( - &self, - from_bcb: BasicCoverageBlock, - candidate_successors: &[BasicCoverageBlock], - ) -> Option<BasicCoverageBlock> { - // If there are no candidates, avoid iterating over the loop stack. - if candidate_successors.is_empty() { - return None; - } - - // Consider each loop on the current traversal context stack, top-down. - for loop_header_node in self.graph.loop_headers_containing(from_bcb) { - // Try to find a candidate edge that doesn't exit this loop. - for &target_bcb in candidate_successors { - // An edge is a reloop edge if its target dominates any BCB that has - // an edge back to the loop header. (Otherwise it's an exit edge.) - let is_reloop_edge = self - .graph - .reloop_predecessors(loop_header_node) - .any(|reloop_bcb| self.graph.dominates(target_bcb, reloop_bcb)); - if is_reloop_edge { - // We found a good out-edge to be given an expression. - return Some(target_bcb); - } - } - - // All of the candidate edges exit this loop, so keep looking - // for a good reloop edge for one of the outer loops. - } - - None - } - - fn into_coverage_counters(self) -> CoverageCounters { - Transcriber::new(&self).transcribe_counters() - } -} - -/// Helper struct for converting `CountersBuilder` into a final `CoverageCounters`. -struct Transcriber<'a> { - old: &'a CountersBuilder<'a>, - new: CoverageCounters, - phys_counter_for_site: FxHashMap<Site, BcbCounter>, -} - -impl<'a> Transcriber<'a> { - fn new(old: &'a CountersBuilder<'a>) -> Self { - Self { - old, - new: CoverageCounters::with_num_bcbs(old.graph.num_nodes()), - phys_counter_for_site: FxHashMap::default(), - } - } - - fn transcribe_counters(mut self) -> CoverageCounters { - for bcb in self.old.bcb_needs_counter.iter() { - let site = Site::Node { bcb }; - let site_counter = self.site_counter(site); - - // Resolve the site counter into flat lists of nodes/edges whose - // physical counts contribute to the counter for this node. - // Distinguish between counts that will be added vs subtracted. - let mut pos = vec![]; - let mut neg = vec![]; - self.push_resolved_sites(site_counter, &mut pos, &mut neg); - - // Simplify by cancelling out sites that appear on both sides. - let (mut pos, mut neg) = sort_and_cancel(pos, neg); - - if pos.is_empty() { - // If we somehow end up with no positive terms after cancellation, - // fall back to creating a physical counter. There's no known way - // for this to happen, but it's hard to confidently rule it out. - debug_assert!(false, "{site:?} has no positive counter terms"); - pos = vec![Some(site)]; - neg = vec![]; - } - - let mut new_counters_for_sites = |sites: Vec<Option<Site>>| { - sites - .into_iter() - .filter_map(|id| try { self.ensure_phys_counter(id?) }) - .collect::<Vec<_>>() - }; - let mut pos = new_counters_for_sites(pos); - let mut neg = new_counters_for_sites(neg); - - pos.sort(); - neg.sort(); - - let pos_counter = self.new.make_sum(&pos).expect("`pos` should not be empty"); - let new_counter = self.new.make_subtracted_sum(pos_counter, &neg); - self.new.set_node_counter(bcb, new_counter); - } - - self.new - } - - fn site_counter(&self, site: Site) -> SiteCounter { - self.old.site_counters.get(&site).copied().unwrap_or_else(|| { - // We should have already created all necessary site counters. - // But if we somehow didn't, avoid crashing in release builds, - // and just use an extra physical counter instead. - debug_assert!(false, "{site:?} should have a counter"); - SiteCounter::Phys { site } - }) - } - - fn ensure_phys_counter(&mut self, site: Site) -> BcbCounter { - *self.phys_counter_for_site.entry(site).or_insert_with(|| self.new.make_phys_counter(site)) - } - - /// Resolves the given counter into flat lists of nodes/edges, whose counters - /// will then be added and subtracted to form a counter expression. - fn push_resolved_sites(&self, counter: SiteCounter, pos: &mut Vec<Site>, neg: &mut Vec<Site>) { - match counter { - SiteCounter::Phys { site } => pos.push(site), - SiteCounter::NodeSumExpr { bcb } => { - for &from_bcb in &self.old.graph.predecessors[bcb] { - let edge_counter = self.site_counter(Site::Edge { from_bcb, to_bcb: bcb }); - self.push_resolved_sites(edge_counter, pos, neg); - } - } - SiteCounter::EdgeDiffExpr { from_bcb, to_bcb } => { - // First, add the count for `from_bcb`. - let node_counter = self.site_counter(Site::Node { bcb: from_bcb }); - self.push_resolved_sites(node_counter, pos, neg); - - // Then subtract the counts for the other out-edges. - for target in sibling_out_edge_targets(self.old.graph, from_bcb, to_bcb) { - let edge_counter = self.site_counter(Site::Edge { from_bcb, to_bcb: target }); - // Swap `neg` and `pos` so that the counter is subtracted. - self.push_resolved_sites(edge_counter, neg, pos); - } - } - } - } -} - -/// Given two lists: -/// - Sorts each list. -/// - Converts each list to `Vec<Option<T>>`. -/// - Scans for values that appear in both lists, and cancels them out by -/// replacing matching pairs of values with `None`. -fn sort_and_cancel<T: Ord>(mut pos: Vec<T>, mut neg: Vec<T>) -> (Vec<Option<T>>, Vec<Option<T>>) { - pos.sort(); - neg.sort(); - - // Convert to `Vec<Option<T>>`. If `T` has a niche, this should be zero-cost. - let mut pos = pos.into_iter().map(Some).collect::<Vec<_>>(); - let mut neg = neg.into_iter().map(Some).collect::<Vec<_>>(); - - // Scan through the lists using two cursors. When either cursor reaches the - // end of its list, there can be no more equal pairs, so stop. - let mut p = 0; - let mut n = 0; - while p < pos.len() && n < neg.len() { - // If the values are equal, remove them and advance both cursors. - // Otherwise, advance whichever cursor points to the lesser value. - // (Choosing which cursor to advance relies on both lists being sorted.) - match pos[p].cmp(&neg[n]) { - Ordering::Less => p += 1, - Ordering::Equal => { - pos[p] = None; - neg[n] = None; - p += 1; - n += 1; - } - Ordering::Greater => n += 1, - } - } - - (pos, neg) } diff --git a/compiler/rustc_mir_transform/src/coverage/counters/balanced_flow.rs b/compiler/rustc_mir_transform/src/coverage/counters/balanced_flow.rs new file mode 100644 index 00000000000..c108f96a564 --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/balanced_flow.rs @@ -0,0 +1,133 @@ +//! A control-flow graph can be said to have “balanced flow” if the flow +//! (execution count) of each node is equal to the sum of its in-edge flows, +//! and also equal to the sum of its out-edge flows. +//! +//! Control-flow graphs typically have one or more nodes that don't satisfy the +//! balanced-flow property, e.g.: +//! - The start node has out-edges, but no in-edges. +//! - Return nodes have in-edges, but no out-edges. +//! - `Yield` nodes can have an out-flow that is less than their in-flow. +//! - Inescapable loops cause the in-flow/out-flow relationship to break down. +//! +//! Balanced-flow graphs are nevertheless useful for analysis, so this module +//! provides a wrapper type ([`BalancedFlowGraph`]) that imposes balanced flow +//! on an underlying graph. This is done by non-destructively adding synthetic +//! nodes and edges as necessary. + +use rustc_data_structures::graph; +use rustc_data_structures::graph::iterate::DepthFirstSearch; +use rustc_data_structures::graph::reversed::ReversedGraph; +use rustc_index::Idx; +use rustc_index::bit_set::DenseBitSet; + +use crate::coverage::counters::iter_nodes::IterNodes; + +/// A view of an underlying graph that has been augmented to have “balanced flow”. +/// This means that the flow (execution count) of each node is equal to the +/// sum of its in-edge flows, and also equal to the sum of its out-edge flows. +/// +/// To achieve this, a synthetic "sink" node is non-destructively added to the +/// graph, with synthetic in-edges from these nodes: +/// - Any node that has no out-edges. +/// - Any node that explicitly requires a sink edge, as indicated by a +/// caller-supplied `force_sink_edge` function. +/// - Any node that would otherwise be unable to reach the sink, because it is +/// part of an inescapable loop. +/// +/// To make the graph fully balanced, there is also a synthetic edge from the +/// sink node back to the start node. +/// +/// --- +/// The benefit of having a balanced-flow graph is that it can be subsequently +/// transformed in ways that are guaranteed to preserve balanced flow +/// (e.g. merging nodes together), which is useful for discovering relationships +/// between the node flows of different nodes in the graph. +pub(crate) struct BalancedFlowGraph<G: graph::DirectedGraph> { + graph: G, + sink_edge_nodes: DenseBitSet<G::Node>, + pub(crate) sink: G::Node, +} + +impl<G: graph::DirectedGraph> BalancedFlowGraph<G> { + /// Creates a balanced view of an underlying graph, by adding a synthetic + /// sink node that has in-edges from nodes that need or request such an edge, + /// and a single out-edge to the start node. + /// + /// Assumes that all nodes in the underlying graph are reachable from the + /// start node. + pub(crate) fn for_graph(graph: G, force_sink_edge: impl Fn(G::Node) -> bool) -> Self + where + G: graph::ControlFlowGraph, + { + let mut sink_edge_nodes = DenseBitSet::new_empty(graph.num_nodes()); + let mut dfs = DepthFirstSearch::new(ReversedGraph::new(&graph)); + + // First, determine the set of nodes that explicitly request or require + // an out-edge to the sink. + for node in graph.iter_nodes() { + if force_sink_edge(node) || graph.successors(node).next().is_none() { + sink_edge_nodes.insert(node); + dfs.push_start_node(node); + } + } + + // Next, find all nodes that are currently not reverse-reachable from + // `sink_edge_nodes`, and add them to the set as well. + dfs.complete_search(); + sink_edge_nodes.union_not(dfs.visited_set()); + + // The sink node is 1 higher than the highest real node. + let sink = G::Node::new(graph.num_nodes()); + + BalancedFlowGraph { graph, sink_edge_nodes, sink } + } +} + +impl<G> graph::DirectedGraph for BalancedFlowGraph<G> +where + G: graph::DirectedGraph, +{ + type Node = G::Node; + + /// Returns the number of nodes in this balanced-flow graph, which is 1 + /// more than the number of nodes in the underlying graph, to account for + /// the synthetic sink node. + fn num_nodes(&self) -> usize { + // The sink node's index is already the size of the underlying graph, + // so just add 1 to that instead. + self.sink.index() + 1 + } +} + +impl<G> graph::StartNode for BalancedFlowGraph<G> +where + G: graph::StartNode, +{ + fn start_node(&self) -> Self::Node { + self.graph.start_node() + } +} + +impl<G> graph::Successors for BalancedFlowGraph<G> +where + G: graph::StartNode + graph::Successors, +{ + fn successors(&self, node: Self::Node) -> impl Iterator<Item = Self::Node> { + let real_edges; + let sink_edge; + + if node == self.sink { + // The sink node has no real out-edges, and one synthetic out-edge + // to the start node. + real_edges = None; + sink_edge = Some(self.graph.start_node()); + } else { + // Real nodes have their real out-edges, and possibly one synthetic + // out-edge to the sink node. + real_edges = Some(self.graph.successors(node)); + sink_edge = self.sink_edge_nodes.contains(node).then_some(self.sink); + } + + real_edges.into_iter().flatten().chain(sink_edge) + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/counters/iter_nodes.rs b/compiler/rustc_mir_transform/src/coverage/counters/iter_nodes.rs new file mode 100644 index 00000000000..9d87f7af1b0 --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/iter_nodes.rs @@ -0,0 +1,16 @@ +use rustc_data_structures::graph; +use rustc_index::Idx; + +pub(crate) trait IterNodes: graph::DirectedGraph { + /// Iterates over all nodes of a graph in ascending numeric order. + /// Assumes that nodes are densely numbered, i.e. every index in + /// `0..num_nodes` is a valid node. + /// + /// FIXME: Can this just be part of [`graph::DirectedGraph`]? + fn iter_nodes( + &self, + ) -> impl Iterator<Item = Self::Node> + DoubleEndedIterator + ExactSizeIterator { + (0..self.num_nodes()).map(<Self::Node as Idx>::new) + } +} +impl<G: graph::DirectedGraph> IterNodes for G {} diff --git a/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs b/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs new file mode 100644 index 00000000000..610498c6c0e --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs @@ -0,0 +1,306 @@ +//! For each node in a control-flow graph, determines whether that node should +//! have a physical counter, or a counter expression that is derived from the +//! physical counters of other nodes. +//! +//! Based on the algorithm given in +//! "Optimal measurement points for program frequency counts" +//! (Knuth & Stevenson, 1973). + +use rustc_data_structures::graph; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::{Idx, IndexVec}; +use rustc_middle::mir::coverage::Op; +use smallvec::SmallVec; + +use crate::coverage::counters::iter_nodes::IterNodes; +use crate::coverage::counters::union_find::{FrozenUnionFind, UnionFind}; + +#[cfg(test)] +mod tests; + +/// View of some underlying graph, in which each node's successors have been +/// merged into a single "supernode". +/// +/// The resulting supernodes have no obvious meaning on their own. +/// However, merging successor nodes means that a node's out-edges can all +/// be combined into a single out-edge, whose flow is the same as the flow +/// (execution count) of its corresponding node in the original graph. +/// +/// With all node flows now in the original graph now represented as edge flows +/// in the merged graph, it becomes possible to analyze the original node flows +/// using techniques for analyzing edge flows. +#[derive(Debug)] +pub(crate) struct MergedNodeFlowGraph<Node: Idx> { + /// Maps each node to the supernode that contains it, indicated by some + /// arbitrary "root" node that is part of that supernode. + supernodes: FrozenUnionFind<Node>, + /// For each node, stores the single supernode that all of its successors + /// have been merged into. + /// + /// (Note that each node in a supernode can potentially have a _different_ + /// successor supernode from its peers.) + succ_supernodes: IndexVec<Node, Node>, +} + +impl<Node: Idx> MergedNodeFlowGraph<Node> { + /// Creates a "merged" view of an underlying graph. + /// + /// The given graph is assumed to have [“balanced flow”](balanced-flow), + /// though it does not necessarily have to be a `BalancedFlowGraph`. + /// + /// [balanced-flow]: `crate::coverage::counters::balanced_flow::BalancedFlowGraph`. + pub(crate) fn for_balanced_graph<G>(graph: G) -> Self + where + G: graph::DirectedGraph<Node = Node> + graph::Successors, + { + let mut supernodes = UnionFind::<G::Node>::new(graph.num_nodes()); + + // For each node, merge its successors into a single supernode, and + // arbitrarily choose one of those successors to represent all of them. + let successors = graph + .iter_nodes() + .map(|node| { + graph + .successors(node) + .reduce(|a, b| supernodes.unify(a, b)) + .expect("each node in a balanced graph must have at least one out-edge") + }) + .collect::<IndexVec<G::Node, G::Node>>(); + + // Now that unification is complete, freeze the supernode forest, + // and resolve each arbitrarily-chosen successor to its canonical root. + // (This avoids having to explicitly resolve them later.) + let supernodes = supernodes.freeze(); + let succ_supernodes = successors.into_iter().map(|succ| supernodes.find(succ)).collect(); + + Self { supernodes, succ_supernodes } + } + + fn num_nodes(&self) -> usize { + self.succ_supernodes.len() + } + + fn is_supernode(&self, node: Node) -> bool { + self.supernodes.find(node) == node + } + + /// Using the information in this merged graph, together with a given + /// permutation of all nodes in the graph, to create physical counters and + /// counter expressions for each node in the underlying graph. + /// + /// The given list must contain exactly one copy of each node in the + /// underlying balanced-flow graph. The order of nodes is used as a hint to + /// influence counter allocation: + /// - Earlier nodes are more likely to receive counter expressions. + /// - Later nodes are more likely to receive physical counters. + pub(crate) fn make_node_counters(&self, all_nodes_permutation: &[Node]) -> NodeCounters<Node> { + let mut builder = SpantreeBuilder::new(self); + + for &node in all_nodes_permutation { + builder.visit_node(node); + } + + NodeCounters { counter_exprs: builder.finish() } + } +} + +/// End result of allocating physical counters and counter expressions for the +/// nodes of a graph. +#[derive(Debug)] +pub(crate) struct NodeCounters<Node: Idx> { + counter_exprs: IndexVec<Node, CounterExprVec<Node>>, +} + +impl<Node: Idx> NodeCounters<Node> { + /// For the given node, returns the finished list of terms that represent + /// its physical counter or counter expression. Always non-empty. + /// + /// If a node was given a physical counter, its "expression" will contain + /// that counter as its sole element. + pub(crate) fn counter_expr(&self, this: Node) -> &[CounterTerm<Node>] { + self.counter_exprs[this].as_slice() + } +} + +#[derive(Debug)] +struct SpantreeEdge<Node> { + /// If true, this edge in the spantree has been reversed an odd number of + /// times, so all physical counters added to its node's counter expression + /// need to be negated. + is_reversed: bool, + /// Each spantree edge is "claimed" by the (regular) node that caused it to + /// be created. When a node with a physical counter traverses this edge, + /// that counter is added to the claiming node's counter expression. + claiming_node: Node, + /// Supernode at the other end of this spantree edge. Transitively points + /// to the "root" of this supernode's spantree component. + span_parent: Node, +} + +/// Part of a node's counter expression, which is a sum of counter terms. +#[derive(Debug)] +pub(crate) struct CounterTerm<Node> { + /// Whether to add or subtract the value of the node's physical counter. + pub(crate) op: Op, + /// The node whose physical counter is represented by this term. + pub(crate) node: Node, +} + +/// Stores the list of counter terms that make up a node's counter expression. +type CounterExprVec<Node> = SmallVec<[CounterTerm<Node>; 2]>; + +#[derive(Debug)] +struct SpantreeBuilder<'a, Node: Idx> { + graph: &'a MergedNodeFlowGraph<Node>, + is_unvisited: DenseBitSet<Node>, + /// Links supernodes to each other, gradually forming a spanning tree of + /// the merged-flow graph. + /// + /// A supernode without a span edge is the root of its component of the + /// spantree. Nodes that aren't supernodes cannot have a spantree edge. + span_edges: IndexVec<Node, Option<SpantreeEdge<Node>>>, + /// Shared path buffer recycled by all calls to `yank_to_spantree_root`. + yank_buffer: Vec<Node>, + /// An in-progress counter expression for each node. Each expression is + /// initially empty, and will be filled in as relevant nodes are visited. + counter_exprs: IndexVec<Node, CounterExprVec<Node>>, +} + +impl<'a, Node: Idx> SpantreeBuilder<'a, Node> { + fn new(graph: &'a MergedNodeFlowGraph<Node>) -> Self { + let num_nodes = graph.num_nodes(); + Self { + graph, + is_unvisited: DenseBitSet::new_filled(num_nodes), + span_edges: IndexVec::from_fn_n(|_| None, num_nodes), + yank_buffer: vec![], + counter_exprs: IndexVec::from_fn_n(|_| SmallVec::new(), num_nodes), + } + } + + /// Given a supernode, finds the supernode that is the "root" of its + /// spantree component. Two nodes that have the same spantree root are + /// connected in the spantree. + fn spantree_root(&self, this: Node) -> Node { + debug_assert!(self.graph.is_supernode(this)); + + match self.span_edges[this] { + None => this, + Some(SpantreeEdge { span_parent, .. }) => self.spantree_root(span_parent), + } + } + + /// Rotates edges in the spantree so that `this` is the root of its + /// spantree component. + fn yank_to_spantree_root(&mut self, this: Node) { + debug_assert!(self.graph.is_supernode(this)); + + // The rotation is done iteratively, by first traversing from `this` to + // its root and storing the path in a buffer, and then traversing the + // path buffer backwards to reverse all the edges. + + // Recycle the same path buffer for all calls to this method. + let path_buf = &mut self.yank_buffer; + path_buf.clear(); + path_buf.push(this); + + // Traverse the spantree until we reach a supernode that has no + // span-parent, which must be the root. + let mut curr = this; + while let &Some(SpantreeEdge { span_parent, .. }) = &self.span_edges[curr] { + path_buf.push(span_parent); + curr = span_parent; + } + + // For each spantree edge `a -> b` in the path that was just traversed, + // reverse it to become `a <- b`, while preserving `claiming_node`. + for &[a, b] in path_buf.array_windows::<2>().rev() { + let SpantreeEdge { is_reversed, claiming_node, span_parent } = self.span_edges[a] + .take() + .expect("all nodes in the path (except the last) have a `span_parent`"); + debug_assert_eq!(span_parent, b); + debug_assert!(self.span_edges[b].is_none()); + self.span_edges[b] = + Some(SpantreeEdge { is_reversed: !is_reversed, claiming_node, span_parent: a }); + } + + // The result of the rotation is that `this` is now a spantree root. + debug_assert!(self.span_edges[this].is_none()); + } + + /// Must be called exactly once for each node in the balanced-flow graph. + fn visit_node(&mut self, this: Node) { + // Assert that this node was unvisited, and mark it visited. + assert!(self.is_unvisited.remove(this), "node has already been visited: {this:?}"); + + // Get the supernode containing `this`, and make it the root of its + // component of the spantree. + let this_supernode = self.graph.supernodes.find(this); + self.yank_to_spantree_root(this_supernode); + + // Get the supernode containing all of this's successors. + let succ_supernode = self.graph.succ_supernodes[this]; + debug_assert!(self.graph.is_supernode(succ_supernode)); + + // If two supernodes are already connected in the spantree, they will + // have the same spantree root. (Each supernode is connected to itself.) + if this_supernode != self.spantree_root(succ_supernode) { + // Adding this node's flow edge to the spantree would cause two + // previously-disconnected supernodes to become connected, so add + // it. That spantree-edge is now "claimed" by this node. + // + // Claiming a spantree-edge means that this node will get a counter + // expression instead of a physical counter. That expression is + // currently empty, but will be built incrementally as the other + // nodes are visited. + self.span_edges[this_supernode] = Some(SpantreeEdge { + is_reversed: false, + claiming_node: this, + span_parent: succ_supernode, + }); + } else { + // This node's flow edge would join two supernodes that are already + // connected in the spantree (or are the same supernode). That would + // create a cycle in the spantree, so don't add an edge. + // + // Instead, create a physical counter for this node, and add that + // counter to all expressions on the path from `succ_supernode` to + // `this_supernode`. + + // Instead of setting `this.measure = true` as in the original paper, + // we just add the node's ID to its own "expression". + self.counter_exprs[this].push(CounterTerm { node: this, op: Op::Add }); + + // Walk the spantree from `this.successor` back to `this`. For each + // spantree edge along the way, add this node's physical counter to + // the counter expression of the node that claimed the spantree edge. + let mut curr = succ_supernode; + while curr != this_supernode { + let &SpantreeEdge { is_reversed, claiming_node, span_parent } = + self.span_edges[curr].as_ref().unwrap(); + let op = if is_reversed { Op::Subtract } else { Op::Add }; + self.counter_exprs[claiming_node].push(CounterTerm { node: this, op }); + + curr = span_parent; + } + } + } + + /// Asserts that all nodes have been visited, and returns the computed + /// counter expressions (made up of physical counters) for each node. + fn finish(self) -> IndexVec<Node, CounterExprVec<Node>> { + let Self { graph, is_unvisited, span_edges, yank_buffer: _, counter_exprs } = self; + assert!(is_unvisited.is_empty(), "some nodes were never visited: {is_unvisited:?}"); + debug_assert!( + span_edges + .iter_enumerated() + .all(|(node, span_edge)| { span_edge.is_some() <= graph.is_supernode(node) }), + "only supernodes can have a span edge", + ); + debug_assert!( + counter_exprs.iter().all(|expr| !expr.is_empty()), + "after visiting all nodes, every node should have a non-empty expression", + ); + counter_exprs + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/counters/node_flow/tests.rs b/compiler/rustc_mir_transform/src/coverage/counters/node_flow/tests.rs new file mode 100644 index 00000000000..9e7f754523d --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/node_flow/tests.rs @@ -0,0 +1,64 @@ +use itertools::Itertools; +use rustc_data_structures::graph; +use rustc_data_structures::graph::vec_graph::VecGraph; +use rustc_index::Idx; +use rustc_middle::mir::coverage::Op; + +use super::{CounterTerm, MergedNodeFlowGraph, NodeCounters}; + +fn merged_node_flow_graph<G: graph::Successors>(graph: G) -> MergedNodeFlowGraph<G::Node> { + MergedNodeFlowGraph::for_balanced_graph(graph) +} + +fn make_graph<Node: Idx + Ord>(num_nodes: usize, edge_pairs: Vec<(Node, Node)>) -> VecGraph<Node> { + VecGraph::new(num_nodes, edge_pairs) +} + +/// Example used in "Optimal Measurement Points for Program Frequency Counts" +/// (Knuth & Stevenson, 1973), but with 0-based node IDs. +#[test] +fn example_driver() { + let graph = make_graph::<u32>(5, vec![ + (0, 1), + (0, 3), + (1, 0), + (1, 2), + (2, 1), + (2, 4), + (3, 3), + (3, 4), + (4, 0), + ]); + + let merged = merged_node_flow_graph(&graph); + let counters = merged.make_node_counters(&[3, 1, 2, 0, 4]); + + assert_eq!(format_counter_expressions(&counters), &[ + // (comment to force vertical formatting for clarity) + "[0]: +c0", + "[1]: +c0 +c2 -c4", + "[2]: +c2", + "[3]: +c3", + "[4]: +c4", + ]); +} + +fn format_counter_expressions<Node: Idx>(counters: &NodeCounters<Node>) -> Vec<String> { + let format_item = |&CounterTerm { node, op }| { + let op = match op { + Op::Subtract => '-', + Op::Add => '+', + }; + format!("{op}c{node:?}") + }; + + counters + .counter_exprs + .indices() + .map(|node| { + let mut expr = counters.counter_expr(node).iter().collect::<Vec<_>>(); + expr.sort_by_key(|item| item.node.index()); + format!("[{node:?}]: {}", expr.into_iter().map(format_item).join(" ")) + }) + .collect() +} diff --git a/compiler/rustc_mir_transform/src/coverage/counters/tests.rs b/compiler/rustc_mir_transform/src/coverage/counters/tests.rs deleted file mode 100644 index 794d4358f82..00000000000 --- a/compiler/rustc_mir_transform/src/coverage/counters/tests.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::fmt::Debug; - -use super::sort_and_cancel; - -fn flatten<T>(input: Vec<Option<T>>) -> Vec<T> { - input.into_iter().flatten().collect() -} - -fn sort_and_cancel_and_flatten<T: Clone + Ord>(pos: Vec<T>, neg: Vec<T>) -> (Vec<T>, Vec<T>) { - let (pos_actual, neg_actual) = sort_and_cancel(pos, neg); - (flatten(pos_actual), flatten(neg_actual)) -} - -#[track_caller] -fn check_test_case<T: Clone + Debug + Ord>( - pos: Vec<T>, - neg: Vec<T>, - pos_expected: Vec<T>, - neg_expected: Vec<T>, -) { - eprintln!("pos = {pos:?}; neg = {neg:?}"); - let output = sort_and_cancel_and_flatten(pos, neg); - assert_eq!(output, (pos_expected, neg_expected)); -} - -#[test] -fn cancellation() { - let cases: &[(Vec<u32>, Vec<u32>, Vec<u32>, Vec<u32>)] = &[ - (vec![], vec![], vec![], vec![]), - (vec![4, 2, 1, 5, 3], vec![], vec![1, 2, 3, 4, 5], vec![]), - (vec![5, 5, 5, 5, 5], vec![5], vec![5, 5, 5, 5], vec![]), - (vec![1, 1, 2, 2, 3, 3], vec![1, 2, 3], vec![1, 2, 3], vec![]), - (vec![1, 1, 2, 2, 3, 3], vec![2, 4, 2], vec![1, 1, 3, 3], vec![4]), - ]; - - for (pos, neg, pos_expected, neg_expected) in cases { - check_test_case(pos.to_vec(), neg.to_vec(), pos_expected.to_vec(), neg_expected.to_vec()); - // Same test case, but with its inputs flipped and its outputs flipped. - check_test_case(neg.to_vec(), pos.to_vec(), neg_expected.to_vec(), pos_expected.to_vec()); - } -} diff --git a/compiler/rustc_mir_transform/src/coverage/counters/union_find.rs b/compiler/rustc_mir_transform/src/coverage/counters/union_find.rs new file mode 100644 index 00000000000..2da4f5f5fce --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/union_find.rs @@ -0,0 +1,116 @@ +use std::cmp::Ordering; +use std::mem; + +use rustc_index::{Idx, IndexVec}; + +#[cfg(test)] +mod tests; + +/// Simple implementation of a union-find data structure, i.e. a disjoint-set +/// forest. +#[derive(Debug)] +pub(crate) struct UnionFind<Key: Idx> { + table: IndexVec<Key, UnionFindEntry<Key>>, +} + +#[derive(Debug)] +struct UnionFindEntry<Key> { + /// Transitively points towards the "root" of the set containing this key. + /// + /// Invariant: A root key is its own parent. + parent: Key, + /// When merging two "root" keys, their ranks determine which key becomes + /// the new root, to prevent the parent tree from becoming unnecessarily + /// tall. See [`UnionFind::unify`] for details. + rank: u32, +} + +impl<Key: Idx> UnionFind<Key> { + /// Creates a new disjoint-set forest containing the keys `0..num_keys`. + /// Initially, every key is part of its own one-element set. + pub(crate) fn new(num_keys: usize) -> Self { + // Initially, every key is the root of its own set, so its parent is itself. + Self { table: IndexVec::from_fn_n(|key| UnionFindEntry { parent: key, rank: 0 }, num_keys) } + } + + /// Returns the "root" key of the disjoint-set containing the given key. + /// If two keys have the same root, they belong to the same set. + /// + /// Also updates internal data structures to make subsequent `find` + /// operations faster. + pub(crate) fn find(&mut self, key: Key) -> Key { + // Loop until we find a key that is its own parent. + let mut curr = key; + while let parent = self.table[curr].parent + && curr != parent + { + // Perform "path compression" by peeking one layer ahead, and + // setting the current key's parent to that value. + // (This works even when `parent` is the root of its set, because + // of the invariant that a root is its own parent.) + let parent_parent = self.table[parent].parent; + self.table[curr].parent = parent_parent; + + // Advance by one step and continue. + curr = parent; + } + curr + } + + /// Merges the set containing `a` and the set containing `b` into one set. + /// + /// Returns the common root of both keys, after the merge. + pub(crate) fn unify(&mut self, a: Key, b: Key) -> Key { + let mut a = self.find(a); + let mut b = self.find(b); + + // If both keys have the same root, they're already in the same set, + // so there's nothing more to do. + if a == b { + return a; + }; + + // Ensure that `a` has strictly greater rank, swapping if necessary. + // If both keys have the same rank, increment the rank of `a` so that + // future unifications will also prefer `a`, leading to flatter trees. + match Ord::cmp(&self.table[a].rank, &self.table[b].rank) { + Ordering::Less => mem::swap(&mut a, &mut b), + Ordering::Equal => self.table[a].rank += 1, + Ordering::Greater => {} + } + + debug_assert!(self.table[a].rank > self.table[b].rank); + debug_assert_eq!(self.table[b].parent, b); + + // Make `a` the parent of `b`. + self.table[b].parent = a; + + a + } + + /// Creates a snapshot of this disjoint-set forest that can no longer be + /// mutated, but can be queried without mutation. + pub(crate) fn freeze(&mut self) -> FrozenUnionFind<Key> { + // Just resolve each key to its actual root. + let roots = self.table.indices().map(|key| self.find(key)).collect(); + FrozenUnionFind { roots } + } +} + +/// Snapshot of a disjoint-set forest that can no longer be mutated, but can be +/// queried in O(1) time without mutation. +/// +/// This is really just a wrapper around a direct mapping from keys to roots, +/// but with a [`Self::find`] method that resembles [`UnionFind::find`]. +#[derive(Debug)] +pub(crate) struct FrozenUnionFind<Key: Idx> { + roots: IndexVec<Key, Key>, +} + +impl<Key: Idx> FrozenUnionFind<Key> { + /// Returns the "root" key of the disjoint-set containing the given key. + /// If two keys have the same root, they belong to the same set. + pub(crate) fn find(&self, key: Key) -> Key { + self.roots[key] + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/counters/union_find/tests.rs b/compiler/rustc_mir_transform/src/coverage/counters/union_find/tests.rs new file mode 100644 index 00000000000..34a4e4f8e6e --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/union_find/tests.rs @@ -0,0 +1,32 @@ +use super::UnionFind; + +#[test] +fn empty() { + let mut sets = UnionFind::<u32>::new(10); + + for i in 1..10 { + assert_eq!(sets.find(i), i); + } +} + +#[test] +fn transitive() { + let mut sets = UnionFind::<u32>::new(10); + + sets.unify(3, 7); + sets.unify(4, 2); + + assert_eq!(sets.find(7), sets.find(3)); + assert_eq!(sets.find(2), sets.find(4)); + assert_ne!(sets.find(3), sets.find(4)); + + sets.unify(7, 4); + + assert_eq!(sets.find(7), sets.find(3)); + assert_eq!(sets.find(2), sets.find(4)); + assert_eq!(sets.find(3), sets.find(4)); + + for i in [0, 1, 5, 6, 8, 9] { + assert_eq!(sets.find(i), i); + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 3fa8b063fa7..25dc7f31227 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -1,7 +1,6 @@ use std::cmp::Ordering; -use std::collections::VecDeque; use std::ops::{Index, IndexMut}; -use std::{iter, mem, slice}; +use std::{mem, slice}; use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashSet; @@ -211,54 +210,6 @@ impl CoverageGraph { self.dominator_order_rank[a].cmp(&self.dominator_order_rank[b]) } - /// Returns the source of this node's sole in-edge, if it has exactly one. - /// That edge can be assumed to have the same execution count as the node - /// itself (in the absence of panics). - pub(crate) fn sole_predecessor( - &self, - to_bcb: BasicCoverageBlock, - ) -> Option<BasicCoverageBlock> { - // Unlike `simple_successor`, there is no need for extra checks here. - if let &[from_bcb] = self.predecessors[to_bcb].as_slice() { Some(from_bcb) } else { None } - } - - /// Returns the target of this node's sole out-edge, if it has exactly - /// one, but only if that edge can be assumed to have the same execution - /// count as the node itself (in the absence of panics). - pub(crate) fn simple_successor( - &self, - from_bcb: BasicCoverageBlock, - ) -> Option<BasicCoverageBlock> { - // If a node's count is the sum of its out-edges, and it has exactly - // one out-edge, then that edge has the same count as the node. - if self.bcbs[from_bcb].is_out_summable - && let &[to_bcb] = self.successors[from_bcb].as_slice() - { - Some(to_bcb) - } else { - None - } - } - - /// For each loop that contains the given node, yields the "loop header" - /// node representing that loop, from innermost to outermost. If the given - /// node is itself a loop header, it is yielded first. - pub(crate) fn loop_headers_containing( - &self, - bcb: BasicCoverageBlock, - ) -> impl Iterator<Item = BasicCoverageBlock> + Captures<'_> { - let self_if_loop_header = self.is_loop_header.contains(bcb).then_some(bcb).into_iter(); - - let mut curr = Some(bcb); - let strictly_enclosing = iter::from_fn(move || { - let enclosing = self.enclosing_loop_header[curr?]; - curr = enclosing; - enclosing - }); - - self_if_loop_header.chain(strictly_enclosing) - } - /// For the given node, yields the subset of its predecessor nodes that /// it dominates. If that subset is non-empty, the node is a "loop header", /// and each of those predecessors represents an in-edge that jumps back to @@ -489,126 +440,3 @@ impl<'a, 'tcx> graph::Successors for CoverageRelevantSubgraph<'a, 'tcx> { self.coverage_successors(bb).into_iter() } } - -/// State of a node in the coverage graph during ready-first traversal. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -enum ReadyState { - /// This node has not yet been added to the fallback queue or ready queue. - Unqueued, - /// This node is currently in the fallback queue. - InFallbackQueue, - /// This node's predecessors have all been visited, so it is in the ready queue. - /// (It might also have a stale entry in the fallback queue.) - InReadyQueue, - /// This node has been visited. - /// (It might also have a stale entry in the fallback queue.) - Visited, -} - -/// Iterator that visits nodes in the coverage graph, in an order that always -/// prefers "ready" nodes whose predecessors have already been visited. -pub(crate) struct ReadyFirstTraversal<'a> { - graph: &'a CoverageGraph, - - /// For each node, the number of its predecessor nodes that haven't been visited yet. - n_unvisited_preds: IndexVec<BasicCoverageBlock, u32>, - /// Indicates whether a node has been visited, or which queue it is in. - state: IndexVec<BasicCoverageBlock, ReadyState>, - - /// Holds unvisited nodes whose predecessors have all been visited. - ready_queue: VecDeque<BasicCoverageBlock>, - /// Holds unvisited nodes with some unvisited predecessors. - /// Also contains stale entries for nodes that were upgraded to ready. - fallback_queue: VecDeque<BasicCoverageBlock>, -} - -impl<'a> ReadyFirstTraversal<'a> { - pub(crate) fn new(graph: &'a CoverageGraph) -> Self { - let num_nodes = graph.num_nodes(); - - let n_unvisited_preds = - IndexVec::from_fn_n(|node| graph.predecessors[node].len() as u32, num_nodes); - let mut state = IndexVec::from_elem_n(ReadyState::Unqueued, num_nodes); - - // We know from coverage graph construction that the start node is the - // only node with no predecessors. - debug_assert!( - n_unvisited_preds.iter_enumerated().all(|(node, &n)| (node == START_BCB) == (n == 0)) - ); - let ready_queue = VecDeque::from(vec![START_BCB]); - state[START_BCB] = ReadyState::InReadyQueue; - - Self { graph, state, n_unvisited_preds, ready_queue, fallback_queue: VecDeque::new() } - } - - /// Returns the next node from the ready queue, or else the next unvisited - /// node from the fallback queue. - fn next_inner(&mut self) -> Option<BasicCoverageBlock> { - // Always prefer to yield a ready node if possible. - if let Some(node) = self.ready_queue.pop_front() { - assert_eq!(self.state[node], ReadyState::InReadyQueue); - return Some(node); - } - - while let Some(node) = self.fallback_queue.pop_front() { - match self.state[node] { - // This entry in the fallback queue is not stale, so yield it. - ReadyState::InFallbackQueue => return Some(node), - // This node was added to the fallback queue, but later became - // ready and was visited via the ready queue. Ignore it here. - ReadyState::Visited => {} - // Unqueued nodes can't be in the fallback queue, by definition. - // We know that the ready queue is empty at this point. - ReadyState::Unqueued | ReadyState::InReadyQueue => unreachable!( - "unexpected state for {node:?} in the fallback queue: {:?}", - self.state[node] - ), - } - } - - None - } - - fn mark_visited_and_enqueue_successors(&mut self, node: BasicCoverageBlock) { - assert!(self.state[node] < ReadyState::Visited); - self.state[node] = ReadyState::Visited; - - // For each of this node's successors, decrease the successor's - // "unvisited predecessors" count, and enqueue it if appropriate. - for &succ in &self.graph.successors[node] { - let is_unqueued = match self.state[succ] { - ReadyState::Unqueued => true, - ReadyState::InFallbackQueue => false, - ReadyState::InReadyQueue => { - unreachable!("nodes in the ready queue have no unvisited predecessors") - } - // The successor was already visited via one of its other predecessors. - ReadyState::Visited => continue, - }; - - self.n_unvisited_preds[succ] -= 1; - if self.n_unvisited_preds[succ] == 0 { - // This node's predecessors have all been visited, so add it to - // the ready queue. If it's already in the fallback queue, that - // fallback entry will be ignored later. - self.state[succ] = ReadyState::InReadyQueue; - self.ready_queue.push_back(succ); - } else if is_unqueued { - // This node has unvisited predecessors, so add it to the - // fallback queue in case we run out of ready nodes later. - self.state[succ] = ReadyState::InFallbackQueue; - self.fallback_queue.push_back(succ); - } - } - } -} - -impl<'a> Iterator for ReadyFirstTraversal<'a> { - type Item = BasicCoverageBlock; - - fn next(&mut self) -> Option<Self::Item> { - let node = self.next_inner()?; - self.mark_visited_and_enqueue_successors(node); - Some(node) - } -} diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 57956448414..19568735df7 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -15,16 +15,13 @@ use rustc_middle::hir::nested_filter; use rustc_middle::mir::coverage::{ CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, }; -use rustc_middle::mir::{ - self, BasicBlock, BasicBlockData, SourceInfo, Statement, StatementKind, Terminator, - TerminatorKind, -}; +use rustc_middle::mir::{self, BasicBlock, Statement, StatementKind, TerminatorKind}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; use rustc_span::def_id::LocalDefId; use tracing::{debug, debug_span, trace}; -use crate::coverage::counters::{CoverageCounters, Site}; +use crate::coverage::counters::CoverageCounters; use crate::coverage::graph::CoverageGraph; use crate::coverage::mappings::ExtractedMappings; @@ -92,8 +89,7 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: return; } - let coverage_counters = - CoverageCounters::make_bcb_counters(&graph, &bcbs_with_counter_mappings); + let coverage_counters = counters::make_bcb_counters(&graph, &bcbs_with_counter_mappings); let mappings = create_mappings(&extracted_mappings, &coverage_counters); if mappings.is_empty() { @@ -242,27 +238,8 @@ fn inject_coverage_statements<'tcx>( coverage_counters: &CoverageCounters, ) { // Inject counter-increment statements into MIR. - for (id, site) in coverage_counters.counter_increment_sites() { - // Determine the block to inject a counter-increment statement into. - // For BCB nodes this is just their first block, but for edges we need - // to create a new block between the two BCBs, and inject into that. - let target_bb = match site { - Site::Node { bcb } => graph[bcb].leader_bb(), - Site::Edge { from_bcb, to_bcb } => { - // Create a new block between the last block of `from_bcb` and - // the first block of `to_bcb`. - let from_bb = graph[from_bcb].last_bb(); - let to_bb = graph[to_bcb].leader_bb(); - - let new_bb = inject_edge_counter_basic_block(mir_body, from_bb, to_bb); - debug!( - "Edge {from_bcb:?} (last {from_bb:?}) -> {to_bcb:?} (leader {to_bb:?}) \ - requires a new MIR BasicBlock {new_bb:?} for counter increment {id:?}", - ); - new_bb - } - }; - + for (id, bcb) in coverage_counters.counter_increment_sites() { + let target_bb = graph[bcb].leader_bb(); inject_statement(mir_body, CoverageKind::CounterIncrement { id }, target_bb); } @@ -335,31 +312,6 @@ fn inject_mcdc_statements<'tcx>( } } -/// Given two basic blocks that have a control-flow edge between them, creates -/// and returns a new block that sits between those blocks. -fn inject_edge_counter_basic_block( - mir_body: &mut mir::Body<'_>, - from_bb: BasicBlock, - to_bb: BasicBlock, -) -> BasicBlock { - let span = mir_body[from_bb].terminator().source_info.span.shrink_to_hi(); - let new_bb = mir_body.basic_blocks_mut().push(BasicBlockData { - statements: vec![], // counter will be injected here - terminator: Some(Terminator { - source_info: SourceInfo::outermost(span), - kind: TerminatorKind::Goto { target: to_bb }, - }), - is_cleanup: false, - }); - let edge_ref = mir_body[from_bb] - .terminator_mut() - .successors_mut() - .find(|successor| **successor == to_bb) - .expect("from_bb should have a successor for to_bb"); - *edge_ref = new_bb; - new_bb -} - fn inject_statement(mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb: BasicBlock) { debug!(" injecting statement {counter_kind:?} for {bb:?}"); let data = &mut mir_body[bb]; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 9459ef99445..db999bea986 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,4 +1,5 @@ // tidy-alphabetical-start +#![feature(array_windows)] #![feature(assert_matches)] #![feature(box_patterns)] #![feature(const_type_name)] diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 0f3943cfe6a..bb603df1129 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -1410,19 +1410,33 @@ impl<'v> RootCollector<'_, 'v> { && !self.tcx.generics_of(id.owner_id).requires_monomorphization(self.tcx) { debug!("RootCollector: ADT drop-glue for `{id:?}`",); + let id_args = + ty::GenericArgs::for_item(self.tcx, id.owner_id.to_def_id(), |param, _| { + match param.kind { + GenericParamDefKind::Lifetime => { + self.tcx.lifetimes.re_erased.into() + } + GenericParamDefKind::Type { .. } + | GenericParamDefKind::Const { .. } => { + unreachable!( + "`own_requires_monomorphization` check means that \ + we should have no type/const params" + ) + } + } + }); // This type is impossible to instantiate, so we should not try to // generate a `drop_in_place` instance for it. if self.tcx.instantiate_and_check_impossible_predicates(( id.owner_id.to_def_id(), - ty::List::empty(), + id_args, )) { return; } - let ty = self.tcx.erase_regions( - self.tcx.type_of(id.owner_id.to_def_id()).instantiate_identity(), - ); + let ty = + self.tcx.type_of(id.owner_id.to_def_id()).instantiate(self.tcx, id_args); assert!(!ty.has_non_region_param()); visit_drop_use(self.tcx, ty, true, DUMMY_SP, self.output); } diff --git a/compiler/rustc_next_trait_solver/Cargo.toml b/compiler/rustc_next_trait_solver/Cargo.toml index 451c215566b..f9168112216 100644 --- a/compiler/rustc_next_trait_solver/Cargo.toml +++ b/compiler/rustc_next_trait_solver/Cargo.toml @@ -13,6 +13,7 @@ rustc_macros = { path = "../rustc_macros", optional = true } rustc_serialize = { path = "../rustc_serialize", optional = true } rustc_type_ir = { path = "../rustc_type_ir", default-features = false } rustc_type_ir_macros = { path = "../rustc_type_ir_macros" } +smallvec = "1.8.1" tracing = "0.1" # tidy-alphabetical-end diff --git a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs index d68fca60829..4faa243c02a 100644 --- a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs +++ b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs @@ -8,6 +8,7 @@ use rustc_type_ir::lang_items::TraitSolverLangItem; use rustc_type_ir::solve::CanonicalResponse; use rustc_type_ir::visit::TypeVisitableExt as _; use rustc_type_ir::{self as ty, Interner, TraitPredicate, TypingMode, Upcast as _, elaborate}; +use smallvec::SmallVec; use tracing::{instrument, trace}; use crate::delegate::SolverDelegate; @@ -225,7 +226,7 @@ where } ecx.probe_and_evaluate_goal_for_constituent_tys( - CandidateSource::BuiltinImpl(BuiltinImplSource::Misc), + CandidateSource::BuiltinImpl(BuiltinImplSource::Trivial), goal, structural_traits::instantiate_constituent_tys_for_sized_trait, ) @@ -741,12 +742,14 @@ where a_data.principal(), )); } else if let Some(a_principal) = a_data.principal() { - for new_a_principal in - elaborate::supertraits(self.cx(), a_principal.with_self_ty(cx, a_ty)).skip(1) + for (idx, new_a_principal) in + elaborate::supertraits(self.cx(), a_principal.with_self_ty(cx, a_ty)) + .enumerate() + .skip(1) { responses.extend(self.consider_builtin_upcast_to_principal( goal, - CandidateSource::BuiltinImpl(BuiltinImplSource::TraitUpcasting), + CandidateSource::BuiltinImpl(BuiltinImplSource::TraitUpcasting(idx)), a_data, a_region, b_data, @@ -1192,7 +1195,30 @@ where }; } - // FIXME: prefer trivial builtin impls + // We prefer trivial builtin candidates, i.e. builtin impls without any + // nested requirements, over all others. This is a fix for #53123 and + // prevents where-bounds from accidentally extending the lifetime of a + // variable. + if candidates + .iter() + .any(|c| matches!(c.source, CandidateSource::BuiltinImpl(BuiltinImplSource::Trivial))) + { + let trivial_builtin_impls: SmallVec<[_; 1]> = candidates + .iter() + .filter(|c| { + matches!(c.source, CandidateSource::BuiltinImpl(BuiltinImplSource::Trivial)) + }) + .map(|c| c.result) + .collect(); + // There should only ever be a single trivial builtin candidate + // as they would otherwise overlap. + assert_eq!(trivial_builtin_impls.len(), 1); + return if let Some(response) = self.try_merge_responses(&trivial_builtin_impls) { + Ok((response, Some(TraitGoalProvenVia::Misc))) + } else { + Ok((self.bail_with_ambiguity(&trivial_builtin_impls), None)) + }; + } // If there are non-global where-bounds, prefer where-bounds // (including global ones) over everything else. diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs index 30f9e698521..60734122e63 100644 --- a/compiler/rustc_passes/src/stability.rs +++ b/compiler/rustc_passes/src/stability.rs @@ -5,8 +5,8 @@ use std::mem::replace; use std::num::NonZero; use rustc_attr_parsing::{ - self as attr, ConstStability, DeprecatedSince, Stability, StabilityLevel, StableSince, - UnstableReason, VERSION_PLACEHOLDER, + self as attr, AllowedThroughUnstableModules, ConstStability, DeprecatedSince, Stability, + StabilityLevel, StableSince, UnstableReason, VERSION_PLACEHOLDER, }; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::unord::{ExtendUnord, UnordMap, UnordSet}; @@ -20,11 +20,16 @@ use rustc_hir::{FieldDef, Item, ItemKind, TraitRef, Ty, TyKind, Variant}; use rustc_middle::hir::nested_filter; use rustc_middle::middle::lib_features::{FeatureStability, LibFeatures}; use rustc_middle::middle::privacy::EffectiveVisibilities; -use rustc_middle::middle::stability::{AllowUnstable, DeprecationEntry, Index}; +use rustc_middle::middle::stability::{ + AllowUnstable, Deprecated, DeprecationEntry, EvalResult, Index, +}; use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::lint; -use rustc_session::lint::builtin::{INEFFECTIVE_UNSTABLE_TRAIT_IMPL, USELESS_DEPRECATED}; +use rustc_session::lint::builtin::{ + DEPRECATED, INEFFECTIVE_UNSTABLE_TRAIT_IMPL, USELESS_DEPRECATED, +}; use rustc_span::{Span, Symbol, sym}; use tracing::{debug, info}; @@ -593,9 +598,11 @@ impl<'tcx> MissingStabilityAnnotations<'tcx> { } fn check_missing_const_stability(&self, def_id: LocalDefId, span: Span) { - let is_const = self.tcx.is_const_fn(def_id.to_def_id()); + let is_const = self.tcx.is_const_fn(def_id.to_def_id()) + || (self.tcx.def_kind(def_id.to_def_id()) == DefKind::Trait + && self.tcx.is_const_trait(def_id.to_def_id())); - // Reachable const fn must have a stability attribute. + // Reachable const fn/trait must have a stability attribute. if is_const && self.effective_visibilities.is_reachable(def_id) && self.tcx.lookup_const_stability(def_id).is_none() @@ -772,7 +779,13 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> { // For implementations of traits, check the stability of each item // individually as it's possible to have a stable trait with unstable // items. - hir::ItemKind::Impl(hir::Impl { of_trait: Some(ref t), self_ty, items, .. }) => { + hir::ItemKind::Impl(hir::Impl { + of_trait: Some(ref t), + self_ty, + items, + constness, + .. + }) => { let features = self.tcx.features(); if features.staged_api() { let attrs = self.tcx.hir().attrs(item.hir_id()); @@ -814,6 +827,16 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> { } } + match constness { + rustc_hir::Constness::Const => { + if let Some(def_id) = t.trait_def_id() { + // FIXME(const_trait_impl): Improve the span here. + self.tcx.check_const_stability(def_id, t.path.span, t.path.span); + } + } + rustc_hir::Constness::NotConst => {} + } + for impl_item_ref in *items { let impl_item = self.tcx.associated_item(impl_item_ref.id.owner_id); @@ -829,6 +852,18 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> { intravisit::walk_item(self, item); } + fn visit_poly_trait_ref(&mut self, t: &'tcx hir::PolyTraitRef<'tcx>) { + match t.modifiers.constness { + hir::BoundConstness::Always(span) | hir::BoundConstness::Maybe(span) => { + if let Some(def_id) = t.trait_ref.trait_def_id() { + self.tcx.check_const_stability(def_id, t.trait_ref.path.span, span); + } + } + hir::BoundConstness::Never => {} + } + intravisit::walk_poly_trait_ref(self, t); + } + fn visit_path(&mut self, path: &hir::Path<'tcx>, id: hir::HirId) { if let Some(def_id) = path.res.opt_def_id() { let method_span = path.segments.last().map(|s| s.ident.span); @@ -844,42 +879,95 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> { }, ); - let is_allowed_through_unstable_modules = |def_id| { - self.tcx.lookup_stability(def_id).is_some_and(|stab| match stab.level { - StabilityLevel::Stable { allowed_through_unstable_modules, .. } => { - allowed_through_unstable_modules + if item_is_allowed { + // The item itself is allowed; check whether the path there is also allowed. + let is_allowed_through_unstable_modules: Option<AllowedThroughUnstableModules> = + self.tcx.lookup_stability(def_id).and_then(|stab| match stab.level { + StabilityLevel::Stable { allowed_through_unstable_modules, .. } => { + allowed_through_unstable_modules + } + _ => None, + }); + + if is_allowed_through_unstable_modules.is_none() { + // Check parent modules stability as well if the item the path refers to is itself + // stable. We only emit warnings for unstable path segments if the item is stable + // or allowed because stability is often inherited, so the most common case is that + // both the segments and the item are unstable behind the same feature flag. + // + // We check here rather than in `visit_path_segment` to prevent visiting the last + // path segment twice + // + // We include special cases via #[rustc_allowed_through_unstable_modules] for items + // that were accidentally stabilized through unstable paths before this check was + // added, such as `core::intrinsics::transmute` + let parents = path.segments.iter().rev().skip(1); + for path_segment in parents { + if let Some(def_id) = path_segment.res.opt_def_id() { + // use `None` for id to prevent deprecation check + self.tcx.check_stability_allow_unstable( + def_id, + None, + path.span, + None, + if is_unstable_reexport(self.tcx, id) { + AllowUnstable::Yes + } else { + AllowUnstable::No + }, + ); + } } - _ => false, - }) - }; - - if item_is_allowed && !is_allowed_through_unstable_modules(def_id) { - // Check parent modules stability as well if the item the path refers to is itself - // stable. We only emit warnings for unstable path segments if the item is stable - // or allowed because stability is often inherited, so the most common case is that - // both the segments and the item are unstable behind the same feature flag. - // - // We check here rather than in `visit_path_segment` to prevent visiting the last - // path segment twice - // - // We include special cases via #[rustc_allowed_through_unstable_modules] for items - // that were accidentally stabilized through unstable paths before this check was - // added, such as `core::intrinsics::transmute` - let parents = path.segments.iter().rev().skip(1); - for path_segment in parents { - if let Some(def_id) = path_segment.res.opt_def_id() { - // use `None` for id to prevent deprecation check - self.tcx.check_stability_allow_unstable( - def_id, - None, - path.span, - None, - if is_unstable_reexport(self.tcx, id) { - AllowUnstable::Yes - } else { - AllowUnstable::No - }, - ); + } else if let Some(AllowedThroughUnstableModules::WithDeprecation(deprecation)) = + is_allowed_through_unstable_modules + { + // Similar to above, but we cannot use `check_stability_allow_unstable` as that would + // immediately show the stability error. We just want to know the result and disaplay + // our own kind of error. + let parents = path.segments.iter().rev().skip(1); + for path_segment in parents { + if let Some(def_id) = path_segment.res.opt_def_id() { + // use `None` for id to prevent deprecation check + let eval_result = self.tcx.eval_stability_allow_unstable( + def_id, + None, + path.span, + None, + if is_unstable_reexport(self.tcx, id) { + AllowUnstable::Yes + } else { + AllowUnstable::No + }, + ); + let is_allowed = matches!(eval_result, EvalResult::Allow); + if !is_allowed { + // Calculating message for lint involves calling `self.def_path_str`, + // which will by default invoke the expensive `visible_parent_map` query. + // Skip all that work if the lint is allowed anyway. + if self.tcx.lint_level_at_node(DEPRECATED, id).0 + == lint::Level::Allow + { + return; + } + // Show a deprecation message. + let def_path = + with_no_trimmed_paths!(self.tcx.def_path_str(def_id)); + let def_kind = self.tcx.def_descr(def_id); + let diag = Deprecated { + sub: None, + kind: def_kind.to_owned(), + path: def_path, + note: Some(deprecation), + since_kind: lint::DeprecatedSinceKind::InEffect, + }; + self.tcx.emit_node_span_lint( + DEPRECATED, + id, + method_span.unwrap_or(path.span), + diag, + ); + } + } } } } diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 7324d3fe786..1a0292ebbde 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -3960,7 +3960,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> { match res { Res::SelfCtor(_) // See #70549. | Res::Def( - DefKind::Ctor(_, CtorKind::Const) | DefKind::Const | DefKind::ConstParam, + DefKind::Ctor(_, CtorKind::Const) | DefKind::Const | DefKind::AssocConst | DefKind::ConstParam, _, ) if is_syntactic_ambiguity => { // Disambiguate in favor of a unit struct/variant or constant pattern. @@ -3969,7 +3969,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> { } Some(res) } - Res::Def(DefKind::Ctor(..) | DefKind::Const | DefKind::Static { .. }, _) => { + Res::Def(DefKind::Ctor(..) | DefKind::Const | DefKind::AssocConst | DefKind::Static { .. }, _) => { // This is unambiguously a fresh binding, either syntactically // (e.g., `IDENT @ PAT` or `ref IDENT`) or because `IDENT` resolves // to something unusable as a pattern (e.g., constructor function), @@ -4005,7 +4005,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> { ); None } - Res::Def(DefKind::Fn, _) | Res::Local(..) | Res::Err => { + Res::Def(DefKind::Fn | DefKind::AssocFn, _) | Res::Local(..) | Res::Err => { // These entities are explicitly allowed to be shadowed by fresh bindings. None } @@ -5019,12 +5019,13 @@ struct ItemInfoCollector<'a, 'ra, 'tcx> { } impl ItemInfoCollector<'_, '_, '_> { - fn collect_fn_info(&mut self, sig: &FnSig, id: NodeId) { + fn collect_fn_info(&mut self, sig: &FnSig, id: NodeId, attrs: &[Attribute]) { let sig = DelegationFnSig { header: sig.header, param_count: sig.decl.inputs.len(), has_self: sig.decl.has_self(), c_variadic: sig.decl.c_variadic(), + target_feature: attrs.iter().any(|attr| attr.has_name(sym::target_feature)), }; self.r.delegation_fn_sigs.insert(self.r.local_def_id(id), sig); } @@ -5043,7 +5044,7 @@ impl<'ast> Visitor<'ast> for ItemInfoCollector<'_, '_, '_> { | ItemKind::Trait(box Trait { ref generics, .. }) | ItemKind::TraitAlias(ref generics, _) => { if let ItemKind::Fn(box Fn { ref sig, .. }) = &item.kind { - self.collect_fn_info(sig, item.id); + self.collect_fn_info(sig, item.id, &item.attrs); } let def_id = self.r.local_def_id(item.id); @@ -5076,7 +5077,7 @@ impl<'ast> Visitor<'ast> for ItemInfoCollector<'_, '_, '_> { fn visit_assoc_item(&mut self, item: &'ast AssocItem, ctxt: AssocCtxt) { if let AssocItemKind::Fn(box Fn { ref sig, .. }) = &item.kind { - self.collect_fn_info(sig, item.id); + self.collect_fn_info(sig, item.id, &item.attrs); } visit::walk_assoc_item(self, item, ctxt); } diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs index b24e343c58d..25e35fead7e 100644 --- a/compiler/rustc_resolve/src/macros.rs +++ b/compiler/rustc_resolve/src/macros.rs @@ -1031,6 +1031,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> { is_soft, span, soft_handler, + stability::UnstableKind::Regular, ); } } diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 5192ad61af2..21afb7df7cb 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -132,13 +132,6 @@ pub enum LtoCli { } /// The different settings that the `-C instrument-coverage` flag can have. -/// -/// Coverage instrumentation now supports combining `-C instrument-coverage` -/// with compiler and linker optimization (enabled with `-O` or `-C opt-level=1` -/// and higher). Nevertheless, there are many variables, depending on options -/// selected, code structure, and enabled attributes. If errors are encountered, -/// either while compiling or when generating `llvm-cov show` reports, consider -/// lowering the optimization level, or including/excluding `-C link-dead-code`. #[derive(Clone, Copy, PartialEq, Hash, Debug)] pub enum InstrumentCoverage { /// `-C instrument-coverage=no` (or `off`, `false` etc.) diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 3af6df63017..63aaa3abc8e 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1638,7 +1638,7 @@ options! { "extra arguments to append to the linker invocation (space separated)"), #[rustc_lint_opt_deny_field_access("use `Session::link_dead_code` instead of this field")] link_dead_code: Option<bool> = (None, parse_opt_bool, [TRACKED], - "keep dead code at link time (useful for code coverage) (default: no)"), + "try to generate and link dead code (default: no)"), link_self_contained: LinkSelfContained = (LinkSelfContained::default(), parse_link_self_contained, [UNTRACKED], "control whether to link Rust provided C objects/libraries or rely \ on a C toolchain or linker installed in the system"), diff --git a/compiler/rustc_smir/src/rustc_internal/internal.rs b/compiler/rustc_smir/src/rustc_internal/internal.rs index c465367b6b9..3bc896dd7ef 100644 --- a/compiler/rustc_smir/src/rustc_internal/internal.rs +++ b/compiler/rustc_smir/src/rustc_internal/internal.rs @@ -472,6 +472,7 @@ impl RustcInternal for Abi { Abi::PtxKernel => rustc_abi::ExternAbi::PtxKernel, Abi::Msp430Interrupt => rustc_abi::ExternAbi::Msp430Interrupt, Abi::X86Interrupt => rustc_abi::ExternAbi::X86Interrupt, + Abi::GpuKernel => rustc_abi::ExternAbi::GpuKernel, Abi::EfiApi => rustc_abi::ExternAbi::EfiApi, Abi::AvrInterrupt => rustc_abi::ExternAbi::AvrInterrupt, Abi::AvrNonBlockingInterrupt => rustc_abi::ExternAbi::AvrNonBlockingInterrupt, diff --git a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs index b39a15a8633..4a03ff4beae 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs @@ -113,6 +113,7 @@ impl<'tcx> Stable<'tcx> for callconv::Conv { Conv::X86VectorCall => CallConvention::X86VectorCall, Conv::X86_64SysV => CallConvention::X86_64SysV, Conv::X86_64Win64 => CallConvention::X86_64Win64, + Conv::GpuKernel => CallConvention::GpuKernel, Conv::AvrInterrupt => CallConvention::AvrInterrupt, Conv::AvrNonBlockingInterrupt => CallConvention::AvrNonBlockingInterrupt, Conv::RiscvInterrupt { .. } => CallConvention::RiscvInterrupt, diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs index e15dad78c69..a7e122639ea 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs @@ -911,6 +911,7 @@ impl<'tcx> Stable<'tcx> for rustc_abi::ExternAbi { ExternAbi::Win64 { unwind } => Abi::Win64 { unwind }, ExternAbi::SysV64 { unwind } => Abi::SysV64 { unwind }, ExternAbi::PtxKernel => Abi::PtxKernel, + ExternAbi::GpuKernel => Abi::GpuKernel, ExternAbi::Msp430Interrupt => Abi::Msp430Interrupt, ExternAbi::X86Interrupt => Abi::X86Interrupt, ExternAbi::EfiApi => Abi::EfiApi, diff --git a/compiler/rustc_span/src/analyze_source_file.rs b/compiler/rustc_span/src/analyze_source_file.rs index 28ce883daee..fba20566580 100644 --- a/compiler/rustc_span/src/analyze_source_file.rs +++ b/compiler/rustc_span/src/analyze_source_file.rs @@ -29,6 +29,7 @@ pub(crate) fn analyze_source_file(src: &str) -> (Vec<RelativeBytePos>, Vec<Multi (lines, multi_byte_chars) } +#[cfg(bootstrap)] cfg_match! { cfg(any(target_arch = "x86", target_arch = "x86_64")) => { fn analyze_source_file_dispatch( @@ -185,6 +186,165 @@ cfg_match! { } } } + +#[cfg(not(bootstrap))] +cfg_match! { + any(target_arch = "x86", target_arch = "x86_64") => { + fn analyze_source_file_dispatch( + src: &str, + lines: &mut Vec<RelativeBytePos>, + multi_byte_chars: &mut Vec<MultiByteChar>, + ) { + if is_x86_feature_detected!("sse2") { + unsafe { + analyze_source_file_sse2(src, lines, multi_byte_chars); + } + } else { + analyze_source_file_generic( + src, + src.len(), + RelativeBytePos::from_u32(0), + lines, + multi_byte_chars, + ); + } + } + + /// Checks 16 byte chunks of text at a time. If the chunk contains + /// something other than printable ASCII characters and newlines, the + /// function falls back to the generic implementation. Otherwise it uses + /// SSE2 intrinsics to quickly find all newlines. + #[target_feature(enable = "sse2")] + unsafe fn analyze_source_file_sse2( + src: &str, + lines: &mut Vec<RelativeBytePos>, + multi_byte_chars: &mut Vec<MultiByteChar>, + ) { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + const CHUNK_SIZE: usize = 16; + + let src_bytes = src.as_bytes(); + + let chunk_count = src.len() / CHUNK_SIZE; + + // This variable keeps track of where we should start decoding a + // chunk. If a multi-byte character spans across chunk boundaries, + // we need to skip that part in the next chunk because we already + // handled it. + let mut intra_chunk_offset = 0; + + for chunk_index in 0..chunk_count { + let ptr = src_bytes.as_ptr() as *const __m128i; + // We don't know if the pointer is aligned to 16 bytes, so we + // use `loadu`, which supports unaligned loading. + let chunk = unsafe { _mm_loadu_si128(ptr.add(chunk_index)) }; + + // For character in the chunk, see if its byte value is < 0, which + // indicates that it's part of a UTF-8 char. + let multibyte_test = unsafe { _mm_cmplt_epi8(chunk, _mm_set1_epi8(0)) }; + // Create a bit mask from the comparison results. + let multibyte_mask = unsafe { _mm_movemask_epi8(multibyte_test) }; + + // If the bit mask is all zero, we only have ASCII chars here: + if multibyte_mask == 0 { + assert!(intra_chunk_offset == 0); + + // Check if there are any control characters in the chunk. All + // control characters that we can encounter at this point have a + // byte value less than 32 or ... + let control_char_test0 = unsafe { _mm_cmplt_epi8(chunk, _mm_set1_epi8(32)) }; + let control_char_mask0 = unsafe { _mm_movemask_epi8(control_char_test0) }; + + // ... it's the ASCII 'DEL' character with a value of 127. + let control_char_test1 = unsafe { _mm_cmpeq_epi8(chunk, _mm_set1_epi8(127)) }; + let control_char_mask1 = unsafe { _mm_movemask_epi8(control_char_test1) }; + + let control_char_mask = control_char_mask0 | control_char_mask1; + + if control_char_mask != 0 { + // Check for newlines in the chunk + let newlines_test = unsafe { _mm_cmpeq_epi8(chunk, _mm_set1_epi8(b'\n' as i8)) }; + let newlines_mask = unsafe { _mm_movemask_epi8(newlines_test) }; + + if control_char_mask == newlines_mask { + // All control characters are newlines, record them + let mut newlines_mask = 0xFFFF0000 | newlines_mask as u32; + let output_offset = RelativeBytePos::from_usize(chunk_index * CHUNK_SIZE + 1); + + loop { + let index = newlines_mask.trailing_zeros(); + + if index >= CHUNK_SIZE as u32 { + // We have arrived at the end of the chunk. + break; + } + + lines.push(RelativeBytePos(index) + output_offset); + + // Clear the bit, so we can find the next one. + newlines_mask &= (!1) << index; + } + + // We are done for this chunk. All control characters were + // newlines and we took care of those. + continue; + } else { + // Some of the control characters are not newlines, + // fall through to the slow path below. + } + } else { + // No control characters, nothing to record for this chunk + continue; + } + } + + // The slow path. + // There are control chars in here, fallback to generic decoding. + let scan_start = chunk_index * CHUNK_SIZE + intra_chunk_offset; + intra_chunk_offset = analyze_source_file_generic( + &src[scan_start..], + CHUNK_SIZE - intra_chunk_offset, + RelativeBytePos::from_usize(scan_start), + lines, + multi_byte_chars, + ); + } + + // There might still be a tail left to analyze + let tail_start = chunk_count * CHUNK_SIZE + intra_chunk_offset; + if tail_start < src.len() { + analyze_source_file_generic( + &src[tail_start..], + src.len() - tail_start, + RelativeBytePos::from_usize(tail_start), + lines, + multi_byte_chars, + ); + } + } + } + _ => { + // The target (or compiler version) does not support SSE2 ... + fn analyze_source_file_dispatch( + src: &str, + lines: &mut Vec<RelativeBytePos>, + multi_byte_chars: &mut Vec<MultiByteChar>, + ) { + analyze_source_file_generic( + src, + src.len(), + RelativeBytePos::from_u32(0), + lines, + multi_byte_chars, + ); + } + } +} + // `scan_len` determines the number of bytes in `src` to scan. Note that the // function can read past `scan_len` if a multi-byte character start within the // range but extends past it. The overflow is returned by the function. diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 0c8c722e4d8..b57b5df2609 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -379,6 +379,7 @@ symbols! { abi_avr_interrupt, abi_c_cmse_nonsecure_call, abi_efiapi, + abi_gpu_kernel, abi_msp430_interrupt, abi_ptx, abi_riscv_interrupt, diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 58c0a05df1f..879f3fac21f 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -19,15 +19,15 @@ pub(super) fn mangle<'tcx>( let def_id = instance.def_id(); // We want to compute the "type" of this item. Unfortunately, some - // kinds of items (e.g., closures) don't have an entry in the - // item-type array. So walk back up the find the closest parent - // that DOES have an entry. + // kinds of items (e.g., synthetic static allocations from const eval) + // don't have a proper implementation for the `type_of` query. So walk + // back up the find the closest parent that DOES have a type. let mut ty_def_id = def_id; let instance_ty; loop { let key = tcx.def_key(ty_def_id); match key.disambiguated_data.data { - DefPathData::TypeNs(_) | DefPathData::ValueNs(_) => { + DefPathData::TypeNs(_) | DefPathData::ValueNs(_) | DefPathData::Closure => { instance_ty = tcx.type_of(ty_def_id).instantiate_identity(); debug!(?instance_ty); break; diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 6a3899e66e7..41b78d9121d 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -546,6 +546,8 @@ pub enum Conv { PtxKernel, + GpuKernel, + X86Fastcall, X86Intr, X86Stdcall, @@ -865,6 +867,7 @@ impl FromStr for Conv { "X86VectorCall" => Ok(Conv::X86VectorCall), "X86_64SysV" => Ok(Conv::X86_64SysV), "X86_64Win64" => Ok(Conv::X86_64Win64), + "GpuKernel" => Ok(Conv::GpuKernel), "AvrInterrupt" => Ok(Conv::AvrInterrupt), "AvrNonBlockingInterrupt" => Ok(Conv::AvrNonBlockingInterrupt), "RiscvInterrupt(machine)" => { diff --git a/compiler/rustc_target/src/callconv/wasm.rs b/compiler/rustc_target/src/callconv/wasm.rs index 3c4cd76a754..d01b59cbb03 100644 --- a/compiler/rustc_target/src/callconv/wasm.rs +++ b/compiler/rustc_target/src/callconv/wasm.rs @@ -1,3 +1,5 @@ +use rustc_abi::{BackendRepr, Float, Integer, Primitive}; + use crate::abi::call::{ArgAbi, FnAbi}; use crate::abi::{HasDataLayout, TyAbiInterface}; @@ -27,6 +29,16 @@ where if ret.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, ret) { ret.make_indirect(); } + + // `long double`, `__int128_t` and `__uint128_t` use an indirect return + if let BackendRepr::Scalar(scalar) = ret.layout.backend_repr { + match scalar.primitive() { + Primitive::Int(Integer::I128, _) | Primitive::Float(Float::F128) => { + ret.make_indirect(); + } + _ => {} + } + } } fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs index b09d8d724ef..961cce5d6b9 100644 --- a/compiler/rustc_target/src/json.rs +++ b/compiler/rustc_target/src/json.rs @@ -113,6 +113,7 @@ impl ToJson for crate::abi::call::Conv { Self::X86VectorCall => "X86VectorCall", Self::X86_64SysV => "X86_64SysV", Self::X86_64Win64 => "X86_64Win64", + Self::GpuKernel => "GpuKernel", Self::AvrInterrupt => "AvrInterrupt", Self::AvrNonBlockingInterrupt => "AvrNonBlockingInterrupt", Self::RiscvInterrupt { kind } => { diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index beb59f1d5c5..1f2df7f0168 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -2860,6 +2860,7 @@ impl Target { } Win64 { .. } | SysV64 { .. } => self.arch == "x86_64", PtxKernel => self.arch == "nvptx64", + GpuKernel => ["amdgpu", "nvptx64"].contains(&&self.arch[..]), Msp430Interrupt => self.arch == "msp430", RiscvInterruptM | RiscvInterruptS => ["riscv32", "riscv64"].contains(&&self.arch[..]), AvrInterrupt | AvrNonBlockingInterrupt => self.arch == "avr", diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs index 53300c95fa7..9778299eb19 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs @@ -620,6 +620,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { }) => { let then_span = self.find_block_span_from_hir_id(then_id); let else_span = self.find_block_span_from_hir_id(else_id); + if let hir::Node::Expr(e) = self.tcx.hir_node(else_id) + && let hir::ExprKind::If(_cond, _then, None) = e.kind + && else_ty.is_unit() + { + // Account for `let x = if a { 1 } else if b { 2 };` + err.note("`if` expressions without `else` evaluate to `()`"); + err.note("consider adding an `else` block that evaluates to the expected type"); + } err.span_label(then_span, "expected because of this"); if let Some(sp) = outer_span { err.span_label(sp, "`if` and `else` have incompatible types"); @@ -824,9 +832,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { fn cmp_fn_sig( &self, sig1: &ty::PolyFnSig<'tcx>, - fn_def1: Option<(DefId, &'tcx [ty::GenericArg<'tcx>])>, + fn_def1: Option<(DefId, Option<&'tcx [ty::GenericArg<'tcx>]>)>, sig2: &ty::PolyFnSig<'tcx>, - fn_def2: Option<(DefId, &'tcx [ty::GenericArg<'tcx>])>, + fn_def2: Option<(DefId, Option<&'tcx [ty::GenericArg<'tcx>]>)>, ) -> (DiagStyledString, DiagStyledString) { let sig1 = &(self.normalize_fn_sig)(*sig1); let sig2 = &(self.normalize_fn_sig)(*sig2); @@ -850,8 +858,20 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // unsafe extern "C" for<'a> fn(&'a T) -> &'a T // ^^^^^^ - values.0.push(sig1.safety.prefix_str(), sig1.safety != sig2.safety); - values.1.push(sig2.safety.prefix_str(), sig1.safety != sig2.safety); + let safety = |fn_def, sig: ty::FnSig<'_>| match fn_def { + None => sig.safety.prefix_str(), + Some((did, _)) => { + if self.tcx.codegen_fn_attrs(did).safe_target_features { + "#[target_features] " + } else { + sig.safety.prefix_str() + } + } + }; + let safety1 = safety(fn_def1, sig1); + let safety2 = safety(fn_def2, sig2); + values.0.push(safety1, safety1 != safety2); + values.1.push(safety2, safety1 != safety2); // unsafe extern "C" for<'a> fn(&'a T) -> &'a T // ^^^^^^^^^^ @@ -932,23 +952,23 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { (values.1).0.extend(x2.0); } - let fmt = |(did, args)| format!(" {{{}}}", self.tcx.def_path_str_with_args(did, args)); + let fmt = |did, args| format!(" {{{}}}", self.tcx.def_path_str_with_args(did, args)); match (fn_def1, fn_def2) { - (None, None) => {} - (Some(fn_def1), Some(fn_def2)) => { - let path1 = fmt(fn_def1); - let path2 = fmt(fn_def2); + (Some((fn_def1, Some(fn_args1))), Some((fn_def2, Some(fn_args2)))) => { + let path1 = fmt(fn_def1, fn_args1); + let path2 = fmt(fn_def2, fn_args2); let same_path = path1 == path2; values.0.push(path1, !same_path); values.1.push(path2, !same_path); } - (Some(fn_def1), None) => { - values.0.push_highlighted(fmt(fn_def1)); + (Some((fn_def1, Some(fn_args1))), None) => { + values.0.push_highlighted(fmt(fn_def1, fn_args1)); } - (None, Some(fn_def2)) => { - values.1.push_highlighted(fmt(fn_def2)); + (None, Some((fn_def2, Some(fn_args2)))) => { + values.1.push_highlighted(fmt(fn_def2, fn_args2)); } + _ => {} } values @@ -1339,17 +1359,22 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { (ty::FnDef(did1, args1), ty::FnDef(did2, args2)) => { let sig1 = self.tcx.fn_sig(*did1).instantiate(self.tcx, args1); let sig2 = self.tcx.fn_sig(*did2).instantiate(self.tcx, args2); - self.cmp_fn_sig(&sig1, Some((*did1, args1)), &sig2, Some((*did2, args2))) + self.cmp_fn_sig( + &sig1, + Some((*did1, Some(args1))), + &sig2, + Some((*did2, Some(args2))), + ) } (ty::FnDef(did1, args1), ty::FnPtr(sig_tys2, hdr2)) => { let sig1 = self.tcx.fn_sig(*did1).instantiate(self.tcx, args1); - self.cmp_fn_sig(&sig1, Some((*did1, args1)), &sig_tys2.with(*hdr2), None) + self.cmp_fn_sig(&sig1, Some((*did1, Some(args1))), &sig_tys2.with(*hdr2), None) } (ty::FnPtr(sig_tys1, hdr1), ty::FnDef(did2, args2)) => { let sig2 = self.tcx.fn_sig(*did2).instantiate(self.tcx, args2); - self.cmp_fn_sig(&sig_tys1.with(*hdr1), None, &sig2, Some((*did2, args2))) + self.cmp_fn_sig(&sig_tys1.with(*hdr1), None, &sig2, Some((*did2, Some(args2)))) } (ty::FnPtr(sig_tys1, hdr1), ty::FnPtr(sig_tys2, hdr2)) => { @@ -1531,7 +1556,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { (false, Mismatch::Fixed("existential projection")) } }; - let Some(vals) = self.values_str(values) else { + let Some(vals) = self.values_str(values, cause) else { // Derived error. Cancel the emitter. // NOTE(eddyb) this was `.cancel()`, but `diag` // is borrowed, so we can't fully defuse it. @@ -1956,7 +1981,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { }) | ObligationCauseCode::BlockTailExpression(.., source)) = code && let hir::MatchSource::TryDesugar(_) = source - && let Some((expected_ty, found_ty, _)) = self.values_str(trace.values) + && let Some((expected_ty, found_ty, _)) = self.values_str(trace.values, &trace.cause) { suggestions.push(TypeErrorAdditionalDiags::TryCannotConvert { found: found_ty.content(), @@ -2085,6 +2110,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { fn values_str( &self, values: ValuePairs<'tcx>, + cause: &ObligationCause<'tcx>, ) -> Option<(DiagStyledString, DiagStyledString, Option<PathBuf>)> { match values { ValuePairs::Regions(exp_found) => self.expected_found_str(exp_found), @@ -2109,7 +2135,19 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if exp_found.references_error() { return None; } - let (exp, fnd) = self.cmp_fn_sig(&exp_found.expected, None, &exp_found.found, None); + let (fn_def1, fn_def2) = if let ObligationCauseCode::CompareImplItem { + impl_item_def_id, + trait_item_def_id, + .. + } = *cause.code() + { + (Some((trait_item_def_id, None)), Some((impl_item_def_id.to_def_id(), None))) + } else { + (None, None) + }; + + let (exp, fnd) = + self.cmp_fn_sig(&exp_found.expected, fn_def1, &exp_found.found, fn_def2); Some((exp, fnd, None)) } } diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs index 2cfccc57c97..1dd09fe7aaf 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs @@ -461,9 +461,11 @@ impl<T> Trait<T> for X { (ty::FnPtr(_, hdr), ty::FnDef(def_id, _)) | (ty::FnDef(def_id, _), ty::FnPtr(_, hdr)) => { if tcx.fn_sig(def_id).skip_binder().safety() < hdr.safety { - diag.note( + if !tcx.codegen_fn_attrs(def_id).safe_target_features { + diag.note( "unsafe functions cannot be coerced into safe function pointers", - ); + ); + } } } (ty::Adt(_, _), ty::Adt(def, args)) diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs index 98b5fb2052f..3acca47025c 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs @@ -221,7 +221,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { infer::Subtype(ref trace) => RegionOriginNote::WithRequirement { span: trace.cause.span, requirement: ObligationCauseAsDiagArg(trace.cause.clone()), - expected_found: self.values_str(trace.values).map(|(e, f, _)| (e, f)), + expected_found: self.values_str(trace.values, &trace.cause).map(|(e, f, _)| (e, f)), } .add_to_diag(err), infer::Reborrow(span) => { @@ -946,8 +946,10 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if let infer::Subtype(ref sup_trace) = sup_origin && let infer::Subtype(ref sub_trace) = sub_origin - && let Some((sup_expected, sup_found, _)) = self.values_str(sup_trace.values) - && let Some((sub_expected, sub_found, _)) = self.values_str(sub_trace.values) + && let Some((sup_expected, sup_found, _)) = + self.values_str(sup_trace.values, &sup_trace.cause) + && let Some((sub_expected, sub_found, _)) = + self.values_str(sub_trace.values, &sup_trace.cause) && sub_expected == sup_expected && sub_found == sup_found { diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs index fc0de13aeab..b4d294a70c0 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/ambiguity.rs @@ -172,14 +172,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { let bound_predicate = predicate.kind(); let mut err = match bound_predicate.skip_binder() { ty::PredicateKind::Clause(ty::ClauseKind::Trait(data)) => { - let trait_ref = bound_predicate.rebind(data.trait_ref); - debug!(?trait_ref); + let trait_pred = bound_predicate.rebind(data); + debug!(?trait_pred); if let Err(e) = predicate.error_reported() { return e; } - if let Err(guar) = self.tcx.ensure().coherent_trait(trait_ref.def_id()) { + if let Err(guar) = self.tcx.ensure().coherent_trait(trait_pred.def_id()) { // Avoid bogus "type annotations needed `Foo: Bar`" errors on `impl Bar for Foo` in case // other `Foo` impls are incoherent. return guar; @@ -200,13 +200,13 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // avoid inundating the user with unnecessary errors, but we now // check upstream for type errors and don't add the obligations to // begin with in those cases. - if self.tcx.is_lang_item(trait_ref.def_id(), LangItem::Sized) { + if self.tcx.is_lang_item(trait_pred.def_id(), LangItem::Sized) { match self.tainted_by_errors() { None => { let err = self.emit_inference_failure_err( obligation.cause.body_id, span, - trait_ref.self_ty().skip_binder().into(), + trait_pred.self_ty().skip_binder().into(), TypeAnnotationNeeded::E0282, false, ); @@ -251,10 +251,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { let mut ambiguities = compute_applicable_impls_for_diagnostics( self.infcx, - &obligation.with(self.tcx, trait_ref), + &obligation.with(self.tcx, trait_pred), ); - let has_non_region_infer = - trait_ref.skip_binder().args.types().any(|t| !t.is_ty_or_numeric_infer()); + let has_non_region_infer = trait_pred + .skip_binder() + .trait_ref + .args + .types() + .any(|t| !t.is_ty_or_numeric_infer()); // It doesn't make sense to talk about applicable impls if there are more than a // handful of them. If there are a lot of them, but only a few of them have no type // params, we only show those, as they are more likely to be useful/intended. @@ -294,7 +298,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if impl_candidates.len() < 40 { self.report_similar_impl_candidates( impl_candidates.as_slice(), - trait_ref, + trait_pred, obligation.cause.body_id, &mut err, false, @@ -306,7 +310,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if let ObligationCauseCode::WhereClause(def_id, _) | ObligationCauseCode::WhereClauseInExpr(def_id, ..) = *obligation.cause.code() { - self.suggest_fully_qualified_path(&mut err, def_id, span, trait_ref.def_id()); + self.suggest_fully_qualified_path(&mut err, def_id, span, trait_pred.def_id()); } if let Some(ty::GenericArgKind::Type(_)) = arg.map(|arg| arg.unpack()) diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs index 6076c999086..c40ba330845 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs @@ -23,9 +23,7 @@ use rustc_middle::ty::print::{ FmtPrinter, Print, PrintPolyTraitPredicateExt, PrintTraitPredicateExt as _, PrintTraitRefExt as _, with_forced_trimmed_paths, }; -use rustc_middle::ty::{ - self, ToPolyTraitRef, TraitRef, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, Upcast, -}; +use rustc_middle::ty::{self, TraitRef, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, Upcast}; use rustc_middle::{bug, span_bug}; use rustc_span::{BytePos, DUMMY_SP, STDLIB_STABLE_CRATES, Span, Symbol, sym}; use tracing::{debug, instrument}; @@ -155,12 +153,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { (leaf_trait_predicate, &obligation) }; - let main_trait_ref = main_trait_predicate.to_poly_trait_ref(); - let leaf_trait_ref = leaf_trait_predicate.to_poly_trait_ref(); - if let Some(guar) = self.emit_specialized_closure_kind_error( &obligation, - leaf_trait_ref, + leaf_trait_predicate, ) { return guar; } @@ -202,14 +197,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } = self.on_unimplemented_note(main_trait_predicate, main_obligation, &mut long_ty_file); let have_alt_message = message.is_some() || label.is_some(); - let is_try_conversion = self.is_try_conversion(span, main_trait_ref.def_id()); + let is_try_conversion = self.is_try_conversion(span, main_trait_predicate.def_id()); let is_unsize = - self.tcx.is_lang_item(leaf_trait_ref.def_id(), LangItem::Unsize); + self.tcx.is_lang_item(leaf_trait_predicate.def_id(), LangItem::Unsize); let (message, notes, append_const_msg) = if is_try_conversion { ( Some(format!( "`?` couldn't convert the error to `{}`", - main_trait_ref.skip_binder().self_ty(), + main_trait_predicate.skip_binder().self_ty(), )), vec![ "the question mark operation (`?`) implicitly performs a \ @@ -230,12 +225,12 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { post_message, ); - let (err_msg, safe_transmute_explanation) = if self.tcx.is_lang_item(main_trait_ref.def_id(), LangItem::TransmuteTrait) + let (err_msg, safe_transmute_explanation) = if self.tcx.is_lang_item(main_trait_predicate.def_id(), LangItem::TransmuteTrait) { // Recompute the safe transmute reason and use that for the error reporting match self.get_safe_transmute_error_and_reason( obligation.clone(), - main_trait_ref, + main_trait_predicate, span, ) { GetSafeTransmuteErrorAndReason::Silent => { @@ -266,7 +261,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } let mut suggested = false; if is_try_conversion { - suggested = self.try_conversion_context(&obligation, main_trait_ref.skip_binder(), &mut err); + suggested = self.try_conversion_context(&obligation, main_trait_predicate, &mut err); } if is_try_conversion && let Some(ret_span) = self.return_type_span(&obligation) { @@ -274,12 +269,12 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { ret_span, format!( "expected `{}` because of this", - main_trait_ref.skip_binder().self_ty() + main_trait_predicate.skip_binder().self_ty() ), ); } - if tcx.is_lang_item(leaf_trait_ref.def_id(), LangItem::Tuple) { + if tcx.is_lang_item(leaf_trait_predicate.def_id(), LangItem::Tuple) { self.add_tuple_trait_message( obligation.cause.code().peel_derives(), &mut err, @@ -319,7 +314,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // If it has a custom `#[rustc_on_unimplemented]` // error message, let's display it as the label! err.span_label(span, s); - if !matches!(leaf_trait_ref.skip_binder().self_ty().kind(), ty::Param(_)) { + if !matches!(leaf_trait_predicate.skip_binder().self_ty().kind(), ty::Param(_)) { // When the self type is a type param We don't need to "the trait // `std::marker::Sized` is not implemented for `T`" as we will point // at the type param with a label to suggest constraining it. @@ -339,7 +334,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if let ObligationCauseCode::Coercion { source, target } = *obligation.cause.code().peel_derives() { - if self.tcx.is_lang_item(leaf_trait_ref.def_id(), LangItem::Sized) { + if self.tcx.is_lang_item(leaf_trait_predicate.def_id(), LangItem::Sized) { self.suggest_borrowing_for_object_cast( &mut err, root_obligation, @@ -368,7 +363,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { err.span_label(tcx.def_span(body), s); } - self.suggest_floating_point_literal(&obligation, &mut err, leaf_trait_ref); + self.suggest_floating_point_literal(&obligation, &mut err, leaf_trait_predicate); self.suggest_dereferencing_index(&obligation, &mut err, leaf_trait_predicate); suggested |= self.suggest_dereferences(&obligation, &mut err, leaf_trait_predicate); suggested |= self.suggest_fn_call(&obligation, &mut err, leaf_trait_predicate); @@ -376,7 +371,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { suggested = if let &[cand] = &impl_candidates[..] { let cand = cand.trait_ref; if let (ty::FnPtr(..), ty::FnDef(..)) = - (cand.self_ty().kind(), main_trait_ref.self_ty().skip_binder().kind()) + (cand.self_ty().kind(), main_trait_predicate.self_ty().skip_binder().kind()) { // Wrap method receivers and `&`-references in parens let suggestion = if self.tcx.sess.source_map().span_look_ahead(span, ".", Some(50)).is_some() { @@ -423,11 +418,11 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { span, leaf_trait_predicate, ); - self.note_version_mismatch(&mut err, leaf_trait_ref); + self.note_version_mismatch(&mut err, leaf_trait_predicate); self.suggest_remove_await(&obligation, &mut err); self.suggest_derive(&obligation, &mut err, leaf_trait_predicate); - if tcx.is_lang_item(leaf_trait_ref.def_id(), LangItem::Try) { + if tcx.is_lang_item(leaf_trait_predicate.def_id(), LangItem::Try) { self.suggest_await_before_try( &mut err, &obligation, @@ -455,9 +450,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { ); } - let is_fn_trait = tcx.is_fn_trait(leaf_trait_ref.def_id()); + let is_fn_trait = tcx.is_fn_trait(leaf_trait_predicate.def_id()); let is_target_feature_fn = if let ty::FnDef(def_id, _) = - *leaf_trait_ref.skip_binder().self_ty().kind() + *leaf_trait_predicate.skip_binder().self_ty().kind() { !self.tcx.codegen_fn_attrs(def_id).target_features.is_empty() } else { @@ -509,7 +504,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } self.explain_hrtb_projection(&mut err, leaf_trait_predicate, obligation.param_env, &obligation.cause); - self.suggest_desugaring_async_fn_in_trait(&mut err, main_trait_ref); + self.suggest_desugaring_async_fn_in_trait(&mut err, main_trait_predicate); // Return early if the trait is Debug or Display and the invocation // originates within a standard library macro, because the output @@ -527,7 +522,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if in_std_macro && matches!( - self.tcx.get_diagnostic_name(leaf_trait_ref.def_id()), + self.tcx.get_diagnostic_name(leaf_trait_predicate.def_id()), Some(sym::Debug | sym::Display) ) { @@ -785,21 +780,22 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { fn emit_specialized_closure_kind_error( &self, obligation: &PredicateObligation<'tcx>, - mut trait_ref: ty::PolyTraitRef<'tcx>, + mut trait_pred: ty::PolyTraitPredicate<'tcx>, ) -> Option<ErrorGuaranteed> { // If `AsyncFnKindHelper` is not implemented, that means that the closure kind // doesn't extend the goal kind. This is worth reporting, but we can only do so // if we actually know which closure this goal comes from, so look at the cause // to see if we can extract that information. - if self.tcx.is_lang_item(trait_ref.def_id(), LangItem::AsyncFnKindHelper) - && let Some(found_kind) = trait_ref.skip_binder().args.type_at(0).to_opt_closure_kind() + if self.tcx.is_lang_item(trait_pred.def_id(), LangItem::AsyncFnKindHelper) + && let Some(found_kind) = + trait_pred.skip_binder().trait_ref.args.type_at(0).to_opt_closure_kind() && let Some(expected_kind) = - trait_ref.skip_binder().args.type_at(1).to_opt_closure_kind() + trait_pred.skip_binder().trait_ref.args.type_at(1).to_opt_closure_kind() && !found_kind.extends(expected_kind) { if let Some((_, Some(parent))) = obligation.cause.code().parent_with_predicate() { // If we have a derived obligation, then the parent will be a `AsyncFn*` goal. - trait_ref = parent.to_poly_trait_ref(); + trait_pred = parent; } else if let &ObligationCauseCode::FunctionArg { arg_hir_id, .. } = obligation.cause.code() && let Some(typeck_results) = &self.typeck_results @@ -820,9 +816,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } } - let self_ty = trait_ref.self_ty().skip_binder(); + let self_ty = trait_pred.self_ty().skip_binder(); - if let Some(expected_kind) = self.tcx.fn_trait_kind_from_def_id(trait_ref.def_id()) { + if let Some(expected_kind) = self.tcx.fn_trait_kind_from_def_id(trait_pred.def_id()) { let (closure_def_id, found_args, by_ref_captures) = match *self_ty.kind() { ty::Closure(def_id, args) => { (def_id, args.as_closure().sig().map_bound(|sig| sig.inputs()[0]), None) @@ -837,7 +833,8 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { _ => return None, }; - let expected_args = trait_ref.map_bound(|trait_ref| trait_ref.args.type_at(1)); + let expected_args = + trait_pred.map_bound(|trait_pred| trait_pred.trait_ref.args.type_at(1)); // Verify that the arguments are compatible. If the signature is // mismatched, then we have a totally different error to report. @@ -909,7 +906,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { fn try_conversion_context( &self, obligation: &PredicateObligation<'tcx>, - trait_ref: ty::TraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, err: &mut Diag<'_>, ) -> bool { let span = obligation.cause.span; @@ -953,8 +950,8 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if !self.tcx.is_diagnostic_item(sym::FromResidual, y.def_id()) { return false; } - let self_ty = trait_ref.self_ty(); - let found_ty = trait_ref.args.get(1).and_then(|a| a.as_type()); + let self_ty = trait_pred.skip_binder().self_ty(); + let found_ty = trait_pred.skip_binder().trait_ref.args.get(1).and_then(|a| a.as_type()); let mut prev_ty = self.resolve_vars_if_possible( typeck.expr_ty_adjusted_opt(expr).unwrap_or(Ty::new_misc_error(self.tcx)), @@ -1223,18 +1220,18 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { goal: ty::TraitPredicate<'tcx>, assumption: ty::PolyTraitPredicate<'tcx>, ) -> bool { + // Fast path if goal.polarity != assumption.polarity() { return false; } - let trait_goal = goal.trait_ref; let trait_assumption = self.instantiate_binder_with_fresh_vars( DUMMY_SP, infer::BoundRegionConversionTime::HigherRankedType, - assumption.to_poly_trait_ref(), + assumption, ); - self.can_eq(ty::ParamEnv::empty(), trait_goal, trait_assumption) + self.can_eq(ty::ParamEnv::empty(), goal.trait_ref, trait_assumption.trait_ref) } fn can_match_projection( @@ -1682,7 +1679,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { pub(super) fn report_similar_impl_candidates( &self, impl_candidates: &[ImplCandidate<'tcx>], - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, body_def_id: LocalDefId, err: &mut Diag<'_>, other: bool, @@ -1727,7 +1724,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // We'll check for the case where the reason for the mismatch is that the trait comes from // one crate version and the type comes from another crate version, even though they both // are from the same crate. - let trait_def_id = trait_ref.def_id(); + let trait_def_id = trait_pred.def_id(); let trait_name = self.tcx.item_name(trait_def_id); let crate_name = self.tcx.crate_name(trait_def_id.krate); if let Some(other_trait_def_id) = self.tcx.all_traits().find(|def_id| { @@ -1739,7 +1736,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // different crate `DefId`. We highlight the traits. let found_type = - if let ty::Adt(def, _) = trait_ref.self_ty().skip_binder().peel_refs().kind() { + if let ty::Adt(def, _) = trait_pred.self_ty().skip_binder().peel_refs().kind() { Some(def.did()) } else { None @@ -1836,7 +1833,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { if self.probe(|_| { let ocx = ObligationCtxt::new(self); - self.enter_forall(trait_ref, |obligation_trait_ref| { + self.enter_forall(trait_pred, |obligation_trait_ref| { let impl_args = self.fresh_args_for_item(DUMMY_SP, single.impl_def_id); let impl_trait_ref = ocx.normalize( &ObligationCause::dummy(), @@ -1864,7 +1861,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { let mut terrs = vec![]; for (obligation_arg, impl_arg) in - std::iter::zip(obligation_trait_ref.args, impl_trait_ref.args) + std::iter::zip(obligation_trait_ref.trait_ref.args, impl_trait_ref.args) { if (obligation_arg, impl_arg).references_error() { return false; @@ -1906,8 +1903,8 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } let traits = self.cmp_traits( - obligation_trait_ref.def_id, - &obligation_trait_ref.args[1..], + obligation_trait_ref.def_id(), + &obligation_trait_ref.trait_ref.args[1..], impl_trait_ref.def_id, &impl_trait_ref.args[1..], ); @@ -1991,7 +1988,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } if let &[cand] = &candidates[..] { let (desc, mention_castable) = - match (cand.self_ty().kind(), trait_ref.self_ty().skip_binder().kind()) { + match (cand.self_ty().kind(), trait_pred.self_ty().skip_binder().kind()) { (ty::FnPtr(..), ty::FnDef(..)) => { (" implemented for fn pointer `", ", cast using `as`") } @@ -2055,7 +2052,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { .filter(|cand| !self.tcx.do_not_recommend_impl(cand.impl_def_id)) .collect::<Vec<_>>(); - let def_id = trait_ref.def_id(); + let def_id = trait_pred.def_id(); if impl_candidates.is_empty() { if self.tcx.trait_is_auto(def_id) || self.tcx.lang_items().iter().any(|(_, id)| id == def_id) @@ -2132,11 +2129,10 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { && !self.tcx.trait_is_auto(def_id) && !self.tcx.lang_items().iter().any(|(_, id)| id == def_id) { - let trait_ref = trait_pred.to_poly_trait_ref(); let impl_candidates = self.find_similar_impl_candidates(trait_pred); self.report_similar_impl_candidates( &impl_candidates, - trait_ref, + trait_pred, body_def_id, err, true, @@ -2173,12 +2169,16 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { /// If the `Self` type of the unsatisfied trait `trait_ref` implements a trait /// with the same path as `trait_ref`, a help message about /// a probable version mismatch is added to `err` - fn note_version_mismatch(&self, err: &mut Diag<'_>, trait_ref: ty::PolyTraitRef<'tcx>) -> bool { + fn note_version_mismatch( + &self, + err: &mut Diag<'_>, + trait_pred: ty::PolyTraitPredicate<'tcx>, + ) -> bool { let get_trait_impls = |trait_def_id| { let mut trait_impls = vec![]; self.tcx.for_each_relevant_impl( trait_def_id, - trait_ref.skip_binder().self_ty(), + trait_pred.skip_binder().self_ty(), |impl_def_id| { trait_impls.push(impl_def_id); }, @@ -2186,11 +2186,11 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { trait_impls }; - let required_trait_path = self.tcx.def_path_str(trait_ref.def_id()); + let required_trait_path = self.tcx.def_path_str(trait_pred.def_id()); let traits_with_same_path: UnordSet<_> = self .tcx .visible_traits() - .filter(|trait_def_id| *trait_def_id != trait_ref.def_id()) + .filter(|trait_def_id| *trait_def_id != trait_pred.def_id()) .map(|trait_def_id| (self.tcx.def_path_str(trait_def_id), trait_def_id)) .filter(|(p, _)| *p == required_trait_path) .collect(); @@ -2374,7 +2374,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { fn get_safe_transmute_error_and_reason( &self, obligation: PredicateObligation<'tcx>, - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, span: Span, ) -> GetSafeTransmuteErrorAndReason { use rustc_transmute::Answer; @@ -2386,19 +2386,19 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } // Erase regions because layout code doesn't particularly care about regions. - let trait_ref = - self.tcx.erase_regions(self.tcx.instantiate_bound_regions_with_erased(trait_ref)); + let trait_pred = + self.tcx.erase_regions(self.tcx.instantiate_bound_regions_with_erased(trait_pred)); let src_and_dst = rustc_transmute::Types { - dst: trait_ref.args.type_at(0), - src: trait_ref.args.type_at(1), + dst: trait_pred.trait_ref.args.type_at(0), + src: trait_pred.trait_ref.args.type_at(1), }; let ocx = ObligationCtxt::new(self); let Ok(assume) = ocx.structurally_normalize_const( &obligation.cause, obligation.param_env, - trait_ref.args.const_at(2), + trait_pred.trait_ref.args.const_at(2), ) else { self.dcx().span_delayed_bug( span, @@ -2417,8 +2417,8 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { return GetSafeTransmuteErrorAndReason::Silent; }; - let dst = trait_ref.args.type_at(0); - let src = trait_ref.args.type_at(1); + let dst = trait_pred.trait_ref.args.type_at(0); + let src = trait_pred.trait_ref.args.type_at(1); let err_msg = format!("`{src}` cannot be safely transmuted into `{dst}`"); match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable( @@ -2566,12 +2566,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { trait_predicate.skip_binder().polarity, ) { - self.add_help_message_for_fn_trait( - trait_predicate.to_poly_trait_ref(), - err, - implemented_kind, - params, - ); + self.add_help_message_for_fn_trait(trait_predicate, err, implemented_kind, params); } else if !trait_predicate.has_non_region_infer() && self.predicate_can_apply(obligation.param_env, trait_predicate) { @@ -2606,7 +2601,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { let impl_candidates = self.find_similar_impl_candidates(trait_predicate); if !self.report_similar_impl_candidates( &impl_candidates, - trait_predicate.to_poly_trait_ref(), + trait_predicate, body_def_id, err, true, @@ -2623,7 +2618,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { self.suggest_convert_to_slice( err, obligation, - trait_predicate.to_poly_trait_ref(), + trait_predicate, impl_candidates.as_slice(), span, ); @@ -2634,7 +2629,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { fn add_help_message_for_fn_trait( &self, - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, err: &mut Diag<'_>, implemented_kind: ty::ClosureKind, params: ty::Binder<'tcx, Ty<'tcx>>, @@ -2647,12 +2642,12 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // to implement. let selected_kind = self .tcx - .fn_trait_kind_from_def_id(trait_ref.def_id()) + .fn_trait_kind_from_def_id(trait_pred.def_id()) .expect("expected to map DefId to ClosureKind"); if !implemented_kind.extends(selected_kind) { err.note(format!( "`{}` implements `{}`, but it must implement `{}`, which is more general", - trait_ref.skip_binder().self_ty(), + trait_pred.skip_binder().self_ty(), implemented_kind, selected_kind )); @@ -2660,7 +2655,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // Note any argument mismatches let given_ty = params.skip_binder(); - let expected_ty = trait_ref.skip_binder().args.type_at(1); + let expected_ty = trait_pred.skip_binder().trait_ref.args.type_at(1); if let ty::Tuple(given) = given_ty.kind() && let ty::Tuple(expected) = expected_ty.kind() { diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs index 51efe39a7bc..2d932e36470 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs @@ -10,7 +10,7 @@ use rustc_hir::{AttrArgs, AttrKind, Attribute}; use rustc_macros::LintDiagnostic; use rustc_middle::bug; use rustc_middle::ty::print::PrintTraitRefExt as _; -use rustc_middle::ty::{self, GenericArgsRef, GenericParamDefKind, ToPolyTraitRef, TyCtxt}; +use rustc_middle::ty::{self, GenericArgsRef, GenericParamDefKind, TyCtxt}; use rustc_parse_format::{ParseMode, Parser, Piece, Position}; use rustc_session::lint::builtin::UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES; use rustc_span::{Span, Symbol, kw, sym}; @@ -42,18 +42,18 @@ static ALLOWED_FORMAT_SYMBOLS: &[Symbol] = &[ impl<'tcx> TypeErrCtxt<'_, 'tcx> { fn impl_similar_to( &self, - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, obligation: &PredicateObligation<'tcx>, ) -> Option<(DefId, GenericArgsRef<'tcx>)> { let tcx = self.tcx; let param_env = obligation.param_env; - self.enter_forall(trait_ref, |trait_ref| { - let trait_self_ty = trait_ref.self_ty(); + self.enter_forall(trait_pred, |trait_pred| { + let trait_self_ty = trait_pred.self_ty(); let mut self_match_impls = vec![]; let mut fuzzy_match_impls = vec![]; - self.tcx.for_each_relevant_impl(trait_ref.def_id, trait_self_ty, |def_id| { + self.tcx.for_each_relevant_impl(trait_pred.def_id(), trait_self_ty, |def_id| { let impl_args = self.fresh_args_for_item(obligation.cause.span, def_id); let impl_trait_ref = tcx.impl_trait_ref(def_id).unwrap().instantiate(tcx, impl_args); @@ -64,7 +64,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { self_match_impls.push((def_id, impl_args)); if iter::zip( - trait_ref.args.types().skip(1), + trait_pred.trait_ref.args.types().skip(1), impl_trait_ref.args.types().skip(1), ) .all(|(u, v)| self.fuzzy_match_tys(u, v, false).is_some()) @@ -117,7 +117,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { } let (def_id, args) = self - .impl_similar_to(trait_pred.to_poly_trait_ref(), obligation) + .impl_similar_to(trait_pred, obligation) .unwrap_or_else(|| (trait_pred.def_id(), trait_pred.skip_binder().trait_ref.args)); let trait_pred = trait_pred.skip_binder(); @@ -205,9 +205,15 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { if self_ty.is_fn() { let fn_sig = self_ty.fn_sig(self.tcx); - let shortname = match fn_sig.safety() { - hir::Safety::Safe => "fn", - hir::Safety::Unsafe => "unsafe fn", + let shortname = if let ty::FnDef(def_id, _) = self_ty.kind() + && self.tcx.codegen_fn_attrs(def_id).safe_target_features + { + "#[target_feature] fn" + } else { + match fn_sig.safety() { + hir::Safety::Safe => "fn", + hir::Safety::Unsafe => "unsafe fn", + } }; flags.push((sym::_Self, Some(shortname.to_owned()))); } diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs index 9d85ca1dd4d..c2e73b732d3 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs @@ -32,9 +32,9 @@ use rustc_middle::ty::print::{ with_forced_trimmed_paths, with_no_trimmed_paths, }; use rustc_middle::ty::{ - self, AdtKind, GenericArgs, InferTy, IsSuggestable, ToPolyTraitRef, Ty, TyCtxt, TypeFoldable, - TypeFolder, TypeSuperFoldable, TypeVisitableExt, TypeckResults, Upcast, - suggest_arbitrary_trait_bound, suggest_constraining_type_param, + self, AdtKind, GenericArgs, InferTy, IsSuggestable, Ty, TyCtxt, TypeFoldable, TypeFolder, + TypeSuperFoldable, TypeVisitableExt, TypeckResults, Upcast, suggest_arbitrary_trait_bound, + suggest_constraining_type_param, }; use rustc_middle::{bug, span_bug}; use rustc_span::def_id::LocalDefId; @@ -218,15 +218,15 @@ pub fn suggest_restriction<'tcx, G: EmissionGuarantee>( (_, None) => predicate_constraint(hir_generics, trait_pred.upcast(tcx)), (None, Some((ident, []))) => ( ident.span.shrink_to_hi(), - format!(": {}", trait_pred.to_poly_trait_ref().print_trait_sugared()), + format!(": {}", trait_pred.print_modifiers_and_trait_path()), ), (_, Some((_, [.., bounds]))) => ( bounds.span().shrink_to_hi(), - format!(" + {}", trait_pred.to_poly_trait_ref().print_trait_sugared()), + format!(" + {}", trait_pred.print_modifiers_and_trait_path()), ), (Some(_), Some((_, []))) => ( hir_generics.span.shrink_to_hi(), - format!(": {}", trait_pred.to_poly_trait_ref().print_trait_sugared()), + format!(": {}", trait_pred.print_modifiers_and_trait_path()), ), }; @@ -3729,7 +3729,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { &self, obligation: &PredicateObligation<'tcx>, err: &mut Diag<'_>, - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, ) { let rhs_span = match obligation.cause.code() { ObligationCauseCode::BinOp { rhs_span: Some(span), rhs_is_lit, .. } if *rhs_is_lit => { @@ -3737,8 +3737,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } _ => return, }; - if let ty::Float(_) = trait_ref.skip_binder().self_ty().kind() - && let ty::Infer(InferTy::IntVar(_)) = trait_ref.skip_binder().args.type_at(1).kind() + if let ty::Float(_) = trait_pred.skip_binder().self_ty().kind() + && let ty::Infer(InferTy::IntVar(_)) = + trait_pred.skip_binder().trait_ref.args.type_at(1).kind() { err.span_suggestion_verbose( rhs_span.shrink_to_hi(), @@ -4448,7 +4449,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { &self, err: &mut Diag<'_>, obligation: &PredicateObligation<'tcx>, - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, candidate_impls: &[ImplCandidate<'tcx>], span: Span, ) { @@ -4464,7 +4465,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { // 1. `[T; _]` (array of T) // 2. `&[T; _]` (reference to array of T) // 3. `&mut [T; _]` (mutable reference to array of T) - let (element_ty, mut mutability) = match *trait_ref.skip_binder().self_ty().kind() { + let (element_ty, mut mutability) = match *trait_pred.skip_binder().self_ty().kind() { ty::Array(element_ty, _) => (element_ty, None), ty::Ref(_, pointee_ty, mutability) => match *pointee_ty.kind() { @@ -4620,14 +4621,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { pub(super) fn suggest_desugaring_async_fn_in_trait( &self, err: &mut Diag<'_>, - trait_ref: ty::PolyTraitRef<'tcx>, + trait_pred: ty::PolyTraitPredicate<'tcx>, ) { // Don't suggest if RTN is active -- we should prefer a where-clause bound instead. if self.tcx.features().return_type_notation() { return; } - let trait_def_id = trait_ref.def_id(); + let trait_def_id = trait_pred.def_id(); // Only suggest specifying auto traits if !self.tcx.trait_is_auto(trait_def_id) { @@ -4635,7 +4636,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { } // Look for an RPITIT - let ty::Alias(ty::Projection, alias_ty) = trait_ref.self_ty().skip_binder().kind() else { + let ty::Alias(ty::Projection, alias_ty) = trait_pred.self_ty().skip_binder().kind() else { return; }; let Some(ty::ImplTraitInTraitData::Trait { fn_def_id, opaque_def_id }) = diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 7db0f2bb5a7..4498beff4ea 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -264,9 +264,10 @@ fn fulfillment_error_for_no_solution<'tcx>( infcx.tcx.type_of(uv.def).instantiate(infcx.tcx, uv.args) } ty::ConstKind::Param(param_ct) => param_ct.find_ty_from_env(obligation.param_env), - _ => span_bug!( + ty::ConstKind::Value(ty, _) => ty, + kind => span_bug!( obligation.cause.span, - "ConstArgHasWrongType failed but we don't know how to compute type" + "ConstArgHasWrongType failed but we don't know how to compute type for {kind:?}" ), }; FulfillmentErrorCode::Select(SelectionError::ConstArgHasWrongType { diff --git a/compiler/rustc_trait_selection/src/solve/select.rs b/compiler/rustc_trait_selection/src/solve/select.rs index 1661852903c..b0b6274907d 100644 --- a/compiler/rustc_trait_selection/src/solve/select.rs +++ b/compiler/rustc_trait_selection/src/solve/select.rs @@ -117,6 +117,10 @@ fn candidate_should_be_dropped_in_favor_of<'tcx>( CandidateSource::BuiltinImpl(BuiltinImplSource::Object(a)), CandidateSource::BuiltinImpl(BuiltinImplSource::Object(b)), ) => a >= b, + ( + CandidateSource::BuiltinImpl(BuiltinImplSource::TraitUpcasting(a)), + CandidateSource::BuiltinImpl(BuiltinImplSource::TraitUpcasting(b)), + ) => a >= b, // Prefer dyn candidates over non-dyn candidates. This is necessary to // handle the unsoundness between `impl<T: ?Sized> Any for T` and `dyn Any: Any`. ( diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs index 971d3a81102..e27143f1396 100644 --- a/compiler/rustc_trait_selection/src/traits/coherence.rs +++ b/compiler/rustc_trait_selection/src/traits/coherence.rs @@ -6,7 +6,7 @@ use std::fmt::Debug; -use rustc_data_structures::fx::FxIndexSet; +use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; use rustc_errors::{Diag, EmissionGuarantee}; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; @@ -116,28 +116,39 @@ pub fn overlapping_impls( return None; } - let _overlap_with_bad_diagnostics = overlap( - tcx, - TrackAmbiguityCauses::No, - skip_leak_check, - impl1_def_id, - impl2_def_id, - overlap_mode, - )?; - - // In the case where we detect an error, run the check again, but - // this time tracking intercrate ambiguity causes for better - // diagnostics. (These take time and can lead to false errors.) - let overlap = overlap( - tcx, - TrackAmbiguityCauses::Yes, - skip_leak_check, - impl1_def_id, - impl2_def_id, - overlap_mode, - ) - .unwrap(); - Some(overlap) + if tcx.next_trait_solver_in_coherence() { + overlap( + tcx, + TrackAmbiguityCauses::Yes, + skip_leak_check, + impl1_def_id, + impl2_def_id, + overlap_mode, + ) + } else { + let _overlap_with_bad_diagnostics = overlap( + tcx, + TrackAmbiguityCauses::No, + skip_leak_check, + impl1_def_id, + impl2_def_id, + overlap_mode, + )?; + + // In the case where we detect an error, run the check again, but + // this time tracking intercrate ambiguity causes for better + // diagnostics. (These take time and can lead to false errors.) + let overlap = overlap( + tcx, + TrackAmbiguityCauses::Yes, + skip_leak_check, + impl1_def_id, + impl2_def_id, + overlap_mode, + ) + .unwrap(); + Some(overlap) + } } fn fresh_impl_header<'tcx>(infcx: &InferCtxt<'tcx>, impl_def_id: DefId) -> ty::ImplHeader<'tcx> { @@ -615,6 +626,7 @@ fn compute_intercrate_ambiguity_causes<'tcx>( } struct AmbiguityCausesVisitor<'a, 'tcx> { + cache: FxHashSet<Goal<'tcx, ty::Predicate<'tcx>>>, causes: &'a mut FxIndexSet<IntercrateAmbiguityCause<'tcx>>, } @@ -624,6 +636,10 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a, 'tcx> { } fn visit_goal(&mut self, goal: &InspectGoal<'_, 'tcx>) { + if !self.cache.insert(goal.goal()) { + return; + } + let infcx = goal.infcx(); for cand in goal.candidates() { cand.visit_nested_in_probe(self); @@ -748,5 +764,10 @@ fn search_ambiguity_causes<'tcx>( goal: Goal<'tcx, ty::Predicate<'tcx>>, causes: &mut FxIndexSet<IntercrateAmbiguityCause<'tcx>>, ) { - infcx.probe(|_| infcx.visit_proof_tree(goal, &mut AmbiguityCausesVisitor { causes })); + infcx.probe(|_| { + infcx.visit_proof_tree(goal, &mut AmbiguityCausesVisitor { + cache: Default::default(), + causes, + }) + }); } diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs index 1dcd0d0dfb8..da16a742099 100644 --- a/compiler/rustc_trait_selection/src/traits/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/mod.rs @@ -714,9 +714,18 @@ pub fn impossible_predicates<'tcx>(tcx: TyCtxt<'tcx>, predicates: Vec<ty::Clause } let errors = ocx.select_all_or_error(); - let result = !errors.is_empty(); - debug!("impossible_predicates = {:?}", result); - result + if !errors.is_empty() { + return true; + } + + // Leak check for any higher-ranked trait mismatches. + // We only need to do this in the old solver, since the new solver already + // leak-checks. + if !infcx.next_trait_solver() && infcx.leak_check(ty::UniverseIndex::ROOT, None).is_err() { + return true; + } + + false } fn instantiate_and_check_impossible_predicates<'tcx>( diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index aae0e34ddf3..d59cf88875e 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -991,7 +991,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>( Err(ErrorGuaranteed { .. }) => true, } } - ImplSource::Builtin(BuiltinImplSource::Misc, _) => { + ImplSource::Builtin(BuiltinImplSource::Misc | BuiltinImplSource::Trivial, _) => { // While a builtin impl may be known to exist, the associated type may not yet // be known. Any type with multiple potential associated types is therefore // not eligible. @@ -1296,7 +1296,7 @@ fn confirm_select_candidate<'cx, 'tcx>( ) -> Progress<'tcx> { match impl_source { ImplSource::UserDefined(data) => confirm_impl_candidate(selcx, obligation, data), - ImplSource::Builtin(BuiltinImplSource::Misc, data) => { + ImplSource::Builtin(BuiltinImplSource::Misc | BuiltinImplSource::Trivial, data) => { let tcx = selcx.tcx(); let trait_def_id = obligation.predicate.trait_def_id(tcx); if tcx.is_lang_item(trait_def_id, LangItem::Coroutine) { diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index 968dc631e50..b370f802052 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -16,7 +16,7 @@ use rustc_infer::traits::{ Obligation, ObligationCause, PolyTraitObligation, PredicateObligations, SelectionError, }; use rustc_middle::ty::fast_reject::DeepRejectCtxt; -use rustc_middle::ty::{self, ToPolyTraitRef, Ty, TypeVisitableExt, TypingMode}; +use rustc_middle::ty::{self, Ty, TypeVisitableExt, TypingMode}; use rustc_middle::{bug, span_bug}; use rustc_type_ir::Interner; use tracing::{debug, instrument, trace}; @@ -186,10 +186,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { } selcx.infcx.probe(|_| { + // We checked the polarity already match selcx.match_normalize_trait_ref( obligation, placeholder_trait_predicate.trait_ref, - bound.to_poly_trait_ref(), + bound.map_bound(|pred| pred.trait_ref), ) { Ok(None) => { candidates.vec.push(ProjectionCandidate(idx)); diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index 3619d16cde2..729ae3f2c2a 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -16,7 +16,7 @@ use rustc_hir::lang_items::LangItem; use rustc_infer::infer::{DefineOpaqueTypes, HigherRankedType, InferOk}; use rustc_infer::traits::ObligationCauseCode; use rustc_middle::traits::{BuiltinImplSource, SignatureMismatchData}; -use rustc_middle::ty::{self, GenericArgsRef, ToPolyTraitRef, Ty, TyCtxt, Upcast}; +use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, Upcast}; use rustc_middle::{bug, span_bug}; use rustc_span::def_id::DefId; use rustc_type_ir::elaborate; @@ -458,8 +458,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { ensure_sufficient_stack(|| { let cause = obligation.derived_cause(ObligationCauseCode::BuiltinDerived); - let poly_trait_ref = obligation.predicate.to_poly_trait_ref(); - let trait_ref = self.infcx.enter_forall_and_leak_universe(poly_trait_ref); + assert_eq!(obligation.predicate.polarity(), ty::PredicatePolarity::Positive); + let trait_ref = + self.infcx.enter_forall_and_leak_universe(obligation.predicate).trait_ref; let trait_obligations = self.impl_or_trait_obligations( &cause, obligation.recursion_depth + 1, @@ -1090,7 +1091,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { )? .expect("did not expect ambiguity during confirmation"); - Ok(ImplSource::Builtin(BuiltinImplSource::TraitUpcasting, nested)) + Ok(ImplSource::Builtin(BuiltinImplSource::TraitUpcasting(idx), nested)) } fn confirm_builtin_unsize_candidate( diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index 9e7da5eb368..5581ea46882 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -1895,6 +1895,18 @@ impl<'tcx> SelectionContext<'_, 'tcx> { Some(None) => {} None => return None, } + // Same for upcasting. + let upcast_bound = candidates + .iter() + .filter_map(|c| { + if let TraitUpcastingUnsizeCandidate(i) = c.candidate { Some(i) } else { None } + }) + .try_reduce(|c1, c2| if has_non_region_infer { None } else { Some(c1.min(c2)) }); + match upcast_bound { + Some(Some(index)) => return Some(TraitUpcastingUnsizeCandidate(index)), + Some(None) => {} + None => return None, + } // Finally, handle overlapping user-written impls. let impls = candidates.iter().filter_map(|c| { diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index b63534880d1..c528179ae0e 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -293,6 +293,7 @@ fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: ExternAbi, c_variadic: bool) -> Conv PtxKernel => Conv::PtxKernel, Msp430Interrupt => Conv::Msp430Intr, X86Interrupt => Conv::X86Intr, + GpuKernel => Conv::GpuKernel, AvrInterrupt => Conv::AvrInterrupt, AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt, RiscvInterruptM => Conv::RiscvInterrupt { kind: RiscvInterruptKind::Machine }, diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index fc76a86f797..d5e1937efaa 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -248,7 +248,7 @@ fn resolve_associated_item<'tcx>( }) } } - traits::ImplSource::Builtin(BuiltinImplSource::Misc, _) => { + traits::ImplSource::Builtin(BuiltinImplSource::Misc | BuiltinImplSource::Trivial, _) => { if tcx.is_lang_item(trait_ref.def_id, LangItem::Clone) { // FIXME(eddyb) use lang items for methods instead of names. let name = tcx.item_name(trait_item_id); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index ab606478c51..17be0bd0ab9 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -105,21 +105,27 @@ fn map_error<'tcx>( // See `tests/ui/layout/trivial-bounds-sized.rs` for an example. assert!(field.layout.is_unsized(), "invalid layout error {err:#?}"); if !field.ty.is_sized(cx.tcx(), cx.typing_env) { - cx.tcx().dcx().delayed_bug(format!( + let guar = cx.tcx().dcx().delayed_bug(format!( "encountered unexpected unsized field in layout of {ty:?}: {field:#?}" )); + LayoutError::ReferencesError(guar) + } else { + LayoutError::Unknown(ty) } - LayoutError::Unknown(ty) } LayoutCalculatorError::EmptyUnion => { // This is always a compile error. - cx.tcx().dcx().delayed_bug(format!("computed layout of empty union: {ty:?}")); - LayoutError::Unknown(ty) + let guar = + cx.tcx().dcx().delayed_bug(format!("computed layout of empty union: {ty:?}")); + LayoutError::ReferencesError(guar) } LayoutCalculatorError::ReprConflict => { // packed enums are the only known trigger of this, but others might arise - cx.tcx().dcx().delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}")); - LayoutError::Unknown(ty) + let guar = cx + .tcx() + .dcx() + .delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}")); + LayoutError::ReferencesError(guar) } }; error(cx, err) @@ -432,8 +438,10 @@ fn layout_of_uncached<'tcx>( ty::Adt(def, args) if def.repr().simd() => { if !def.is_struct() { // Should have yielded E0517 by now. - tcx.dcx().delayed_bug("#[repr(simd)] was applied to an ADT that is not a struct"); - return Err(error(cx, LayoutError::Unknown(ty))); + let guar = tcx + .dcx() + .delayed_bug("#[repr(simd)] was applied to an ADT that is not a struct"); + return Err(error(cx, LayoutError::ReferencesError(guar))); } let fields = &def.non_enum_variant().fields; @@ -459,10 +467,10 @@ fn layout_of_uncached<'tcx>( // (should be caught by typeck) for fi in fields { if fi.ty(tcx, args) != f0_ty { - tcx.dcx().delayed_bug( + let guar = tcx.dcx().delayed_bug( "#[repr(simd)] was applied to an ADT with heterogeneous field type", ); - return Err(error(cx, LayoutError::Unknown(ty))); + return Err(error(cx, LayoutError::ReferencesError(guar))); } } @@ -567,11 +575,11 @@ fn layout_of_uncached<'tcx>( if def.is_union() { if def.repr().pack.is_some() && def.repr().align.is_some() { - tcx.dcx().span_delayed_bug( + let guar = tcx.dcx().span_delayed_bug( tcx.def_span(def.did()), "union cannot be packed and aligned", ); - return Err(error(cx, LayoutError::Unknown(ty))); + return Err(error(cx, LayoutError::ReferencesError(guar))); } return Ok(tcx.mk_layout( diff --git a/compiler/rustc_type_ir/src/binder.rs b/compiler/rustc_type_ir/src/binder.rs index cb59bc608c2..0d0092ea1aa 100644 --- a/compiler/rustc_type_ir/src/binder.rs +++ b/compiler/rustc_type_ir/src/binder.rs @@ -804,7 +804,7 @@ impl<'a, I: Interner> ArgFolder<'a, I> { #[inline(never)] fn region_param_out_of_range(&self, ebr: I::EarlyParamRegion, r: I::Region) -> ! { panic!( - "const parameter `{:?}` ({:?}/{}) out of range when instantiating args={:?}", + "region parameter `{:?}` ({:?}/{}) out of range when instantiating args={:?}", ebr, r, ebr.index(), diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 1ae904d50e0..c06004d4d0f 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -169,6 +169,9 @@ pub enum CandidateSource<I: Interner> { #[derive(Clone, Copy, Hash, PartialEq, Eq, Debug)] #[cfg_attr(feature = "nightly", derive(HashStable_NoContext, TyEncodable, TyDecodable))] pub enum BuiltinImplSource { + /// A built-in impl that is considered trivial, without any nested requirements. They + /// are preferred over where-clauses, and we want to track them explicitly. + Trivial, /// Some built-in impl we don't need to differentiate. This should be used /// unless more specific information is necessary. Misc, @@ -177,8 +180,9 @@ pub enum BuiltinImplSource { /// A built-in implementation of `Upcast` for trait objects to other trait objects. /// /// This can be removed when `feature(dyn_upcasting)` is stabilized, since we only - /// use it to detect when upcasting traits in hir typeck. - TraitUpcasting, + /// use it to detect when upcasting traits in hir typeck. The index is only used + /// for winnowing. + TraitUpcasting(usize), /// Unsizing a tuple like `(A, B, ..., X)` to `(A, B, ..., Y)` if `X` unsizes to `Y`. /// /// This can be removed when `feature(tuple_unsizing)` is stabilized, since we only diff --git a/compiler/stable_mir/src/abi.rs b/compiler/stable_mir/src/abi.rs index 17e6a852022..861b6692b53 100644 --- a/compiler/stable_mir/src/abi.rs +++ b/compiler/stable_mir/src/abi.rs @@ -442,6 +442,8 @@ pub enum CallConvention { PtxKernel, + GpuKernel, + X86Fastcall, X86Intr, X86Stdcall, diff --git a/compiler/stable_mir/src/ty.rs b/compiler/stable_mir/src/ty.rs index d7eb435e13f..3434597e7b0 100644 --- a/compiler/stable_mir/src/ty.rs +++ b/compiler/stable_mir/src/ty.rs @@ -1077,6 +1077,7 @@ pub enum Abi { PtxKernel, Msp430Interrupt, X86Interrupt, + GpuKernel, EfiApi, AvrInterrupt, AvrNonBlockingInterrupt, |