diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
| -rw-r--r-- | compiler/rustc_mir_transform/src/coroutine.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coroutine/by_move_body.rs | 35 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/counters.rs | 39 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/mod.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/cross_crate_inline.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/lib.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/pass_manager.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/shim.rs | 121 |
8 files changed, 50 insertions, 177 deletions
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 54b13a40e92..0d18d4fd69e 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -1964,15 +1964,21 @@ fn check_must_not_suspend_ty<'tcx>( debug!("Checking must_not_suspend for {}", ty); match *ty.kind() { - ty::Adt(..) if ty.is_box() => { - let boxed_ty = ty.boxed_ty(); - let descr_pre = &format!("{}boxed ", data.descr_pre); + ty::Adt(_, args) if ty.is_box() => { + let boxed_ty = args.type_at(0); + let allocator_ty = args.type_at(1); check_must_not_suspend_ty( tcx, boxed_ty, hir_id, param_env, - SuspendCheckData { descr_pre, ..data }, + SuspendCheckData { descr_pre: &format!("{}boxed ", data.descr_pre), ..data }, + ) || check_must_not_suspend_ty( + tcx, + allocator_ty, + hir_id, + param_env, + SuspendCheckData { descr_pre: &format!("{}allocator ", data.descr_pre), ..data }, ) } ty::Adt(def, _) => check_must_not_suspend_def(tcx, def.did(), hir_id, data), diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index e40f4520671..000b96ee801 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -67,45 +67,10 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody { by_move_body.source = mir::MirSource { instance: InstanceDef::CoroutineKindShim { coroutine_def_id: coroutine_def_id.to_def_id(), - target_kind: ty::ClosureKind::FnOnce, }, promoted: None, }; body.coroutine.as_mut().unwrap().by_move_body = Some(by_move_body); - - // If this is coming from an `AsyncFn` coroutine-closure, we must also create a by-mut body. - // This is actually just a copy of the by-ref body, but with a different self type. - // FIXME(async_closures): We could probably unify this with the by-ref body somehow. - if coroutine_kind == ty::ClosureKind::Fn { - let by_mut_coroutine_ty = Ty::new_coroutine( - tcx, - coroutine_def_id.to_def_id(), - ty::CoroutineArgs::new( - tcx, - ty::CoroutineArgsParts { - parent_args: args.as_coroutine().parent_args(), - kind_ty: Ty::from_closure_kind(tcx, ty::ClosureKind::FnMut), - resume_ty: args.as_coroutine().resume_ty(), - yield_ty: args.as_coroutine().yield_ty(), - return_ty: args.as_coroutine().return_ty(), - witness: args.as_coroutine().witness(), - tupled_upvars_ty: args.as_coroutine().tupled_upvars_ty(), - }, - ) - .args, - ); - let mut by_mut_body = body.clone(); - by_mut_body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = by_mut_coroutine_ty; - dump_mir(tcx, false, "coroutine_by_mut", &0, &by_mut_body, |_, _| Ok(())); - by_mut_body.source = mir::MirSource { - instance: InstanceDef::CoroutineKindShim { - coroutine_def_id: coroutine_def_id.to_def_id(), - target_kind: ty::ClosureKind::FnMut, - }, - promoted: None, - }; - body.coroutine.as_mut().unwrap().by_mut_body = Some(by_mut_body); - } } } diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index 9a1d8bae6b4..69dc4f2ddea 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -1,13 +1,12 @@ +use std::fmt::{self, Debug}; + use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::WithNumNodes; -use rustc_index::bit_set::BitSet; use rustc_index::IndexVec; -use rustc_middle::mir::coverage::*; +use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op}; -use super::graph::{BasicCoverageBlock, CoverageGraph, TraverseCoverageGraphWithLoops}; - -use std::fmt::{self, Debug}; +use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, TraverseCoverageGraphWithLoops}; /// The coverage counter or counter expression associated with a particular /// BCB node or BCB edge. @@ -18,10 +17,6 @@ pub(super) enum BcbCounter { } impl BcbCounter { - fn is_expression(&self) -> bool { - matches!(self, Self::Expression { .. }) - } - pub(super) fn as_term(&self) -> CovTerm { match *self { BcbCounter::Counter { id, .. } => CovTerm::Counter(id), @@ -60,10 +55,6 @@ pub(super) struct CoverageCounters { /// We currently don't iterate over this map, but if we do in the future, /// switch it back to `FxIndexMap` to avoid query stability hazards. bcb_edge_counters: FxHashMap<(BasicCoverageBlock, BasicCoverageBlock), BcbCounter>, - /// Tracks which BCBs have a counter associated with some incoming edge. - /// Only used by assertions, to verify that BCBs with incoming edge - /// counters do not have their own physical counters (expressions are allowed). - bcb_has_incoming_edge_counters: BitSet<BasicCoverageBlock>, /// Table of expression data, associating each expression ID with its /// corresponding operator (+ or -) and its LHS/RHS operands. expressions: IndexVec<ExpressionId, Expression>, @@ -83,7 +74,6 @@ impl CoverageCounters { counter_increment_sites: IndexVec::new(), bcb_counters: IndexVec::from_elem_n(None, num_bcbs), bcb_edge_counters: FxHashMap::default(), - bcb_has_incoming_edge_counters: BitSet::new_empty(num_bcbs), expressions: IndexVec::new(), }; @@ -122,14 +112,6 @@ impl CoverageCounters { } fn set_bcb_counter(&mut self, bcb: BasicCoverageBlock, counter_kind: BcbCounter) -> BcbCounter { - assert!( - // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also - // have an expression (to be injected into an existing `BasicBlock` represented by this - // `BasicCoverageBlock`). - counter_kind.is_expression() || !self.bcb_has_incoming_edge_counters.contains(bcb), - "attempt to add a `Counter` to a BCB target with existing incoming edge counters" - ); - if let Some(replaced) = self.bcb_counters[bcb].replace(counter_kind) { bug!( "attempt to set a BasicCoverageBlock coverage counter more than once; \ @@ -146,19 +128,6 @@ impl CoverageCounters { to_bcb: BasicCoverageBlock, counter_kind: BcbCounter, ) -> BcbCounter { - // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also - // have an expression (to be injected into an existing `BasicBlock` represented by this - // `BasicCoverageBlock`). - if let Some(node_counter) = self.bcb_counter(to_bcb) - && !node_counter.is_expression() - { - bug!( - "attempt to add an incoming edge counter from {from_bcb:?} \ - when the target BCB already has {node_counter:?}" - ); - } - - self.bcb_has_incoming_edge_counters.insert(to_bcb); if let Some(replaced) = self.bcb_edge_counters.insert((from_bcb, to_bcb), counter_kind) { bug!( "attempt to set an edge counter more than once; from_bcb: \ diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index b2407c54507..83189c6a50a 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -13,7 +13,6 @@ use self::spans::{BcbMapping, BcbMappingKind, CoverageSpans}; use crate::MirPass; -use rustc_middle::hir; use rustc_middle::mir::coverage::*; use rustc_middle::mir::{ self, BasicBlock, BasicBlockData, Coverage, SourceInfo, Statement, StatementKind, Terminator, @@ -368,8 +367,7 @@ fn extract_hir_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> ExtractedHir // to HIR for it. let hir_node = tcx.hir_node_by_def_id(def_id); - let (_, fn_body_id) = - hir::map::associated_body(hir_node).expect("HIR node is a function with body"); + let fn_body_id = hir_node.body_id().expect("HIR node is a function with body"); let hir_body = tcx.hir().body(fn_body_id); let maybe_fn_sig = hir_node.fn_sig(); diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 07e6ecccaa4..483fd753e70 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -23,10 +23,6 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { return false; } - if tcx.intrinsic(def_id).is_some_and(|i| i.must_be_overridden) { - return false; - } - // This just reproduces the logic from Instance::requires_inline. match tcx.def_kind(def_id) { DefKind::Ctor(..) | DefKind::Closure => return true, diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 74e7d51ea96..24bc263e5a7 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -8,7 +8,6 @@ #![feature(is_sorted)] #![feature(let_chains)] #![feature(map_try_insert)] -#![cfg_attr(bootstrap, feature(min_specialization))] #![feature(never_type)] #![feature(option_get_or_insert_default)] #![feature(round_char_boundary)] @@ -638,12 +637,6 @@ fn optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> &Body<'_> { } fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { - if tcx.intrinsic(did).is_some_and(|i| i.must_be_overridden) { - span_bug!( - tcx.def_span(did), - "this intrinsic must be overridden by the codegen backend, it has no meaningful body", - ) - } if tcx.is_constructor(did.to_def_id()) { // There's no reason to run all of the MIR passes on constructors when // we can just output the MIR we want directly. This also saves const diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 77478cc741d..17a1c3c7157 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -186,9 +186,6 @@ fn run_passes_inner<'tcx>( if let Some(by_move_body) = coroutine.by_move_body.as_mut() { run_passes_inner(tcx, by_move_body, passes, phase_change, validate_each); } - if let Some(by_mut_body) = coroutine.by_mut_body.as_mut() { - run_passes_inner(tcx, by_mut_body, passes, phase_change, validate_each); - } } } diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index dd2b24ad669..71de64c6d26 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -3,8 +3,8 @@ use rustc_hir::def_id::DefId; use rustc_hir::lang_items::LangItem; use rustc_middle::mir::*; use rustc_middle::query::Providers; +use rustc_middle::ty::GenericArgs; use rustc_middle::ty::{self, CoroutineArgs, EarlyBinder, Ty, TyCtxt}; -use rustc_middle::ty::{GenericArgs, CAPTURE_STRUCT_LOCAL}; use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT}; use rustc_index::{Idx, IndexVec}; @@ -72,37 +72,12 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' ty::InstanceDef::ConstructCoroutineInClosureShim { coroutine_closure_def_id, - target_kind, - } => match target_kind { - ty::ClosureKind::Fn => unreachable!("shouldn't be building shim for Fn"), - ty::ClosureKind::FnMut => { - // No need to optimize the body, it has already been optimized - // since we steal it from the `AsyncFn::call` body and just fix - // the return type. - return build_construct_coroutine_by_mut_shim(tcx, coroutine_closure_def_id); - } - ty::ClosureKind::FnOnce => { - build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id) - } - }, + receiver_by_ref, + } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref), - ty::InstanceDef::CoroutineKindShim { coroutine_def_id, target_kind } => match target_kind { - ty::ClosureKind::Fn => unreachable!(), - ty::ClosureKind::FnMut => { - return tcx - .optimized_mir(coroutine_def_id) - .coroutine_by_mut_body() - .unwrap() - .clone(); - } - ty::ClosureKind::FnOnce => { - return tcx - .optimized_mir(coroutine_def_id) - .coroutine_by_move_body() - .unwrap() - .clone(); - } - }, + ty::InstanceDef::CoroutineKindShim { coroutine_def_id } => { + return tcx.optimized_mir(coroutine_def_id).coroutine_by_move_body().unwrap().clone(); + } ty::InstanceDef::DropGlue(def_id, ty) => { // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end @@ -123,21 +98,11 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() { coroutine_body.coroutine_drop().unwrap() } else { - match args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap() { - ty::ClosureKind::Fn => { - unreachable!() - } - ty::ClosureKind::FnMut => coroutine_body - .coroutine_by_mut_body() - .unwrap() - .coroutine_drop() - .unwrap(), - ty::ClosureKind::FnOnce => coroutine_body - .coroutine_by_move_body() - .unwrap() - .coroutine_drop() - .unwrap(), - } + assert_eq!( + args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(), + ty::ClosureKind::FnOnce + ); + coroutine_body.coroutine_by_move_body().unwrap().coroutine_drop().unwrap() }; let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args); @@ -1053,12 +1018,26 @@ fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'t fn build_construct_coroutine_by_move_shim<'tcx>( tcx: TyCtxt<'tcx>, coroutine_closure_def_id: DefId, + receiver_by_ref: bool, ) -> Body<'tcx> { - let self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity(); + let mut self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity(); let ty::CoroutineClosure(_, args) = *self_ty.kind() else { bug!(); }; + // We use `*mut Self` here because we only need to emit an ABI-compatible shim body, + // rather than match the signature exactly. + // + // The self type here is a coroutine-closure, not a coroutine, and we never read from + // it because it never has any captures, because this is only true in the Fn/FnMut + // implementation, not the AsyncFn/AsyncFnMut implementation, which is implemented only + // if the coroutine-closure has no captures. + if receiver_by_ref { + // Triple-check that there's no captures here. + assert_eq!(args.as_coroutine_closure().tupled_upvars_ty(), tcx.types.unit); + self_ty = Ty::new_mut_ptr(tcx, self_ty); + } + let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| { tcx.mk_fn_sig( [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()), @@ -1114,49 +1093,19 @@ fn build_construct_coroutine_by_move_shim<'tcx>( let source = MirSource::from_instance(ty::InstanceDef::ConstructCoroutineInClosureShim { coroutine_closure_def_id, - target_kind: ty::ClosureKind::FnOnce, + receiver_by_ref, }); let body = new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span); - dump_mir(tcx, false, "coroutine_closure_by_move", &0, &body, |_, _| Ok(())); - - body -} - -fn build_construct_coroutine_by_mut_shim<'tcx>( - tcx: TyCtxt<'tcx>, - coroutine_closure_def_id: DefId, -) -> Body<'tcx> { - let mut body = tcx.optimized_mir(coroutine_closure_def_id).clone(); - let coroutine_closure_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity(); - let ty::CoroutineClosure(_, args) = *coroutine_closure_ty.kind() else { - bug!(); - }; - let args = args.as_coroutine_closure(); - - body.local_decls[RETURN_PLACE].ty = - tcx.instantiate_bound_regions_with_erased(args.coroutine_closure_sig().map_bound(|sig| { - sig.to_coroutine_given_kind_and_upvars( - tcx, - args.parent_args(), - tcx.coroutine_for_closure(coroutine_closure_def_id), - ty::ClosureKind::FnMut, - tcx.lifetimes.re_erased, - args.tupled_upvars_ty(), - args.coroutine_captures_by_ref_ty(), - ) - })); - body.local_decls[CAPTURE_STRUCT_LOCAL].ty = - Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_closure_ty); - - body.source = MirSource::from_instance(ty::InstanceDef::ConstructCoroutineInClosureShim { - coroutine_closure_def_id, - target_kind: ty::ClosureKind::FnMut, - }); - - body.pass_count = 0; - dump_mir(tcx, false, "coroutine_closure_by_mut", &0, &body, |_, _| Ok(())); + dump_mir( + tcx, + false, + if receiver_by_ref { "coroutine_closure_by_ref" } else { "coroutine_closure_by_move" }, + &0, + &body, + |_, _| Ok(()), + ); body } |