diff options
Diffstat (limited to 'compiler/rustc_middle/src')
30 files changed, 607 insertions, 311 deletions
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs index d1bbb0598fe..2dcb0de92b7 100644 --- a/compiler/rustc_middle/src/arena.rs +++ b/compiler/rustc_middle/src/arena.rs @@ -9,6 +9,7 @@ macro_rules! arena_types { ($macro:path) => ( $macro!([ [] layout: rustc_abi::LayoutData<rustc_abi::FieldIdx, rustc_abi::VariantIdx>, + [] proxy_coroutine_layout: rustc_middle::mir::CoroutineLayout<'tcx>, [] fn_abi: rustc_target::callconv::FnAbi<'tcx, rustc_middle::ty::Ty<'tcx>>, // AdtDef are interned and compared by address [decode] adt_def: rustc_middle::ty::AdtDefData, diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index 644cdac5d55..0c998a2cbb3 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -13,8 +13,11 @@ use crate::ty::TyCtxt; macro_rules! define_dep_nodes { ( - $($(#[$attr:meta])* - [$($modifiers:tt)*] fn $variant:ident($($K:tt)*) -> $V:ty,)*) => { + $( + $(#[$attr:meta])* + [$($modifiers:tt)*] fn $variant:ident($($K:tt)*) -> $V:ty, + )* + ) => { #[macro_export] macro_rules! make_dep_kind_array { @@ -83,7 +86,9 @@ macro_rules! define_dep_nodes { }; } -rustc_query_append!(define_dep_nodes![ +// Create various data structures for each query, and also for a few things +// that aren't queries. +rustc_with_all_queries!(define_dep_nodes![ /// We use this for most things when incr. comp. is turned off. [] fn Null() -> (), /// We use this to create a forever-red node. diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index df025aeebf0..711036865bb 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -61,6 +61,7 @@ #![feature(try_trait_v2_yeet)] #![feature(type_alias_impl_trait)] #![feature(yeet_expr)] +#![recursion_limit = "256"] // tidy-alphabetical-end #[cfg(test)] diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs index 0bfbd398797..3c62017072e 100644 --- a/compiler/rustc_middle/src/middle/exported_symbols.rs +++ b/compiler/rustc_middle/src/middle/exported_symbols.rs @@ -44,6 +44,7 @@ pub enum ExportedSymbol<'tcx> { Generic(DefId, GenericArgsRef<'tcx>), DropGlue(Ty<'tcx>), AsyncDropGlueCtorShim(Ty<'tcx>), + AsyncDropGlue(DefId, Ty<'tcx>), ThreadLocalShim(DefId), NoDefId(ty::SymbolName<'tcx>), } @@ -63,6 +64,9 @@ impl<'tcx> ExportedSymbol<'tcx> { ExportedSymbol::AsyncDropGlueCtorShim(ty) => { tcx.symbol_name(ty::Instance::resolve_async_drop_in_place(tcx, ty)) } + ExportedSymbol::AsyncDropGlue(def_id, ty) => { + tcx.symbol_name(ty::Instance::resolve_async_drop_in_place_poll(tcx, def_id, ty)) + } ExportedSymbol::ThreadLocalShim(def_id) => tcx.symbol_name(ty::Instance { def: ty::InstanceKind::ThreadLocalShim(def_id), args: ty::GenericArgs::empty(), diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs index 4222a68e544..e5d1dda3aa0 100644 --- a/compiler/rustc_middle/src/mir/interpret/queries.rs +++ b/compiler/rustc_middle/src/mir/interpret/queries.rs @@ -115,15 +115,16 @@ impl<'tcx> TyCtxt<'tcx> { // @lcnr believes that successfully evaluating even though there are // used generic parameters is a bug of evaluation, so checking for it // here does feel somewhat sensible. - if !self.features().generic_const_exprs() && ct.args.has_non_region_param() { - let def_kind = self.def_kind(instance.def_id()); - assert!( - matches!( - def_kind, - DefKind::InlineConst | DefKind::AnonConst | DefKind::AssocConst - ), - "{cid:?} is {def_kind:?}", - ); + if !self.features().generic_const_exprs() + && ct.args.has_non_region_param() + // We only FCW for anon consts as repeat expr counts with anon consts are the only place + // that we have a back compat hack for. We don't need to check this is a const argument + // as only anon consts as const args should get evaluated "for the type system". + // + // If we don't *only* FCW anon consts we can wind up incorrectly FCW'ing uses of assoc + // consts in pattern positions. #140447 + && self.def_kind(instance.def_id()) == DefKind::AnonConst + { let mir_body = self.mir_for_ctfe(instance.def_id()); if mir_body.is_polymorphic { let Some(local_def_id) = ct.def.as_local() else { return }; diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index db19c858e7c..adc100941a3 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -200,7 +200,13 @@ pub struct CoroutineInfo<'tcx> { /// Coroutine drop glue. This field is populated after the state transform pass. pub coroutine_drop: Option<Body<'tcx>>, - /// The layout of a coroutine. This field is populated after the state transform pass. + /// Coroutine async drop glue. + pub coroutine_drop_async: Option<Body<'tcx>>, + + /// When coroutine has sync drop, this is async proxy calling `coroutine_drop` sync impl. + pub coroutine_drop_proxy_async: Option<Body<'tcx>>, + + /// The layout of a coroutine. Produced by the state transformation. pub coroutine_layout: Option<CoroutineLayout<'tcx>>, /// If this is a coroutine then record the type of source expression that caused this coroutine @@ -220,6 +226,8 @@ impl<'tcx> CoroutineInfo<'tcx> { yield_ty: Some(yield_ty), resume_ty: Some(resume_ty), coroutine_drop: None, + coroutine_drop_async: None, + coroutine_drop_proxy_async: None, coroutine_layout: None, } } @@ -588,6 +596,26 @@ impl<'tcx> Body<'tcx> { } #[inline] + pub fn coroutine_drop_async(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop_async.as_ref()) + } + + #[inline] + pub fn coroutine_requires_async_drop(&self) -> bool { + self.coroutine_drop_async().is_some() + } + + #[inline] + pub fn future_drop_poll(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref().and_then(|coroutine| { + coroutine + .coroutine_drop_async + .as_ref() + .or(coroutine.coroutine_drop_proxy_async.as_ref()) + }) + } + + #[inline] pub fn coroutine_kind(&self) -> Option<CoroutineKind> { self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind) } diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 1ffe958dbdd..6b413a7383a 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -530,6 +530,8 @@ impl<'tcx> CodegenUnit<'tcx> { | InstanceKind::CloneShim(..) | InstanceKind::ThreadLocalShim(..) | InstanceKind::FnPtrAddrShim(..) + | InstanceKind::AsyncDropGlue(..) + | InstanceKind::FutureDropPollShim(..) | InstanceKind::AsyncDropGlueCtorShim(..) => None, } } diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index d57019fee0f..57ae7dc55c5 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -253,9 +253,7 @@ fn dump_path<'tcx>( })); s } - ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) => { - // Unfortunately, pretty-printed typed are not very filename-friendly. - // We dome some filtering. + ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) => { let mut s = ".".to_owned(); s.extend(ty.to_string().chars().filter_map(|c| match c { ' ' => None, @@ -264,6 +262,34 @@ fn dump_path<'tcx>( })); s } + ty::InstanceKind::AsyncDropGlue(_, ty) => { + let ty::Coroutine(_, args) = ty.kind() else { + bug!(); + }; + let ty = args.first().unwrap().expect_ty(); + let mut s = ".".to_owned(); + s.extend(ty.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s + } + ty::InstanceKind::FutureDropPollShim(_, proxy_cor, impl_cor) => { + let mut s = ".".to_owned(); + s.extend(proxy_cor.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s.push('.'); + s.extend(impl_cor.to_string().chars().filter_map(|c| match c { + ' ' => None, + ':' | '<' | '>' => Some('_'), + c => Some(c), + })); + s + } _ => String::new(), }; @@ -1050,7 +1076,13 @@ impl<'tcx> TerminatorKind<'tcx> { Call { target: None, unwind: _, .. } => vec![], Yield { drop: Some(_), .. } => vec!["resume".into(), "drop".into()], Yield { drop: None, .. } => vec!["resume".into()], - Drop { unwind: UnwindAction::Cleanup(_), .. } => vec!["return".into(), "unwind".into()], + Drop { unwind: UnwindAction::Cleanup(_), drop: Some(_), .. } => { + vec!["return".into(), "unwind".into(), "drop".into()] + } + Drop { unwind: UnwindAction::Cleanup(_), drop: None, .. } => { + vec!["return".into(), "unwind".into()] + } + Drop { unwind: _, drop: Some(_), .. } => vec!["return".into(), "drop".into()], Drop { unwind: _, .. } => vec!["return".into()], Assert { unwind: UnwindAction::Cleanup(_), .. } => { vec!["success".into(), "unwind".into()] diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 304b3caa6e1..bb068f3821d 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -77,6 +77,8 @@ pub enum MirPhase { /// exception is fields of packed structs. In analysis MIR, `Drop(P)` for a `P` that might be /// misaligned for this reason implicitly moves `P` to a temporary before dropping. Runtime /// MIR has no such rules, and dropping a misaligned place is simply UB. + /// - Async drops: after drop elaboration some drops may become async (`drop`, `async_fut` fields). + /// StateTransform pass will expand those async drops or reset to sync. /// - Unwinding: in analysis MIR, unwinding from a function which may not unwind aborts. In /// runtime MIR, this is UB. /// - Retags: If `-Zmir-emit-retag` is enabled, analysis MIR has "implicit" retags in the same @@ -768,7 +770,34 @@ pub enum TerminatorKind<'tcx> { /// The `replace` flag indicates whether this terminator was created as part of an assignment. /// This should only be used for diagnostic purposes, and does not have any operational /// meaning. - Drop { place: Place<'tcx>, target: BasicBlock, unwind: UnwindAction, replace: bool }, + /// + /// Async drop processing: + /// In compiler/rustc_mir_build/src/build/scope.rs we detect possible async drop: + /// drop of object with `needs_async_drop`. + /// Async drop later, in StateTransform pass, may be expanded into additional yield-point + /// for poll-loop of async drop future. + /// So we need prepared 'drop' target block in the similar way as for `Yield` terminator + /// (see `drops.build_mir::<CoroutineDrop>` in scopes.rs). + /// In compiler/rustc_mir_transform/src/elaborate_drops.rs for object implementing `AsyncDrop` trait + /// we need to prepare async drop feature - resolve `AsyncDrop::drop` and codegen call. + /// `async_fut` is set to the corresponding local. + /// For coroutine drop we don't need this logic because coroutine drop works with the same + /// layout object as coroutine itself. So `async_fut` will be `None` for coroutine drop. + /// Both `drop` and `async_fut` fields are only used in compiler/rustc_mir_transform/src/coroutine.rs, + /// StateTransform pass. In `expand_async_drops` async drops are expanded + /// into one or two yield points with poll ready/pending switch. + /// When a coroutine has any internal async drop, the coroutine drop function will be async + /// (generated by `create_coroutine_drop_shim_async`, not `create_coroutine_drop_shim`). + Drop { + place: Place<'tcx>, + target: BasicBlock, + unwind: UnwindAction, + replace: bool, + /// Cleanup to be done if the coroutine is dropped at this suspend point (for async drop). + drop: Option<BasicBlock>, + /// Prepared async future local (for async drop) + async_fut: Option<Local>, + }, /// Roughly speaking, evaluates the `func` operand and the arguments, and starts execution of /// the referred to function. The operand types must match the argument types of the function. @@ -1043,6 +1072,7 @@ pub enum AssertKind<O> { RemainderByZero(O), ResumedAfterReturn(CoroutineKind), ResumedAfterPanic(CoroutineKind), + ResumedAfterDrop(CoroutineKind), MisalignedPointerDereference { required: O, found: O }, NullPointerDereference, } diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs index 82e8422c52d..8a1ead7d19d 100644 --- a/compiler/rustc_middle/src/mir/terminator.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -208,6 +208,16 @@ impl<O> AssertKind<O> { LangItem::PanicGenFnNonePanic } NullPointerDereference => LangItem::PanicNullPointerDereference, + ResumedAfterDrop(CoroutineKind::Coroutine(_)) => LangItem::PanicCoroutineResumedDrop, + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::Async, _)) => { + LangItem::PanicAsyncFnResumedDrop + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::AsyncGen, _)) => { + LangItem::PanicAsyncGenFnResumedDrop + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::Gen, _)) => { + LangItem::PanicGenFnNoneDrop + } BoundsCheck { .. } | MisalignedPointerDereference { .. } => { bug!("Unexpected AssertKind") @@ -298,6 +308,18 @@ impl<O> AssertKind<O> { ResumedAfterPanic(CoroutineKind::Desugared(CoroutineDesugaring::Gen, _)) => { write!(f, "\"`gen fn` should just keep returning `None` after panicking\"") } + ResumedAfterDrop(CoroutineKind::Coroutine(_)) => { + write!(f, "\"coroutine resumed after async drop\"") + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::Async, _)) => { + write!(f, "\"`async fn` resumed after async drop\"") + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::AsyncGen, _)) => { + write!(f, "\"`async gen fn` resumed after async drop\"") + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::Gen, _)) => { + write!(f, "\"`gen fn` resumed after drop\"") + } } } @@ -345,6 +367,19 @@ impl<O> AssertKind<O> { middle_assert_coroutine_resume_after_panic } NullPointerDereference => middle_assert_null_ptr_deref, + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::Async, _)) => { + middle_assert_async_resume_after_drop + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::AsyncGen, _)) => { + todo!() + } + ResumedAfterDrop(CoroutineKind::Desugared(CoroutineDesugaring::Gen, _)) => { + middle_assert_gen_resume_after_drop + } + ResumedAfterDrop(CoroutineKind::Coroutine(_)) => { + middle_assert_coroutine_resume_after_drop + } + MisalignedPointerDereference { .. } => middle_assert_misaligned_ptr_deref, } } @@ -377,7 +412,10 @@ impl<O> AssertKind<O> { add!("left", format!("{left:#?}")); add!("right", format!("{right:#?}")); } - ResumedAfterReturn(_) | ResumedAfterPanic(_) | NullPointerDereference => {} + ResumedAfterReturn(_) + | ResumedAfterPanic(_) + | NullPointerDereference + | ResumedAfterDrop(_) => {} MisalignedPointerDereference { required, found } => { add!("required", format!("{required:#?}")); add!("found", format!("{found:#?}")); @@ -457,7 +495,7 @@ mod helper { #[define_opaque(Successors)] pub fn successors_for_value(&self, value: u128) -> Successors<'_> { let target = self.target_for_value(value); - (&[]).into_iter().copied().chain(Some(target)) + (&[]).into_iter().copied().chain(Some(target).into_iter().chain(None)) } } @@ -467,13 +505,23 @@ mod helper { pub fn successors(&self) -> Successors<'_> { use self::TerminatorKind::*; match *self { + // 3-successors for async drop: target, unwind, dropline (parent coroutine drop) + Drop { target: ref t, unwind: UnwindAction::Cleanup(u), drop: Some(d), .. } => { + slice::from_ref(t) + .into_iter() + .copied() + .chain(Some(u).into_iter().chain(Some(d))) + } + // 2-successors Call { target: Some(ref t), unwind: UnwindAction::Cleanup(u), .. } | Yield { resume: ref t, drop: Some(u), .. } - | Drop { target: ref t, unwind: UnwindAction::Cleanup(u), .. } + | Drop { target: ref t, unwind: UnwindAction::Cleanup(u), drop: None, .. } + | Drop { target: ref t, unwind: _, drop: Some(u), .. } | Assert { target: ref t, unwind: UnwindAction::Cleanup(u), .. } | FalseUnwind { real_target: ref t, unwind: UnwindAction::Cleanup(u) } => { - slice::from_ref(t).into_iter().copied().chain(Some(u)) + slice::from_ref(t).into_iter().copied().chain(Some(u).into_iter().chain(None)) } + // single successor Goto { target: ref t } | Call { target: None, unwind: UnwindAction::Cleanup(ref t), .. } | Call { target: Some(ref t), unwind: _, .. } @@ -481,23 +529,33 @@ mod helper { | Drop { target: ref t, unwind: _, .. } | Assert { target: ref t, unwind: _, .. } | FalseUnwind { real_target: ref t, unwind: _ } => { - slice::from_ref(t).into_iter().copied().chain(None) + slice::from_ref(t).into_iter().copied().chain(None.into_iter().chain(None)) } + // No successors UnwindResume | UnwindTerminate(_) | CoroutineDrop | Return | Unreachable | TailCall { .. } - | Call { target: None, unwind: _, .. } => (&[]).into_iter().copied().chain(None), + | Call { target: None, unwind: _, .. } => { + (&[]).into_iter().copied().chain(None.into_iter().chain(None)) + } + // Multiple successors InlineAsm { ref targets, unwind: UnwindAction::Cleanup(u), .. } => { - targets.iter().copied().chain(Some(u)) + targets.iter().copied().chain(Some(u).into_iter().chain(None)) + } + InlineAsm { ref targets, unwind: _, .. } => { + targets.iter().copied().chain(None.into_iter().chain(None)) } - InlineAsm { ref targets, unwind: _, .. } => targets.iter().copied().chain(None), - SwitchInt { ref targets, .. } => targets.targets.iter().copied().chain(None), - FalseEdge { ref real_target, imaginary_target } => { - slice::from_ref(real_target).into_iter().copied().chain(Some(imaginary_target)) + SwitchInt { ref targets, .. } => { + targets.targets.iter().copied().chain(None.into_iter().chain(None)) } + // FalseEdge + FalseEdge { ref real_target, imaginary_target } => slice::from_ref(real_target) + .into_iter() + .copied() + .chain(Some(imaginary_target).into_iter().chain(None)), } } @@ -506,16 +564,31 @@ mod helper { pub fn successors_mut(&mut self) -> SuccessorsMut<'_> { use self::TerminatorKind::*; match *self { + // 3-successors for async drop: target, unwind, dropline (parent coroutine drop) + Drop { + target: ref mut t, + unwind: UnwindAction::Cleanup(ref mut u), + drop: Some(ref mut d), + .. + } => slice::from_mut(t).into_iter().chain(Some(u).into_iter().chain(Some(d))), + // 2-successors Call { target: Some(ref mut t), unwind: UnwindAction::Cleanup(ref mut u), .. } | Yield { resume: ref mut t, drop: Some(ref mut u), .. } - | Drop { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. } + | Drop { + target: ref mut t, + unwind: UnwindAction::Cleanup(ref mut u), + drop: None, + .. + } + | Drop { target: ref mut t, unwind: _, drop: Some(ref mut u), .. } | Assert { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. } | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), - } => slice::from_mut(t).into_iter().chain(Some(u)), + } => slice::from_mut(t).into_iter().chain(Some(u).into_iter().chain(None)), + // single successor Goto { target: ref mut t } | Call { target: None, unwind: UnwindAction::Cleanup(ref mut t), .. } | Call { target: Some(ref mut t), unwind: _, .. } @@ -523,22 +596,33 @@ mod helper { | Drop { target: ref mut t, unwind: _, .. } | Assert { target: ref mut t, unwind: _, .. } | FalseUnwind { real_target: ref mut t, unwind: _ } => { - slice::from_mut(t).into_iter().chain(None) + slice::from_mut(t).into_iter().chain(None.into_iter().chain(None)) } + // No successors UnwindResume | UnwindTerminate(_) | CoroutineDrop | Return | Unreachable | TailCall { .. } - | Call { target: None, unwind: _, .. } => (&mut []).into_iter().chain(None), + | Call { target: None, unwind: _, .. } => { + (&mut []).into_iter().chain(None.into_iter().chain(None)) + } + // Multiple successors InlineAsm { ref mut targets, unwind: UnwindAction::Cleanup(ref mut u), .. } => { - targets.iter_mut().chain(Some(u)) + targets.iter_mut().chain(Some(u).into_iter().chain(None)) + } + InlineAsm { ref mut targets, unwind: _, .. } => { + targets.iter_mut().chain(None.into_iter().chain(None)) + } + SwitchInt { ref mut targets, .. } => { + targets.targets.iter_mut().chain(None.into_iter().chain(None)) } - InlineAsm { ref mut targets, unwind: _, .. } => targets.iter_mut().chain(None), - SwitchInt { ref mut targets, .. } => targets.targets.iter_mut().chain(None), + // FalseEdge FalseEdge { ref mut real_target, ref mut imaginary_target } => { - slice::from_mut(real_target).into_iter().chain(Some(imaginary_target)) + slice::from_mut(real_target) + .into_iter() + .chain(Some(imaginary_target).into_iter().chain(None)) } } } @@ -671,8 +755,10 @@ impl<'tcx> TerminatorKind<'tcx> { Goto { target } => TerminatorEdges::Single(target), + // FIXME: Maybe we need also TerminatorEdges::Trio for async drop + // (target + unwind + dropline) Assert { target, unwind, expected: _, msg: _, cond: _ } - | Drop { target, unwind, place: _, replace: _ } + | Drop { target, unwind, place: _, replace: _, drop: _, async_fut: _ } | FalseUnwind { real_target: target, unwind } => match unwind { UnwindAction::Cleanup(unwind) => TerminatorEdges::Double(target, unwind), UnwindAction::Continue | UnwindAction::Terminate(_) | UnwindAction::Unreachable => { diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index de4d5140e85..1777756174b 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -353,17 +353,21 @@ macro_rules! make_mir_visitor { coroutine_closure_def_id: _def_id, receiver_by_ref: _, } - | ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, None) | ty::InstanceKind::DropGlue(_def_id, None) => {} ty::InstanceKind::FnPtrShim(_def_id, ty) | ty::InstanceKind::DropGlue(_def_id, Some(ty)) | ty::InstanceKind::CloneShim(_def_id, ty) | ty::InstanceKind::FnPtrAddrShim(_def_id, ty) - | ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, Some(ty)) => { + | ty::InstanceKind::AsyncDropGlue(_def_id, ty) + | ty::InstanceKind::AsyncDropGlueCtorShim(_def_id, ty) => { // FIXME(eddyb) use a better `TyContext` here. self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); } + ty::InstanceKind::FutureDropPollShim(_def_id, proxy_ty, impl_ty) => { + self.visit_ty($(& $mutability)? *proxy_ty, TyContext::Location(location)); + self.visit_ty($(& $mutability)? *impl_ty, TyContext::Location(location)); + } } self.visit_args(callee_args, location); } @@ -521,7 +525,14 @@ macro_rules! make_mir_visitor { self.visit_operand(discr, location); } - TerminatorKind::Drop { place, target: _, unwind: _, replace: _ } => { + TerminatorKind::Drop { + place, + target: _, + unwind: _, + replace: _, + drop: _, + async_fut: _, + } => { self.visit_place( place, PlaceContext::MutatingUse(MutatingUseContext::Drop), @@ -634,7 +645,7 @@ macro_rules! make_mir_visitor { OverflowNeg(op) | DivisionByZero(op) | RemainderByZero(op) => { self.visit_operand(op, location); } - ResumedAfterReturn(_) | ResumedAfterPanic(_) | NullPointerDereference => { + ResumedAfterReturn(_) | ResumedAfterPanic(_) | NullPointerDereference | ResumedAfterDrop(_) => { // Nothing to visit } MisalignedPointerDereference { required, found } => { diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 3432648feab..88f4c4ae4d3 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -289,7 +289,7 @@ rustc_queries! { /// Returns whether the type alias given by `DefId` is lazy. /// - /// I.e., if the type alias expands / ought to expand to a [weak] [alias type] + /// I.e., if the type alias expands / ought to expand to a [free] [alias type] /// instead of the underyling aliased type. /// /// Relevant for features `lazy_type_alias` and `type_alias_impl_trait`. @@ -298,7 +298,7 @@ rustc_queries! { /// /// This query *may* panic if the given definition is not a type alias. /// - /// [weak]: rustc_middle::ty::Weak + /// [free]: rustc_middle::ty::Free /// [alias type]: rustc_middle::ty::AliasTy query type_alias_is_lazy(key: DefId) -> bool { desc { |tcx| @@ -1359,7 +1359,11 @@ rustc_queries! { /// Generates a MIR body for the shim. query mir_shims(key: ty::InstanceKind<'tcx>) -> &'tcx mir::Body<'tcx> { arena_cache - desc { |tcx| "generating MIR shim for `{}`", tcx.def_path_str(key.def_id()) } + desc { + |tcx| "generating MIR shim for `{}`, instance={:?}", + tcx.def_path_str(key.def_id()), + key + } } /// The `symbol_name` query provides the symbol name for calling a @@ -1590,6 +1594,10 @@ rustc_queries! { query is_unpin_raw(env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool { desc { "computing whether `{}` is `Unpin`", env.value } } + /// Query backing `Ty::is_async_drop`. + query is_async_drop_raw(env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool { + desc { "computing whether `{}` is `AsyncDrop`", env.value } + } /// Query backing `Ty::needs_drop`. query needs_drop_raw(env: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool { desc { "computing whether `{}` needs drop", env.value } @@ -1622,6 +1630,14 @@ rustc_queries! { cache_on_disk_if { true } } + /// A list of types where the ADT requires async drop if and only if any of + /// those types require async drop. If the ADT is known to always need async drop + /// then `Err(AlwaysRequiresDrop)` is returned. + query adt_async_drop_tys(def_id: DefId) -> Result<&'tcx ty::List<Ty<'tcx>>, AlwaysRequiresDrop> { + desc { |tcx| "computing when `{}` needs async drop", tcx.def_path_str(def_id) } + cache_on_disk_if { true } + } + /// A list of types where the ADT requires drop if and only if any of those types /// has significant drop. A type marked with the attribute `rustc_insignificant_dtor` /// is considered to not be significant. A drop is significant if it is implemented @@ -2280,7 +2296,7 @@ rustc_queries! { /// Do not call this query directly: Invoke `normalize` instead. /// /// </div> - query normalize_canonicalized_weak_ty( + query normalize_canonicalized_free_alias( goal: CanonicalAliasGoal<'tcx> ) -> Result< &'tcx Canonical<'tcx, canonical::QueryResponse<'tcx, NormalizationResult<'tcx>>>, @@ -2562,5 +2578,5 @@ rustc_queries! { } } -rustc_query_append! { define_callbacks! } +rustc_with_all_queries! { define_callbacks! } rustc_feedable_queries! { define_feedable! } diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 69b6f88d72b..769df1ffd6f 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -313,8 +313,11 @@ macro_rules! separate_provide_extern_default { macro_rules! define_callbacks { ( - $($(#[$attr:meta])* - [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { + $( + $(#[$attr:meta])* + [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty, + )* + ) => { #[allow(unused_lifetimes)] pub mod queries { diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs index c168142fb1e..086ec529f33 100644 --- a/compiler/rustc_middle/src/thir.rs +++ b/compiler/rustc_middle/src/thir.rs @@ -799,7 +799,12 @@ pub enum PatKind<'tcx> { /// Deref pattern, written `box P` for now. DerefPattern { subpattern: Box<Pat<'tcx>>, - mutability: hir::Mutability, + /// Whether the pattern scrutinee needs to be borrowed in order to call `Deref::deref` or + /// `DerefMut::deref_mut`, and if so, which. This is `ByRef::No` for deref patterns on + /// boxes; they are lowered using a built-in deref rather than a method call, thus they + /// don't borrow the scrutinee. + #[type_visitable(ignore)] + borrow: ByRef, }, /// One of the following: diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs index 7e6151745e2..27079af06fc 100644 --- a/compiler/rustc_middle/src/traits/mod.rs +++ b/compiler/rustc_middle/src/traits/mod.rs @@ -404,7 +404,7 @@ pub enum ObligationCauseCode<'tcx> { /// Requirement for a `const N: Ty` to implement `Ty: ConstParamTy` ConstParam(Ty<'tcx>), - /// Obligations emitted during the normalization of a weak type alias. + /// Obligations emitted during the normalization of a free type alias. TypeAlias(ObligationCauseCodeHandle<'tcx>, Span, DefId), } diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs index 76f3d2bab9c..3f6faa1a572 100644 --- a/compiler/rustc_middle/src/traits/query.rs +++ b/compiler/rustc_middle/src/traits/query.rs @@ -181,7 +181,7 @@ pub struct MethodAutoderefBadTy<'tcx> { pub ty: Canonical<'tcx, QueryResponse<'tcx, Ty<'tcx>>>, } -/// Result of the `normalize_canonicalized_{{,inherent_}projection,weak}_ty` queries. +/// Result of the `normalize_canonicalized_{{,inherent_}projection,free}_ty` queries. #[derive(Clone, Debug, HashStable, TypeFoldable, TypeVisitable)] pub struct NormalizationResult<'tcx> { /// Result of the normalization. diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs index 23927c112bc..5ff87959a80 100644 --- a/compiler/rustc_middle/src/ty/codec.rs +++ b/compiler/rustc_middle/src/ty/codec.rs @@ -442,6 +442,15 @@ impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for ty::List<ty::BoundVaria } } +impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for ty::List<ty::Pattern<'tcx>> { + fn decode(decoder: &mut D) -> &'tcx Self { + let len = decoder.read_usize(); + decoder.interner().mk_patterns_from_iter( + (0..len).map::<ty::Pattern<'tcx>, _>(|_| Decodable::decode(decoder)), + ) + } +} + impl<'tcx, D: TyDecoder<'tcx>> RefDecodable<'tcx, D> for ty::List<ty::Const<'tcx>> { fn decode(decoder: &mut D) -> &'tcx Self { let len = decoder.read_usize(); @@ -503,6 +512,7 @@ impl_decodable_via_ref! { &'tcx mir::Body<'tcx>, &'tcx mir::ConcreteOpaqueTypes<'tcx>, &'tcx ty::List<ty::BoundVariableKind>, + &'tcx ty::List<ty::Pattern<'tcx>>, &'tcx ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>, &'tcx ty::List<FieldIdx>, &'tcx ty::List<(VariantIdx, FieldIdx)>, diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index e8dad1e056c..3ea285d3d8e 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -21,6 +21,7 @@ use rustc_data_structures::defer; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::intern::Interned; +use rustc_data_structures::jobserver::Proxy; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -136,6 +137,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { type AllocId = crate::mir::interpret::AllocId; type Pat = Pattern<'tcx>; + type PatList = &'tcx List<Pattern<'tcx>>; type Safety = hir::Safety; type Abi = ExternAbi; type Const = ty::Const<'tcx>; @@ -226,7 +228,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { } } DefKind::OpaqueTy => ty::Opaque, - DefKind::TyAlias => ty::Weak, + DefKind::TyAlias => ty::Free, kind => bug!("unexpected DefKind in AliasTy: {kind:?}"), } } @@ -242,7 +244,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { } } DefKind::OpaqueTy => ty::AliasTermKind::OpaqueTy, - DefKind::TyAlias => ty::AliasTermKind::WeakTy, + DefKind::TyAlias => ty::AliasTermKind::FreeTy, DefKind::AssocConst => ty::AliasTermKind::ProjectionConst, DefKind::AnonConst | DefKind::Const | DefKind::Ctor(_, CtorKind::Const) => { ty::AliasTermKind::UnevaluatedConst @@ -717,7 +719,6 @@ macro_rules! bidirectional_lang_item_map { bidirectional_lang_item_map! { // tidy-alphabetical-start - AsyncDestruct, AsyncFn, AsyncFnKindHelper, AsyncFnKindUpvars, @@ -843,6 +844,7 @@ pub struct CtxtInterners<'tcx> { captures: InternedSet<'tcx, List<&'tcx ty::CapturedPlace<'tcx>>>, offset_of: InternedSet<'tcx, List<(VariantIdx, FieldIdx)>>, valtree: InternedSet<'tcx, ty::ValTreeKind<'tcx>>, + patterns: InternedSet<'tcx, List<ty::Pattern<'tcx>>>, } impl<'tcx> CtxtInterners<'tcx> { @@ -879,6 +881,7 @@ impl<'tcx> CtxtInterners<'tcx> { captures: InternedSet::with_capacity(N), offset_of: InternedSet::with_capacity(N), valtree: InternedSet::with_capacity(N), + patterns: InternedSet::with_capacity(N), } } @@ -1439,6 +1442,9 @@ pub struct GlobalCtxt<'tcx> { pub(crate) alloc_map: interpret::AllocMap<'tcx>, current_gcx: CurrentGcx, + + /// A jobserver reference used to release then acquire a token while waiting on a query. + pub jobserver_proxy: Arc<Proxy>, } impl<'tcx> GlobalCtxt<'tcx> { @@ -1643,6 +1649,7 @@ impl<'tcx> TyCtxt<'tcx> { query_system: QuerySystem<'tcx>, hooks: crate::hooks::Providers, current_gcx: CurrentGcx, + jobserver_proxy: Arc<Proxy>, f: impl FnOnce(TyCtxt<'tcx>) -> T, ) -> T { let data_layout = s.target.parse_data_layout().unwrap_or_else(|err| { @@ -1677,6 +1684,7 @@ impl<'tcx> TyCtxt<'tcx> { data_layout, alloc_map: interpret::AllocMap::new(), current_gcx, + jobserver_proxy, }); // This is a separate function to work around a crash with parallel rustc (#135870) @@ -1715,6 +1723,10 @@ impl<'tcx> TyCtxt<'tcx> { self.coroutine_kind(def_id).is_some() } + pub fn is_async_drop_in_place_coroutine(self, def_id: DefId) -> bool { + self.is_lang_item(self.parent(def_id), LangItem::AsyncDropInPlace) + } + /// Returns the movability of the coroutine of `def_id`, or panics /// if given a `def_id` that is not a coroutine. pub fn coroutine_movability(self, def_id: DefId) -> hir::Movability { @@ -2659,6 +2671,7 @@ slice_interners!( local_def_ids: intern_local_def_ids(LocalDefId), captures: intern_captures(&'tcx ty::CapturedPlace<'tcx>), offset_of: pub mk_offset_of((VariantIdx, FieldIdx)), + patterns: pub mk_patterns(Pattern<'tcx>), ); impl<'tcx> TyCtxt<'tcx> { @@ -2932,6 +2945,14 @@ impl<'tcx> TyCtxt<'tcx> { self.intern_local_def_ids(def_ids) } + pub fn mk_patterns_from_iter<I, T>(self, iter: I) -> T::Output + where + I: Iterator<Item = T>, + T: CollectAndApply<ty::Pattern<'tcx>, &'tcx List<ty::Pattern<'tcx>>>, + { + T::collect_and_apply(iter, |xs| self.mk_patterns(xs)) + } + pub fn mk_local_def_ids_from_iter<I, T>(self, iter: I) -> T::Output where I: Iterator<Item = T>, diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 5ecf1174def..13723874ad3 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -205,7 +205,7 @@ impl<'tcx> Ty<'tcx> { ty::Placeholder(..) => "higher-ranked type".into(), ty::Bound(..) => "bound type variable".into(), ty::Alias(ty::Projection | ty::Inherent, _) => "associated type".into(), - ty::Alias(ty::Weak, _) => "type alias".into(), + ty::Alias(ty::Free, _) => "type alias".into(), ty::Param(_) => "type parameter".into(), ty::Alias(ty::Opaque, ..) => "opaque type".into(), } diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs index 32988965a35..d8bab58545f 100644 --- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs +++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs @@ -127,7 +127,7 @@ impl<'tcx> Ty<'tcx> { InhabitedPredicate::True } Never => InhabitedPredicate::False, - Param(_) | Alias(ty::Projection | ty::Weak, _) => InhabitedPredicate::GenericType(self), + Param(_) | Alias(ty::Projection | ty::Free, _) => InhabitedPredicate::GenericType(self), Alias(ty::Opaque, alias_ty) => { match alias_ty.def_id.as_local() { // Foreign opaque is considered inhabited. diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index faad0a82acb..d660e7d0d60 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -147,6 +147,9 @@ pub enum InstanceKind<'tcx> { /// native support. ThreadLocalShim(DefId), + /// Proxy shim for async drop of future (def_id, proxy_cor_ty, impl_cor_ty) + FutureDropPollShim(DefId, Ty<'tcx>, Ty<'tcx>), + /// `core::ptr::drop_in_place::<T>`. /// /// The `DefId` is for `core::ptr::drop_in_place`. @@ -173,7 +176,13 @@ pub enum InstanceKind<'tcx> { /// /// The `DefId` is for `core::future::async_drop::async_drop_in_place`, the `Ty` /// is the type `T`. - AsyncDropGlueCtorShim(DefId, Option<Ty<'tcx>>), + AsyncDropGlueCtorShim(DefId, Ty<'tcx>), + + /// `core::future::async_drop::async_drop_in_place::<'_, T>::{closure}`. + /// + /// async_drop_in_place poll function implementation (for generated coroutine). + /// `Ty` here is `async_drop_in_place<T>::{closure}` coroutine type, not just `T` + AsyncDropGlue(DefId, Ty<'tcx>), } impl<'tcx> Instance<'tcx> { @@ -221,7 +230,9 @@ impl<'tcx> Instance<'tcx> { .upstream_monomorphizations_for(def) .and_then(|monos| monos.get(&self.args).cloned()), InstanceKind::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args), - InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => { + InstanceKind::AsyncDropGlue(_, _) => None, + InstanceKind::FutureDropPollShim(_, _, _) => None, + InstanceKind::AsyncDropGlueCtorShim(_, _) => { tcx.upstream_async_drop_glue_for(self.args) } _ => None, @@ -248,6 +259,8 @@ impl<'tcx> InstanceKind<'tcx> { | InstanceKind::DropGlue(def_id, _) | InstanceKind::CloneShim(def_id, _) | InstanceKind::FnPtrAddrShim(def_id, _) + | InstanceKind::FutureDropPollShim(def_id, _, _) + | InstanceKind::AsyncDropGlue(def_id, _) | InstanceKind::AsyncDropGlueCtorShim(def_id, _) => def_id, } } @@ -257,7 +270,9 @@ impl<'tcx> InstanceKind<'tcx> { match self { ty::InstanceKind::Item(def) => Some(def), ty::InstanceKind::DropGlue(def_id, Some(_)) - | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_)) + | InstanceKind::AsyncDropGlueCtorShim(def_id, _) + | InstanceKind::AsyncDropGlue(def_id, _) + | InstanceKind::FutureDropPollShim(def_id, ..) | InstanceKind::ThreadLocalShim(def_id) => Some(def_id), InstanceKind::VTableShim(..) | InstanceKind::ReifyShim(..) @@ -267,7 +282,6 @@ impl<'tcx> InstanceKind<'tcx> { | InstanceKind::ClosureOnceShim { .. } | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } | InstanceKind::DropGlue(..) - | InstanceKind::AsyncDropGlueCtorShim(..) | InstanceKind::CloneShim(..) | InstanceKind::FnPtrAddrShim(..) => None, } @@ -292,7 +306,9 @@ impl<'tcx> InstanceKind<'tcx> { let def_id = match *self { ty::InstanceKind::Item(def) => def, ty::InstanceKind::DropGlue(_, Some(_)) => return false, - ty::InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => return false, + ty::InstanceKind::AsyncDropGlueCtorShim(_, ty) => return ty.is_coroutine(), + ty::InstanceKind::FutureDropPollShim(_, _, _) => return false, + ty::InstanceKind::AsyncDropGlue(_, _) => return false, ty::InstanceKind::ThreadLocalShim(_) => return false, _ => return true, }; @@ -325,11 +341,12 @@ impl<'tcx> InstanceKind<'tcx> { | InstanceKind::FnPtrAddrShim(..) | InstanceKind::FnPtrShim(..) | InstanceKind::DropGlue(_, Some(_)) - | InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => false, + | InstanceKind::FutureDropPollShim(..) + | InstanceKind::AsyncDropGlue(_, _) => false, + InstanceKind::AsyncDropGlueCtorShim(_, _) => false, InstanceKind::ClosureOnceShim { .. } | InstanceKind::ConstructCoroutineInClosureShim { .. } | InstanceKind::DropGlue(..) - | InstanceKind::AsyncDropGlueCtorShim(..) | InstanceKind::Item(_) | InstanceKind::Intrinsic(..) | InstanceKind::ReifyShim(..) @@ -406,8 +423,11 @@ pub fn fmt_instance( InstanceKind::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), InstanceKind::CloneShim(_, ty) => write!(f, " - shim({ty})"), InstanceKind::FnPtrAddrShim(_, ty) => write!(f, " - shim({ty})"), - InstanceKind::AsyncDropGlueCtorShim(_, None) => write!(f, " - shim(None)"), - InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), + InstanceKind::FutureDropPollShim(_, proxy_ty, impl_ty) => { + write!(f, " - dropshim({proxy_ty}-{impl_ty})") + } + InstanceKind::AsyncDropGlue(_, ty) => write!(f, " - shim({ty})"), + InstanceKind::AsyncDropGlueCtorShim(_, ty) => write!(f, " - shim(Some({ty}))"), } } @@ -425,6 +445,51 @@ impl<'tcx> fmt::Display for Instance<'tcx> { } } +// async_drop_in_place<T>::coroutine.poll, when T is a standart coroutine, +// should be resolved to this coroutine's future_drop_poll (through FutureDropPollShim proxy). +// async_drop_in_place<async_drop_in_place<T>::coroutine>::coroutine.poll, +// when T is a standart coroutine, should be resolved to this coroutine's future_drop_poll. +// async_drop_in_place<async_drop_in_place<T>::coroutine>::coroutine.poll, +// when T is not a coroutine, should be resolved to the innermost +// async_drop_in_place<T>::coroutine's poll function (through FutureDropPollShim proxy) +fn resolve_async_drop_poll<'tcx>(mut cor_ty: Ty<'tcx>) -> Instance<'tcx> { + let first_cor = cor_ty; + let ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { + bug!(); + }; + let poll_def_id = *poll_def_id; + let mut child_ty = cor_ty; + loop { + if let ty::Coroutine(child_def, child_args) = child_ty.kind() { + cor_ty = child_ty; + if *child_def == poll_def_id { + child_ty = child_args.first().unwrap().expect_ty(); + continue; + } else { + return Instance { + def: ty::InstanceKind::FutureDropPollShim(poll_def_id, first_cor, cor_ty), + args: proxy_args, + }; + } + } else { + let ty::Coroutine(_, child_args) = cor_ty.kind() else { + bug!(); + }; + if first_cor != cor_ty { + return Instance { + def: ty::InstanceKind::FutureDropPollShim(poll_def_id, first_cor, cor_ty), + args: proxy_args, + }; + } else { + return Instance { + def: ty::InstanceKind::AsyncDropGlue(poll_def_id, cor_ty), + args: child_args, + }; + } + } + } +} + impl<'tcx> Instance<'tcx> { pub fn new(def_id: DefId, args: GenericArgsRef<'tcx>) -> Instance<'tcx> { assert!( @@ -736,6 +801,15 @@ impl<'tcx> Instance<'tcx> { ) } + pub fn resolve_async_drop_in_place_poll( + tcx: TyCtxt<'tcx>, + def_id: DefId, + ty: Ty<'tcx>, + ) -> ty::Instance<'tcx> { + let args = tcx.mk_args(&[ty.into()]); + Instance::expect_resolve(tcx, ty::TypingEnv::fully_monomorphized(), def_id, args, DUMMY_SP) + } + #[instrument(level = "debug", skip(tcx), ret)] pub fn fn_once_adapter_instance( tcx: TyCtxt<'tcx>, @@ -800,6 +874,9 @@ impl<'tcx> Instance<'tcx> { }; if tcx.is_lang_item(trait_item_id, coroutine_callable_item) { + if tcx.is_async_drop_in_place_coroutine(coroutine_def_id) { + return Some(resolve_async_drop_poll(rcvr_args.type_at(0))); + } let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind() else { bug!() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index be00c0e116d..2f4c03f0953 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -17,7 +17,7 @@ use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::num::NonZero; use std::ptr::NonNull; -use std::{fmt, str}; +use std::{fmt, iter, str}; pub use adt::*; pub use assoc::*; @@ -39,6 +39,7 @@ use rustc_hir::LangItem; use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res}; use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap}; use rustc_index::IndexVec; +use rustc_index::bit_set::BitMatrix; use rustc_macros::{ Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable, extension, @@ -49,6 +50,7 @@ use rustc_session::lint::LintBuffer; pub use rustc_session::lint::RegisteredTools; use rustc_span::hygiene::MacroKind; use rustc_span::{DUMMY_SP, ExpnId, ExpnKind, Ident, Span, Symbol, kw, sym}; +pub use rustc_type_ir::data_structures::DelayedSet; pub use rustc_type_ir::relate::VarianceDiagInfo; pub use rustc_type_ir::*; use tracing::{debug, instrument}; @@ -103,7 +105,7 @@ pub use self::visit::*; use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason}; use crate::metadata::ModChild; use crate::middle::privacy::EffectiveVisibilities; -use crate::mir::{Body, CoroutineLayout}; +use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, SourceInfo}; use crate::query::{IntoQueryParam, Providers}; use crate::ty; use crate::ty::codec::{TyDecoder, TyEncoder}; @@ -1739,11 +1741,13 @@ impl<'tcx> TyCtxt<'tcx> { | ty::InstanceKind::Virtual(..) | ty::InstanceKind::ClosureOnceShim { .. } | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::FutureDropPollShim(..) | ty::InstanceKind::DropGlue(..) | ty::InstanceKind::CloneShim(..) | ty::InstanceKind::ThreadLocalShim(..) | ty::InstanceKind::FnPtrAddrShim(..) - | ty::InstanceKind::AsyncDropGlueCtorShim(..) => self.mir_shims(instance), + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::AsyncDropGlue(..) => self.mir_shims(instance), } } @@ -1859,12 +1863,12 @@ impl<'tcx> TyCtxt<'tcx> { self.def_kind(trait_def_id) == DefKind::TraitAlias } - /// Returns layout of a coroutine. Layout might be unavailable if the + /// Returns layout of a non-async-drop coroutine. Layout might be unavailable if the /// coroutine is tainted by errors. /// /// Takes `coroutine_kind` which can be acquired from the `CoroutineArgs::kind_ty`, /// e.g. `args.as_coroutine().kind_ty()`. - pub fn coroutine_layout( + fn ordinary_coroutine_layout( self, def_id: DefId, coroutine_kind_ty: Ty<'tcx>, @@ -1898,6 +1902,51 @@ impl<'tcx> TyCtxt<'tcx> { } } + /// Returns layout of a `async_drop_in_place::{closure}` coroutine + /// (returned from `async fn async_drop_in_place<T>(..)`). + /// Layout might be unavailable if the coroutine is tainted by errors. + fn async_drop_coroutine_layout( + self, + def_id: DefId, + args: GenericArgsRef<'tcx>, + ) -> Option<&'tcx CoroutineLayout<'tcx>> { + let instance = InstanceKind::AsyncDropGlue(def_id, Ty::new_coroutine(self, def_id, args)); + self.mir_shims(instance).coroutine_layout_raw() + } + + /// Returns layout of a coroutine. Layout might be unavailable if the + /// coroutine is tainted by errors. + pub fn coroutine_layout( + self, + def_id: DefId, + args: GenericArgsRef<'tcx>, + ) -> Option<&'tcx CoroutineLayout<'tcx>> { + if self.is_async_drop_in_place_coroutine(def_id) { + // layout of `async_drop_in_place<T>::{closure}` in case, + // when T is a coroutine, contains this internal coroutine's ptr in upvars + // and doesn't require any locals. Here is an `empty coroutine's layout` + let arg_cor_ty = args.first().unwrap().expect_ty(); + if arg_cor_ty.is_coroutine() { + let span = self.def_span(def_id); + let source_info = SourceInfo::outermost(span); + let variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, CoroutineSavedLocal>> = + iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + let proxy_layout = CoroutineLayout { + field_tys: [].into(), + field_names: [].into(), + variant_fields, + variant_source_info: [source_info].into(), + storage_conflicts: BitMatrix::new(0, 0), + }; + return Some(self.arena.alloc(proxy_layout)); + } else { + self.async_drop_coroutine_layout(def_id, args) + } + } else { + self.ordinary_coroutine_layout(def_id, args.as_coroutine().kind_ty()) + } + } + /// Given the `DefId` of an impl, returns the `DefId` of the trait it implements. /// If it implements no trait, returns `None`. pub fn trait_id_of_impl(self, def_id: DefId) -> Option<DefId> { diff --git a/compiler/rustc_middle/src/ty/pattern.rs b/compiler/rustc_middle/src/ty/pattern.rs index 758adc42e3e..5af9b17dd77 100644 --- a/compiler/rustc_middle/src/ty/pattern.rs +++ b/compiler/rustc_middle/src/ty/pattern.rs @@ -23,6 +23,13 @@ impl<'tcx> Flags for Pattern<'tcx> { FlagComputation::for_const_kind(&start.kind()).flags | FlagComputation::for_const_kind(&end.kind()).flags } + ty::PatternKind::Or(pats) => { + let mut flags = pats[0].flags(); + for pat in pats[1..].iter() { + flags |= pat.flags(); + } + flags + } } } @@ -31,6 +38,13 @@ impl<'tcx> Flags for Pattern<'tcx> { ty::PatternKind::Range { start, end } => { start.outer_exclusive_binder().max(end.outer_exclusive_binder()) } + ty::PatternKind::Or(pats) => { + let mut idx = pats[0].outer_exclusive_binder(); + for pat in pats[1..].iter() { + idx = idx.max(pat.outer_exclusive_binder()); + } + idx + } } } } @@ -77,6 +91,19 @@ impl<'tcx> IrPrint<PatternKind<'tcx>> for TyCtxt<'tcx> { write!(f, "..={end}") } + PatternKind::Or(patterns) => { + write!(f, "(")?; + let mut first = true; + for pat in patterns { + if first { + first = false + } else { + write!(f, " | ")?; + } + write!(f, "{pat:?}")?; + } + write!(f, ")") + } } } diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index 1df3bff5244..ad8677f7c7d 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -820,7 +820,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write { ty::Foreign(def_id) => { p!(print_def_path(def_id, &[])); } - ty::Alias(ty::Projection | ty::Inherent | ty::Weak, ref data) => { + ty::Alias(ty::Projection | ty::Inherent | ty::Free, ref data) => { p!(print(data)) } ty::Placeholder(placeholder) => match placeholder.bound.kind { @@ -3205,7 +3205,7 @@ define_print! { p!(print_def_path(self.def_id, self.args)); } } - | ty::AliasTermKind::WeakTy + | ty::AliasTermKind::FreeTy | ty::AliasTermKind::OpaqueTy | ty::AliasTermKind::UnevaluatedConst | ty::AliasTermKind::ProjectionConst => { diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs index b1dfcb80bde..6ad4e5276b2 100644 --- a/compiler/rustc_middle/src/ty/relate.rs +++ b/compiler/rustc_middle/src/ty/relate.rs @@ -49,6 +49,7 @@ impl<'tcx> Relate<TyCtxt<'tcx>> for ty::Pattern<'tcx> { a: Self, b: Self, ) -> RelateResult<'tcx, Self> { + let tcx = relation.cx(); match (&*a, &*b) { ( &ty::PatternKind::Range { start: start_a, end: end_a }, @@ -56,8 +57,17 @@ impl<'tcx> Relate<TyCtxt<'tcx>> for ty::Pattern<'tcx> { ) => { let start = relation.relate(start_a, start_b)?; let end = relation.relate(end_a, end_b)?; - Ok(relation.cx().mk_pat(ty::PatternKind::Range { start, end })) + Ok(tcx.mk_pat(ty::PatternKind::Range { start, end })) + } + (&ty::PatternKind::Or(a), &ty::PatternKind::Or(b)) => { + if a.len() != b.len() { + return Err(TypeError::Mismatch); + } + let v = iter::zip(a, b).map(|(a, b)| relation.relate(a, b)); + let patterns = tcx.mk_patterns_from_iter(v)?; + Ok(tcx.mk_pat(ty::PatternKind::Or(patterns))) } + (ty::PatternKind::Range { .. } | ty::PatternKind::Or(_), _) => Err(TypeError::Mismatch), } } } diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 26861666c1d..58f7bc75054 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -13,7 +13,6 @@ use rustc_type_ir::{ConstKind, TypeFolder, VisitorResult, try_visit}; use super::print::PrettyPrinter; use super::{GenericArg, GenericArgKind, Pattern, Region}; -use crate::infer::canonical::CanonicalVarInfos; use crate::mir::PlaceElem; use crate::ty::print::{FmtPrinter, Printer, with_no_trimmed_paths}; use crate::ty::{ @@ -779,5 +778,5 @@ list_fold! { ty::Clauses<'tcx> : mk_clauses, &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> : mk_poly_existential_predicates, &'tcx ty::List<PlaceElem<'tcx>> : mk_place_elems, - CanonicalVarInfos<'tcx> : mk_canonical_var_infos, + &'tcx ty::List<ty::Pattern<'tcx>> : mk_patterns, } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index affb7b31ae1..2165cf186bd 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -4,7 +4,6 @@ use std::assert_matches::debug_assert_matches; use std::borrow::Cow; -use std::iter; use std::ops::{ControlFlow, Range}; use hir::def::{CtorKind, DefKind}; @@ -19,7 +18,7 @@ use rustc_type_ir::TyKind::*; use rustc_type_ir::walk::TypeWalker; use rustc_type_ir::{self as ir, BoundVar, CollectAndApply, DynKind, TypeVisitableExt, elaborate}; use tracing::instrument; -use ty::util::{AsyncDropGlueMorphology, IntTypeExt}; +use ty::util::IntTypeExt; use super::GenericParamDefKind; use crate::infer::canonical::Canonical; @@ -78,8 +77,7 @@ impl<'tcx> ty::CoroutineArgs<TyCtxt<'tcx>> { #[inline] fn variant_range(&self, def_id: DefId, tcx: TyCtxt<'tcx>) -> Range<VariantIdx> { // FIXME requires optimized MIR - FIRST_VARIANT - ..tcx.coroutine_layout(def_id, tcx.types.unit).unwrap().variant_fields.next_index() + FIRST_VARIANT..tcx.coroutine_layout(def_id, self.args).unwrap().variant_fields.next_index() } /// The discriminant for the given variant. Panics if the `variant_index` is @@ -139,10 +137,14 @@ impl<'tcx> ty::CoroutineArgs<TyCtxt<'tcx>> { def_id: DefId, tcx: TyCtxt<'tcx>, ) -> impl Iterator<Item: Iterator<Item = Ty<'tcx>>> { - let layout = tcx.coroutine_layout(def_id, self.kind_ty()).unwrap(); + let layout = tcx.coroutine_layout(def_id, self.args).unwrap(); layout.variant_fields.iter().map(move |variant| { variant.iter().map(move |field| { - ty::EarlyBinder::bind(layout.field_tys[*field].ty).instantiate(tcx, self.args) + if tcx.is_async_drop_in_place_coroutine(def_id) { + layout.field_tys[*field].ty + } else { + ty::EarlyBinder::bind(layout.field_tys[*field].ty).instantiate(tcx, self.args) + } }) }) } @@ -489,7 +491,7 @@ impl<'tcx> Ty<'tcx> { (kind, tcx.def_kind(alias_ty.def_id)), (ty::Opaque, DefKind::OpaqueTy) | (ty::Projection | ty::Inherent, DefKind::AssocTy) - | (ty::Weak, DefKind::TyAlias) + | (ty::Free, DefKind::TyAlias) ); Ty::new(tcx, Alias(kind, alias_ty)) } @@ -1046,10 +1048,6 @@ impl<'tcx> rustc_type_ir::inherent::Ty<TyCtxt<'tcx>> for Ty<'tcx> { self.discriminant_ty(interner) } - fn async_destructor_ty(self, interner: TyCtxt<'tcx>) -> Ty<'tcx> { - self.async_destructor_ty(interner) - } - fn has_unsafe_fields(self) -> bool { Ty::has_unsafe_fields(self) } @@ -1419,6 +1417,34 @@ impl<'tcx> Ty<'tcx> { cf.is_break() } + /// Returns the deepest `async_drop_in_place::{closure}` implementation. + /// + /// `async_drop_in_place<T>::{closure}`, when T is a coroutine, is a proxy-impl + /// to call async drop poll from impl coroutine. + pub fn find_async_drop_impl_coroutine<F: FnMut(Ty<'tcx>)>( + self, + tcx: TyCtxt<'tcx>, + mut f: F, + ) -> Ty<'tcx> { + assert!(self.is_coroutine()); + let mut cor_ty = self; + let mut ty = cor_ty; + loop { + if let ty::Coroutine(def_id, args) = ty.kind() { + cor_ty = ty; + f(ty); + if tcx.is_async_drop_in_place_coroutine(*def_id) { + ty = args.first().unwrap().expect_ty(); + continue; + } else { + return cor_ty; + } + } else { + return cor_ty; + } + } + } + /// Returns the type and mutability of `*ty`. /// /// The parameter `explicit` indicates if this is an *explicit* dereference. @@ -1560,125 +1586,6 @@ impl<'tcx> Ty<'tcx> { } } - /// Returns the type of the async destructor of this type. - pub fn async_destructor_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { - match self.async_drop_glue_morphology(tcx) { - AsyncDropGlueMorphology::Noop => { - return Ty::async_destructor_combinator(tcx, LangItem::AsyncDropNoop) - .instantiate_identity(); - } - AsyncDropGlueMorphology::DeferredDropInPlace => { - let drop_in_place = - Ty::async_destructor_combinator(tcx, LangItem::AsyncDropDeferredDropInPlace) - .instantiate(tcx, &[self.into()]); - return Ty::async_destructor_combinator(tcx, LangItem::AsyncDropFuse) - .instantiate(tcx, &[drop_in_place.into()]); - } - AsyncDropGlueMorphology::Custom => (), - } - - match *self.kind() { - ty::Param(_) | ty::Alias(..) | ty::Infer(ty::TyVar(_)) => { - let assoc_items = tcx - .associated_item_def_ids(tcx.require_lang_item(LangItem::AsyncDestruct, None)); - Ty::new_projection(tcx, assoc_items[0], [self]) - } - - ty::Array(elem_ty, _) | ty::Slice(elem_ty) => { - let dtor = Ty::async_destructor_combinator(tcx, LangItem::AsyncDropSlice) - .instantiate(tcx, &[elem_ty.into()]); - Ty::async_destructor_combinator(tcx, LangItem::AsyncDropFuse) - .instantiate(tcx, &[dtor.into()]) - } - - ty::Adt(adt_def, args) if adt_def.is_enum() || adt_def.is_struct() => self - .adt_async_destructor_ty( - tcx, - adt_def.variants().iter().map(|v| v.fields.iter().map(|f| f.ty(tcx, args))), - ), - ty::Tuple(tys) => self.adt_async_destructor_ty(tcx, iter::once(tys)), - ty::Closure(_, args) => { - self.adt_async_destructor_ty(tcx, iter::once(args.as_closure().upvar_tys())) - } - ty::CoroutineClosure(_, args) => self - .adt_async_destructor_ty(tcx, iter::once(args.as_coroutine_closure().upvar_tys())), - - ty::Adt(adt_def, _) => { - assert!(adt_def.is_union()); - - let surface_drop = self.surface_async_dropper_ty(tcx).unwrap(); - - Ty::async_destructor_combinator(tcx, LangItem::AsyncDropFuse) - .instantiate(tcx, &[surface_drop.into()]) - } - - ty::Bound(..) - | ty::Foreign(_) - | ty::Placeholder(_) - | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => { - bug!("`async_destructor_ty` applied to unexpected type: {self:?}") - } - - _ => bug!("`async_destructor_ty` is not yet implemented for type: {self:?}"), - } - } - - fn adt_async_destructor_ty<I>(self, tcx: TyCtxt<'tcx>, variants: I) -> Ty<'tcx> - where - I: Iterator + ExactSizeIterator, - I::Item: IntoIterator<Item = Ty<'tcx>>, - { - debug_assert_eq!(self.async_drop_glue_morphology(tcx), AsyncDropGlueMorphology::Custom); - - let defer = Ty::async_destructor_combinator(tcx, LangItem::AsyncDropDefer); - let chain = Ty::async_destructor_combinator(tcx, LangItem::AsyncDropChain); - - let noop = - Ty::async_destructor_combinator(tcx, LangItem::AsyncDropNoop).instantiate_identity(); - let either = Ty::async_destructor_combinator(tcx, LangItem::AsyncDropEither); - - let variants_dtor = variants - .into_iter() - .map(|variant| { - variant - .into_iter() - .map(|ty| defer.instantiate(tcx, &[ty.into()])) - .reduce(|acc, next| chain.instantiate(tcx, &[acc.into(), next.into()])) - .unwrap_or(noop) - }) - .reduce(|other, matched| { - either.instantiate(tcx, &[other.into(), matched.into(), self.into()]) - }) - .unwrap(); - - let dtor = if let Some(dropper_ty) = self.surface_async_dropper_ty(tcx) { - Ty::async_destructor_combinator(tcx, LangItem::AsyncDropChain) - .instantiate(tcx, &[dropper_ty.into(), variants_dtor.into()]) - } else { - variants_dtor - }; - - Ty::async_destructor_combinator(tcx, LangItem::AsyncDropFuse) - .instantiate(tcx, &[dtor.into()]) - } - - fn surface_async_dropper_ty(self, tcx: TyCtxt<'tcx>) -> Option<Ty<'tcx>> { - let adt_def = self.ty_adt_def()?; - let dropper = adt_def - .async_destructor(tcx) - .map(|_| LangItem::SurfaceAsyncDropInPlace) - .or_else(|| adt_def.destructor(tcx).map(|_| LangItem::AsyncDropSurfaceDropInPlace))?; - Some(Ty::async_destructor_combinator(tcx, dropper).instantiate(tcx, &[self.into()])) - } - - fn async_destructor_combinator( - tcx: TyCtxt<'tcx>, - lang_item: LangItem, - ) -> ty::EarlyBinder<'tcx, Ty<'tcx>> { - tcx.fn_sig(tcx.require_lang_item(lang_item, None)) - .map_bound(|fn_sig| fn_sig.output().no_bound_vars().unwrap()) - } - /// Returns the type of metadata for (potentially wide) pointers to this type, /// or the struct tail if the metadata type cannot be determined. pub fn ptr_metadata_ty_or_tail( @@ -2020,6 +1927,13 @@ impl<'tcx> Ty<'tcx> { } } + pub fn is_async_drop_in_place_coroutine(self, tcx: TyCtxt<'_>) -> bool { + match self.kind() { + ty::Coroutine(def, ..) => tcx.is_async_drop_in_place_coroutine(*def), + _ => false, + } + } + /// Returns `true` when the outermost type cannot be further normalized, /// resolved, or instantiated. This includes all primitive types, but also /// things like ADTs and trait objects, since even if their arguments or diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs index 4c5c669771f..c6a45f84686 100644 --- a/compiler/rustc_middle/src/ty/typeck_results.rs +++ b/compiler/rustc_middle/src/ty/typeck_results.rs @@ -475,6 +475,21 @@ impl<'tcx> TypeckResults<'tcx> { has_ref_mut } + /// How should a deref pattern find the place for its inner pattern to match on? + /// + /// In most cases, if the pattern recursively contains a `ref mut` binding, we find the inner + /// pattern's scrutinee by calling `DerefMut::deref_mut`, and otherwise we call `Deref::deref`. + /// However, for boxes we can use a built-in deref instead, which doesn't borrow the scrutinee; + /// in this case, we return `ByRef::No`. + pub fn deref_pat_borrow_mode(&self, pointer_ty: Ty<'_>, inner: &hir::Pat<'_>) -> ByRef { + if pointer_ty.is_box() { + ByRef::No + } else { + let mutable = self.pat_has_ref_mut_binding(inner); + ByRef::Yes(if mutable { Mutability::Mut } else { Mutability::Not }) + } + } + /// For a given closure, returns the iterator of `ty::CapturedPlace`s that are captured /// by the closure. pub fn closure_min_captures_flattened( @@ -701,6 +716,8 @@ pub type CanonicalUserTypeAnnotations<'tcx> = #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] pub struct CanonicalUserTypeAnnotation<'tcx> { + #[type_foldable(identity)] + #[type_visitable(ignore)] pub user_ty: Box<CanonicalUserType<'tcx>>, pub span: Span, pub inferred_ty: Ty<'tcx>, diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 08cee1101d0..f5158edffcf 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -468,25 +468,6 @@ impl<'tcx> TyCtxt<'tcx> { Some(ty::AsyncDestructor { impl_did: dtor_candidate? }) } - /// Returns async drop glue morphology for a definition. To get async drop - /// glue morphology for a type see [`Ty::async_drop_glue_morphology`]. - // - // FIXME: consider making this a query - pub fn async_drop_glue_morphology(self, did: DefId) -> AsyncDropGlueMorphology { - let ty: Ty<'tcx> = self.type_of(did).instantiate_identity(); - - // Async drop glue morphology is an internal detail, so - // using `TypingMode::PostAnalysis` probably should be fine. - let typing_env = ty::TypingEnv::fully_monomorphized(); - if ty.needs_async_drop(self, typing_env) { - AsyncDropGlueMorphology::Custom - } else if ty.needs_drop(self, typing_env) { - AsyncDropGlueMorphology::DeferredDropInPlace - } else { - AsyncDropGlueMorphology::Noop - } - } - /// Returns the set of types that are required to be alive in /// order to run the destructor of `def` (see RFCs 769 and /// 1238). @@ -911,7 +892,7 @@ impl<'tcx> TyCtxt<'tcx> { || self.extern_crate(key).is_some_and(|e| e.is_direct()) } - /// Expand any [weak alias types][weak] contained within the given `value`. + /// Expand any [free alias types][free] contained within the given `value`. /// /// This should be used over other normalization routines in situations where /// it's important not to normalize other alias types and where the predicates @@ -926,19 +907,19 @@ impl<'tcx> TyCtxt<'tcx> { /// <div class="warning"> /// This delays a bug on overflow! Therefore you need to be certain that the /// contained types get fully normalized at a later stage. Note that even on - /// overflow all well-behaved weak alias types get expanded correctly, so the + /// overflow all well-behaved free alias types get expanded correctly, so the /// result is still useful. /// </div> /// - /// [weak]: ty::Weak - pub fn expand_weak_alias_tys<T: TypeFoldable<TyCtxt<'tcx>>>(self, value: T) -> T { - value.fold_with(&mut WeakAliasTypeExpander { tcx: self, depth: 0 }) + /// [free]: ty::Free + pub fn expand_free_alias_tys<T: TypeFoldable<TyCtxt<'tcx>>>(self, value: T) -> T { + value.fold_with(&mut FreeAliasTypeExpander { tcx: self, depth: 0 }) } - /// Peel off all [weak alias types] in this type until there are none left. + /// Peel off all [free alias types] in this type until there are none left. /// - /// This only expands weak alias types in “head” / outermost positions. It can - /// be used over [expand_weak_alias_tys] as an optimization in situations where + /// This only expands free alias types in “head” / outermost positions. It can + /// be used over [expand_free_alias_tys] as an optimization in situations where /// one only really cares about the *kind* of the final aliased type but not /// the types the other constituent types alias. /// @@ -947,17 +928,17 @@ impl<'tcx> TyCtxt<'tcx> { /// type gets fully normalized at a later stage. /// </div> /// - /// [weak]: ty::Weak - /// [expand_weak_alias_tys]: Self::expand_weak_alias_tys - pub fn peel_off_weak_alias_tys(self, mut ty: Ty<'tcx>) -> Ty<'tcx> { - let ty::Alias(ty::Weak, _) = ty.kind() else { return ty }; + /// [free]: ty::Free + /// [expand_free_alias_tys]: Self::expand_free_alias_tys + pub fn peel_off_free_alias_tys(self, mut ty: Ty<'tcx>) -> Ty<'tcx> { + let ty::Alias(ty::Free, _) = ty.kind() else { return ty }; let limit = self.recursion_limit(); let mut depth = 0; - while let ty::Alias(ty::Weak, alias) = ty.kind() { + while let ty::Alias(ty::Free, alias) = ty.kind() { if !limit.value_within_limit(depth) { - let guar = self.dcx().delayed_bug("overflow expanding weak alias type"); + let guar = self.dcx().delayed_bug("overflow expanding free alias type"); return Ty::new_error(self, guar); } @@ -985,7 +966,7 @@ impl<'tcx> TyCtxt<'tcx> { } ty::AliasTermKind::OpaqueTy => Some(self.variances_of(def_id)), ty::AliasTermKind::InherentTy - | ty::AliasTermKind::WeakTy + | ty::AliasTermKind::FreeTy | ty::AliasTermKind::UnevaluatedConst | ty::AliasTermKind::ProjectionConst => None, } @@ -1078,25 +1059,25 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for OpaqueTypeExpander<'tcx> { } } -struct WeakAliasTypeExpander<'tcx> { +struct FreeAliasTypeExpander<'tcx> { tcx: TyCtxt<'tcx>, depth: usize, } -impl<'tcx> TypeFolder<TyCtxt<'tcx>> for WeakAliasTypeExpander<'tcx> { +impl<'tcx> TypeFolder<TyCtxt<'tcx>> for FreeAliasTypeExpander<'tcx> { fn cx(&self) -> TyCtxt<'tcx> { self.tcx } fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> { - if !ty.has_type_flags(ty::TypeFlags::HAS_TY_WEAK) { + if !ty.has_type_flags(ty::TypeFlags::HAS_TY_FREE_ALIAS) { return ty; } - let ty::Alias(ty::Weak, alias) = ty.kind() else { + let ty::Alias(ty::Free, alias) = ty.kind() else { return ty.super_fold_with(self); }; if !self.tcx.recursion_limit().value_within_limit(self.depth) { - let guar = self.tcx.dcx().delayed_bug("overflow expanding weak alias type"); + let guar = self.tcx.dcx().delayed_bug("overflow expanding free alias type"); return Ty::new_error(self.tcx, guar); } @@ -1107,25 +1088,13 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for WeakAliasTypeExpander<'tcx> { } fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> { - if !ct.has_type_flags(ty::TypeFlags::HAS_TY_WEAK) { + if !ct.has_type_flags(ty::TypeFlags::HAS_TY_FREE_ALIAS) { return ct; } ct.super_fold_with(self) } } -/// Indicates the form of `AsyncDestruct::Destructor`. Used to simplify async -/// drop glue for types not using async drop. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum AsyncDropGlueMorphology { - /// Async destructor simply does nothing - Noop, - /// Async destructor simply runs `drop_in_place` - DeferredDropInPlace, - /// Async destructor has custom logic - Custom, -} - impl<'tcx> Ty<'tcx> { /// Returns the `Size` for primitive types (bool, uint, int, char, float). pub fn primitive_size(self, tcx: TyCtxt<'tcx>) -> Size { @@ -1295,16 +1264,17 @@ impl<'tcx> Ty<'tcx> { } } - /// Get morphology of the async drop glue, needed for types which do not - /// use async drop. To get async drop glue morphology for a definition see - /// [`TyCtxt::async_drop_glue_morphology`]. Used for `AsyncDestruct::Destructor` - /// type construction. - // - // FIXME: implement optimization to not instantiate a certain morphology of - // async drop glue too soon to allow per type optimizations, see array case - // for more info. Perhaps then remove this method and use `needs_(async_)drop` - // instead. - pub fn async_drop_glue_morphology(self, tcx: TyCtxt<'tcx>) -> AsyncDropGlueMorphology { + /// Checks whether values of this type `T` implement the `AsyncDrop` trait. + pub fn is_async_drop(self, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool { + !self.is_trivially_not_async_drop() + && tcx.is_async_drop_raw(typing_env.as_query_input(self)) + } + + /// Fast path helper for testing if a type is `AsyncDrop`. + /// + /// Returning true means the type is known to be `!AsyncDrop`. Returning + /// `false` means nothing -- could be `AsyncDrop`, might not be. + fn is_trivially_not_async_drop(self) -> bool { match self.kind() { ty::Int(_) | ty::Uint(_) @@ -1316,46 +1286,26 @@ impl<'tcx> Ty<'tcx> { | ty::Ref(..) | ty::RawPtr(..) | ty::FnDef(..) - | ty::FnPtr(..) - | ty::Infer(ty::FreshIntTy(_)) - | ty::Infer(ty::FreshFloatTy(_)) => AsyncDropGlueMorphology::Noop, - + | ty::Error(_) + | ty::FnPtr(..) => true, // FIXME(unsafe_binders): ty::UnsafeBinder(_) => todo!(), - - ty::Tuple(tys) if tys.is_empty() => AsyncDropGlueMorphology::Noop, - ty::Adt(adt_def, _) if adt_def.is_manually_drop() => AsyncDropGlueMorphology::Noop, - - // Foreign types can never have destructors. - ty::Foreign(_) => AsyncDropGlueMorphology::Noop, - - // FIXME: implement dynamic types async drops - ty::Error(_) | ty::Dynamic(..) => AsyncDropGlueMorphology::DeferredDropInPlace, - - ty::Tuple(_) | ty::Array(_, _) | ty::Slice(_) => { - // Assume worst-case scenario, because we can instantiate async - // destructors in different orders: - // - // 1. Instantiate [T; N] with T = String and N = 0 - // 2. Instantiate <[String; 0] as AsyncDestruct>::Destructor - // - // And viceversa, thus we cannot rely on String not using async - // drop or array having zero (0) elements - AsyncDropGlueMorphology::Custom - } - ty::Pat(ty, _) => ty.async_drop_glue_morphology(tcx), - - ty::Adt(adt_def, _) => tcx.async_drop_glue_morphology(adt_def.did()), - - ty::Closure(did, _) - | ty::CoroutineClosure(did, _) - | ty::Coroutine(did, _) - | ty::CoroutineWitness(did, _) => tcx.async_drop_glue_morphology(*did), - - ty::Alias(..) | ty::Param(_) | ty::Bound(..) | ty::Placeholder(..) | ty::Infer(_) => { - // No specifics, but would usually mean forwarding async drop glue - AsyncDropGlueMorphology::Custom + ty::Tuple(fields) => fields.iter().all(Self::is_trivially_not_async_drop), + ty::Pat(elem_ty, _) | ty::Slice(elem_ty) | ty::Array(elem_ty, _) => { + elem_ty.is_trivially_not_async_drop() } + ty::Adt(..) + | ty::Bound(..) + | ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Dynamic(..) + | ty::Foreign(_) + | ty::Coroutine(..) + | ty::CoroutineWitness(..) + | ty::Infer(_) + | ty::Alias(..) + | ty::Param(_) + | ty::Placeholder(_) => false, } } @@ -1401,9 +1351,6 @@ impl<'tcx> Ty<'tcx> { /// (Note that this implies that if `ty` has an async destructor attached, /// then `needs_async_drop` will definitely return `true` for `ty`.) /// - /// When constructing `AsyncDestruct::Destructor` type, use - /// [`Ty::async_drop_glue_morphology`] instead. - // // FIXME(zetanumbers): Note that this method is used to check eligible types // in unions. #[inline] diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs index 44c7b6a7c9e..3853a804a92 100644 --- a/compiler/rustc_middle/src/ty/visit.rs +++ b/compiler/rustc_middle/src/ty/visit.rs @@ -139,7 +139,7 @@ impl<'tcx> TyCtxt<'tcx> { { let mut collector = LateBoundRegionsCollector::new(just_constrained); let value = value.skip_binder(); - let value = if just_constrained { self.expand_weak_alias_tys(value) } else { value }; + let value = if just_constrained { self.expand_free_alias_tys(value) } else { value }; value.visit_with(&mut collector); collector.regions } @@ -182,8 +182,8 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for LateBoundRegionsCollector { ty::Alias(ty::Projection | ty::Inherent | ty::Opaque, _) => { return; } - // All weak alias types should've been expanded beforehand. - ty::Alias(ty::Weak, _) => bug!("unexpected weak alias type"), + // All free alias types should've been expanded beforehand. + ty::Alias(ty::Free, _) => bug!("unexpected free alias type"), _ => {} } } |