diff options
180 files changed, 3643 insertions, 366 deletions
diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl index e0dc227ca4c..37e45379ba9 100644 --- a/compiler/rustc_ast_lowering/messages.ftl +++ b/compiler/rustc_ast_lowering/messages.ftl @@ -123,9 +123,6 @@ ast_lowering_never_pattern_with_guard = a guard on a never pattern will never be run .suggestion = remove this guard -ast_lowering_not_supported_for_lifetime_binder_async_closure = - `for<...>` binders on `async` closures are not currently supported - ast_lowering_previously_used_here = previously used here ast_lowering_register1 = register `{$reg1_name}` diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs index 7658dfa5d5f..ec92afee47a 100644 --- a/compiler/rustc_ast_lowering/src/errors.rs +++ b/compiler/rustc_ast_lowering/src/errors.rs @@ -326,13 +326,6 @@ pub struct MisplacedRelaxTraitBound { pub span: Span, } -#[derive(Diagnostic, Clone, Copy)] -#[diag(ast_lowering_not_supported_for_lifetime_binder_async_closure)] -pub struct NotSupportedForLifetimeBinderAsyncClosure { - #[primary_span] - pub span: Span, -} - #[derive(Diagnostic)] #[diag(ast_lowering_match_arm_with_no_body)] pub struct MatchArmWithNoBody { diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index 3b00a84e67e..c4798887637 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -1,9 +1,10 @@ +use std::assert_matches::assert_matches; + use super::errors::{ AsyncCoroutinesNotSupported, AwaitOnlyInAsyncFnAndBlocks, BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters, FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody, - NeverPatternWithBody, NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure, - UnderscoreExprLhsAssign, + NeverPatternWithBody, NeverPatternWithGuard, UnderscoreExprLhsAssign, }; use super::ResolverAstLoweringExt; use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs}; @@ -1028,30 +1029,27 @@ impl<'hir> LoweringContext<'_, 'hir> { fn_decl_span: Span, fn_arg_span: Span, ) -> hir::ExprKind<'hir> { - if let &ClosureBinder::For { span, .. } = binder { - self.dcx().emit_err(NotSupportedForLifetimeBinderAsyncClosure { span }); - } - let (binder_clause, generic_params) = self.lower_closure_binder(binder); + assert_matches!( + coroutine_kind, + CoroutineKind::Async { .. }, + "only async closures are supported currently" + ); + let body = self.with_new_scopes(fn_decl_span, |this| { + let inner_decl = + FnDecl { inputs: decl.inputs.clone(), output: FnRetTy::Default(fn_decl_span) }; + // Transform `async |x: u8| -> X { ... }` into // `|x: u8| || -> X { ... }`. let body_id = this.lower_body(|this| { - let async_ret_ty = if let FnRetTy::Ty(ty) = &decl.output { - let itctx = ImplTraitContext::Disallowed(ImplTraitPosition::AsyncBlock); - Some(hir::FnRetTy::Return(this.lower_ty(ty, &itctx))) - } else { - None - }; - let (parameters, expr) = this.lower_coroutine_body_with_moved_arguments( - decl, + &inner_decl, |this| this.with_new_scopes(fn_decl_span, |this| this.lower_expr_mut(body)), body.span, coroutine_kind, hir::CoroutineSource::Closure, - async_ret_ty, ); let hir_id = this.lower_node_id(coroutine_kind.closure_id()); @@ -1062,15 +1060,12 @@ impl<'hir> LoweringContext<'_, 'hir> { body_id }); - let outer_decl = - FnDecl { inputs: decl.inputs.clone(), output: FnRetTy::Default(fn_decl_span) }; - let bound_generic_params = self.lower_lifetime_binder(closure_id, generic_params); // We need to lower the declaration outside the new scope, because we // have to conserve the state of being inside a loop condition for the // closure argument types. let fn_decl = - self.lower_fn_decl(&outer_decl, closure_id, fn_decl_span, FnDeclKind::Closure, None); + self.lower_fn_decl(&decl, closure_id, fn_decl_span, FnDeclKind::Closure, None); let c = self.arena.alloc(hir::Closure { def_id: self.local_def_id(closure_id), @@ -1081,7 +1076,10 @@ impl<'hir> LoweringContext<'_, 'hir> { body, fn_decl_span: self.lower_span(fn_decl_span), fn_arg_span: Some(self.lower_span(fn_arg_span)), - kind: hir::ClosureKind::Closure, + // Lower this as a `CoroutineClosure`. That will ensure that HIR typeck + // knows that a `FnDecl` output type like `-> &str` actually means + // "coroutine that returns &str", rather than directly returning a `&str`. + kind: hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async), constness: hir::Constness::NotConst, }); hir::ExprKind::Closure(c) diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index 987f74af0a4..2a3e172f771 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -1091,7 +1091,6 @@ impl<'hir> LoweringContext<'_, 'hir> { body.span, coroutine_kind, hir::CoroutineSource::Fn, - None, ); // FIXME(async_fn_track_caller): Can this be moved above? @@ -1113,7 +1112,6 @@ impl<'hir> LoweringContext<'_, 'hir> { body_span: Span, coroutine_kind: CoroutineKind, coroutine_source: hir::CoroutineSource, - return_type_hint: Option<hir::FnRetTy<'hir>>, ) -> (&'hir [hir::Param<'hir>], hir::Expr<'hir>) { let mut parameters: Vec<hir::Param<'_>> = Vec::new(); let mut statements: Vec<hir::Stmt<'_>> = Vec::new(); @@ -1283,12 +1281,13 @@ impl<'hir> LoweringContext<'_, 'hir> { }; let closure_id = coroutine_kind.closure_id(); let coroutine_expr = self.make_desugared_coroutine_expr( - // FIXME(async_closures): This should only move locals, - // and not upvars. Capturing closure upvars by ref doesn't - // work right now anyways, so whatever. - CaptureBy::Value { move_kw: rustc_span::DUMMY_SP }, + // The default capture mode here is by-ref. Later on during upvar analysis, + // we will force the captured arguments to by-move, but for async closures, + // we want to make sure that we avoid unnecessarily moving captures, or else + // all async closures would default to `FnOnce` as their calling mode. + CaptureBy::Ref, closure_id, - return_type_hint, + None, body_span, desugaring_kind, coroutine_source, diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index c0b6922fc05..d6abaa8428b 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -33,6 +33,7 @@ #![allow(internal_features)] #![feature(rustdoc_internals)] #![doc(rust_logo)] +#![feature(assert_matches)] #![feature(box_patterns)] #![feature(let_chains)] #![deny(rustc::untranslatable_diagnostic)] @@ -298,7 +299,6 @@ enum ImplTraitPosition { Path, Variable, Trait, - AsyncBlock, Bound, Generic, ExternFnParam, @@ -325,7 +325,6 @@ impl std::fmt::Display for ImplTraitPosition { ImplTraitPosition::Path => "paths", ImplTraitPosition::Variable => "the type of variable bindings", ImplTraitPosition::Trait => "traits", - ImplTraitPosition::AsyncBlock => "async blocks", ImplTraitPosition::Bound => "bounds", ImplTraitPosition::Generic => "generics", ImplTraitPosition::ExternFnParam => "`extern fn` parameters", diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs index b0b7cc076ba..6debb3362b0 100644 --- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs @@ -858,7 +858,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { use crate::session_diagnostics::CaptureVarCause::*; match kind { hir::ClosureKind::Coroutine(_) => MoveUseInCoroutine { var_span }, - hir::ClosureKind::Closure => MoveUseInClosure { var_span }, + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { + MoveUseInClosure { var_span } + } } }); @@ -905,7 +907,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { hir::ClosureKind::Coroutine(_) => { BorrowUsePlaceCoroutine { place: desc_place, var_span, is_single_var: true } } - hir::ClosureKind::Closure => { + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { BorrowUsePlaceClosure { place: desc_place, var_span, is_single_var: true } } } @@ -1056,7 +1058,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { var_span, is_single_var: true, }, - hir::ClosureKind::Closure => BorrowUsePlaceClosure { + hir::ClosureKind::Closure + | hir::ClosureKind::CoroutineClosure(_) => BorrowUsePlaceClosure { place: desc_place, var_span, is_single_var: true, @@ -1140,7 +1143,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { var_span, is_single_var: false, }, - hir::ClosureKind::Closure => { + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { BorrowUsePlaceClosure { place: desc_place, var_span, is_single_var: false } } } @@ -1158,7 +1161,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { hir::ClosureKind::Coroutine(_) => { FirstBorrowUsePlaceCoroutine { place: borrow_place_desc, var_span } } - hir::ClosureKind::Closure => { + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { FirstBorrowUsePlaceClosure { place: borrow_place_desc, var_span } } } @@ -1175,7 +1178,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { hir::ClosureKind::Coroutine(_) => { SecondBorrowUsePlaceCoroutine { place: desc_place, var_span } } - hir::ClosureKind::Closure => { + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { SecondBorrowUsePlaceClosure { place: desc_place, var_span } } } @@ -2942,7 +2945,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { use crate::session_diagnostics::CaptureVarCause::*; match kind { hir::ClosureKind::Coroutine(_) => BorrowUseInCoroutine { var_span }, - hir::ClosureKind::Closure => BorrowUseInClosure { var_span }, + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { + BorrowUseInClosure { var_span } + } } }); @@ -2958,7 +2963,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { use crate::session_diagnostics::CaptureVarCause::*; match kind { hir::ClosureKind::Coroutine(_) => BorrowUseInCoroutine { var_span }, - hir::ClosureKind::Closure => BorrowUseInClosure { var_span }, + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { + BorrowUseInClosure { var_span } + } } }); diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs index b35d4e16ecc..59f3aa706ed 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mod.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs @@ -614,7 +614,7 @@ impl UseSpans<'_> { PartialAssignment => AssignPartInCoroutine { path_span }, }); } - hir::ClosureKind::Closure => { + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { err.subdiagnostic(match action { Borrow => BorrowInClosure { path_span }, MatchOn | Use => UseInClosure { path_span }, @@ -1253,7 +1253,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { hir::ClosureKind::Coroutine(_) => { CaptureVarCause::PartialMoveUseInCoroutine { var_span, is_partial } } - hir::ClosureKind::Closure => { + hir::ClosureKind::Closure | hir::ClosureKind::CoroutineClosure(_) => { CaptureVarCause::PartialMoveUseInClosure { var_span, is_partial } } }) diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs index 3fddf67f55b..55649ec2f16 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs @@ -1472,7 +1472,7 @@ fn suggest_ampmut<'tcx>( } fn is_closure_or_coroutine(ty: Ty<'_>) -> bool { - ty.is_closure() || ty.is_coroutine() + ty.is_closure() || ty.is_coroutine() || ty.is_coroutine_closure() } /// Given a field that needs to be mutable, returns a span where the " mut " could go. diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs index 15e1066e983..c91b8eaab05 100644 --- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs +++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs @@ -324,9 +324,13 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> { ty::BoundRegionKind::BrEnv => { let def_ty = self.regioncx.universal_regions().defining_ty; - let DefiningTy::Closure(_, args) = def_ty else { - // Can't have BrEnv in functions, constants or coroutines. - bug!("BrEnv outside of closure."); + let closure_kind = match def_ty { + DefiningTy::Closure(_, args) => args.as_closure().kind(), + DefiningTy::CoroutineClosure(_, args) => args.as_coroutine_closure().kind(), + _ => { + // Can't have BrEnv in functions, constants or coroutines. + bug!("BrEnv outside of closure."); + } }; let hir::ExprKind::Closure(&hir::Closure { fn_decl_span, .. }) = tcx.hir().expect_expr(self.mir_hir_id()).kind @@ -334,21 +338,18 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> { bug!("Closure is not defined by a closure expr"); }; let region_name = self.synthesize_region_name(); - - let closure_kind_ty = args.as_closure().kind_ty(); - let note = match closure_kind_ty.to_opt_closure_kind() { - Some(ty::ClosureKind::Fn) => { + let note = match closure_kind { + ty::ClosureKind::Fn => { "closure implements `Fn`, so references to captured variables \ can't escape the closure" } - Some(ty::ClosureKind::FnMut) => { + ty::ClosureKind::FnMut => { "closure implements `FnMut`, so references to captured variables \ can't escape the closure" } - Some(ty::ClosureKind::FnOnce) => { + ty::ClosureKind::FnOnce => { bug!("BrEnv in a `FnOnce` closure"); } - None => bug!("Closure kind not inferred in borrow check"), }; Some(RegionName { @@ -692,7 +693,10 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> { hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( hir::CoroutineDesugaring::Async, hir::CoroutineSource::Closure, - )) => " of async closure", + )) + | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async) => { + " of async closure" + } hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( hir::CoroutineDesugaring::Async, @@ -719,7 +723,10 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> { hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( hir::CoroutineDesugaring::Gen, hir::CoroutineSource::Closure, - )) => " of gen closure", + )) + | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Gen) => { + " of gen closure" + } hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( hir::CoroutineDesugaring::Gen, @@ -743,7 +750,10 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> { hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( hir::CoroutineDesugaring::AsyncGen, hir::CoroutineSource::Closure, - )) => " of async gen closure", + )) + | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::AsyncGen) => { + " of async gen closure" + } hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( hir::CoroutineDesugaring::AsyncGen, diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 8b5e548345c..bb64571889b 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -3,6 +3,7 @@ #![allow(internal_features)] #![feature(rustdoc_internals)] #![doc(rust_logo)] +#![feature(assert_matches)] #![feature(associated_type_bounds)] #![feature(box_patterns)] #![feature(let_chains)] @@ -1303,7 +1304,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { // moved into the closure and subsequently used by the closure, // in order to populate our used_mut set. match **aggregate_kind { - AggregateKind::Closure(def_id, _) | AggregateKind::Coroutine(def_id, _) => { + AggregateKind::Closure(def_id, _) + | AggregateKind::CoroutineClosure(def_id, _) + | AggregateKind::Coroutine(def_id, _) => { let def_id = def_id.expect_local(); let BorrowCheckResult { used_mut_upvars, .. } = self.infcx.tcx.mir_borrowck(def_id); @@ -1609,6 +1612,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { | ty::FnPtr(_) | ty::Dynamic(_, _, _) | ty::Closure(_, _) + | ty::CoroutineClosure(_, _) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never @@ -1633,7 +1637,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { return; } } - ty::Closure(_, _) | ty::Coroutine(_, _) | ty::Tuple(_) => (), + ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Coroutine(_, _) + | ty::Tuple(_) => (), ty::Bool | ty::Char | ty::Int(_) diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs index 2d997dfadf0..4cfde47664e 100644 --- a/compiler/rustc_borrowck/src/path_utils.rs +++ b/compiler/rustc_borrowck/src/path_utils.rs @@ -164,7 +164,7 @@ pub(crate) fn is_upvar_field_projection<'tcx>( match place_ref.last_projection() { Some((place_base, ProjectionElem::Field(field, _ty))) => { let base_ty = place_base.ty(body, tcx).ty; - if (base_ty.is_closure() || base_ty.is_coroutine()) + if (base_ty.is_closure() || base_ty.is_coroutine() || base_ty.is_coroutine_closure()) && (!by_ref || upvars[field.index()].is_by_ref()) { Some(field) diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs index 59518f68ab1..a5a7ce4ea3e 100644 --- a/compiler/rustc_borrowck/src/type_check/input_output.rs +++ b/compiler/rustc_borrowck/src/type_check/input_output.rs @@ -7,13 +7,18 @@ //! `RETURN_PLACE` the MIR arguments) are always fully normalized (and //! contain revealed `impl Trait` values). +use std::assert_matches::assert_matches; + use itertools::Itertools; -use rustc_infer::infer::BoundRegionConversionTime; +use rustc_hir as hir; +use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind}; +use rustc_infer::infer::{BoundRegionConversionTime, RegionVariableOrigin}; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty}; use rustc_span::Span; -use crate::universal_regions::UniversalRegions; +use crate::renumber::RegionCtxt; +use crate::universal_regions::{DefiningTy, UniversalRegions}; use super::{Locations, TypeChecker}; @@ -23,9 +28,11 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { #[instrument(skip(self, body), level = "debug")] pub(super) fn check_signature_annotation(&mut self, body: &Body<'tcx>) { let mir_def_id = body.source.def_id().expect_local(); + if !self.tcx().is_closure_or_coroutine(mir_def_id.to_def_id()) { return; } + let user_provided_poly_sig = self.tcx().closure_user_provided_sig(mir_def_id); // Instantiate the canonicalized variables from user-provided signature @@ -34,12 +41,75 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // so that they represent the view from "inside" the closure. let user_provided_sig = self .instantiate_canonical_with_fresh_inference_vars(body.span, &user_provided_poly_sig); - let user_provided_sig = self.infcx.instantiate_binder_with_fresh_vars( + let mut user_provided_sig = self.infcx.instantiate_binder_with_fresh_vars( body.span, BoundRegionConversionTime::FnCall, user_provided_sig, ); + // FIXME(async_closures): It's kind of wacky that we must apply this + // transformation here, since we do the same thing in HIR typeck. + // Maybe we could just fix up the canonicalized signature during HIR typeck? + if let DefiningTy::CoroutineClosure(_, args) = + self.borrowck_context.universal_regions.defining_ty + { + assert_matches!( + self.tcx().coroutine_kind(self.tcx().coroutine_for_closure(mir_def_id)), + Some(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Async, + hir::CoroutineSource::Closure + )), + "this needs to be modified if we're lowering non-async closures" + ); + // Make sure to use the args from `DefiningTy` so the right NLL region vids are prepopulated + // into the type. + let args = args.as_coroutine_closure(); + let tupled_upvars_ty = ty::CoroutineClosureSignature::tupled_upvars_by_closure_kind( + self.tcx(), + args.kind(), + Ty::new_tup(self.tcx(), user_provided_sig.inputs()), + args.tupled_upvars_ty(), + args.coroutine_captures_by_ref_ty(), + self.infcx.next_region_var(RegionVariableOrigin::MiscVariable(body.span), || { + RegionCtxt::Unknown + }), + ); + + let next_ty_var = || { + self.infcx.next_ty_var(TypeVariableOrigin { + span: body.span, + kind: TypeVariableOriginKind::MiscVariable, + }) + }; + let output_ty = Ty::new_coroutine( + self.tcx(), + self.tcx().coroutine_for_closure(mir_def_id), + ty::CoroutineArgs::new( + self.tcx(), + ty::CoroutineArgsParts { + parent_args: args.parent_args(), + kind_ty: Ty::from_closure_kind(self.tcx(), args.kind()), + return_ty: user_provided_sig.output(), + tupled_upvars_ty, + // For async closures, none of these can be annotated, so just fill + // them with fresh ty vars. + resume_ty: next_ty_var(), + yield_ty: next_ty_var(), + witness: next_ty_var(), + }, + ) + .args, + ); + + user_provided_sig = self.tcx().mk_fn_sig( + user_provided_sig.inputs().iter().copied(), + output_ty, + user_provided_sig.c_variadic, + user_provided_sig.unsafety, + user_provided_sig.abi, + ); + } + let is_coroutine_with_implicit_resume_ty = self.tcx().is_coroutine(mir_def_id.to_def_id()) && user_provided_sig.inputs().is_empty(); diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 59c4d9a6c78..cdb187e0776 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -808,6 +808,14 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { }), }; } + ty::CoroutineClosure(_, args) => { + return match args.as_coroutine_closure().upvar_tys().get(field.index()) { + Some(&ty) => Ok(ty), + None => Err(FieldAccessError::OutOfRange { + field_count: args.as_coroutine_closure().upvar_tys().len(), + }), + }; + } ty::Coroutine(_, args) => { // Only prefix fields (upvars and current state) are // accessible without a variant index. @@ -1875,6 +1883,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { }), } } + AggregateKind::CoroutineClosure(_, args) => { + match args.as_coroutine_closure().upvar_tys().get(field_index.as_usize()) { + Some(ty) => Ok(*ty), + None => Err(FieldAccessError::OutOfRange { + field_count: args.as_coroutine_closure().upvar_tys().len(), + }), + } + } AggregateKind::Array(ty) => Ok(ty), AggregateKind::Tuple => { unreachable!("This should have been covered in check_rvalues"); @@ -2478,6 +2494,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { AggregateKind::Tuple => None, AggregateKind::Closure(_, _) => None, AggregateKind::Coroutine(_, _) => None, + AggregateKind::CoroutineClosure(_, _) => None, }, } } @@ -2705,7 +2722,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // desugaring. A closure gets desugared to a struct, and // these extra requirements are basically like where // clauses on the struct. - AggregateKind::Closure(def_id, args) | AggregateKind::Coroutine(def_id, args) => ( + AggregateKind::Closure(def_id, args) + | AggregateKind::CoroutineClosure(def_id, args) + | AggregateKind::Coroutine(def_id, args) => ( def_id, self.prove_closure_bounds( tcx, @@ -2754,10 +2773,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { let typeck_root_args = ty::GenericArgs::identity_for_item(tcx, typeck_root_def_id); let parent_args = match tcx.def_kind(def_id) { - DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => { - args.as_coroutine().parent_args() + // We don't want to dispatch on 3 different kind of closures here, so take + // advantage of the fact that the `parent_args` is the same length as the + // `typeck_root_args`. + DefKind::Closure => { + // FIXME(async_closures): It may be useful to add a debug assert here + // to actually call `type_of` and check the `parent_args` are the same + // length as the `typeck_root_args`. + &args[..typeck_root_args.len()] } - DefKind::Closure => args.as_closure().parent_args(), DefKind::InlineConst => args.as_inline_const().parent_args(), other => bug!("unexpected item {:?}", other), }; diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs index ae8a135f090..111eaaf60f7 100644 --- a/compiler/rustc_borrowck/src/universal_regions.rs +++ b/compiler/rustc_borrowck/src/universal_regions.rs @@ -97,6 +97,13 @@ pub enum DefiningTy<'tcx> { /// `ClosureArgs::coroutine_return_ty`. Coroutine(DefId, GenericArgsRef<'tcx>), + /// The MIR is a special kind of closure that returns coroutines. + /// + /// See the documentation on `CoroutineClosureSignature` for details + /// on how to construct the callable signature of the coroutine from + /// its args. + CoroutineClosure(DefId, GenericArgsRef<'tcx>), + /// The MIR is a fn item with the given `DefId` and args. The signature /// of the function can be bound then with the `fn_sig` query. FnDef(DefId, GenericArgsRef<'tcx>), @@ -119,6 +126,7 @@ impl<'tcx> DefiningTy<'tcx> { pub fn upvar_tys(self) -> &'tcx ty::List<Ty<'tcx>> { match self { DefiningTy::Closure(_, args) => args.as_closure().upvar_tys(), + DefiningTy::CoroutineClosure(_, args) => args.as_coroutine_closure().upvar_tys(), DefiningTy::Coroutine(_, args) => args.as_coroutine().upvar_tys(), DefiningTy::FnDef(..) | DefiningTy::Const(..) | DefiningTy::InlineConst(..) => { ty::List::empty() @@ -131,7 +139,9 @@ impl<'tcx> DefiningTy<'tcx> { /// user's code. pub fn implicit_inputs(self) -> usize { match self { - DefiningTy::Closure(..) | DefiningTy::Coroutine(..) => 1, + DefiningTy::Closure(..) + | DefiningTy::CoroutineClosure(..) + | DefiningTy::Coroutine(..) => 1, DefiningTy::FnDef(..) | DefiningTy::Const(..) | DefiningTy::InlineConst(..) => 0, } } @@ -147,6 +157,7 @@ impl<'tcx> DefiningTy<'tcx> { pub fn def_id(&self) -> DefId { match *self { DefiningTy::Closure(def_id, ..) + | DefiningTy::CoroutineClosure(def_id, ..) | DefiningTy::Coroutine(def_id, ..) | DefiningTy::FnDef(def_id, ..) | DefiningTy::Const(def_id, ..) @@ -355,6 +366,9 @@ impl<'tcx> UniversalRegions<'tcx> { err.note(format!("late-bound region is {:?}", self.to_region_vid(r))); }); } + DefiningTy::CoroutineClosure(..) => { + todo!() + } DefiningTy::Coroutine(def_id, args) => { let v = with_no_trimmed_paths!( args[tcx.generics_of(def_id).parent_count..] @@ -568,6 +582,9 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { match *defining_ty.kind() { ty::Closure(def_id, args) => DefiningTy::Closure(def_id, args), ty::Coroutine(def_id, args) => DefiningTy::Coroutine(def_id, args), + ty::CoroutineClosure(def_id, args) => { + DefiningTy::CoroutineClosure(def_id, args) + } ty::FnDef(def_id, args) => DefiningTy::FnDef(def_id, args), _ => span_bug!( tcx.def_span(self.mir_def), @@ -623,6 +640,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { let identity_args = GenericArgs::identity_for_item(tcx, typeck_root_def_id); let fr_args = match defining_ty { DefiningTy::Closure(_, args) + | DefiningTy::CoroutineClosure(_, args) | DefiningTy::Coroutine(_, args) | DefiningTy::InlineConst(_, args) => { // In the case of closures, we rely on the fact that @@ -702,6 +720,55 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { ty::Binder::dummy(inputs_and_output) } + // Construct the signature of the CoroutineClosure for the purposes of borrowck. + // This is pretty straightforward -- we: + // 1. first grab the `coroutine_closure_sig`, + // 2. compute the self type (`&`/`&mut`/no borrow), + // 3. flatten the tupled_input_tys, + // 4. construct the correct generator type to return with + // `CoroutineClosureSignature::to_coroutine_given_kind_and_upvars`. + // Then we wrap it all up into a list of inputs and output. + DefiningTy::CoroutineClosure(def_id, args) => { + assert_eq!(self.mir_def.to_def_id(), def_id); + let closure_sig = args.as_coroutine_closure().coroutine_closure_sig(); + let bound_vars = tcx.mk_bound_variable_kinds_from_iter( + closure_sig + .bound_vars() + .iter() + .chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))), + ); + let br = ty::BoundRegion { + var: ty::BoundVar::from_usize(bound_vars.len() - 1), + kind: ty::BrEnv, + }; + let env_region = ty::Region::new_bound(tcx, ty::INNERMOST, br); + let closure_kind = args.as_coroutine_closure().kind(); + + let closure_ty = tcx.closure_env_ty( + Ty::new_coroutine_closure(tcx, def_id, args), + closure_kind, + env_region, + ); + + let inputs = closure_sig.skip_binder().tupled_inputs_ty.tuple_fields(); + let output = closure_sig.skip_binder().to_coroutine_given_kind_and_upvars( + tcx, + args.as_coroutine_closure().parent_args(), + tcx.coroutine_for_closure(def_id), + closure_kind, + env_region, + args.as_coroutine_closure().tupled_upvars_ty(), + args.as_coroutine_closure().coroutine_captures_by_ref_ty(), + ); + + ty::Binder::bind_with_vars( + tcx.mk_type_list_from_iter( + iter::once(closure_ty).chain(inputs).chain(iter::once(output)), + ), + bound_vars, + ) + } + DefiningTy::FnDef(def_id, _) => { let sig = tcx.fn_sig(def_id).instantiate_identity(); let sig = indices.fold_to_region_vids(tcx, sig); diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs index e5c0b2de4ca..25149b80201 100644 --- a/compiler/rustc_codegen_gcc/src/type_of.rs +++ b/compiler/rustc_codegen_gcc/src/type_of.rs @@ -87,7 +87,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str if !cx.sess().fewer_names() => { let mut name = with_no_trimmed_paths!(layout.ty.to_string()); diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index e88f4217c9d..219c7025311 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -33,7 +33,7 @@ fn uncached_llvm_type<'a, 'tcx>( // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str // For performance reasons we use names only when emitting LLVM IR. if !cx.sess().fewer_names() => { diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs index 4f9f70648bd..5bd7442822a 100644 --- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs +++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs @@ -398,7 +398,9 @@ fn push_debuginfo_type_name<'tcx>( // processing visited.remove(&t); } - ty::Closure(def_id, args) | ty::Coroutine(def_id, args, ..) => { + ty::Closure(def_id, args) + | ty::CoroutineClosure(def_id, args) + | ty::Coroutine(def_id, args, ..) => { // Name will be "{closure_env#0}<T1, T2, ...>", "{coroutine_env#0}<T1, T2, ...>", or // "{async_fn_env#0}<T1, T2, ...>", etc. // In the case of cpp-like debuginfo, the name additionally gets wrapped inside of @@ -768,6 +770,8 @@ fn push_closure_or_coroutine_name<'tcx>( // Truncate the args to the length of the above generics. This will cut off // anything closure- or coroutine-specific. + // FIXME(async_closures): This is probably not going to be correct w.r.t. + // multiple coroutine flavors. Maybe truncate to (parent + 1)? let args = args.truncate_to(tcx, generics); push_generic_params_internal(tcx, args, enclosing_fn_def_id, output, visited); } diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index 12544f5b029..5c2bf4626c4 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -172,6 +172,7 @@ pub(crate) fn const_to_valtree_inner<'tcx>( | ty::Infer(_) // FIXME(oli-obk): we can probably encode closures just like structs | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) => Err(ValTreeCreationError::NonSupportedType), } @@ -301,6 +302,7 @@ pub fn valtree_to_const_value<'tcx>( | ty::Placeholder(..) | ty::Infer(_) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::FnPtr(_) diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index c14bd142efa..dd989ab80fd 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -1007,6 +1007,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { | ty::CoroutineWitness(..) | ty::Array(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Never | ty::Error(_) => true, diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 1e9e7d94596..7991f90b815 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -85,6 +85,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>( | ty::FnPtr(_) | ty::Dynamic(_, _, _) | ty::Closure(_, _) + | ty::CoroutineClosure(_, _) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index b7ffb4a16fc..85a2e4778d2 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -545,6 +545,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ty::InstanceDef::VTableShim(..) | ty::InstanceDef::ReifyShim(..) | ty::InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } + | ty::InstanceDef::CoroutineKindShim { .. } | ty::InstanceDef::FnPtrShim(..) | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::CloneShim(..) diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index b5cd3259520..811c2c3c208 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -644,6 +644,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' | ty::Str | ty::Dynamic(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) => Ok(false), // Some types only occur during typechecking, they have no layout. // We should not see them here and we could not check them anyway. diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index 21bdb66a276..c4542aaa7b2 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -58,6 +58,7 @@ impl<'tcx> MirPass<'tcx> for Validator { let body_abi = match body_ty.kind() { ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), ty::Closure(..) => Abi::RustCall, + ty::CoroutineClosure(..) => Abi::RustCall, ty::Coroutine(..) => Abi::Rust, _ => { span_bug!(body.span, "unexpected body ty: {:?} phase {:?}", body_ty, mir_phase) @@ -665,6 +666,14 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; check_equal(self, location, f_ty); } + ty::CoroutineClosure(_, args) => { + let args = args.as_coroutine_closure(); + let Some(&f_ty) = args.upvar_tys().get(f.as_usize()) else { + fail_out_of_bounds(self, location); + return; + }; + check_equal(self, location, f_ty); + } &ty::Coroutine(def_id, args) => { let f_ty = if let Some(var) = parent_ty.variant_index { let gen_body = if def_id == self.body.source.def_id() { @@ -861,6 +870,20 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } } + AggregateKind::CoroutineClosure(_, args) => { + let upvars = args.as_coroutine_closure().upvar_tys(); + if upvars.len() != fields.len() { + self.fail( + location, + "coroutine-closure has the wrong number of initialized fields", + ); + } + for (src, dest) in std::iter::zip(fields, upvars) { + if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest) { + self.fail(location, "coroutine-closure field has the wrong type"); + } + } + } }, Rvalue::Ref(_, BorrowKind::Fake, _) => { if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { diff --git a/compiler/rustc_const_eval/src/util/type_name.rs b/compiler/rustc_const_eval/src/util/type_name.rs index 976e42ad768..2b80623ab45 100644 --- a/compiler/rustc_const_eval/src/util/type_name.rs +++ b/compiler/rustc_const_eval/src/util/type_name.rs @@ -51,6 +51,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { | ty::FnDef(def_id, args) | ty::Alias(ty::Projection | ty::Opaque, ty::AliasTy { def_id, args, .. }) | ty::Closure(def_id, args) + | ty::CoroutineClosure(def_id, args) | ty::Coroutine(def_id, args) => self.print_def_path(def_id, args), ty::Foreign(def_id) => self.print_def_path(def_id, &[]), diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index de1b28acb12..ff50086ff8f 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -952,6 +952,11 @@ pub enum ClosureKind { /// usage (e.g. `let x = || { yield (); }`) or from a desugared expression /// (e.g. `async` and `gen` blocks). Coroutine(CoroutineKind), + /// This is a coroutine-closure, which is a special sugared closure that + /// returns one of the sugared coroutine (`async`/`gen`/`async gen`). It + /// additionally allows capturing the coroutine's upvars by ref, and therefore + /// needs to be specially treated during analysis and borrowck. + CoroutineClosure(CoroutineDesugaring), } /// A block of statements `{ .. }`, which may have a label (in this case the @@ -3698,6 +3703,7 @@ impl<'hir> Node<'hir> { expect_generic_param, &'hir GenericParam<'hir>, Node::GenericParam(n), n; expect_crate, &'hir Mod<'hir>, Node::Crate(n), n; expect_infer, &'hir InferArg, Node::Infer(n), n; + expect_closure, &'hir Closure<'hir>, Node::Expr(Expr { kind: ExprKind::Closure(n), .. }), n; } } diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 8e327c029df..9c9e72d6e65 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -209,6 +209,7 @@ language_item_table! { AsyncFn, sym::async_fn, async_fn_trait, Target::Trait, GenericRequirement::Exact(1); AsyncFnMut, sym::async_fn_mut, async_fn_mut_trait, Target::Trait, GenericRequirement::Exact(1); AsyncFnOnce, sym::async_fn_once, async_fn_once_trait, Target::Trait, GenericRequirement::Exact(1); + AsyncFnKindHelper, sym::async_fn_kind_helper,async_fn_kind_helper, Target::Trait, GenericRequirement::Exact(1); FnOnceOutput, sym::fn_once_output, fn_once_output, Target::AssocTy, GenericRequirement::None; diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs index abef365c3ca..0df5a57bc2c 100644 --- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs +++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs @@ -171,6 +171,7 @@ impl<'tcx> InherentCollect<'tcx> { } ty::FnDef(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Bound(..) diff --git a/compiler/rustc_hir_analysis/src/coherence/orphan.rs b/compiler/rustc_hir_analysis/src/coherence/orphan.rs index 1736de760d5..45641be52d2 100644 --- a/compiler/rustc_hir_analysis/src/coherence/orphan.rs +++ b/compiler/rustc_hir_analysis/src/coherence/orphan.rs @@ -244,6 +244,7 @@ fn do_orphan_check_impl<'tcx>( | ty::Tuple(..) => (LocalImpl::Allow, NonlocalImpl::DisallowOther), ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Bound(..) diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs index 8d862d5eb71..fbcebb7c87c 100644 --- a/compiler/rustc_hir_analysis/src/collect.rs +++ b/compiler/rustc_hir_analysis/src/collect.rs @@ -81,6 +81,7 @@ pub fn provide(providers: &mut Providers) { impl_trait_ref, impl_polarity, coroutine_kind, + coroutine_for_closure, collect_mod_item_types, is_type_alias_impl_trait, ..*providers @@ -1531,6 +1532,29 @@ fn coroutine_kind(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<hir::CoroutineK } } +fn coroutine_for_closure(tcx: TyCtxt<'_>, def_id: LocalDefId) -> DefId { + let &rustc_hir::Closure { kind: hir::ClosureKind::CoroutineClosure(_), body, .. } = + tcx.hir_node_by_def_id(def_id).expect_closure() + else { + bug!() + }; + + let &hir::Expr { + kind: + hir::ExprKind::Closure(&rustc_hir::Closure { + def_id, + kind: hir::ClosureKind::Coroutine(_), + .. + }), + .. + } = tcx.hir().body(body).value + else { + bug!() + }; + + def_id.to_def_id() +} + fn is_type_alias_impl_trait<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> bool { match tcx.hir_node_by_def_id(def_id) { Node::Item(hir::Item { kind: hir::ItemKind::OpaqueTy(opaque), .. }) => { diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs index c29d4131843..1dabb6feb5e 100644 --- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs @@ -349,6 +349,13 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics { ClosureKind::Coroutine(_) => { &["<resume_ty>", "<yield_ty>", "<return_ty>", "<witness>", "<upvars>"][..] } + ClosureKind::CoroutineClosure(_) => &[ + "<closure_kind>", + "<closure_signature_parts>", + "<upvars>", + "<bound_captures_by_ref>", + "<witness>", + ][..], }; params.extend(dummy_args.iter().map(|&arg| ty::GenericParamDef { diff --git a/compiler/rustc_hir_analysis/src/variance/constraints.rs b/compiler/rustc_hir_analysis/src/variance/constraints.rs index f09594cbbc6..580cdb4a3a2 100644 --- a/compiler/rustc_hir_analysis/src/variance/constraints.rs +++ b/compiler/rustc_hir_analysis/src/variance/constraints.rs @@ -235,8 +235,8 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> { // leaf type -- noop } - ty::FnDef(..) | ty::Coroutine(..) | ty::Closure(..) => { - bug!("Unexpected closure type in variance computation"); + ty::FnDef(..) | ty::Coroutine(..) | ty::Closure(..) | ty::CoroutineClosure(..) => { + bug!("Unexpected coroutine/closure type in variance computation"); } ty::Ref(region, ty, mutbl) => { diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs index b263c985534..fbe6f454dbc 100644 --- a/compiler/rustc_hir_typeck/src/callee.rs +++ b/compiler/rustc_hir_typeck/src/callee.rs @@ -141,33 +141,77 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { return Some(CallStep::Builtin(adjusted_ty)); } - ty::Closure(def_id, args) => { + // Check whether this is a call to a closure where we + // haven't yet decided on whether the closure is fn vs + // fnmut vs fnonce. If so, we have to defer further processing. + ty::Closure(def_id, args) if self.closure_kind(adjusted_ty).is_none() => { let def_id = def_id.expect_local(); + let closure_sig = args.as_closure().sig(); + let closure_sig = self.instantiate_binder_with_fresh_vars( + call_expr.span, + infer::FnCall, + closure_sig, + ); + let adjustments = self.adjust_steps(autoderef); + self.record_deferred_call_resolution( + def_id, + DeferredCallResolution { + call_expr, + callee_expr, + closure_ty: adjusted_ty, + adjustments, + fn_sig: closure_sig, + }, + ); + return Some(CallStep::DeferredClosure(def_id, closure_sig)); + } - // Check whether this is a call to a closure where we - // haven't yet decided on whether the closure is fn vs - // fnmut vs fnonce. If so, we have to defer further processing. - if self.closure_kind(args).is_none() { - let closure_sig = args.as_closure().sig(); - let closure_sig = self.instantiate_binder_with_fresh_vars( - call_expr.span, - infer::FnCall, - closure_sig, - ); - let adjustments = self.adjust_steps(autoderef); - self.record_deferred_call_resolution( - def_id, - DeferredCallResolution { - call_expr, - callee_expr, - adjusted_ty, - adjustments, - fn_sig: closure_sig, - closure_args: args, - }, - ); - return Some(CallStep::DeferredClosure(def_id, closure_sig)); - } + // When calling a `CoroutineClosure` that is local to the body, we will + // not know what its `closure_kind` is yet. Instead, just fill in the + // signature with an infer var for the `tupled_upvars_ty` of the coroutine, + // and record a deferred call resolution which will constrain that var + // as part of `AsyncFn*` trait confirmation. + ty::CoroutineClosure(def_id, args) if self.closure_kind(adjusted_ty).is_none() => { + let def_id = def_id.expect_local(); + let closure_args = args.as_coroutine_closure(); + let coroutine_closure_sig = self.instantiate_binder_with_fresh_vars( + call_expr.span, + infer::FnCall, + closure_args.coroutine_closure_sig(), + ); + let tupled_upvars_ty = self.next_ty_var(TypeVariableOrigin { + kind: TypeVariableOriginKind::TypeInference, + span: callee_expr.span, + }); + let call_sig = self.tcx.mk_fn_sig( + [coroutine_closure_sig.tupled_inputs_ty], + coroutine_closure_sig.to_coroutine( + self.tcx, + closure_args.parent_args(), + // Inherit the kind ty of the closure, since we're calling this + // coroutine with the most relaxed `AsyncFn*` trait that we can. + // We don't necessarily need to do this here, but it saves us + // computing one more infer var that will get constrained later. + closure_args.kind_ty(), + self.tcx.coroutine_for_closure(def_id), + tupled_upvars_ty, + ), + coroutine_closure_sig.c_variadic, + coroutine_closure_sig.unsafety, + coroutine_closure_sig.abi, + ); + let adjustments = self.adjust_steps(autoderef); + self.record_deferred_call_resolution( + def_id, + DeferredCallResolution { + call_expr, + callee_expr, + closure_ty: adjusted_ty, + adjustments, + fn_sig: call_sig, + }, + ); + return Some(CallStep::DeferredClosure(def_id, call_sig)); } // Hack: we know that there are traits implementing Fn for &F @@ -886,10 +930,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { pub struct DeferredCallResolution<'tcx> { call_expr: &'tcx hir::Expr<'tcx>, callee_expr: &'tcx hir::Expr<'tcx>, - adjusted_ty: Ty<'tcx>, + closure_ty: Ty<'tcx>, adjustments: Vec<Adjustment<'tcx>>, fn_sig: ty::FnSig<'tcx>, - closure_args: GenericArgsRef<'tcx>, } impl<'a, 'tcx> DeferredCallResolution<'tcx> { @@ -898,10 +941,10 @@ impl<'a, 'tcx> DeferredCallResolution<'tcx> { // we should not be invoked until the closure kind has been // determined by upvar inference - assert!(fcx.closure_kind(self.closure_args).is_some()); + assert!(fcx.closure_kind(self.closure_ty).is_some()); // We may now know enough to figure out fn vs fnmut etc. - match fcx.try_overloaded_call_traits(self.call_expr, self.adjusted_ty, None) { + match fcx.try_overloaded_call_traits(self.call_expr, self.closure_ty, None) { Some((autoref, method_callee)) => { // One problem is that when we get here, we are going // to have a newly instantiated function signature @@ -937,7 +980,7 @@ impl<'a, 'tcx> DeferredCallResolution<'tcx> { span_bug!( self.call_expr.span, "Expected to find a suitable `Fn`/`FnMut`/`FnOnce` implementation for `{}`", - self.adjusted_ty + self.closure_ty ) } } diff --git a/compiler/rustc_hir_typeck/src/cast.rs b/compiler/rustc_hir_typeck/src/cast.rs index a0ac839f3dd..58823ea30ce 100644 --- a/compiler/rustc_hir_typeck/src/cast.rs +++ b/compiler/rustc_hir_typeck/src/cast.rs @@ -133,6 +133,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { | ty::FnDef(..) | ty::FnPtr(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::Adt(..) | ty::Never diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs index f51cc97b45d..a985fa201d0 100644 --- a/compiler/rustc_hir_typeck/src/closure.rs +++ b/compiler/rustc_hir_typeck/src/closure.rs @@ -63,7 +63,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { None => (None, None), }; - let ClosureSignatures { bound_sig, liberated_sig } = + let ClosureSignatures { bound_sig, mut liberated_sig } = self.sig_of_closure(expr_def_id, closure.fn_decl, closure.kind, expected_sig); debug!(?bound_sig, ?liberated_sig); @@ -125,7 +125,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Gen, _) | hir::CoroutineKind::Coroutine(_) => { let yield_ty = self.next_ty_var(TypeVariableOrigin { - kind: TypeVariableOriginKind::TypeInference, + kind: TypeVariableOriginKind::ClosureSynthetic, span: expr_span, }); self.require_type_is_sized(yield_ty, expr_span, traits::SizedYieldType); @@ -137,7 +137,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // not a problem. hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::AsyncGen, _) => { let yield_ty = self.next_ty_var(TypeVariableOrigin { - kind: TypeVariableOriginKind::TypeInference, + kind: TypeVariableOriginKind::ClosureSynthetic, span: expr_span, }); self.require_type_is_sized(yield_ty, expr_span, traits::SizedYieldType); @@ -166,8 +166,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let resume_ty = liberated_sig.inputs().get(0).copied().unwrap_or(tcx.types.unit); let interior = self.next_ty_var(TypeVariableOrigin { - kind: TypeVariableOriginKind::MiscVariable, - span: body.value.span, + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, }); self.deferred_coroutine_interiors.borrow_mut().push(( expr_def_id, @@ -175,10 +175,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { interior, )); + // Coroutines that come from coroutine closures have not yet determined + // their kind ty, so make a fresh infer var which will be constrained + // later during upvar analysis. Regular coroutines always have the kind + // ty of `().` + let kind_ty = match kind { + hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure) => self + .next_ty_var(TypeVariableOrigin { + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, + }), + _ => tcx.types.unit, + }; + let coroutine_args = ty::CoroutineArgs::new( tcx, ty::CoroutineArgsParts { parent_args, + kind_ty, resume_ty, yield_ty, return_ty: liberated_sig.output(), @@ -192,6 +206,94 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { Some(CoroutineTypes { resume_ty, yield_ty }), ) } + hir::ClosureKind::CoroutineClosure(kind) => { + // async closures always return the type ascribed after the `->` (if present), + // and yield `()`. + let (bound_return_ty, bound_yield_ty) = match kind { + hir::CoroutineDesugaring::Async => { + (bound_sig.skip_binder().output(), tcx.types.unit) + } + hir::CoroutineDesugaring::Gen | hir::CoroutineDesugaring::AsyncGen => { + todo!("`gen` and `async gen` closures not supported yet") + } + }; + // Compute all of the variables that will be used to populate the coroutine. + let resume_ty = self.next_ty_var(TypeVariableOrigin { + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, + }); + let interior = self.next_ty_var(TypeVariableOrigin { + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, + }); + let closure_kind_ty = self.next_ty_var(TypeVariableOrigin { + // FIXME(eddyb) distinguish closure kind inference variables from the rest. + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, + }); + let coroutine_captures_by_ref_ty = self.next_ty_var(TypeVariableOrigin { + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, + }); + let closure_args = ty::CoroutineClosureArgs::new( + tcx, + ty::CoroutineClosureArgsParts { + parent_args, + closure_kind_ty, + signature_parts_ty: Ty::new_fn_ptr( + tcx, + bound_sig.map_bound(|sig| { + tcx.mk_fn_sig( + [ + resume_ty, + Ty::new_tup_from_iter(tcx, sig.inputs().iter().copied()), + ], + Ty::new_tup(tcx, &[bound_yield_ty, bound_return_ty]), + sig.c_variadic, + sig.unsafety, + sig.abi, + ) + }), + ), + tupled_upvars_ty, + coroutine_captures_by_ref_ty, + coroutine_witness_ty: interior, + }, + ); + + let coroutine_upvars_ty = self.next_ty_var(TypeVariableOrigin { + kind: TypeVariableOriginKind::ClosureSynthetic, + span: expr_span, + }); + + // We need to turn the liberated signature that we got from HIR, which + // looks something like `|Args...| -> T`, into a signature that is suitable + // for type checking the inner body of the closure, which always returns a + // coroutine. To do so, we use the `CoroutineClosureSignature` to compute + // the coroutine type, filling in the tupled_upvars_ty and kind_ty with infer + // vars which will get constrained during upvar analysis. + let coroutine_output_ty = tcx.liberate_late_bound_regions( + expr_def_id.to_def_id(), + closure_args.coroutine_closure_sig().map_bound(|sig| { + sig.to_coroutine( + tcx, + parent_args, + closure_kind_ty, + tcx.coroutine_for_closure(expr_def_id), + coroutine_upvars_ty, + ) + }), + ); + liberated_sig = tcx.mk_fn_sig( + liberated_sig.inputs().iter().copied(), + coroutine_output_ty, + liberated_sig.c_variadic, + liberated_sig.unsafety, + liberated_sig.abi, + ); + + (Ty::new_coroutine_closure(tcx, expr_def_id.to_def_id(), closure_args.args), None) + } }; check_fn( @@ -690,7 +792,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { _, )) | hir::ClosureKind::Coroutine(hir::CoroutineKind::Coroutine(_)) - | hir::ClosureKind::Closure => astconv.ty_infer(None, decl.output.span()), + | hir::ClosureKind::Closure + | hir::ClosureKind::CoroutineClosure(_) => { + astconv.ty_infer(None, decl.output.span()) + } }, }; diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs index 904961d9eba..2a1c417a16b 100644 --- a/compiler/rustc_hir_typeck/src/method/suggest.rs +++ b/compiler/rustc_hir_typeck/src/method/suggest.rs @@ -57,6 +57,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { match ty.kind() { // Not all of these (e.g., unsafe fns) implement `FnOnce`, // so we look for these beforehand. + // FIXME(async_closures): These don't impl `FnOnce` by default. ty::Closure(..) | ty::FnDef(..) | ty::FnPtr(_) => true, // If it's not a simple function, look for things which implement `FnOnce`. _ => { diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs index 82c5e566f16..c4773a88521 100644 --- a/compiler/rustc_hir_typeck/src/upvar.rs +++ b/compiler/rustc_hir_typeck/src/upvar.rs @@ -170,9 +170,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ) { // Extract the type of the closure. let ty = self.node_ty(closure_hir_id); - let (closure_def_id, args) = match *ty.kind() { - ty::Closure(def_id, args) => (def_id, UpvarArgs::Closure(args)), - ty::Coroutine(def_id, args) => (def_id, UpvarArgs::Coroutine(args)), + let (closure_def_id, args, infer_kind) = match *ty.kind() { + ty::Closure(def_id, args) => { + (def_id, UpvarArgs::Closure(args), self.closure_kind(ty).is_none()) + } + ty::CoroutineClosure(def_id, args) => { + (def_id, UpvarArgs::CoroutineClosure(args), self.closure_kind(ty).is_none()) + } + ty::Coroutine(def_id, args) => (def_id, UpvarArgs::Coroutine(args), false), ty::Error(_) => { // #51714: skip analysis when we have already encountered type errors return; @@ -188,18 +193,63 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { }; let closure_def_id = closure_def_id.expect_local(); - let infer_kind = if let UpvarArgs::Closure(closure_args) = args { - self.closure_kind(closure_args).is_none().then_some(closure_args) - } else { - None - }; - assert_eq!(self.tcx.hir().body_owner_def_id(body.id()), closure_def_id); let mut delegate = InferBorrowKind { closure_def_id, capture_information: Default::default(), fake_reads: Default::default(), }; + + // As noted in `lower_coroutine_body_with_moved_arguments`, we default the capture mode + // to `ByRef` for the `async {}` block internal to async fns/closure. This means + // that we would *not* be moving all of the parameters into the async block by default. + // + // We force all of these arguments to be captured by move before we do expr use analysis. + // + // FIXME(async_closures): This could be cleaned up. It's a bit janky that we're just + // moving all of the `LocalSource::AsyncFn` locals here. + if let Some(hir::CoroutineKind::Desugared( + _, + hir::CoroutineSource::Fn | hir::CoroutineSource::Closure, + )) = self.tcx.coroutine_kind(closure_def_id) + { + let hir::ExprKind::Block(block, _) = body.value.kind else { + bug!(); + }; + for stmt in block.stmts { + let hir::StmtKind::Local(hir::Local { + init: Some(init), + source: hir::LocalSource::AsyncFn, + pat, + .. + }) = stmt.kind + else { + bug!(); + }; + let hir::PatKind::Binding(hir::BindingAnnotation(hir::ByRef::No, _), _, _, _) = + pat.kind + else { + // Complex pattern, skip the non-upvar local. + continue; + }; + let hir::ExprKind::Path(hir::QPath::Resolved(_, path)) = init.kind else { + bug!(); + }; + let hir::def::Res::Local(local_id) = path.res else { + bug!(); + }; + let place = self.place_for_root_variable(closure_def_id, local_id); + delegate.capture_information.push(( + place, + ty::CaptureInfo { + capture_kind_expr_id: Some(init.hir_id), + path_expr_id: Some(init.hir_id), + capture_kind: UpvarCapture::ByValue, + }, + )); + } + } + euv::ExprUseVisitor::new( &mut delegate, &self.infcx, @@ -257,10 +307,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let before_feature_tys = self.final_upvar_tys(closure_def_id); - if let Some(closure_args) = infer_kind { + if infer_kind { // Unify the (as yet unbound) type variable in the closure // args with the kind we inferred. - let closure_kind_ty = closure_args.as_closure().kind_ty(); + let closure_kind_ty = match args { + UpvarArgs::Closure(args) => args.as_closure().kind_ty(), + UpvarArgs::CoroutineClosure(args) => args.as_coroutine_closure().kind_ty(), + UpvarArgs::Coroutine(_) => unreachable!("coroutines don't have an inferred kind"), + }; self.demand_eqtype( span, Ty::from_closure_kind(self.tcx, closure_kind), @@ -282,6 +336,84 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } + // For coroutine-closures, we additionally must compute the + // `coroutine_captures_by_ref_ty` type, which is used to generate the by-ref + // version of the coroutine-closure's output coroutine. + if let UpvarArgs::CoroutineClosure(args) = args { + let closure_env_region: ty::Region<'_> = ty::Region::new_bound( + self.tcx, + ty::INNERMOST, + ty::BoundRegion { + var: ty::BoundVar::from_usize(0), + kind: ty::BoundRegionKind::BrEnv, + }, + ); + let tupled_upvars_ty_for_borrow = Ty::new_tup_from_iter( + self.tcx, + self.typeck_results + .borrow() + .closure_min_captures_flattened( + self.tcx.coroutine_for_closure(closure_def_id).expect_local(), + ) + // Skip the captures that are just moving the closure's args + // into the coroutine. These are always by move, and we append + // those later in the `CoroutineClosureSignature` helper functions. + .skip( + args.as_coroutine_closure() + .coroutine_closure_sig() + .skip_binder() + .tupled_inputs_ty + .tuple_fields() + .len(), + ) + .map(|captured_place| { + let upvar_ty = captured_place.place.ty(); + let capture = captured_place.info.capture_kind; + // Not all upvars are captured by ref, so use + // `apply_capture_kind_on_capture_ty` to ensure that we + // compute the right captured type. + apply_capture_kind_on_capture_ty( + self.tcx, + upvar_ty, + capture, + Some(closure_env_region), + ) + }), + ); + let coroutine_captures_by_ref_ty = Ty::new_fn_ptr( + self.tcx, + ty::Binder::bind_with_vars( + self.tcx.mk_fn_sig( + [], + tupled_upvars_ty_for_borrow, + false, + hir::Unsafety::Normal, + rustc_target::spec::abi::Abi::Rust, + ), + self.tcx.mk_bound_variable_kinds(&[ty::BoundVariableKind::Region( + ty::BoundRegionKind::BrEnv, + )]), + ), + ); + self.demand_eqtype( + span, + args.as_coroutine_closure().coroutine_captures_by_ref_ty(), + coroutine_captures_by_ref_ty, + ); + + // Additionally, we can now constrain the coroutine's kind type. + let ty::Coroutine(_, coroutine_args) = + *self.typeck_results.borrow().expr_ty(body.value).kind() + else { + bug!(); + }; + self.demand_eqtype( + span, + coroutine_args.as_coroutine().kind_ty(), + Ty::from_closure_kind(self.tcx, closure_kind), + ); + } + self.log_closure_min_capture_info(closure_def_id, span); // Now that we've analyzed the closure, we know how each @@ -551,7 +683,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { }; // Go through each entry in the current list of min_captures - // - if ancestor is found, update it's capture kind to account for current place's + // - if ancestor is found, update its capture kind to account for current place's // capture information. // // - if descendant is found, remove it from the list, and update the current place's diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs index e4b37f05b77..d825a2920ee 100644 --- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs +++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs @@ -414,6 +414,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { } ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Bool diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs index b8344310d5d..cf9d9333783 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs @@ -2839,7 +2839,11 @@ impl<'tcx> ObligationCauseExt<'tcx> for ObligationCause<'tcx> { // say, also take a look at the error code, maybe we can // tailor to that. _ => match terr { - TypeError::CyclicTy(ty) if ty.is_closure() || ty.is_coroutine() => Error0644, + TypeError::CyclicTy(ty) + if ty.is_closure() || ty.is_coroutine() || ty.is_coroutine_closure() => + { + Error0644 + } TypeError::IntrinsicCast => Error0308, _ => Error0308, }, @@ -2886,7 +2890,9 @@ impl<'tcx> ObligationCauseExt<'tcx> for ObligationCause<'tcx> { // say, also take a look at the error code, maybe we can // tailor to that. _ => match terr { - TypeError::CyclicTy(ty) if ty.is_closure() || ty.is_coroutine() => { + TypeError::CyclicTy(ty) + if ty.is_closure() || ty.is_coroutine() || ty.is_coroutine_closure() => + { ObligationCauseFailureCode::ClosureSelfref { span } } TypeError::IntrinsicCast => { diff --git a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs index 7287fc26053..c637ab31cd2 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs @@ -883,7 +883,10 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> { GenericArgKind::Type(ty) => { if matches!( ty.kind(), - ty::Alias(ty::Opaque, ..) | ty::Closure(..) | ty::Coroutine(..) + ty::Alias(ty::Opaque, ..) + | ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Coroutine(..) ) { // Opaque types can't be named by the user right now. // diff --git a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs index 0452d4fe6c8..f884ca83073 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs @@ -228,7 +228,10 @@ impl<T> Trait<T> for X { #traits-as-parameters", ); } - (ty::Param(p), ty::Closure(..) | ty::Coroutine(..)) => { + ( + ty::Param(p), + ty::Closure(..) | ty::CoroutineClosure(..) | ty::Coroutine(..), + ) => { let generics = tcx.generics_of(body_owner_def_id); if let Some(param) = generics.opt_type_param(p, tcx) { let p_span = tcx.def_span(param.def_id); @@ -497,7 +500,7 @@ impl<T> Trait<T> for X { } CyclicTy(ty) => { // Watch out for various cases of cyclic types and try to explain. - if ty.is_closure() || ty.is_coroutine() { + if ty.is_closure() || ty.is_coroutine() || ty.is_coroutine_closure() { diag.note( "closures cannot capture themselves or take themselves as argument;\n\ this error may be the result of a recent compiler bug-fix,\n\ diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index 0a39fe007fd..101735b2961 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -1538,9 +1538,13 @@ impl<'tcx> InferCtxt<'tcx> { /// Obtains the latest type of the given closure; this may be a /// closure in the current function, in which case its /// `ClosureKind` may not yet be known. - pub fn closure_kind(&self, closure_args: GenericArgsRef<'tcx>) -> Option<ty::ClosureKind> { - let closure_kind_ty = closure_args.as_closure().kind_ty(); - let closure_kind_ty = self.shallow_resolve(closure_kind_ty); + pub fn closure_kind(&self, closure_ty: Ty<'tcx>) -> Option<ty::ClosureKind> { + let unresolved_kind_ty = match *closure_ty.kind() { + ty::Closure(_, args) => args.as_closure().kind_ty(), + ty::CoroutineClosure(_, args) => args.as_coroutine_closure().kind_ty(), + _ => bug!("unexpected type {closure_ty}"), + }; + let closure_kind_ty = self.shallow_resolve(unresolved_kind_ty); closure_kind_ty.to_opt_closure_kind() } diff --git a/compiler/rustc_infer/src/infer/opaque_types.rs b/compiler/rustc_infer/src/infer/opaque_types.rs index db46b39ce25..5ee0a606a5f 100644 --- a/compiler/rustc_infer/src/infer/opaque_types.rs +++ b/compiler/rustc_infer/src/infer/opaque_types.rs @@ -456,6 +456,17 @@ where args.as_closure().sig_as_fn_ptr_ty().visit_with(self); } + ty::CoroutineClosure(_, args) => { + // Skip lifetime parameters of the enclosing item(s) + + for upvar in args.as_coroutine_closure().upvar_tys() { + upvar.visit_with(self); + } + + // FIXME(async_closures): Is this the right signature to visit here? + args.as_coroutine_closure().signature_parts_ty().visit_with(self); + } + ty::Coroutine(_, args) => { // Skip lifetime parameters of the enclosing item(s) // Also skip the witness type, because that has no free regions. diff --git a/compiler/rustc_infer/src/infer/outlives/components.rs b/compiler/rustc_infer/src/infer/outlives/components.rs index fc3d8375873..7dd1ec32542 100644 --- a/compiler/rustc_infer/src/infer/outlives/components.rs +++ b/compiler/rustc_infer/src/infer/outlives/components.rs @@ -103,6 +103,11 @@ fn compute_components<'tcx>( compute_components(tcx, tupled_ty, out, visited); } + ty::CoroutineClosure(_, args) => { + let tupled_ty = args.as_coroutine_closure().tupled_upvars_ty(); + compute_components(tcx, tupled_ty, out, visited); + } + ty::Coroutine(_, args) => { // Same as the closure case let tupled_ty = args.as_coroutine().tupled_upvars_ty(); diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs index e3d3150b36e..1205395b890 100644 --- a/compiler/rustc_lint/src/types.rs +++ b/compiler/rustc_lint/src/types.rs @@ -1435,6 +1435,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { | ty::Bound(..) | ty::Error(_) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Placeholder(..) diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs index ea3747dfac4..9f670893b27 100644 --- a/compiler/rustc_lint/src/unused.rs +++ b/compiler/rustc_lint/src/unused.rs @@ -355,7 +355,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults { Some(len) => is_ty_must_use(cx, ty, expr, span) .map(|inner| MustUsePath::Array(Box::new(inner), len)), }, - ty::Closure(..) => Some(MustUsePath::Closure(span)), + ty::Closure(..) | ty::CoroutineClosure(..) => Some(MustUsePath::Closure(span)), ty::Coroutine(def_id, ..) => { // async fn should be treated as "implementor of `Future`" let must_use = if cx.tcx.coroutine_is_async(def_id) { diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs index f92c72c8a58..a4e193ba2c9 100644 --- a/compiler/rustc_middle/src/middle/lang_items.rs +++ b/compiler/rustc_middle/src/middle/lang_items.rs @@ -23,7 +23,7 @@ impl<'tcx> TyCtxt<'tcx> { }) } - /// Given a [`DefId`] of a [`Fn`], [`FnMut`] or [`FnOnce`] traits, + /// Given a [`DefId`] of one of the [`Fn`], [`FnMut`] or [`FnOnce`] traits, /// returns a corresponding [`ty::ClosureKind`]. /// For any other [`DefId`] return `None`. pub fn fn_trait_kind_from_def_id(self, id: DefId) -> Option<ty::ClosureKind> { @@ -36,6 +36,19 @@ impl<'tcx> TyCtxt<'tcx> { } } + /// Given a [`DefId`] of one of the `AsyncFn`, `AsyncFnMut` or `AsyncFnOnce` traits, + /// returns a corresponding [`ty::ClosureKind`]. + /// For any other [`DefId`] return `None`. + pub fn async_fn_trait_kind_from_def_id(self, id: DefId) -> Option<ty::ClosureKind> { + let items = self.lang_items(); + match Some(id) { + x if x == items.async_fn_trait() => Some(ty::ClosureKind::Fn), + x if x == items.async_fn_mut_trait() => Some(ty::ClosureKind::FnMut), + x if x == items.async_fn_once_trait() => Some(ty::ClosureKind::FnOnce), + _ => None, + } + } + /// Given a [`ty::ClosureKind`], get the [`DefId`] of its corresponding `Fn`-family /// trait, if it is defined. pub fn fn_trait_kind_to_def_id(self, kind: ty::ClosureKind) -> Option<DefId> { diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index c9e69253701..9475b89aa15 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -262,6 +262,25 @@ pub struct CoroutineInfo<'tcx> { /// Coroutine drop glue. This field is populated after the state transform pass. pub coroutine_drop: Option<Body<'tcx>>, + /// The body of the coroutine, modified to take its upvars by move rather than by ref. + /// + /// This is used by coroutine-closures, which must return a different flavor of coroutine + /// when called using `AsyncFnOnce::call_once`. It is produced by the `ByMoveBody` pass which + /// is run right after building the initial MIR, and will only be populated for coroutines + /// which come out of the async closure desugaring. + /// + /// This body should be processed in lockstep with the containing body -- any optimization + /// passes, etc, should be applied to this body as well. This is done automatically if + /// using `run_passes`. + pub by_move_body: Option<Body<'tcx>>, + + /// The body of the coroutine, modified to take its upvars by mutable ref rather than by + /// immutable ref. + /// + /// FIXME(async_closures): This is literally the same body as the parent body. Find a better + /// way to represent the by-mut signature (or cap the closure-kind of the coroutine). + pub by_mut_body: Option<Body<'tcx>>, + /// The layout of a coroutine. This field is populated after the state transform pass. pub coroutine_layout: Option<CoroutineLayout<'tcx>>, @@ -281,6 +300,8 @@ impl<'tcx> CoroutineInfo<'tcx> { coroutine_kind, yield_ty: Some(yield_ty), resume_ty: Some(resume_ty), + by_move_body: None, + by_mut_body: None, coroutine_drop: None, coroutine_layout: None, } @@ -591,6 +612,14 @@ impl<'tcx> Body<'tcx> { self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref()) } + pub fn coroutine_by_move_body(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref()?.by_move_body.as_ref() + } + + pub fn coroutine_by_mut_body(&self) -> Option<&Body<'tcx>> { + self.coroutine.as_ref()?.by_mut_body.as_ref() + } + #[inline] pub fn coroutine_kind(&self) -> Option<CoroutineKind> { self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind) diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index 91fdf0b3129..6937df7bb18 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -402,6 +402,8 @@ impl<'tcx> CodegenUnit<'tcx> { | InstanceDef::FnPtrShim(..) | InstanceDef::Virtual(..) | InstanceDef::ClosureOnceShim { .. } + | InstanceDef::ConstructCoroutineInClosureShim { .. } + | InstanceDef::CoroutineKindShim { .. } | InstanceDef::DropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::ThreadLocalShim(..) diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 3b60eba2dfe..6f587fdd53c 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -990,7 +990,8 @@ impl<'tcx> Debug for Rvalue<'tcx> { }) } - AggregateKind::Closure(def_id, args) => ty::tls::with(|tcx| { + AggregateKind::Closure(def_id, args) + | AggregateKind::CoroutineClosure(def_id, args) => ty::tls::with(|tcx| { let name = if tcx.sess.opts.unstable_opts.span_free_formats { let args = tcx.lift(args).unwrap(); format!("{{closure@{}}}", tcx.def_path_str_with_args(def_id, args),) diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index a4b6c4f9c3f..ca56e1fd92c 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -1350,6 +1350,7 @@ pub enum AggregateKind<'tcx> { Closure(DefId, GenericArgsRef<'tcx>), Coroutine(DefId, GenericArgsRef<'tcx>), + CoroutineClosure(DefId, GenericArgsRef<'tcx>), } #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)] diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs index 5597609c7d7..4780042a510 100644 --- a/compiler/rustc_middle/src/mir/tcx.rs +++ b/compiler/rustc_middle/src/mir/tcx.rs @@ -202,6 +202,9 @@ impl<'tcx> Rvalue<'tcx> { AggregateKind::Adt(did, _, args, _, _) => tcx.type_of(did).instantiate(tcx, args), AggregateKind::Closure(did, args) => Ty::new_closure(tcx, did, args), AggregateKind::Coroutine(did, args) => Ty::new_coroutine(tcx, did, args), + AggregateKind::CoroutineClosure(did, args) => { + Ty::new_coroutine_closure(tcx, did, args) + } }, Rvalue::ShallowInitBox(_, ty) => Ty::new_box(tcx, ty), Rvalue::CopyForDeref(ref place) => place.ty(local_decls, tcx).ty, diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 4696f54c897..2c5ca82a4cd 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -345,6 +345,8 @@ macro_rules! make_mir_visitor { ty::InstanceDef::Virtual(_def_id, _) | ty::InstanceDef::ThreadLocalShim(_def_id) | ty::InstanceDef::ClosureOnceShim { call_once: _def_id, track_caller: _ } | + ty::InstanceDef::ConstructCoroutineInClosureShim { coroutine_closure_def_id: _def_id, target_kind: _ } | + ty::InstanceDef::CoroutineKindShim { coroutine_def_id: _def_id, target_kind: _ } | ty::InstanceDef::DropGlue(_def_id, None) => {} ty::InstanceDef::FnPtrShim(_def_id, ty) | @@ -739,6 +741,12 @@ macro_rules! make_mir_visitor { ) => { self.visit_args(coroutine_args, location); } + AggregateKind::CoroutineClosure( + _, + coroutine_closure_args, + ) => { + self.visit_args(coroutine_closure_args, location); + } } for operand in operands { diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 2438f826441..938fba0ed09 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -755,6 +755,11 @@ rustc_queries! { separate_provide_extern } + query coroutine_for_closure(def_id: DefId) -> DefId { + desc { |_tcx| "Given a coroutine-closure def id, return the def id of the coroutine returned by it" } + separate_provide_extern + } + /// Gets a map with the variance of every item; use `variances_of` instead. query crate_variances(_: ()) -> &'tcx ty::CrateVariancesMap<'tcx> { arena_cache diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs index 64f4af08e12..28eba133c76 100644 --- a/compiler/rustc_middle/src/traits/select.rs +++ b/compiler/rustc_middle/src/traits/select.rs @@ -134,6 +134,15 @@ pub enum SelectionCandidate<'tcx> { is_const: bool, }, + /// Implementation of an `AsyncFn`-family trait by one of the anonymous types + /// generated for an `async ||` expression. + AsyncClosureCandidate, + + /// Implementation of the the `AsyncFnKindHelper` helper trait, which + /// is used internally to delay computation for async closures until after + /// upvar analysis is performed in HIR typeck. + AsyncFnKindHelperCandidate, + /// Implementation of a `Coroutine` trait by one of the anonymous types /// generated for a coroutine. CoroutineCandidate, diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 0d53870a0ba..14d2a93e167 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1544,6 +1544,7 @@ impl<'tcx> TyCtxt<'tcx> { CoroutineWitness, Dynamic, Closure, + CoroutineClosure, Tuple, Bound, Param, diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 0e44878524b..80b763d1469 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -299,7 +299,7 @@ impl<'tcx> Ty<'tcx> { }, ty::FnPtr(_) => "fn pointer".into(), ty::Dynamic(..) => "trait object".into(), - ty::Closure(..) => "closure".into(), + ty::Closure(..) | ty::CoroutineClosure(..) => "closure".into(), ty::Coroutine(def_id, ..) => { format!("{:#}", tcx.coroutine_kind(def_id).unwrap()).into() } diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs index b71919adc58..adc153c4dfd 100644 --- a/compiler/rustc_middle/src/ty/fast_reject.rs +++ b/compiler/rustc_middle/src/ty/fast_reject.rs @@ -128,7 +128,9 @@ pub fn simplify_type<'tcx>( _ => Some(SimplifiedType::MarkerTraitObject), }, ty::Ref(_, _, mutbl) => Some(SimplifiedType::Ref(mutbl)), - ty::FnDef(def_id, _) | ty::Closure(def_id, _) => Some(SimplifiedType::Closure(def_id)), + ty::FnDef(def_id, _) | ty::Closure(def_id, _) | ty::CoroutineClosure(def_id, _) => { + Some(SimplifiedType::Closure(def_id)) + } ty::Coroutine(def_id, _) => Some(SimplifiedType::Coroutine(def_id)), ty::CoroutineWitness(def_id, _) => Some(SimplifiedType::CoroutineWitness(def_id)), ty::Never => Some(SimplifiedType::Never), @@ -236,6 +238,7 @@ impl DeepRejectCtxt { | ty::Foreign(..) => debug_assert!(impl_ty.is_known_rigid()), ty::FnDef(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Placeholder(..) @@ -312,7 +315,7 @@ impl DeepRejectCtxt { }, // Impls cannot contain these types as these cannot be named directly. - ty::FnDef(..) | ty::Closure(..) | ty::Coroutine(..) => false, + ty::FnDef(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Coroutine(..) => false, // Placeholder types don't unify with anything on their own ty::Placeholder(..) | ty::Bound(..) => false, diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs index 0c1d1091414..0f4b5fe228c 100644 --- a/compiler/rustc_middle/src/ty/flags.rs +++ b/compiler/rustc_middle/src/ty/flags.rs @@ -136,6 +136,22 @@ impl FlagComputation { self.add_ty(args.tupled_upvars_ty()); } + &ty::CoroutineClosure(_, args) => { + let args = args.as_coroutine_closure(); + let should_remove_further_specializable = + !self.flags.contains(TypeFlags::STILL_FURTHER_SPECIALIZABLE); + self.add_args(args.parent_args()); + if should_remove_further_specializable { + self.flags -= TypeFlags::STILL_FURTHER_SPECIALIZABLE; + } + + self.add_ty(args.kind_ty()); + self.add_ty(args.signature_parts_ty()); + self.add_ty(args.tupled_upvars_ty()); + self.add_ty(args.coroutine_captures_by_ref_ty()); + self.add_ty(args.coroutine_witness_ty()); + } + &ty::Bound(debruijn, _) => { self.add_bound_var(debruijn); self.add_flags(TypeFlags::HAS_TY_BOUND); diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs index 69ad6810c6f..b9fff660a03 100644 --- a/compiler/rustc_middle/src/ty/generic_args.rs +++ b/compiler/rustc_middle/src/ty/generic_args.rs @@ -2,7 +2,7 @@ use crate::ty::codec::{TyDecoder, TyEncoder}; use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder, TypeSuperFoldable}; -use crate::ty::sty::{ClosureArgs, CoroutineArgs, InlineConstArgs}; +use crate::ty::sty::{ClosureArgs, CoroutineArgs, CoroutineClosureArgs, InlineConstArgs}; use crate::ty::visit::{TypeVisitable, TypeVisitableExt, TypeVisitor}; use crate::ty::{self, Lift, List, ParamConst, Ty, TyCtxt}; @@ -288,6 +288,14 @@ impl<'tcx> GenericArgs<'tcx> { ClosureArgs { args: self } } + /// Interpret these generic args as the args of a coroutine-closure type. + /// Coroutine-closure args have a particular structure controlled by the + /// compiler that encodes information like the signature and closure kind; + /// see `ty::CoroutineClosureArgs` struct for more comments. + pub fn as_coroutine_closure(&'tcx self) -> CoroutineClosureArgs<'tcx> { + CoroutineClosureArgs { args: self } + } + /// Interpret these generic args as the args of a coroutine type. /// Coroutine args have a particular structure controlled by the /// compiler that encodes information like the signature and coroutine kind; diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index 293fdb026b6..9c1f4b20d2c 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -82,11 +82,33 @@ pub enum InstanceDef<'tcx> { /// details on that). Virtual(DefId, usize), - /// `<[FnMut closure] as FnOnce>::call_once`. + /// `<[FnMut/Fn closure] as FnOnce>::call_once`. /// /// The `DefId` is the ID of the `call_once` method in `FnOnce`. + /// + /// This generates a body that will just borrow the (owned) self type, + /// and dispatch to the `FnMut::call_mut` instance for the closure. ClosureOnceShim { call_once: DefId, track_caller: bool }, + /// `<[FnMut/Fn coroutine-closure] as FnOnce>::call_once` or + /// `<[Fn coroutine-closure] as FnMut>::call_mut`. + /// + /// The body generated here differs significantly from the `ClosureOnceShim`, + /// since we need to generate a distinct coroutine type that will move the + /// closure's upvars *out* of the closure. + ConstructCoroutineInClosureShim { + coroutine_closure_def_id: DefId, + target_kind: ty::ClosureKind, + }, + + /// `<[coroutine] as Future>::poll`, but for coroutines produced when `AsyncFnOnce` + /// is called on a coroutine-closure whose closure kind greater than `FnOnce`, or + /// similarly for `AsyncFnMut`. + /// + /// This will select the body that is produced by the `ByMoveBody` transform, and thus + /// take and use all of its upvars by-move rather than by-ref. + CoroutineKindShim { coroutine_def_id: DefId, target_kind: ty::ClosureKind }, + /// Compiler-generated accessor for thread locals which returns a reference to the thread local /// the `DefId` defines. This is used to export thread locals from dylibs on platforms lacking /// native support. @@ -168,6 +190,11 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::Intrinsic(def_id) | InstanceDef::ThreadLocalShim(def_id) | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ } + | ty::InstanceDef::ConstructCoroutineInClosureShim { + coroutine_closure_def_id: def_id, + target_kind: _, + } + | ty::InstanceDef::CoroutineKindShim { coroutine_def_id: def_id, target_kind: _ } | InstanceDef::DropGlue(def_id, _) | InstanceDef::CloneShim(def_id, _) | InstanceDef::FnPtrAddrShim(def_id, _) => def_id, @@ -187,6 +214,8 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::Virtual(..) | InstanceDef::Intrinsic(..) | InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } + | ty::InstanceDef::CoroutineKindShim { .. } | InstanceDef::DropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::FnPtrAddrShim(..) => None, @@ -282,6 +311,8 @@ impl<'tcx> InstanceDef<'tcx> { | InstanceDef::FnPtrShim(..) | InstanceDef::DropGlue(_, Some(_)) => false, InstanceDef::ClosureOnceShim { .. } + | InstanceDef::ConstructCoroutineInClosureShim { .. } + | InstanceDef::CoroutineKindShim { .. } | InstanceDef::DropGlue(..) | InstanceDef::Item(_) | InstanceDef::Intrinsic(..) @@ -319,6 +350,8 @@ fn fmt_instance( InstanceDef::Virtual(_, num) => write!(f, " - virtual#{num}"), InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({ty})"), InstanceDef::ClosureOnceShim { .. } => write!(f, " - shim"), + InstanceDef::ConstructCoroutineInClosureShim { .. } => write!(f, " - shim"), + InstanceDef::CoroutineKindShim { .. } => write!(f, " - shim"), InstanceDef::DropGlue(_, None) => write!(f, " - shim(None)"), InstanceDef::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), InstanceDef::CloneShim(_, ty) => write!(f, " - shim({ty})"), @@ -610,7 +643,24 @@ impl<'tcx> Instance<'tcx> { }; if tcx.lang_items().get(coroutine_callable_item) == Some(trait_item_id) { - Some(Instance { def: ty::InstanceDef::Item(coroutine_def_id), args: args }) + let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind() + else { + bug!() + }; + + // If the closure's kind ty disagrees with the identity closure's kind ty, + // then this must be a coroutine generated by one of the `ConstructCoroutineInClosureShim`s. + if args.as_coroutine().kind_ty() == id_args.as_coroutine().kind_ty() { + Some(Instance { def: ty::InstanceDef::Item(coroutine_def_id), args }) + } else { + Some(Instance { + def: ty::InstanceDef::CoroutineKindShim { + coroutine_def_id, + target_kind: args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(), + }, + args, + }) + } } else { // All other methods should be defaulted methods of the built-in trait. // This is important for `Iterator`'s combinators, but also useful for diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 73b0e324f13..8d8d06b7c0b 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -906,6 +906,12 @@ where i, ), + ty::CoroutineClosure(_, args) => field_ty_or_layout( + TyAndLayout { ty: args.as_coroutine_closure().tupled_upvars_ty(), ..this }, + cx, + i, + ), + ty::Coroutine(def_id, args) => match this.variants { Variants::Single { index } => TyMaybeWithLayout::Ty( args.as_coroutine() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 8e51db82f95..c9137f374a2 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -105,9 +105,10 @@ pub use self::region::{ pub use self::rvalue_scopes::RvalueScopes; pub use self::sty::{ AliasTy, Article, Binder, BoundTy, BoundTyKind, BoundVariableKind, CanonicalPolyFnSig, - ClosureArgs, ClosureArgsParts, CoroutineArgs, CoroutineArgsParts, FnSig, GenSig, - InlineConstArgs, InlineConstArgsParts, ParamConst, ParamTy, PolyFnSig, TyKind, TypeAndMut, - UpvarArgs, VarianceDiagInfo, + ClosureArgs, ClosureArgsParts, CoroutineArgs, CoroutineArgsParts, CoroutineClosureArgs, + CoroutineClosureArgsParts, CoroutineClosureSignature, FnSig, GenSig, InlineConstArgs, + InlineConstArgsParts, ParamConst, ParamTy, PolyFnSig, TyKind, TypeAndMut, UpvarArgs, + VarianceDiagInfo, }; pub use self::trait_def::TraitDef; pub use self::typeck_results::{ @@ -1679,6 +1680,8 @@ impl<'tcx> TyCtxt<'tcx> { | ty::InstanceDef::FnPtrShim(..) | ty::InstanceDef::Virtual(..) | ty::InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } + | ty::InstanceDef::CoroutineKindShim { .. } | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::CloneShim(..) | ty::InstanceDef::ThreadLocalShim(..) diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs index f32b7b0852a..19f8ba124f1 100644 --- a/compiler/rustc_middle/src/ty/print/mod.rs +++ b/compiler/rustc_middle/src/ty/print/mod.rs @@ -3,6 +3,7 @@ use crate::ty::{self, Ty, TyCtxt}; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::sso::SsoHashSet; +use rustc_hir as hir; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId}; use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData}; @@ -130,8 +131,24 @@ pub trait Printer<'tcx>: Sized { parent_args = &args[..generics.parent_count.min(args.len())]; match key.disambiguated_data.data { - // Closures' own generics are only captures, don't print them. - DefPathData::Closure => {} + DefPathData::Closure => { + // FIXME(async_closures): This is somewhat ugly. + // We need to additionally print the `kind` field of a closure if + // it is desugared from a coroutine-closure. + if let Some(hir::CoroutineKind::Desugared( + _, + hir::CoroutineSource::Closure, + )) = self.tcx().coroutine_kind(def_id) + && args.len() >= parent_args.len() + 1 + { + return self.path_generic_args( + |cx| cx.print_def_path(def_id, parent_args), + &args[..parent_args.len() + 1][..1], + ); + } else { + // Closures' own generics are only captures, don't print them. + } + } // This covers both `DefKind::AnonConst` and `DefKind::InlineConst`. // Anon consts doesn't have their own generics, and inline consts' own // generics are their inferred types, so don't print them. @@ -259,6 +276,7 @@ fn characteristic_def_id_of_type_cached<'a>( ty::FnDef(def_id, _) | ty::Closure(def_id, _) + | ty::CoroutineClosure(def_id, _) | ty::Coroutine(def_id, _) | ty::CoroutineWitness(def_id, _) | ty::Foreign(def_id) => Some(def_id), diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index bac5068a69b..c0bfd2380ad 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -874,6 +874,48 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write { } p!("}}"); } + ty::CoroutineClosure(did, args) => { + p!(write("{{")); + if !self.should_print_verbose() { + p!(write("coroutine-closure")); + // FIXME(eddyb) should use `def_span`. + if let Some(did) = did.as_local() { + if self.tcx().sess.opts.unstable_opts.span_free_formats { + p!("@", print_def_path(did.to_def_id(), args)); + } else { + let span = self.tcx().def_span(did); + let preference = if with_forced_trimmed_paths() { + FileNameDisplayPreference::Short + } else { + FileNameDisplayPreference::Remapped + }; + p!(write( + "@{}", + // This may end up in stderr diagnostics but it may also be emitted + // into MIR. Hence we use the remapped path if available + self.tcx().sess.source_map().span_to_string(span, preference) + )); + } + } else { + p!(write("@"), print_def_path(did, args)); + } + } else { + p!(print_def_path(did, args)); + p!( + " closure_kind_ty=", + print(args.as_coroutine_closure().kind_ty()), + " signature_parts_ty=", + print(args.as_coroutine_closure().signature_parts_ty()), + " upvar_tys=", + print(args.as_coroutine_closure().tupled_upvars_ty()), + " coroutine_captures_by_ref_ty=", + print(args.as_coroutine_closure().coroutine_captures_by_ref_ty()), + " coroutine_witness_ty=", + print(args.as_coroutine_closure().coroutine_witness_ty()) + ); + } + p!("}}"); + } ty::Array(ty, sz) => p!("[", print(ty), "; ", print(sz), "]"), ty::Slice(ty) => p!("[", print(ty), "]"), } diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs index 8543bd0bbdd..f2321e7e1d2 100644 --- a/compiler/rustc_middle/src/ty/relate.rs +++ b/compiler/rustc_middle/src/ty/relate.rs @@ -481,6 +481,13 @@ pub fn structurally_relate_tys<'tcx, R: TypeRelation<'tcx>>( Ok(Ty::new_closure(tcx, a_id, args)) } + (&ty::CoroutineClosure(a_id, a_args), &ty::CoroutineClosure(b_id, b_args)) + if a_id == b_id => + { + let args = relate_args_invariantly(relation, a_args, b_args)?; + Ok(Ty::new_coroutine_closure(tcx, a_id, args)) + } + (&ty::RawPtr(a_mt), &ty::RawPtr(b_mt)) => { let mt = relate_type_and_mut(relation, a_mt, b_mt, a)?; Ok(Ty::new_ptr(tcx, mt)) diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 11b579a1f85..c6805ba9323 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -584,6 +584,9 @@ impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for Ty<'tcx> { ty::CoroutineWitness(did, args.try_fold_with(folder)?) } ty::Closure(did, args) => ty::Closure(did, args.try_fold_with(folder)?), + ty::CoroutineClosure(did, args) => { + ty::CoroutineClosure(did, args.try_fold_with(folder)?) + } ty::Alias(kind, data) => ty::Alias(kind, data.try_fold_with(folder)?), ty::Bool @@ -632,6 +635,7 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for Ty<'tcx> { ty::Coroutine(_did, ref args) => args.visit_with(visitor), ty::CoroutineWitness(_did, ref args) => args.visit_with(visitor), ty::Closure(_did, ref args) => args.visit_with(visitor), + ty::CoroutineClosure(_did, ref args) => args.visit_with(visitor), ty::Alias(_, ref data) => data.visit_with(visitor), ty::Bool diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index f5fdf210592..927924452f9 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -36,6 +36,7 @@ use rustc_type_ir::TyKind as IrTyKind; use rustc_type_ir::TyKind::*; use rustc_type_ir::TypeAndMut as IrTypeAndMut; +use super::fold::FnMutDelegate; use super::GenericParamDefKind; // Re-export and re-parameterize some `I = TyCtxt<'tcx>` types here @@ -269,6 +270,275 @@ impl<'tcx> ClosureArgs<'tcx> { } } +#[derive(Copy, Clone, PartialEq, Eq, Debug, TypeFoldable, TypeVisitable, Lift)] +pub struct CoroutineClosureArgs<'tcx> { + pub args: GenericArgsRef<'tcx>, +} + +/// See docs for explanation of how each argument is used. +/// +/// See [`CoroutineClosureSignature`] for how these arguments are put together +/// to make a callable [`FnSig`] suitable for typeck and borrowck. +pub struct CoroutineClosureArgsParts<'tcx> { + /// This is the args of the typeck root. + pub parent_args: &'tcx [GenericArg<'tcx>], + /// Represents the maximum calling capability of the closure. + pub closure_kind_ty: Ty<'tcx>, + /// Represents all of the relevant parts of the coroutine returned by this + /// coroutine-closure. This signature parts type will have the general + /// shape of `fn(tupled_inputs, resume_ty) -> (return_ty, yield_ty)`, where + /// `resume_ty`, `return_ty`, and `yield_ty` are the respective types for the + /// coroutine returned by the coroutine-closure. + /// + /// Use `coroutine_closure_sig` to break up this type rather than using it + /// yourself. + pub signature_parts_ty: Ty<'tcx>, + /// The upvars captured by the closure. Remains an inference variable + /// until the upvar analysis, which happens late in HIR typeck. + pub tupled_upvars_ty: Ty<'tcx>, + /// a function pointer that has the shape `for<'env> fn() -> (&'env T, ...)`. + /// This allows us to represent the binder of the self-captures of the closure. + /// + /// For example, if the coroutine returned by the closure borrows `String` + /// from the closure's upvars, this will be `for<'env> fn() -> (&'env String,)`, + /// while the `tupled_upvars_ty`, representing the by-move version of the same + /// captures, will be `(String,)`. + pub coroutine_captures_by_ref_ty: Ty<'tcx>, + /// Witness type returned by the generator produced by this coroutine-closure. + pub coroutine_witness_ty: Ty<'tcx>, +} + +impl<'tcx> CoroutineClosureArgs<'tcx> { + pub fn new( + tcx: TyCtxt<'tcx>, + parts: CoroutineClosureArgsParts<'tcx>, + ) -> CoroutineClosureArgs<'tcx> { + CoroutineClosureArgs { + args: tcx.mk_args_from_iter(parts.parent_args.iter().copied().chain([ + parts.closure_kind_ty.into(), + parts.signature_parts_ty.into(), + parts.tupled_upvars_ty.into(), + parts.coroutine_captures_by_ref_ty.into(), + parts.coroutine_witness_ty.into(), + ])), + } + } + + fn split(self) -> CoroutineClosureArgsParts<'tcx> { + match self.args[..] { + [ + ref parent_args @ .., + closure_kind_ty, + signature_parts_ty, + tupled_upvars_ty, + coroutine_captures_by_ref_ty, + coroutine_witness_ty, + ] => CoroutineClosureArgsParts { + parent_args, + closure_kind_ty: closure_kind_ty.expect_ty(), + signature_parts_ty: signature_parts_ty.expect_ty(), + tupled_upvars_ty: tupled_upvars_ty.expect_ty(), + coroutine_captures_by_ref_ty: coroutine_captures_by_ref_ty.expect_ty(), + coroutine_witness_ty: coroutine_witness_ty.expect_ty(), + }, + _ => bug!("closure args missing synthetics"), + } + } + + pub fn parent_args(self) -> &'tcx [GenericArg<'tcx>] { + self.split().parent_args + } + + #[inline] + pub fn upvar_tys(self) -> &'tcx List<Ty<'tcx>> { + match self.tupled_upvars_ty().kind() { + TyKind::Error(_) => ty::List::empty(), + TyKind::Tuple(..) => self.tupled_upvars_ty().tuple_fields(), + TyKind::Infer(_) => bug!("upvar_tys called before capture types are inferred"), + ty => bug!("Unexpected representation of upvar types tuple {:?}", ty), + } + } + + #[inline] + pub fn tupled_upvars_ty(self) -> Ty<'tcx> { + self.split().tupled_upvars_ty + } + + pub fn kind_ty(self) -> Ty<'tcx> { + self.split().closure_kind_ty + } + + pub fn kind(self) -> ty::ClosureKind { + self.kind_ty().to_opt_closure_kind().unwrap() + } + + pub fn signature_parts_ty(self) -> Ty<'tcx> { + self.split().signature_parts_ty + } + + pub fn coroutine_closure_sig(self) -> ty::Binder<'tcx, CoroutineClosureSignature<'tcx>> { + let interior = self.coroutine_witness_ty(); + let ty::FnPtr(sig) = self.signature_parts_ty().kind() else { bug!() }; + sig.map_bound(|sig| { + let [resume_ty, tupled_inputs_ty] = *sig.inputs() else { + bug!(); + }; + let [yield_ty, return_ty] = **sig.output().tuple_fields() else { bug!() }; + CoroutineClosureSignature { + interior, + tupled_inputs_ty, + resume_ty, + yield_ty, + return_ty, + c_variadic: sig.c_variadic, + unsafety: sig.unsafety, + abi: sig.abi, + } + }) + } + + pub fn coroutine_captures_by_ref_ty(self) -> Ty<'tcx> { + self.split().coroutine_captures_by_ref_ty + } + + pub fn coroutine_witness_ty(self) -> Ty<'tcx> { + self.split().coroutine_witness_ty + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug, TypeFoldable, TypeVisitable)] +pub struct CoroutineClosureSignature<'tcx> { + pub interior: Ty<'tcx>, + pub tupled_inputs_ty: Ty<'tcx>, + pub resume_ty: Ty<'tcx>, + pub yield_ty: Ty<'tcx>, + pub return_ty: Ty<'tcx>, + + // Like the `fn_sig_as_fn_ptr_ty` of a regular closure, these types + // never actually differ. But we save them rather than recreating them + // from scratch just for good measure. + /// Always false + pub c_variadic: bool, + /// Always [`hir::Unsafety::Normal`] + pub unsafety: hir::Unsafety, + /// Always [`abi::Abi::RustCall`] + pub abi: abi::Abi, +} + +impl<'tcx> CoroutineClosureSignature<'tcx> { + /// Construct a coroutine from the closure signature. Since a coroutine signature + /// is agnostic to the type of generator that is returned (by-ref/by-move), + /// the caller must specify what "flavor" of generator that they'd like to + /// create. Additionally, they must manually compute the upvars of the closure. + /// + /// This helper is not really meant to be used directly except for early on + /// during typeck, when we want to put inference vars into the kind and upvars tys. + /// When the kind and upvars are known, use the other helper functions. + pub fn to_coroutine( + self, + tcx: TyCtxt<'tcx>, + parent_args: &'tcx [GenericArg<'tcx>], + coroutine_kind_ty: Ty<'tcx>, + coroutine_def_id: DefId, + tupled_upvars_ty: Ty<'tcx>, + ) -> Ty<'tcx> { + let coroutine_args = ty::CoroutineArgs::new( + tcx, + ty::CoroutineArgsParts { + parent_args, + kind_ty: coroutine_kind_ty, + resume_ty: self.resume_ty, + yield_ty: self.yield_ty, + return_ty: self.return_ty, + witness: self.interior, + tupled_upvars_ty, + }, + ); + + Ty::new_coroutine(tcx, coroutine_def_id, coroutine_args.args) + } + + /// Given known upvars and a [`ClosureKind`](ty::ClosureKind), compute the coroutine + /// returned by that corresponding async fn trait. + /// + /// This function expects the upvars to have been computed already, and doesn't check + /// that the `ClosureKind` is actually supported by the coroutine-closure. + pub fn to_coroutine_given_kind_and_upvars( + self, + tcx: TyCtxt<'tcx>, + parent_args: &'tcx [GenericArg<'tcx>], + coroutine_def_id: DefId, + goal_kind: ty::ClosureKind, + env_region: ty::Region<'tcx>, + closure_tupled_upvars_ty: Ty<'tcx>, + coroutine_captures_by_ref_ty: Ty<'tcx>, + ) -> Ty<'tcx> { + let tupled_upvars_ty = Self::tupled_upvars_by_closure_kind( + tcx, + goal_kind, + self.tupled_inputs_ty, + closure_tupled_upvars_ty, + coroutine_captures_by_ref_ty, + env_region, + ); + + self.to_coroutine( + tcx, + parent_args, + Ty::from_closure_kind(tcx, goal_kind), + coroutine_def_id, + tupled_upvars_ty, + ) + } + + /// Compute the tupled upvars that a coroutine-closure's output coroutine + /// would return for the given `ClosureKind`. + /// + /// When `ClosureKind` is `FnMut`/`Fn`, then this will use the "captures by ref" + /// to return a set of upvars which are borrowed with the given `env_region`. + /// + /// This ensures that the `AsyncFn::call` will return a coroutine whose upvars' + /// lifetimes are related to the lifetime of the borrow on the closure made for + /// the call. This allows borrowck to enforce the self-borrows correctly. + pub fn tupled_upvars_by_closure_kind( + tcx: TyCtxt<'tcx>, + kind: ty::ClosureKind, + tupled_inputs_ty: Ty<'tcx>, + closure_tupled_upvars_ty: Ty<'tcx>, + coroutine_captures_by_ref_ty: Ty<'tcx>, + env_region: ty::Region<'tcx>, + ) -> Ty<'tcx> { + match kind { + ty::ClosureKind::Fn | ty::ClosureKind::FnMut => { + let ty::FnPtr(sig) = *coroutine_captures_by_ref_ty.kind() else { + bug!(); + }; + let coroutine_captures_by_ref_ty = tcx.replace_escaping_bound_vars_uncached( + sig.output().skip_binder(), + FnMutDelegate { + consts: &mut |c, t| ty::Const::new_bound(tcx, ty::INNERMOST, c, t), + types: &mut |t| Ty::new_bound(tcx, ty::INNERMOST, t), + regions: &mut |_| env_region, + }, + ); + Ty::new_tup_from_iter( + tcx, + tupled_inputs_ty + .tuple_fields() + .iter() + .chain(coroutine_captures_by_ref_ty.tuple_fields()), + ) + } + ty::ClosureKind::FnOnce => Ty::new_tup_from_iter( + tcx, + tupled_inputs_ty + .tuple_fields() + .iter() + .chain(closure_tupled_upvars_ty.tuple_fields()), + ), + } + } +} /// Similar to `ClosureArgs`; see the above documentation for more. #[derive(Copy, Clone, PartialEq, Eq, Debug, TypeFoldable, TypeVisitable)] pub struct CoroutineArgs<'tcx> { @@ -278,13 +548,27 @@ pub struct CoroutineArgs<'tcx> { pub struct CoroutineArgsParts<'tcx> { /// This is the args of the typeck root. pub parent_args: &'tcx [GenericArg<'tcx>], + + /// The coroutines returned by a coroutine-closure's `AsyncFnOnce`/`AsyncFnMut` + /// implementations must be distinguished since the former takes the closure's + /// upvars by move, and the latter takes the closure's upvars by ref. + /// + /// This field distinguishes these fields so that codegen can select the right + /// body for the coroutine. This has the same type representation as the closure + /// kind: `i8`/`i16`/`i32`. + /// + /// For regular coroutines, this field will always just be `()`. + pub kind_ty: Ty<'tcx>, + pub resume_ty: Ty<'tcx>, pub yield_ty: Ty<'tcx>, pub return_ty: Ty<'tcx>, + /// The interior type of the coroutine. /// Represents all types that are stored in locals /// in the coroutine's body. pub witness: Ty<'tcx>, + /// The upvars captured by the closure. Remains an inference variable /// until the upvar analysis, which happens late in HIR typeck. pub tupled_upvars_ty: Ty<'tcx>, @@ -296,6 +580,7 @@ impl<'tcx> CoroutineArgs<'tcx> { pub fn new(tcx: TyCtxt<'tcx>, parts: CoroutineArgsParts<'tcx>) -> CoroutineArgs<'tcx> { CoroutineArgs { args: tcx.mk_args_from_iter(parts.parent_args.iter().copied().chain([ + parts.kind_ty.into(), parts.resume_ty.into(), parts.yield_ty.into(), parts.return_ty.into(), @@ -309,16 +594,23 @@ impl<'tcx> CoroutineArgs<'tcx> { /// The ordering assumed here must match that used by `CoroutineArgs::new` above. fn split(self) -> CoroutineArgsParts<'tcx> { match self.args[..] { - [ref parent_args @ .., resume_ty, yield_ty, return_ty, witness, tupled_upvars_ty] => { - CoroutineArgsParts { - parent_args, - resume_ty: resume_ty.expect_ty(), - yield_ty: yield_ty.expect_ty(), - return_ty: return_ty.expect_ty(), - witness: witness.expect_ty(), - tupled_upvars_ty: tupled_upvars_ty.expect_ty(), - } - } + [ + ref parent_args @ .., + kind_ty, + resume_ty, + yield_ty, + return_ty, + witness, + tupled_upvars_ty, + ] => CoroutineArgsParts { + parent_args, + kind_ty: kind_ty.expect_ty(), + resume_ty: resume_ty.expect_ty(), + yield_ty: yield_ty.expect_ty(), + return_ty: return_ty.expect_ty(), + witness: witness.expect_ty(), + tupled_upvars_ty: tupled_upvars_ty.expect_ty(), + }, _ => bug!("coroutine args missing synthetics"), } } @@ -328,6 +620,11 @@ impl<'tcx> CoroutineArgs<'tcx> { self.split().parent_args } + // Returns the kind of the coroutine. See docs on the `kind_ty` field. + pub fn kind_ty(self) -> Ty<'tcx> { + self.split().kind_ty + } + /// This describes the types that can be contained in a coroutine. /// It will be a type variable initially and unified in the last stages of typeck of a body. /// It contains a tuple of all the types that could end up on a coroutine frame. @@ -479,6 +776,7 @@ impl<'tcx> CoroutineArgs<'tcx> { pub enum UpvarArgs<'tcx> { Closure(GenericArgsRef<'tcx>), Coroutine(GenericArgsRef<'tcx>), + CoroutineClosure(GenericArgsRef<'tcx>), } impl<'tcx> UpvarArgs<'tcx> { @@ -490,6 +788,7 @@ impl<'tcx> UpvarArgs<'tcx> { let tupled_tys = match self { UpvarArgs::Closure(args) => args.as_closure().tupled_upvars_ty(), UpvarArgs::Coroutine(args) => args.as_coroutine().tupled_upvars_ty(), + UpvarArgs::CoroutineClosure(args) => args.as_coroutine_closure().tupled_upvars_ty(), }; match tupled_tys.kind() { @@ -505,6 +804,7 @@ impl<'tcx> UpvarArgs<'tcx> { match self { UpvarArgs::Closure(args) => args.as_closure().tupled_upvars_ty(), UpvarArgs::Coroutine(args) => args.as_coroutine().tupled_upvars_ty(), + UpvarArgs::CoroutineClosure(args) => args.as_coroutine_closure().tupled_upvars_ty(), } } } @@ -1394,6 +1694,20 @@ impl<'tcx> Ty<'tcx> { } #[inline] + pub fn new_coroutine_closure( + tcx: TyCtxt<'tcx>, + def_id: DefId, + closure_args: GenericArgsRef<'tcx>, + ) -> Ty<'tcx> { + debug_assert_eq!( + closure_args.len(), + tcx.generics_of(tcx.typeck_root_def_id(def_id)).count() + 5, + "closure constructed with incorrect substitutions" + ); + Ty::new(tcx, CoroutineClosure(def_id, closure_args)) + } + + #[inline] pub fn new_coroutine( tcx: TyCtxt<'tcx>, def_id: DefId, @@ -1401,7 +1715,7 @@ impl<'tcx> Ty<'tcx> { ) -> Ty<'tcx> { debug_assert_eq!( coroutine_args.len(), - tcx.generics_of(tcx.typeck_root_def_id(def_id)).count() + 5, + tcx.generics_of(tcx.typeck_root_def_id(def_id)).count() + 6, "coroutine constructed with incorrect number of substitutions" ); Ty::new(tcx, Coroutine(def_id, coroutine_args)) @@ -1728,6 +2042,11 @@ impl<'tcx> Ty<'tcx> { } #[inline] + pub fn is_coroutine_closure(self) -> bool { + matches!(self.kind(), CoroutineClosure(..)) + } + + #[inline] pub fn is_integral(self) -> bool { matches!(self.kind(), Infer(IntVar(_)) | Int(_) | Uint(_)) } @@ -1795,7 +2114,7 @@ impl<'tcx> Ty<'tcx> { type BreakTy = (); fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> { - if let ty::Closure(_, _) = t.kind() { + if let ty::Closure(..) = t.kind() { ControlFlow::Break(()) } else { t.super_visit_with(self) @@ -1942,6 +2261,7 @@ impl<'tcx> Ty<'tcx> { | ty::FnPtr(..) | ty::Dynamic(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::CoroutineWitness(..) | ty::Never | ty::Tuple(_) @@ -1980,6 +2300,7 @@ impl<'tcx> Ty<'tcx> { | ty::CoroutineWitness(..) | ty::Array(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Never | ty::Error(_) // Extern types have metadata = (). @@ -2034,7 +2355,7 @@ impl<'tcx> Ty<'tcx> { // "Bound" types appear in canonical queries when the // closure type is not yet known - Bound(..) | Infer(_) => None, + Bound(..) | Param(_) | Infer(_) => None, Error(_) => Some(ty::ClosureKind::Fn), @@ -2077,6 +2398,7 @@ impl<'tcx> Ty<'tcx> { | ty::CoroutineWitness(..) | ty::Array(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Never | ty::Error(_) => true, @@ -2140,7 +2462,7 @@ impl<'tcx> Ty<'tcx> { ty::Coroutine(..) | ty::CoroutineWitness(..) => false, // Might be, but not "trivial" so just giving the safe answer. - ty::Adt(..) | ty::Closure(..) => false, + ty::Adt(..) | ty::Closure(..) | ty::CoroutineClosure(..) => false, // Needs normalization or revealing to determine, so no is the safe answer. ty::Alias(..) => false, @@ -2212,6 +2534,7 @@ impl<'tcx> Ty<'tcx> { | FnPtr(_) | Dynamic(_, _, _) | Closure(_, _) + | CoroutineClosure(_, _) | Coroutine(_, _) | CoroutineWitness(..) | Never diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 8cc8abbe718..4feaeb0dd05 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -1119,6 +1119,7 @@ impl<'tcx> Ty<'tcx> { ty::Adt(..) | ty::Bound(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Dynamic(..) | ty::Foreign(_) | ty::Coroutine(..) @@ -1158,6 +1159,7 @@ impl<'tcx> Ty<'tcx> { ty::Adt(..) | ty::Bound(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Dynamic(..) | ty::Foreign(_) | ty::Coroutine(..) @@ -1280,7 +1282,11 @@ impl<'tcx> Ty<'tcx> { // Conservatively return `false` for all others... // Anonymous function types - ty::FnDef(..) | ty::Closure(..) | ty::Dynamic(..) | ty::Coroutine(..) => false, + ty::FnDef(..) + | ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Dynamic(..) + | ty::Coroutine(..) => false, // Generic or inferred types // @@ -1424,6 +1430,7 @@ pub fn needs_drop_components<'tcx>( | ty::Placeholder(..) | ty::Infer(_) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) => Ok(smallvec![ty]), } @@ -1456,7 +1463,11 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool { // Not trivial because they have components, and instead of looking inside, // we'll just perform trait selection. - ty::Closure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Adt(..) => false, + ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Coroutine(..) + | ty::CoroutineWitness(..) + | ty::Adt(..) => false, ty::Array(ty, _) | ty::Slice(ty) => is_trivially_const_drop(ty), diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs index 9050716db9d..46c26241c3e 100644 --- a/compiler/rustc_middle/src/ty/walk.rs +++ b/compiler/rustc_middle/src/ty/walk.rs @@ -189,6 +189,7 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>) } ty::Adt(_, args) | ty::Closure(_, args) + | ty::CoroutineClosure(_, args) | ty::Coroutine(_, args) | ty::CoroutineWitness(_, args) | ty::FnDef(_, args) => { diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs index 6e8af7bb6df..c77f4a06d05 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs @@ -483,6 +483,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { UpvarArgs::Closure(args) => { Box::new(AggregateKind::Closure(closure_id.to_def_id(), args)) } + UpvarArgs::CoroutineClosure(args) => { + Box::new(AggregateKind::CoroutineClosure(closure_id.to_def_id(), args)) + } }; block.and(Rvalue::Aggregate(result, operands)) } diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index 714c5f2686e..9a2f2ceced1 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -495,7 +495,7 @@ fn construct_fn<'tcx>( args.as_coroutine().yield_ty(), args.as_coroutine().resume_ty(), ))), - ty::Closure(..) | ty::FnDef(..) => None, + ty::Closure(..) | ty::CoroutineClosure(..) | ty::FnDef(..) => None, ty => span_bug!(span_with_body, "unexpected type of body: {ty:?}"), }; @@ -822,6 +822,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let upvar_args = match closure_ty.kind() { ty::Closure(_, args) => ty::UpvarArgs::Closure(args), ty::Coroutine(_, args) => ty::UpvarArgs::Coroutine(args), + ty::CoroutineClosure(_, args) => ty::UpvarArgs::CoroutineClosure(args), _ => return, }; diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs index 22094c112fc..35adcc33480 100644 --- a/compiler/rustc_mir_build/src/thir/cx/expr.rs +++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs @@ -556,6 +556,9 @@ impl<'tcx> Cx<'tcx> { ty::Coroutine(def_id, args) => { (def_id, UpvarArgs::Coroutine(args), Some(tcx.coroutine_movability(def_id))) } + ty::CoroutineClosure(def_id, args) => { + (def_id, UpvarArgs::CoroutineClosure(args), None) + } _ => { span_bug!(expr.span, "closure expr w/o closure type: {:?}", closure_ty); } diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs index 5d0bb3954cc..f4f591d15b9 100644 --- a/compiler/rustc_mir_build/src/thir/cx/mod.rs +++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs @@ -127,10 +127,24 @@ impl<'tcx> Cx<'tcx> { ty::Coroutine(..) => { Param { ty: closure_ty, pat: None, ty_span: None, self_kind: None, hir_id: None } } - ty::Closure(_, closure_args) => { + ty::Closure(_, args) => { let closure_env_ty = self.tcx.closure_env_ty( closure_ty, - closure_args.as_closure().kind(), + args.as_closure().kind(), + self.tcx.lifetimes.re_erased, + ); + Param { + ty: closure_env_ty, + pat: None, + ty_span: None, + self_kind: None, + hir_id: None, + } + } + ty::CoroutineClosure(_, args) => { + let closure_env_ty = self.tcx.closure_env_ty( + closure_ty, + args.as_coroutine_closure().kind(), self.tcx.lifetimes.re_erased, ); Param { diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs index 862876f53c7..1b2f2cd9477 100644 --- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs +++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs @@ -861,6 +861,9 @@ where let ty = self.place_ty(self.place); match ty.kind() { ty::Closure(_, args) => self.open_drop_for_tuple(args.as_closure().upvar_tys()), + ty::CoroutineClosure(_, args) => { + self.open_drop_for_tuple(args.as_coroutine_closure().upvar_tys()) + } // Note that `elaborate_drops` only drops the upvars of a coroutine, // and this is ok because `open_drop` here can only be reached // within that own coroutine's resume function. diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index 38ee26c5a87..30dd915521c 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -154,7 +154,8 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> { | ty::FnDef(_, _) | ty::FnPtr(_) | ty::Dynamic(_, _, _) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never @@ -177,7 +178,10 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> { union_path.get_or_insert(base); } } - ty::Closure(_, _) | ty::Coroutine(_, _) | ty::Tuple(_) => (), + ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Coroutine(_, _) + | ty::Tuple(_) => (), ty::Bool | ty::Char | ty::Int(_) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 2802f5491be..2b2af6ee7da 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -1149,6 +1149,12 @@ pub fn iter_fields<'tcx>( ty::Closure(_, args) => { iter_fields(args.as_closure().tupled_upvars_ty(), tcx, param_env, f); } + ty::Coroutine(_, args) => { + iter_fields(args.as_coroutine().tupled_upvars_ty(), tcx, param_env, f); + } + ty::CoroutineClosure(_, args) => { + iter_fields(args.as_coroutine_closure().tupled_upvars_ty(), tcx, param_env, f); + } _ => (), } } diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs index dfc7a9891f9..451d3be255f 100644 --- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs +++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs @@ -39,6 +39,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls { let body_abi = match body_ty.kind() { ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), ty::Closure(..) => Abi::RustCall, + ty::CoroutineClosure(..) => Abi::RustCall, ty::Coroutine(..) => Abi::Rust, _ => span_bug!(body.span, "unexpected body ty: {:?}", body_ty), }; diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 6c4c3917cb5..fbb62695383 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -128,7 +128,9 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { ), } } - &AggregateKind::Closure(def_id, _) | &AggregateKind::Coroutine(def_id, _) => { + &AggregateKind::Closure(def_id, _) + | &AggregateKind::CoroutineClosure(def_id, _) + | &AggregateKind::Coroutine(def_id, _) => { let def_id = def_id.expect_local(); let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } = self.tcx.mir_unsafety_check_result(def_id); diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index aa22b8c7c58..f2448ee3d44 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -607,7 +607,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { AggregateKind::Array(_) | AggregateKind::Tuple | AggregateKind::Closure(_, _) - | AggregateKind::Coroutine(_, _) => VariantIdx::new(0), + | AggregateKind::Coroutine(_, _) + | AggregateKind::CoroutineClosure(_, _) => VariantIdx::new(0), }, }, diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index bde879f6067..297b2fa143d 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -50,6 +50,9 @@ //! For coroutines with state 1 (returned) and state 2 (poisoned) it does nothing. //! Otherwise it drops all the values in scope at the last suspension point. +mod by_move_body; +pub use by_move_body::ByMoveBody; + use crate::abort_unwinding_calls; use crate::deref_separator::deref_finder; use crate::errors; diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs new file mode 100644 index 00000000000..fcd4715b9e8 --- /dev/null +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -0,0 +1,156 @@ +//! A MIR pass which duplicates a coroutine's body and removes any derefs which +//! would be present for upvars that are taken by-ref. The result of which will +//! be a coroutine body that takes all of its upvars by-move, and which we stash +//! into the `CoroutineInfo` for all coroutines returned by coroutine-closures. + +use rustc_data_structures::fx::FxIndexSet; +use rustc_hir as hir; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::{self, dump_mir, MirPass}; +use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt}; +use rustc_target::abi::FieldIdx; + +pub struct ByMoveBody; + +impl<'tcx> MirPass<'tcx> for ByMoveBody { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { + let Some(coroutine_def_id) = body.source.def_id().as_local() else { + return; + }; + let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) = + tcx.coroutine_kind(coroutine_def_id) + else { + return; + }; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; + let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!() }; + + let coroutine_kind = args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(); + if coroutine_kind == ty::ClosureKind::FnOnce { + return; + } + + let mut by_ref_fields = FxIndexSet::default(); + let by_move_upvars = Ty::new_tup_from_iter( + tcx, + tcx.closure_captures(coroutine_def_id).iter().enumerate().map(|(idx, capture)| { + if capture.is_by_ref() { + by_ref_fields.insert(FieldIdx::from_usize(idx)); + } + capture.place.ty() + }), + ); + let by_move_coroutine_ty = Ty::new_coroutine( + tcx, + coroutine_def_id.to_def_id(), + ty::CoroutineArgs::new( + tcx, + ty::CoroutineArgsParts { + parent_args: args.as_coroutine().parent_args(), + kind_ty: Ty::from_closure_kind(tcx, ty::ClosureKind::FnOnce), + resume_ty: args.as_coroutine().resume_ty(), + yield_ty: args.as_coroutine().yield_ty(), + return_ty: args.as_coroutine().return_ty(), + witness: args.as_coroutine().witness(), + tupled_upvars_ty: by_move_upvars, + }, + ) + .args, + ); + + let mut by_move_body = body.clone(); + MakeByMoveBody { tcx, by_ref_fields, by_move_coroutine_ty }.visit_body(&mut by_move_body); + dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(())); + by_move_body.source = mir::MirSource { + instance: InstanceDef::CoroutineKindShim { + coroutine_def_id: coroutine_def_id.to_def_id(), + target_kind: ty::ClosureKind::FnOnce, + }, + promoted: None, + }; + body.coroutine.as_mut().unwrap().by_move_body = Some(by_move_body); + + // If this is coming from an `AsyncFn` coroutine-closure, we must also create a by-mut body. + // This is actually just a copy of the by-ref body, but with a different self type. + // FIXME(async_closures): We could probably unify this with the by-ref body somehow. + if coroutine_kind == ty::ClosureKind::Fn { + let by_mut_coroutine_ty = Ty::new_coroutine( + tcx, + coroutine_def_id.to_def_id(), + ty::CoroutineArgs::new( + tcx, + ty::CoroutineArgsParts { + parent_args: args.as_coroutine().parent_args(), + kind_ty: Ty::from_closure_kind(tcx, ty::ClosureKind::FnMut), + resume_ty: args.as_coroutine().resume_ty(), + yield_ty: args.as_coroutine().yield_ty(), + return_ty: args.as_coroutine().return_ty(), + witness: args.as_coroutine().witness(), + tupled_upvars_ty: args.as_coroutine().tupled_upvars_ty(), + }, + ) + .args, + ); + let mut by_mut_body = body.clone(); + by_mut_body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = by_mut_coroutine_ty; + dump_mir(tcx, false, "coroutine_by_mut", &0, &by_mut_body, |_, _| Ok(())); + by_mut_body.source = mir::MirSource { + instance: InstanceDef::CoroutineKindShim { + coroutine_def_id: coroutine_def_id.to_def_id(), + target_kind: ty::ClosureKind::FnMut, + }, + promoted: None, + }; + body.coroutine.as_mut().unwrap().by_mut_body = Some(by_mut_body); + } + } +} + +struct MakeByMoveBody<'tcx> { + tcx: TyCtxt<'tcx>, + by_ref_fields: FxIndexSet<FieldIdx>, + by_move_coroutine_ty: Ty<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place( + &mut self, + place: &mut mir::Place<'tcx>, + context: mir::visit::PlaceContext, + location: mir::Location, + ) { + if place.local == ty::CAPTURE_STRUCT_LOCAL + && !place.projection.is_empty() + && let mir::ProjectionElem::Field(idx, ty) = place.projection[0] + && self.by_ref_fields.contains(&idx) + { + let (begin, end) = place.projection[1..].split_first().unwrap(); + // FIXME(async_closures): I'm actually a bit surprised to see that we always + // initially deref the by-ref upvars. If this is not actually true, then we + // will at least get an ICE that explains why this isn't true :^) + assert_eq!(*begin, mir::ProjectionElem::Deref); + // Peel one ref off of the ty. + let peeled_ty = ty.builtin_deref(true).unwrap().ty; + *place = mir::Place { + local: place.local, + projection: self.tcx.mk_place_elems_from_iter( + [mir::ProjectionElem::Field(idx, peeled_ty)] + .into_iter() + .chain(end.iter().copied()), + ), + }; + } + self.super_place(place, context, location); + } + + fn visit_local_decl(&mut self, local: mir::Local, local_decl: &mut mir::LocalDecl<'tcx>) { + // Replace the type of the self arg. + if local == ty::CAPTURE_STRUCT_LOCAL { + local_decl.ty = self.by_move_coroutine_ty; + } + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 5b4d58836b4..01fae7c0bec 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -156,7 +156,9 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( fn is_closure_or_coroutine(statement: &Statement<'_>) -> bool { match statement.kind { StatementKind::Assign(box (_, Rvalue::Aggregate(box ref agg_kind, _))) => match agg_kind { - AggregateKind::Closure(_, _) | AggregateKind::Coroutine(_, _) => true, + AggregateKind::Closure(_, _) + | AggregateKind::Coroutine(_, _) + | AggregateKind::CoroutineClosure(..) => true, _ => false, }, _ => false, diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index ad12bce9b02..6a37047a693 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -696,6 +696,7 @@ fn try_write_constant<'tcx>( | ty::Bound(..) | ty::Placeholder(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::Dynamic(..) => throw_machine_stop_str!("unsupported type"), diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs index db7dfc5b43e..b0d758bcbfe 100644 --- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs +++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs @@ -57,6 +57,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool { let body_abi = match body_ty.kind() { ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), ty::Closure(..) => Abi::RustCall, + ty::CoroutineClosure(..) => Abi::RustCall, ty::Coroutine(..) => Abi::Rust, _ => span_bug!(body.span, "unexpected body ty: {:?}", body_ty), }; diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 390ec3e1a36..f9798bc4e70 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -861,9 +861,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { let tcx = self.tcx; if fields.is_empty() { let is_zst = match *kind { - AggregateKind::Array(..) | AggregateKind::Tuple | AggregateKind::Closure(..) => { - true - } + AggregateKind::Array(..) + | AggregateKind::Tuple + | AggregateKind::Closure(..) + | AggregateKind::CoroutineClosure(..) => true, // Only enums can be non-ZST. AggregateKind::Adt(did, ..) => tcx.def_kind(did) != DefKind::Enum, // Coroutines are never ZST, as they at least contain the implicit states. @@ -885,7 +886,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { assert!(!fields.is_empty()); (AggregateTy::Tuple, FIRST_VARIANT) } - AggregateKind::Closure(did, substs) | AggregateKind::Coroutine(did, substs) => { + AggregateKind::Closure(did, substs) + | AggregateKind::CoroutineClosure(did, substs) + | AggregateKind::Coroutine(did, substs) => { (AggregateTy::Def(did, substs), FIRST_VARIANT) } AggregateKind::Adt(did, variant_index, substs, _, None) => { diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 67668a216de..e77553a03d6 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -317,6 +317,8 @@ impl<'tcx> Inliner<'tcx> { | InstanceDef::ReifyShim(_) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } + | InstanceDef::ConstructCoroutineInClosureShim { .. } + | InstanceDef::CoroutineKindShim { .. } | InstanceDef::DropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::ThreadLocalShim(..) diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index d30e0bad813..5b03bc361dd 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -87,6 +87,8 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( | InstanceDef::ReifyShim(_) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } + | InstanceDef::ConstructCoroutineInClosureShim { .. } + | InstanceDef::CoroutineKindShim { .. } | InstanceDef::ThreadLocalShim { .. } | InstanceDef::CloneShim(..) => {} diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 69f93fa3a0e..031515ea958 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -307,6 +307,10 @@ fn mir_const(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> { &Lint(check_packed_ref::CheckPackedRef), &Lint(check_const_item_mutation::CheckConstItemMutation), &Lint(function_item_references::FunctionItemReferences), + // If this is an async closure's output coroutine, generate + // by-move and by-mut bodies if needed. We do this first so + // they can be optimized in lockstep with their parent bodies. + &coroutine::ByMoveBody, // What we need to do constant evaluation. &simplify::SimplifyCfg::Initial, &rustc_peek::SanityCheck, // Just a lint diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index c1ef2b9f887..605e1ad46d7 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -189,6 +189,15 @@ fn run_passes_inner<'tcx>( body.pass_count = 1; } + + if let Some(coroutine) = body.coroutine.as_mut() { + if let Some(by_move_body) = coroutine.by_move_body.as_mut() { + run_passes_inner(tcx, by_move_body, passes, phase_change, validate_each); + } + if let Some(by_mut_body) = coroutine.by_mut_body.as_mut() { + run_passes_inner(tcx, by_mut_body, passes, phase_change, validate_each); + } + } } pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index 34d57a45301..9a94cae3382 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -46,6 +46,7 @@ fn maybe_zst(ty: Ty<'_>) -> bool { ty::Adt(..) | ty::Array(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Tuple(..) | ty::Alias(ty::Opaque, ..) => true, // definitely ZST diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 89414ce940e..7b6de3a5439 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -3,8 +3,8 @@ use rustc_hir::def_id::DefId; use rustc_hir::lang_items::LangItem; use rustc_middle::mir::*; use rustc_middle::query::Providers; -use rustc_middle::ty::GenericArgs; use rustc_middle::ty::{self, CoroutineArgs, EarlyBinder, Ty, TyCtxt}; +use rustc_middle::ty::{GenericArgs, CAPTURE_STRUCT_LOCAL}; use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT}; use rustc_index::{Idx, IndexVec}; @@ -66,11 +66,76 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut)) } + ty::InstanceDef::ConstructCoroutineInClosureShim { + coroutine_closure_def_id, + target_kind, + } => match target_kind { + ty::ClosureKind::Fn => unreachable!("shouldn't be building shim for Fn"), + ty::ClosureKind::FnMut => { + // No need to optimize the body, it has already been optimized + // since we steal it from the `AsyncFn::call` body and just fix + // the return type. + return build_construct_coroutine_by_mut_shim(tcx, coroutine_closure_def_id); + } + ty::ClosureKind::FnOnce => { + build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id) + } + }, + + ty::InstanceDef::CoroutineKindShim { coroutine_def_id, target_kind } => match target_kind { + ty::ClosureKind::Fn => unreachable!(), + ty::ClosureKind::FnMut => { + return tcx + .optimized_mir(coroutine_def_id) + .coroutine_by_mut_body() + .unwrap() + .clone(); + } + ty::ClosureKind::FnOnce => { + return tcx + .optimized_mir(coroutine_def_id) + .coroutine_by_move_body() + .unwrap() + .clone(); + } + }, + ty::InstanceDef::DropGlue(def_id, ty) => { // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end // of this function. Is this intentional? if let Some(ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) { - let body = tcx.optimized_mir(*coroutine_def_id).coroutine_drop().unwrap(); + let coroutine_body = tcx.optimized_mir(*coroutine_def_id); + + let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind() + else { + bug!() + }; + + // If this is a regular coroutine, grab its drop shim. If this is a coroutine + // that comes from a coroutine-closure, and the kind ty differs from the "maximum" + // kind that it supports, then grab the appropriate drop shim. This ensures that + // the future returned by `<[coroutine-closure] as AsyncFnOnce>::call_once` will + // drop the coroutine-closure's upvars. + let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() { + coroutine_body.coroutine_drop().unwrap() + } else { + match args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap() { + ty::ClosureKind::Fn => { + unreachable!() + } + ty::ClosureKind::FnMut => coroutine_body + .coroutine_by_mut_body() + .unwrap() + .coroutine_drop() + .unwrap(), + ty::ClosureKind::FnOnce => coroutine_body + .coroutine_by_move_body() + .unwrap() + .coroutine_drop() + .unwrap(), + } + }; + let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args); debug!("make_shim({:?}) = {:?}", instance, body); @@ -981,3 +1046,114 @@ fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'t let source = MirSource::from_instance(ty::InstanceDef::FnPtrAddrShim(def_id, self_ty)); new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span) } + +fn build_construct_coroutine_by_move_shim<'tcx>( + tcx: TyCtxt<'tcx>, + coroutine_closure_def_id: DefId, +) -> Body<'tcx> { + let self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity(); + let ty::CoroutineClosure(_, args) = *self_ty.kind() else { + bug!(); + }; + + let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| { + tcx.mk_fn_sig( + [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()), + sig.to_coroutine_given_kind_and_upvars( + tcx, + args.as_coroutine_closure().parent_args(), + tcx.coroutine_for_closure(coroutine_closure_def_id), + ty::ClosureKind::FnOnce, + tcx.lifetimes.re_erased, + args.as_coroutine_closure().tupled_upvars_ty(), + args.as_coroutine_closure().coroutine_captures_by_ref_ty(), + ), + sig.c_variadic, + sig.unsafety, + sig.abi, + ) + }); + let sig = tcx.liberate_late_bound_regions(coroutine_closure_def_id, poly_sig); + let ty::Coroutine(coroutine_def_id, coroutine_args) = *sig.output().kind() else { + bug!(); + }; + + let span = tcx.def_span(coroutine_closure_def_id); + let locals = local_decls_for_sig(&sig, span); + + let mut fields = vec![]; + for idx in 1..sig.inputs().len() { + fields.push(Operand::Move(Local::from_usize(idx + 1).into())); + } + for (idx, ty) in args.as_coroutine_closure().upvar_tys().iter().enumerate() { + fields.push(Operand::Move(tcx.mk_place_field( + Local::from_usize(1).into(), + FieldIdx::from_usize(idx), + ty, + ))); + } + + let source_info = SourceInfo::outermost(span); + let rvalue = Rvalue::Aggregate( + Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)), + IndexVec::from_raw(fields), + ); + let stmt = Statement { + source_info, + kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))), + }; + let statements = vec![stmt]; + let start_block = BasicBlockData { + statements, + terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }), + is_cleanup: false, + }; + + let source = MirSource::from_instance(ty::InstanceDef::ConstructCoroutineInClosureShim { + coroutine_closure_def_id, + target_kind: ty::ClosureKind::FnOnce, + }); + + let body = + new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span); + dump_mir(tcx, false, "coroutine_closure_by_move", &0, &body, |_, _| Ok(())); + + body +} + +fn build_construct_coroutine_by_mut_shim<'tcx>( + tcx: TyCtxt<'tcx>, + coroutine_closure_def_id: DefId, +) -> Body<'tcx> { + let mut body = tcx.optimized_mir(coroutine_closure_def_id).clone(); + let coroutine_closure_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity(); + let ty::CoroutineClosure(_, args) = *coroutine_closure_ty.kind() else { + bug!(); + }; + let args = args.as_coroutine_closure(); + + body.local_decls[RETURN_PLACE].ty = + tcx.instantiate_bound_regions_with_erased(args.coroutine_closure_sig().map_bound(|sig| { + sig.to_coroutine_given_kind_and_upvars( + tcx, + args.parent_args(), + tcx.coroutine_for_closure(coroutine_closure_def_id), + ty::ClosureKind::FnMut, + tcx.lifetimes.re_erased, + args.tupled_upvars_ty(), + args.coroutine_captures_by_ref_ty(), + ) + })); + body.local_decls[CAPTURE_STRUCT_LOCAL].ty = + Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_closure_ty); + + body.source = MirSource::from_instance(ty::InstanceDef::ConstructCoroutineInClosureShim { + coroutine_closure_def_id, + target_kind: ty::ClosureKind::FnMut, + }); + + body.pass_count = 0; + dump_mir(tcx, false, "coroutine_closure_by_mut", &0, &body, |_, _| Ok(())); + + body +} diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index b86244e5a4b..3376af98653 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -983,6 +983,8 @@ fn visit_instance_use<'tcx>( | ty::InstanceDef::VTableShim(..) | ty::InstanceDef::ReifyShim(..) | ty::InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } + | ty::InstanceDef::CoroutineKindShim { .. } | ty::InstanceDef::Item(..) | ty::InstanceDef::FnPtrShim(..) | ty::InstanceDef::CloneShim(..) diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index 7ff182381b8..4f6ea66df7b 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -620,6 +620,8 @@ fn characteristic_def_id_of_mono_item<'tcx>( | ty::InstanceDef::ReifyShim(..) | ty::InstanceDef::FnPtrShim(..) | ty::InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } + | ty::InstanceDef::CoroutineKindShim { .. } | ty::InstanceDef::Intrinsic(..) | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::Virtual(..) @@ -783,6 +785,8 @@ fn mono_item_visibility<'tcx>( | InstanceDef::Virtual(..) | InstanceDef::Intrinsic(..) | InstanceDef::ClosureOnceShim { .. } + | InstanceDef::ConstructCoroutineInClosureShim { .. } + | InstanceDef::CoroutineKindShim { .. } | InstanceDef::DropGlue(..) | InstanceDef::CloneShim(..) | InstanceDef::FnPtrAddrShim(..) => return Visibility::Hidden, diff --git a/compiler/rustc_next_trait_solver/src/canonicalizer.rs b/compiler/rustc_next_trait_solver/src/canonicalizer.rs index db1aee11903..42edbeaa622 100644 --- a/compiler/rustc_next_trait_solver/src/canonicalizer.rs +++ b/compiler/rustc_next_trait_solver/src/canonicalizer.rs @@ -332,7 +332,8 @@ impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I> | ty::FnDef(_, _) | ty::FnPtr(_) | ty::Dynamic(_, _, _) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs index 99f8186d554..f7c382bcd7a 100644 --- a/compiler/rustc_passes/src/liveness.rs +++ b/compiler/rustc_passes/src/liveness.rs @@ -719,6 +719,11 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> { ty::ClosureKind::FnMut => {} ty::ClosureKind::FnOnce => return succ, }, + ty::CoroutineClosure(_def_id, args) => match args.as_coroutine_closure().kind() { + ty::ClosureKind::Fn => {} + ty::ClosureKind::FnMut => {} + ty::ClosureKind::FnOnce => return succ, + }, ty::Coroutine(..) => return succ, _ => { span_bug!( diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs index c02a24048f9..b60f32f71c2 100644 --- a/compiler/rustc_pattern_analysis/src/rustc.rs +++ b/compiler/rustc_pattern_analysis/src/rustc.rs @@ -423,7 +423,8 @@ impl<'p, 'tcx> RustcMatchCheckCtxt<'p, 'tcx> { | ty::FnDef(_, _) | ty::FnPtr(_) | ty::Dynamic(_, _, _) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::Alias(_, _) | ty::Param(_) diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs index a64b06e70d6..a37d8822480 100644 --- a/compiler/rustc_privacy/src/lib.rs +++ b/compiler/rustc_privacy/src/lib.rs @@ -182,6 +182,7 @@ where | ty::Foreign(def_id) | ty::FnDef(def_id, ..) | ty::Closure(def_id, ..) + | ty::CoroutineClosure(def_id, ..) | ty::Coroutine(def_id, ..) => { self.def_id_visitor.visit_def_id(def_id, "type", &ty)?; if V::SHALLOW { diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs index e433460e2ad..41a4edfc03b 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs @@ -538,6 +538,9 @@ impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> { tables.tcx.coroutine_movability(*def_id).stable(tables), ) } + mir::AggregateKind::CoroutineClosure(..) => { + todo!("FIXME(async_closures): Lower these to SMIR") + } } } } diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs index ba957843bb0..959a17d24b7 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs @@ -383,6 +383,7 @@ impl<'tcx> Stable<'tcx> for ty::TyKind<'tcx> { tables.closure_def(*def_id), generic_args.stable(tables), )), + ty::CoroutineClosure(..) => todo!("FIXME(async_closures): Lower these to SMIR"), ty::Coroutine(def_id, generic_args) => TyKind::RigidTy(RigidTy::Coroutine( tables.coroutine_def(*def_id), generic_args.stable(tables), @@ -798,6 +799,8 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> { | ty::InstanceDef::ReifyShim(..) | ty::InstanceDef::FnPtrAddrShim(..) | ty::InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } + | ty::InstanceDef::CoroutineKindShim { .. } | ty::InstanceDef::ThreadLocalShim(..) | ty::InstanceDef::DropGlue(..) | ty::InstanceDef::CloneShim(..) diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 2dca9808ada..ed6e69d9199 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -167,6 +167,9 @@ symbols! { Break, C, CStr, + CallFuture, + CallMutFuture, + CallOnceFuture, Capture, Center, Cleanup, @@ -318,6 +321,7 @@ symbols! { TyCtxt, TyKind, Unknown, + Upvars, Vec, VecDeque, Wrapper, @@ -420,6 +424,7 @@ symbols! { async_closure, async_fn, async_fn_in_trait, + async_fn_kind_helper, async_fn_mut, async_fn_once, async_fn_track_caller, diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 4a5f58443bc..646649293fc 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -64,16 +64,29 @@ pub(super) fn mangle<'tcx>( ) .unwrap(); - if let ty::InstanceDef::ThreadLocalShim(..) = instance.def { - let _ = printer.write_str("{{tls-shim}}"); - } - - if let ty::InstanceDef::VTableShim(..) = instance.def { - let _ = printer.write_str("{{vtable-shim}}"); - } - - if let ty::InstanceDef::ReifyShim(..) = instance.def { - let _ = printer.write_str("{{reify-shim}}"); + match instance.def { + ty::InstanceDef::ThreadLocalShim(..) => { + printer.write_str("{{tls-shim}}").unwrap(); + } + ty::InstanceDef::VTableShim(..) => { + printer.write_str("{{vtable-shim}}").unwrap(); + } + ty::InstanceDef::ReifyShim(..) => { + printer.write_str("{{reify-shim}}").unwrap(); + } + // FIXME(async_closures): This shouldn't be needed when we fix + // `Instance::ty`/`Instance::def_id`. + ty::InstanceDef::ConstructCoroutineInClosureShim { target_kind, .. } + | ty::InstanceDef::CoroutineKindShim { target_kind, .. } => match target_kind { + ty::ClosureKind::Fn => unreachable!(), + ty::ClosureKind::FnMut => { + printer.write_str("{{fn-mut-shim}}").unwrap(); + } + ty::ClosureKind::FnOnce => { + printer.write_str("{{fn-once-shim}}").unwrap(); + } + }, + _ => {} } printer.path.finish(hash) @@ -211,6 +224,7 @@ impl<'tcx> Printer<'tcx> for SymbolPrinter<'tcx> { ty::FnDef(def_id, args) | ty::Alias(ty::Projection | ty::Opaque, ty::AliasTy { def_id, args, .. }) | ty::Closure(def_id, args) + | ty::CoroutineClosure(def_id, args) | ty::Coroutine(def_id, args) => self.print_def_path(def_id, args), // The `pretty_print_type` formatting of array size depends on @@ -281,7 +295,11 @@ impl<'tcx> Printer<'tcx> for SymbolPrinter<'tcx> { // Similar to `pretty_path_qualified`, but for the other // types that are printed as paths (see `print_type` above). match self_ty.kind() { - ty::FnDef(..) | ty::Alias(..) | ty::Closure(..) | ty::Coroutine(..) + ty::FnDef(..) + | ty::Alias(..) + | ty::Closure(..) + | ty::CoroutineClosure(..) + | ty::Coroutine(..) if trait_ref.is_none() => { self.print_type(self_ty) diff --git a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs index 0cc82ac7506..9d1b92e1068 100644 --- a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs +++ b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs @@ -628,7 +628,9 @@ fn encode_ty<'tcx>( } // Function types - ty::FnDef(def_id, args) | ty::Closure(def_id, args) => { + ty::FnDef(def_id, args) + | ty::Closure(def_id, args) + | ty::CoroutineClosure(def_id, args) => { // u<length><name>[I<element-type1..element-typeN>E], where <element-type> is <subst>, // as vendor extended type. let mut s = String::new(); @@ -895,6 +897,10 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio ty = Ty::new_closure(tcx, *def_id, transform_args(tcx, args, options)); } + ty::CoroutineClosure(def_id, args) => { + ty = Ty::new_coroutine_closure(tcx, *def_id, transform_args(tcx, args, options)); + } + ty::Coroutine(def_id, args) => { ty = Ty::new_coroutine(tcx, *def_id, transform_args(tcx, args, options)); } diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs index 16ebda55a7a..530221555c5 100644 --- a/compiler/rustc_symbol_mangling/src/v0.rs +++ b/compiler/rustc_symbol_mangling/src/v0.rs @@ -46,6 +46,13 @@ pub(super) fn mangle<'tcx>( ty::InstanceDef::VTableShim(_) => Some("vtable"), ty::InstanceDef::ReifyShim(_) => Some("reify"), + ty::InstanceDef::ConstructCoroutineInClosureShim { target_kind, .. } + | ty::InstanceDef::CoroutineKindShim { target_kind, .. } => match target_kind { + ty::ClosureKind::Fn => unreachable!(), + ty::ClosureKind::FnMut => Some("fn_mut"), + ty::ClosureKind::FnOnce => Some("fn_once"), + }, + _ => None, }; @@ -386,6 +393,7 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> { | ty::FnDef(def_id, args) | ty::Alias(ty::Projection | ty::Opaque, ty::AliasTy { def_id, args, .. }) | ty::Closure(def_id, args) + | ty::CoroutineClosure(def_id, args) | ty::Coroutine(def_id, args) => { self.print_def_path(def_id, args)?; } diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs index 915d722dd02..7052fd776b0 100644 --- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs @@ -182,6 +182,22 @@ pub(super) trait GoalKind<'tcx>: kind: ty::ClosureKind, ) -> QueryResult<'tcx>; + /// An async closure is known to implement the `AsyncFn<A>` family of traits + /// where `A` is given by the signature of the type. + fn consider_builtin_async_fn_trait_candidates( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + kind: ty::ClosureKind, + ) -> QueryResult<'tcx>; + + /// Compute the built-in logic of the `AsyncFnKindHelper` helper trait, which + /// is used internally to delay computation for async closures until after + /// upvar analysis is performed in HIR typeck. + fn consider_builtin_async_fn_kind_helper_candidate( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + ) -> QueryResult<'tcx>; + /// `Tuple` is implemented if the `Self` type is a tuple. fn consider_builtin_tuple_candidate( ecx: &mut EvalCtxt<'_, 'tcx>, @@ -340,7 +356,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { | ty::FnDef(_, _) | ty::FnPtr(_) | ty::Dynamic(_, _, _) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::Never | ty::Tuple(_) => { @@ -460,6 +477,10 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { G::consider_builtin_fn_ptr_trait_candidate(self, goal) } else if let Some(kind) = self.tcx().fn_trait_kind_from_def_id(trait_def_id) { G::consider_builtin_fn_trait_candidates(self, goal, kind) + } else if let Some(kind) = self.tcx().async_fn_trait_kind_from_def_id(trait_def_id) { + G::consider_builtin_async_fn_trait_candidates(self, goal, kind) + } else if lang_items.async_fn_kind_helper() == Some(trait_def_id) { + G::consider_builtin_async_fn_kind_helper_candidate(self, goal) } else if lang_items.tuple_trait() == Some(trait_def_id) { G::consider_builtin_tuple_candidate(self, goal) } else if lang_items.pointee_trait() == Some(trait_def_id) { @@ -538,6 +559,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { | ty::FnPtr(_) | ty::Dynamic(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Never @@ -694,6 +716,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { | ty::FnPtr(_) | ty::Alias(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Never diff --git a/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs index 274a75a125c..d02578c4846 100644 --- a/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs +++ b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs @@ -1,12 +1,14 @@ //! Code which is used by built-in goals that match "structurally", such a auto //! traits, `Copy`/`Clone`. use rustc_data_structures::fx::FxHashMap; +use rustc_hir::LangItem; use rustc_hir::{def_id::DefId, Movability, Mutability}; use rustc_infer::traits::query::NoSolution; use rustc_middle::traits::solve::Goal; use rustc_middle::ty::{ - self, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt, + self, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt, }; +use rustc_span::sym; use crate::solve::EvalCtxt; @@ -57,6 +59,8 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_auto_trait<'tcx>( ty::Closure(_, args) => Ok(vec![args.as_closure().tupled_upvars_ty()]), + ty::CoroutineClosure(_, args) => Ok(vec![args.as_coroutine_closure().tupled_upvars_ty()]), + ty::Coroutine(_, args) => { let coroutine_args = args.as_coroutine(); Ok(vec![coroutine_args.tupled_upvars_ty(), coroutine_args.witness()]) @@ -128,6 +132,7 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_sized_trait<'tcx>( | ty::CoroutineWitness(..) | ty::Array(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Never | ty::Dynamic(_, _, ty::DynStar) | ty::Error(_) => Ok(vec![]), @@ -193,6 +198,8 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>( ty::Closure(_, args) => Ok(vec![args.as_closure().tupled_upvars_ty()]), + ty::CoroutineClosure(..) => Err(NoSolution), + ty::Coroutine(def_id, args) => match ecx.tcx().coroutine_movability(def_id) { Movability::Static => Err(NoSolution), Movability::Movable => { @@ -267,6 +274,131 @@ pub(in crate::solve) fn extract_tupled_inputs_and_output_from_callable<'tcx>( } Ok(Some(closure_args.sig().map_bound(|sig| (sig.inputs()[0], sig.output())))) } + + // Coroutine-closures don't implement `Fn` traits the normal way. + ty::CoroutineClosure(..) => Err(NoSolution), + + ty::Bool + | ty::Char + | ty::Int(_) + | ty::Uint(_) + | ty::Float(_) + | ty::Adt(_, _) + | ty::Foreign(_) + | ty::Str + | ty::Array(_, _) + | ty::Slice(_) + | ty::RawPtr(_) + | ty::Ref(_, _, _) + | ty::Dynamic(_, _, _) + | ty::Coroutine(_, _) + | ty::CoroutineWitness(..) + | ty::Never + | ty::Tuple(_) + | ty::Alias(_, _) + | ty::Param(_) + | ty::Placeholder(..) + | ty::Infer(ty::IntVar(_) | ty::FloatVar(_)) + | ty::Error(_) => Err(NoSolution), + + ty::Bound(..) + | ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => { + bug!("unexpected type `{self_ty}`") + } + } +} + +// Returns a binder of the tupled inputs types, output type, and coroutine type +// from a builtin coroutine-closure type. If we don't yet know the closure kind of +// the coroutine-closure, emit an additional trait predicate for `AsyncFnKindHelper` +// which enforces the closure is actually callable with the given trait. When we +// know the kind already, we can short-circuit this check. +pub(in crate::solve) fn extract_tupled_inputs_and_output_from_async_callable<'tcx>( + tcx: TyCtxt<'tcx>, + self_ty: Ty<'tcx>, + goal_kind: ty::ClosureKind, + env_region: ty::Region<'tcx>, +) -> Result< + (ty::Binder<'tcx, (Ty<'tcx>, Ty<'tcx>, Ty<'tcx>)>, Option<ty::Predicate<'tcx>>), + NoSolution, +> { + match *self_ty.kind() { + ty::CoroutineClosure(def_id, args) => { + let args = args.as_coroutine_closure(); + let kind_ty = args.kind_ty(); + + if let Some(closure_kind) = kind_ty.to_opt_closure_kind() { + if !closure_kind.extends(goal_kind) { + return Err(NoSolution); + } + Ok(( + args.coroutine_closure_sig().map_bound(|sig| { + let coroutine_ty = sig.to_coroutine_given_kind_and_upvars( + tcx, + args.parent_args(), + tcx.coroutine_for_closure(def_id), + goal_kind, + env_region, + args.tupled_upvars_ty(), + args.coroutine_captures_by_ref_ty(), + ); + (sig.tupled_inputs_ty, sig.return_ty, coroutine_ty) + }), + None, + )) + } else { + let async_fn_kind_trait_def_id = + tcx.require_lang_item(LangItem::AsyncFnKindHelper, None); + let upvars_projection_def_id = tcx + .associated_items(async_fn_kind_trait_def_id) + .filter_by_name_unhygienic(sym::Upvars) + .next() + .unwrap() + .def_id; + // When we don't know the closure kind (and therefore also the closure's upvars, + // which are computed at the same time), we must delay the computation of the + // generator's upvars. We do this using the `AsyncFnKindHelper`, which as a trait + // goal functions similarly to the old `ClosureKind` predicate, and ensures that + // the goal kind <= the closure kind. As a projection `AsyncFnKindHelper::Upvars` + // will project to the right upvars for the generator, appending the inputs and + // coroutine upvars respecting the closure kind. + Ok(( + args.coroutine_closure_sig().map_bound(|sig| { + let tupled_upvars_ty = Ty::new_projection( + tcx, + upvars_projection_def_id, + [ + ty::GenericArg::from(kind_ty), + Ty::from_closure_kind(tcx, goal_kind).into(), + env_region.into(), + sig.tupled_inputs_ty.into(), + args.tupled_upvars_ty().into(), + args.coroutine_captures_by_ref_ty().into(), + ], + ); + let coroutine_ty = sig.to_coroutine( + tcx, + args.parent_args(), + Ty::from_closure_kind(tcx, goal_kind), + tcx.coroutine_for_closure(def_id), + tupled_upvars_ty, + ); + (sig.tupled_inputs_ty, sig.return_ty, coroutine_ty) + }), + Some( + ty::TraitRef::new( + tcx, + async_fn_kind_trait_def_id, + [kind_ty, Ty::from_closure_kind(tcx, goal_kind)], + ) + .to_predicate(tcx), + ), + )) + } + } + + ty::FnDef(..) | ty::FnPtr(..) | ty::Closure(..) => Err(NoSolution), + ty::Bool | ty::Char | ty::Int(_) diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs index 9f1b4a09a20..47ba549022d 100644 --- a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs @@ -366,6 +366,119 @@ impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> { Self::consider_implied_clause(ecx, goal, pred, [goal.with(tcx, output_is_sized_pred)]) } + fn consider_builtin_async_fn_trait_candidates( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + goal_kind: ty::ClosureKind, + ) -> QueryResult<'tcx> { + let tcx = ecx.tcx(); + + let env_region = match goal_kind { + ty::ClosureKind::Fn | ty::ClosureKind::FnMut => goal.predicate.alias.args.region_at(2), + // Doesn't matter what this region is + ty::ClosureKind::FnOnce => tcx.lifetimes.re_static, + }; + let (tupled_inputs_and_output_and_coroutine, nested_preds) = + structural_traits::extract_tupled_inputs_and_output_from_async_callable( + tcx, + goal.predicate.self_ty(), + goal_kind, + env_region, + )?; + let output_is_sized_pred = + tupled_inputs_and_output_and_coroutine.map_bound(|(_, output, _)| { + ty::TraitRef::from_lang_item(tcx, LangItem::Sized, DUMMY_SP, [output]) + }); + + let pred = tupled_inputs_and_output_and_coroutine + .map_bound(|(inputs, output, coroutine)| { + let (projection_ty, term) = match tcx.item_name(goal.predicate.def_id()) { + sym::CallOnceFuture => ( + ty::AliasTy::new( + tcx, + goal.predicate.def_id(), + [goal.predicate.self_ty(), inputs], + ), + coroutine.into(), + ), + sym::CallMutFuture | sym::CallFuture => ( + ty::AliasTy::new( + tcx, + goal.predicate.def_id(), + [ + ty::GenericArg::from(goal.predicate.self_ty()), + inputs.into(), + env_region.into(), + ], + ), + coroutine.into(), + ), + sym::Output => ( + ty::AliasTy::new( + tcx, + goal.predicate.def_id(), + [ty::GenericArg::from(goal.predicate.self_ty()), inputs.into()], + ), + output.into(), + ), + name => bug!("no such associated type: {name}"), + }; + ty::ProjectionPredicate { projection_ty, term } + }) + .to_predicate(tcx); + + // A built-in `AsyncFn` impl only holds if the output is sized. + // (FIXME: technically we only need to check this if the type is a fn ptr...) + Self::consider_implied_clause( + ecx, + goal, + pred, + [goal.with(tcx, output_is_sized_pred)] + .into_iter() + .chain(nested_preds.into_iter().map(|pred| goal.with(tcx, pred))), + ) + } + + fn consider_builtin_async_fn_kind_helper_candidate( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + ) -> QueryResult<'tcx> { + let [ + closure_fn_kind_ty, + goal_kind_ty, + borrow_region, + tupled_inputs_ty, + tupled_upvars_ty, + coroutine_captures_by_ref_ty, + ] = **goal.predicate.alias.args + else { + bug!(); + }; + + let Some(closure_kind) = closure_fn_kind_ty.expect_ty().to_opt_closure_kind() else { + // We don't need to worry about the self type being an infer var. + return Err(NoSolution); + }; + let Some(goal_kind) = goal_kind_ty.expect_ty().to_opt_closure_kind() else { + return Err(NoSolution); + }; + if !closure_kind.extends(goal_kind) { + return Err(NoSolution); + } + + let upvars_ty = ty::CoroutineClosureSignature::tupled_upvars_by_closure_kind( + ecx.tcx(), + goal_kind, + tupled_inputs_ty.expect_ty(), + tupled_upvars_ty.expect_ty(), + coroutine_captures_by_ref_ty.expect_ty(), + borrow_region.expect_region(), + ); + + ecx.eq(goal.param_env, goal.predicate.term.ty().unwrap(), upvars_ty)?; + ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + } + fn consider_builtin_tuple_candidate( _ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, @@ -391,6 +504,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> { | ty::FnDef(..) | ty::FnPtr(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Infer(ty::IntVar(..) | ty::FloatVar(..)) | ty::Coroutine(..) | ty::CoroutineWitness(..) @@ -627,6 +741,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> { | ty::FnDef(..) | ty::FnPtr(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Infer(ty::IntVar(..) | ty::FloatVar(..)) | ty::Coroutine(..) | ty::CoroutineWitness(..) diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs index b185e4e5f8e..fd09a6b671d 100644 --- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs @@ -303,6 +303,66 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> { Self::consider_implied_clause(ecx, goal, pred, [goal.with(tcx, output_is_sized_pred)]) } + fn consider_builtin_async_fn_trait_candidates( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + goal_kind: ty::ClosureKind, + ) -> QueryResult<'tcx> { + if goal.predicate.polarity != ty::ImplPolarity::Positive { + return Err(NoSolution); + } + + let tcx = ecx.tcx(); + let (tupled_inputs_and_output_and_coroutine, nested_preds) = + structural_traits::extract_tupled_inputs_and_output_from_async_callable( + tcx, + goal.predicate.self_ty(), + goal_kind, + // This region doesn't matter because we're throwing away the coroutine type + tcx.lifetimes.re_static, + )?; + let output_is_sized_pred = + tupled_inputs_and_output_and_coroutine.map_bound(|(_, output, _)| { + ty::TraitRef::from_lang_item(tcx, LangItem::Sized, DUMMY_SP, [output]) + }); + + let pred = tupled_inputs_and_output_and_coroutine + .map_bound(|(inputs, _, _)| { + ty::TraitRef::new(tcx, goal.predicate.def_id(), [goal.predicate.self_ty(), inputs]) + }) + .to_predicate(tcx); + // A built-in `AsyncFn` impl only holds if the output is sized. + // (FIXME: technically we only need to check this if the type is a fn ptr...) + Self::consider_implied_clause( + ecx, + goal, + pred, + [goal.with(tcx, output_is_sized_pred)] + .into_iter() + .chain(nested_preds.into_iter().map(|pred| goal.with(tcx, pred))), + ) + } + + fn consider_builtin_async_fn_kind_helper_candidate( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + ) -> QueryResult<'tcx> { + let [closure_fn_kind_ty, goal_kind_ty] = **goal.predicate.trait_ref.args else { + bug!(); + }; + + let Some(closure_kind) = closure_fn_kind_ty.expect_ty().to_opt_closure_kind() else { + // We don't need to worry about the self type being an infer var. + return Err(NoSolution); + }; + let goal_kind = goal_kind_ty.expect_ty().to_opt_closure_kind().unwrap(); + if closure_kind.extends(goal_kind) { + ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + } else { + Err(NoSolution) + } + } + fn consider_builtin_tuple_candidate( ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, @@ -950,7 +1010,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { | ty::Ref(_, _, _) | ty::FnDef(_, _) | ty::FnPtr(_) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs index c49185a52c7..4b20de26219 100644 --- a/compiler/rustc_trait_selection/src/traits/coherence.rs +++ b/compiler/rustc_trait_selection/src/traits/coherence.rs @@ -863,7 +863,7 @@ where } } ty::Error(_) => ControlFlow::Break(OrphanCheckEarlyExit::LocalTy(ty)), - ty::Closure(did, ..) | ty::Coroutine(did, ..) => { + ty::Closure(did, ..) | ty::CoroutineClosure(did, ..) | ty::Coroutine(did, ..) => { if self.def_id_is_local(did) { ControlFlow::Break(OrphanCheckEarlyExit::LocalTy(ty)) } else { diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs index a8715b0764f..1ac0f172ef4 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs @@ -959,9 +959,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> { obligation: &PredicateObligation<'tcx>, trait_ref: ty::PolyTraitRef<'tcx>, ) -> Option<ErrorGuaranteed> { - if let ty::Closure(closure_def_id, closure_args) = *trait_ref.self_ty().skip_binder().kind() + let self_ty = trait_ref.self_ty().skip_binder(); + if let ty::Closure(closure_def_id, closure_args) = *self_ty.kind() && let Some(expected_kind) = self.tcx.fn_trait_kind_from_def_id(trait_ref.def_id()) - && let Some(found_kind) = self.closure_kind(closure_args) + && let Some(found_kind) = self.closure_kind(self_ty) && !found_kind.extends(expected_kind) && let sig = closure_args.as_closure().sig() && self.can_sub( @@ -1875,6 +1876,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> { ty::Coroutine(..) => Some(18), ty::Foreign(..) => Some(19), ty::CoroutineWitness(..) => Some(20), + ty::CoroutineClosure(..) => Some(21), ty::Placeholder(..) | ty::Bound(..) | ty::Infer(..) | ty::Error(_) => None, } } @@ -1924,45 +1926,50 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> { fn describe_closure(&self, kind: hir::ClosureKind) -> &'static str { match kind { hir::ClosureKind::Closure => "a closure", - hir::ClosureKind::Coroutine(kind) => match kind { - hir::CoroutineKind::Coroutine(_) => "a coroutine", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::Async, - hir::CoroutineSource::Block, - ) => "an async block", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::Async, - hir::CoroutineSource::Fn, - ) => "an async function", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::Async, - hir::CoroutineSource::Closure, - ) => "an async closure", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::AsyncGen, - hir::CoroutineSource::Block, - ) => "an async gen block", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::AsyncGen, - hir::CoroutineSource::Fn, - ) => "an async gen function", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::AsyncGen, - hir::CoroutineSource::Closure, - ) => "an async gen closure", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::Gen, - hir::CoroutineSource::Block, - ) => "a gen block", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::Gen, - hir::CoroutineSource::Fn, - ) => "a gen function", - hir::CoroutineKind::Desugared( - hir::CoroutineDesugaring::Gen, - hir::CoroutineSource::Closure, - ) => "a gen closure", - }, + hir::ClosureKind::Coroutine(hir::CoroutineKind::Coroutine(_)) => "a coroutine", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Async, + hir::CoroutineSource::Block, + )) => "an async block", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Async, + hir::CoroutineSource::Fn, + )) => "an async function", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Async, + hir::CoroutineSource::Closure, + )) + | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async) => { + "an async closure" + } + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::AsyncGen, + hir::CoroutineSource::Block, + )) => "an async gen block", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::AsyncGen, + hir::CoroutineSource::Fn, + )) => "an async gen function", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::AsyncGen, + hir::CoroutineSource::Closure, + )) + | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::AsyncGen) => { + "an async gen closure" + } + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Gen, + hir::CoroutineSource::Block, + )) => "a gen block", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Gen, + hir::CoroutineSource::Fn, + )) => "a gen function", + hir::ClosureKind::Coroutine(hir::CoroutineKind::Desugared( + hir::CoroutineDesugaring::Gen, + hir::CoroutineSource::Closure, + )) + | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Gen) => "a gen closure", } } diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index abbc2066eac..955c81eee6b 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -1833,10 +1833,28 @@ fn assemble_candidates_from_impls<'cx, 'tcx>( lang_items.fn_trait(), lang_items.fn_mut_trait(), lang_items.fn_once_trait(), + lang_items.async_fn_trait(), + lang_items.async_fn_mut_trait(), + lang_items.async_fn_once_trait(), ].contains(&Some(trait_ref.def_id)) { true - }else if lang_items.discriminant_kind_trait() == Some(trait_ref.def_id) { + } else if lang_items.async_fn_kind_helper() == Some(trait_ref.def_id) { + // FIXME(async_closures): Validity constraints here could be cleaned up. + if obligation.predicate.args.type_at(0).is_ty_var() + || obligation.predicate.args.type_at(4).is_ty_var() + || obligation.predicate.args.type_at(5).is_ty_var() + { + candidate_set.mark_ambiguous(); + true + } else if obligation.predicate.args.type_at(0).to_opt_closure_kind().is_some() + && obligation.predicate.args.type_at(1).to_opt_closure_kind().is_some() + { + true + } else { + false + } + } else if lang_items.discriminant_kind_trait() == Some(trait_ref.def_id) { match self_ty.kind() { ty::Bool | ty::Char @@ -1854,6 +1872,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>( | ty::FnPtr(..) | ty::Dynamic(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Never @@ -1903,6 +1922,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>( | ty::FnPtr(..) | ty::Dynamic(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Never @@ -2059,6 +2079,10 @@ fn confirm_select_candidate<'cx, 'tcx>( } else { confirm_fn_pointer_candidate(selcx, obligation, data) } + } else if selcx.tcx().async_fn_trait_kind_from_def_id(trait_def_id).is_some() { + confirm_async_closure_candidate(selcx, obligation, data) + } else if lang_items.async_fn_kind_helper() == Some(trait_def_id) { + confirm_async_fn_kind_helper_candidate(selcx, obligation, data) } else { confirm_builtin_candidate(selcx, obligation, data) } @@ -2419,6 +2443,173 @@ fn confirm_callable_candidate<'cx, 'tcx>( confirm_param_env_candidate(selcx, obligation, predicate, true) } +fn confirm_async_closure_candidate<'cx, 'tcx>( + selcx: &mut SelectionContext<'cx, 'tcx>, + obligation: &ProjectionTyObligation<'tcx>, + mut nested: Vec<PredicateObligation<'tcx>>, +) -> Progress<'tcx> { + let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty()); + let ty::CoroutineClosure(def_id, args) = *self_ty.kind() else { + unreachable!( + "expected coroutine-closure self type for coroutine-closure candidate, found {self_ty}" + ) + }; + let args = args.as_coroutine_closure(); + let kind_ty = args.kind_ty(); + + let tcx = selcx.tcx(); + let goal_kind = + tcx.async_fn_trait_kind_from_def_id(obligation.predicate.trait_def_id(tcx)).unwrap(); + + let async_fn_kind_helper_trait_def_id = + tcx.require_lang_item(LangItem::AsyncFnKindHelper, None); + nested.push(obligation.with( + tcx, + ty::TraitRef::new( + tcx, + async_fn_kind_helper_trait_def_id, + [kind_ty, Ty::from_closure_kind(tcx, goal_kind)], + ), + )); + + let env_region = match goal_kind { + ty::ClosureKind::Fn | ty::ClosureKind::FnMut => obligation.predicate.args.region_at(2), + ty::ClosureKind::FnOnce => tcx.lifetimes.re_static, + }; + + let upvars_projection_def_id = tcx + .associated_items(async_fn_kind_helper_trait_def_id) + .filter_by_name_unhygienic(sym::Upvars) + .next() + .unwrap() + .def_id; + + // FIXME(async_closures): Confirmation is kind of a mess here. Ideally, + // we'd short-circuit when we know that the goal_kind >= closure_kind, and not + // register a nested predicate or create a new projection ty here. But I'm too + // lazy to make this more efficient atm, and we can always tweak it later, + // since all this does is make the solver do more work. + // + // The code duplication due to the different length args is kind of weird, too. + // + // See the logic in `structural_traits` in the new solver to understand a bit + // more clearly how this *should* look. + let poly_cache_entry = args.coroutine_closure_sig().map_bound(|sig| { + let (projection_ty, term) = match tcx.item_name(obligation.predicate.def_id) { + sym::CallOnceFuture => { + let tupled_upvars_ty = Ty::new_projection( + tcx, + upvars_projection_def_id, + [ + ty::GenericArg::from(kind_ty), + Ty::from_closure_kind(tcx, goal_kind).into(), + env_region.into(), + sig.tupled_inputs_ty.into(), + args.tupled_upvars_ty().into(), + args.coroutine_captures_by_ref_ty().into(), + ], + ); + let coroutine_ty = sig.to_coroutine( + tcx, + args.parent_args(), + Ty::from_closure_kind(tcx, goal_kind), + tcx.coroutine_for_closure(def_id), + tupled_upvars_ty, + ); + ( + ty::AliasTy::new( + tcx, + obligation.predicate.def_id, + [self_ty, sig.tupled_inputs_ty], + ), + coroutine_ty.into(), + ) + } + sym::CallMutFuture | sym::CallFuture => { + let tupled_upvars_ty = Ty::new_projection( + tcx, + upvars_projection_def_id, + [ + ty::GenericArg::from(kind_ty), + Ty::from_closure_kind(tcx, goal_kind).into(), + env_region.into(), + sig.tupled_inputs_ty.into(), + args.tupled_upvars_ty().into(), + args.coroutine_captures_by_ref_ty().into(), + ], + ); + let coroutine_ty = sig.to_coroutine( + tcx, + args.parent_args(), + Ty::from_closure_kind(tcx, goal_kind), + tcx.coroutine_for_closure(def_id), + tupled_upvars_ty, + ); + ( + ty::AliasTy::new( + tcx, + obligation.predicate.def_id, + [ + ty::GenericArg::from(self_ty), + sig.tupled_inputs_ty.into(), + env_region.into(), + ], + ), + coroutine_ty.into(), + ) + } + sym::Output => ( + ty::AliasTy::new(tcx, obligation.predicate.def_id, [self_ty, sig.tupled_inputs_ty]), + sig.return_ty.into(), + ), + name => bug!("no such associated type: {name}"), + }; + ty::ProjectionPredicate { projection_ty, term } + }); + + confirm_param_env_candidate(selcx, obligation, poly_cache_entry, true) + .with_addl_obligations(nested) +} + +fn confirm_async_fn_kind_helper_candidate<'cx, 'tcx>( + selcx: &mut SelectionContext<'cx, 'tcx>, + obligation: &ProjectionTyObligation<'tcx>, + nested: Vec<PredicateObligation<'tcx>>, +) -> Progress<'tcx> { + let [ + // We already checked that the goal_kind >= closure_kind + _closure_kind_ty, + goal_kind_ty, + borrow_region, + tupled_inputs_ty, + tupled_upvars_ty, + coroutine_captures_by_ref_ty, + ] = **obligation.predicate.args + else { + bug!(); + }; + + let predicate = ty::ProjectionPredicate { + projection_ty: ty::AliasTy::new( + selcx.tcx(), + obligation.predicate.def_id, + obligation.predicate.args, + ), + term: ty::CoroutineClosureSignature::tupled_upvars_by_closure_kind( + selcx.tcx(), + goal_kind_ty.expect_ty().to_opt_closure_kind().unwrap(), + tupled_inputs_ty.expect_ty(), + tupled_upvars_ty.expect_ty(), + coroutine_captures_by_ref_ty.expect_ty(), + borrow_region.expect_region(), + ) + .into(), + }; + + confirm_param_env_candidate(selcx, obligation, ty::Binder::dummy(predicate), false) + .with_addl_obligations(nested) +} + fn confirm_param_env_candidate<'cx, 'tcx>( selcx: &mut SelectionContext<'cx, 'tcx>, obligation: &ProjectionTyObligation<'tcx>, diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs index 79f03242c58..6c8834f11f1 100644 --- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs +++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs @@ -48,7 +48,11 @@ pub fn trivial_dropck_outlives<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { // (T1..Tn) and closures have same properties as T1..Tn -- // check if *all* of them are trivial. ty::Tuple(tys) => tys.iter().all(|t| trivial_dropck_outlives(tcx, t)), + ty::Closure(_, args) => trivial_dropck_outlives(tcx, args.as_closure().tupled_upvars_ty()), + ty::CoroutineClosure(_, args) => { + trivial_dropck_outlives(tcx, args.as_coroutine_closure().tupled_upvars_ty()) + } ty::Adt(def, _) => { if Some(def.did()) == tcx.lang_items().manually_drop() { @@ -239,6 +243,22 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>( Ok::<_, NoSolution>(()) })?, + ty::CoroutineClosure(_, args) => { + rustc_data_structures::stack::ensure_sufficient_stack(|| { + for ty in args.as_coroutine_closure().upvar_tys() { + dtorck_constraint_for_ty_inner( + tcx, + param_env, + span, + depth + 1, + ty, + constraints, + )?; + } + Ok::<_, NoSolution>(()) + })? + } + ty::Coroutine(_, args) => { // rust-lang/rust#49918: types can be constructed, stored // in the interior, and sit idle when coroutine yields diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index 12aea88e9b6..2258e796103 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -117,10 +117,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { self.assemble_iterator_candidates(obligation, &mut candidates); } else if lang_items.async_iterator_trait() == Some(def_id) { self.assemble_async_iterator_candidates(obligation, &mut candidates); + } else if lang_items.async_fn_kind_helper() == Some(def_id) { + self.assemble_async_fn_kind_helper_candidates(obligation, &mut candidates); } + // FIXME: Put these into `else if` blocks above, since they're built-in. self.assemble_closure_candidates(obligation, &mut candidates); + self.assemble_async_closure_candidates(obligation, &mut candidates); self.assemble_fn_pointer_candidates(obligation, &mut candidates); + self.assemble_candidates_from_impls(obligation, &mut candidates); self.assemble_candidates_from_object_ty(obligation, &mut candidates); } @@ -306,11 +311,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { // Okay to skip binder because the args on closure types never // touch bound regions, they just capture the in-scope // type/region parameters - match *obligation.self_ty().skip_binder().kind() { - ty::Closure(def_id, closure_args) => { + let self_ty = obligation.self_ty().skip_binder(); + match *self_ty.kind() { + ty::Closure(def_id, _) => { let is_const = self.tcx().is_const_fn_raw(def_id); debug!(?kind, ?obligation, "assemble_unboxed_candidates"); - match self.infcx.closure_kind(closure_args) { + match self.infcx.closure_kind(self_ty) { Some(closure_kind) => { debug!(?closure_kind, "assemble_unboxed_candidates"); if closure_kind.extends(kind) { @@ -334,6 +340,61 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { } } + fn assemble_async_closure_candidates( + &mut self, + obligation: &PolyTraitObligation<'tcx>, + candidates: &mut SelectionCandidateSet<'tcx>, + ) { + let Some(goal_kind) = + self.tcx().async_fn_trait_kind_from_def_id(obligation.predicate.def_id()) + else { + return; + }; + + match *obligation.self_ty().skip_binder().kind() { + ty::CoroutineClosure(_, args) => { + if let Some(closure_kind) = + args.as_coroutine_closure().kind_ty().to_opt_closure_kind() + && !closure_kind.extends(goal_kind) + { + return; + } + candidates.vec.push(AsyncClosureCandidate); + } + ty::Infer(ty::TyVar(_)) => { + candidates.ambiguous = true; + } + _ => {} + } + } + + fn assemble_async_fn_kind_helper_candidates( + &mut self, + obligation: &PolyTraitObligation<'tcx>, + candidates: &mut SelectionCandidateSet<'tcx>, + ) { + let self_ty = obligation.self_ty().skip_binder(); + let target_kind_ty = obligation.predicate.skip_binder().trait_ref.args.type_at(1); + + // `to_opt_closure_kind` is kind of ICEy when it sees non-int types. + if !(self_ty.is_integral() || self_ty.is_ty_var()) { + return; + } + if !(target_kind_ty.is_integral() || self_ty.is_ty_var()) { + return; + } + + // Check that the self kind extends the goal kind. If it does, + // then there's nothing else to check. + if let Some(closure_kind) = self_ty.to_opt_closure_kind() + && let Some(goal_kind) = target_kind_ty.to_opt_closure_kind() + { + if closure_kind.extends(goal_kind) { + candidates.vec.push(AsyncFnKindHelperCandidate); + } + } + } + /// Implements one of the `Fn()` family for a fn pointer. fn assemble_fn_pointer_candidates( &mut self, @@ -488,7 +549,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | ty::Slice(_) | ty::RawPtr(_) | ty::Ref(_, _, _) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never @@ -623,7 +685,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | ty::Ref(..) | ty::FnDef(..) | ty::FnPtr(_) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::Never | ty::Tuple(_) @@ -1000,6 +1063,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | ty::Array(..) | ty::Slice(_) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::Tuple(_) | ty::CoroutineWitness(..) => { @@ -1076,7 +1140,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | ty::FnDef(_, _) | ty::FnPtr(_) | ty::Dynamic(_, _, _) - | ty::Closure(_, _) + | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(_, _) | ty::CoroutineWitness(..) | ty::Never @@ -1139,6 +1204,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | ty::Placeholder(..) | ty::Dynamic(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Never diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index 74f388e53a3..c9d06b0f675 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -83,6 +83,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { ImplSource::Builtin(BuiltinImplSource::Misc, vtable_closure) } + AsyncClosureCandidate => { + let vtable_closure = self.confirm_async_closure_candidate(obligation)?; + ImplSource::Builtin(BuiltinImplSource::Misc, vtable_closure) + } + + // No nested obligations or confirmation process. The checks that we do in + // candidate assembly are sufficient. + AsyncFnKindHelperCandidate => ImplSource::Builtin(BuiltinImplSource::Misc, vec![]), + CoroutineCandidate => { let vtable_coroutine = self.confirm_coroutine_candidate(obligation)?; ImplSource::Builtin(BuiltinImplSource::Misc, vtable_coroutine) @@ -869,6 +878,49 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { Ok(nested) } + #[instrument(skip(self), level = "debug")] + fn confirm_async_closure_candidate( + &mut self, + obligation: &PolyTraitObligation<'tcx>, + ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> { + // Okay to skip binder because the args on closure types never + // touch bound regions, they just capture the in-scope + // type/region parameters. + let self_ty = self.infcx.shallow_resolve(obligation.self_ty().skip_binder()); + let ty::CoroutineClosure(closure_def_id, args) = *self_ty.kind() else { + bug!("async closure candidate for non-coroutine-closure {:?}", obligation); + }; + + let trait_ref = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| { + ty::TraitRef::new( + self.tcx(), + obligation.predicate.def_id(), + [self_ty, sig.tupled_inputs_ty], + ) + }); + + let mut nested = self.confirm_poly_trait_refs(obligation, trait_ref)?; + + let goal_kind = + self.tcx().async_fn_trait_kind_from_def_id(obligation.predicate.def_id()).unwrap(); + nested.push(obligation.with( + self.tcx(), + ty::TraitRef::from_lang_item( + self.tcx(), + LangItem::AsyncFnKindHelper, + obligation.cause.span, + [ + args.as_coroutine_closure().kind_ty(), + Ty::from_closure_kind(self.tcx(), goal_kind), + ], + ), + )); + + debug!(?closure_def_id, ?trait_ref, ?nested, "confirm closure candidate obligations"); + + Ok(nested) + } + /// In the case of closure types and fn pointers, /// we currently treat the input type parameters on the trait as /// outputs. This means that when we have a match we have only diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index 6a6adcbb680..7f41c73b72f 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -1864,6 +1864,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ImplCandidate(..) | AutoImplCandidate | ClosureCandidate { .. } + | AsyncClosureCandidate + | AsyncFnKindHelperCandidate | CoroutineCandidate | FutureCandidate | IteratorCandidate @@ -1894,6 +1896,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ImplCandidate(_) | AutoImplCandidate | ClosureCandidate { .. } + | AsyncClosureCandidate + | AsyncFnKindHelperCandidate | CoroutineCandidate | FutureCandidate | IteratorCandidate @@ -1930,6 +1934,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ImplCandidate(..) | AutoImplCandidate | ClosureCandidate { .. } + | AsyncClosureCandidate + | AsyncFnKindHelperCandidate | CoroutineCandidate | FutureCandidate | IteratorCandidate @@ -1946,6 +1952,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ImplCandidate(..) | AutoImplCandidate | ClosureCandidate { .. } + | AsyncClosureCandidate + | AsyncFnKindHelperCandidate | CoroutineCandidate | FutureCandidate | IteratorCandidate @@ -2054,6 +2062,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { ( ImplCandidate(_) | ClosureCandidate { .. } + | AsyncClosureCandidate + | AsyncFnKindHelperCandidate | CoroutineCandidate | FutureCandidate | IteratorCandidate @@ -2066,6 +2076,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> { | TraitAliasCandidate, ImplCandidate(_) | ClosureCandidate { .. } + | AsyncClosureCandidate + | AsyncFnKindHelperCandidate | CoroutineCandidate | FutureCandidate | IteratorCandidate @@ -2106,6 +2118,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> { | ty::CoroutineWitness(..) | ty::Array(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Never | ty::Dynamic(_, _, ty::DynStar) | ty::Error(_) => { @@ -2227,6 +2240,9 @@ impl<'tcx> SelectionContext<'_, 'tcx> { } } + // FIXME(async_closures): These are never clone, for now. + ty::CoroutineClosure(_, _) => None, + ty::Adt(..) | ty::Alias(..) | ty::Param(..) | ty::Placeholder(..) => { // Fallback to whatever user-defined impls exist in this case. None @@ -2305,6 +2321,11 @@ impl<'tcx> SelectionContext<'_, 'tcx> { t.rebind(vec![ty]) } + ty::CoroutineClosure(_, args) => { + let ty = self.infcx.shallow_resolve(args.as_coroutine_closure().tupled_upvars_ty()); + t.rebind(vec![ty]) + } + ty::Coroutine(_, args) => { let ty = self.infcx.shallow_resolve(args.as_coroutine().tupled_upvars_ty()); let witness = args.as_coroutine().witness(); diff --git a/compiler/rustc_trait_selection/src/traits/structural_match.rs b/compiler/rustc_trait_selection/src/traits/structural_match.rs index d5a37e63d87..89459f377dd 100644 --- a/compiler/rustc_trait_selection/src/traits/structural_match.rs +++ b/compiler/rustc_trait_selection/src/traits/structural_match.rs @@ -79,6 +79,9 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for Search<'tcx> { ty::Closure(..) => { return ControlFlow::Break(ty); } + ty::CoroutineClosure(..) => { + return ControlFlow::Break(ty); + } ty::Coroutine(..) | ty::CoroutineWitness(..) => { return ControlFlow::Break(ty); } diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs index 0f8d9c6bf4b..b4f13ee95a6 100644 --- a/compiler/rustc_trait_selection/src/traits/wf.rs +++ b/compiler/rustc_trait_selection/src/traits/wf.rs @@ -727,6 +727,14 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> { self.out.extend(obligations); } + ty::CoroutineClosure(did, args) => { + // See the above comments. The same apply to coroutine-closures. + walker.skip_current_subtree(); + self.compute(args.as_coroutine_closure().tupled_upvars_ty().into()); + let obligations = self.nominal_obligations(did, args); + self.out.extend(obligations); + } + ty::FnPtr(_) => { // let the loop iterate into the argument/return // types appearing in the fn signature diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index af26616831f..0542ef4ecf9 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -101,6 +101,49 @@ fn fn_sig_for_fn_abi<'tcx>( bound_vars, ) } + ty::CoroutineClosure(def_id, args) => { + let sig = args.as_coroutine_closure().coroutine_closure_sig(); + let bound_vars = tcx.mk_bound_variable_kinds_from_iter( + sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))), + ); + let br = ty::BoundRegion { + var: ty::BoundVar::from_usize(bound_vars.len() - 1), + kind: ty::BoundRegionKind::BrEnv, + }; + let env_region = ty::Region::new_bound(tcx, ty::INNERMOST, br); + + // When this `CoroutineClosure` comes from a `ConstructCoroutineInClosureShim`, + // make sure we respect the `target_kind` in that shim. + // FIXME(async_closures): This shouldn't be needed, and we should be populating + // a separate def-id for these bodies. + let mut kind = args.as_coroutine_closure().kind(); + if let InstanceDef::ConstructCoroutineInClosureShim { target_kind, .. } = instance.def { + kind = target_kind; + } + + let env_ty = + tcx.closure_env_ty(Ty::new_coroutine_closure(tcx, def_id, args), kind, env_region); + + let sig = sig.skip_binder(); + ty::Binder::bind_with_vars( + tcx.mk_fn_sig( + iter::once(env_ty).chain([sig.tupled_inputs_ty]), + sig.to_coroutine_given_kind_and_upvars( + tcx, + args.as_coroutine_closure().parent_args(), + tcx.coroutine_for_closure(def_id), + kind, + env_region, + args.as_coroutine_closure().tupled_upvars_ty(), + args.as_coroutine_closure().coroutine_captures_by_ref_ty(), + ), + sig.c_variadic, + sig.unsafety, + sig.abi, + ), + bound_vars, + ) + } ty::Coroutine(did, args) => { let coroutine_kind = tcx.coroutine_kind(did).unwrap(); let sig = args.as_coroutine().sig(); @@ -112,6 +155,40 @@ fn fn_sig_for_fn_abi<'tcx>( var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BoundRegionKind::BrEnv, }; + + let mut ty = ty; + // When this `Closure` comes from a `CoroutineKindShim`, + // make sure we respect the `target_kind` in that shim. + // FIXME(async_closures): This shouldn't be needed, and we should be populating + // a separate def-id for these bodies. + if let InstanceDef::CoroutineKindShim { target_kind, .. } = instance.def { + // Grab the parent coroutine-closure. It has the same args for the purposes + // of substitution, so this will be okay to do. + let ty::CoroutineClosure(_, coroutine_closure_args) = *tcx + .instantiate_and_normalize_erasing_regions( + args, + param_env, + tcx.type_of(tcx.parent(did)), + ) + .kind() + else { + bug!("CoroutineKindShim comes from calling a coroutine-closure"); + }; + let coroutine_closure_args = coroutine_closure_args.as_coroutine_closure(); + ty = tcx.instantiate_bound_regions_with_erased( + coroutine_closure_args.coroutine_closure_sig().map_bound(|sig| { + sig.to_coroutine_given_kind_and_upvars( + tcx, + coroutine_closure_args.parent_args(), + did, + target_kind, + tcx.lifetimes.re_erased, + coroutine_closure_args.tupled_upvars_ty(), + coroutine_closure_args.coroutine_captures_by_ref_ty(), + ) + }), + ); + } let env_ty = Ty::new_mut_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), ty); let pin_did = tcx.require_lang_item(LangItem::Pin, None); diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 2d76cf994e4..9faad10dd14 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -38,6 +38,7 @@ fn resolve_instance<'tcx>( debug!(" => nontrivial drop glue"); match *ty.kind() { ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::Tuple(..) | ty::Adt(..) @@ -215,6 +216,7 @@ fn resolve_associated_item<'tcx>( ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Tuple(..) => {} _ => return Ok(None), }; @@ -281,6 +283,34 @@ fn resolve_associated_item<'tcx>( tcx.item_name(trait_item_id) ), } + } else if let Some(target_kind) = tcx.async_fn_trait_kind_from_def_id(trait_ref.def_id) + { + match *rcvr_args.type_at(0).kind() { + ty::CoroutineClosure(coroutine_closure_def_id, args) => { + // If we're computing `AsyncFnOnce`/`AsyncFnMut` for a by-ref closure, + // or `AsyncFnOnce` for a by-mut closure, then construct a new body that + // has the right return types. + // + // Specifically, `AsyncFnMut` for a by-ref coroutine-closure just needs + // to have its input and output types fixed (`&mut self` and returning + // `i16` coroutine kind). + if target_kind > args.as_coroutine_closure().kind() { + Some(Instance { + def: ty::InstanceDef::ConstructCoroutineInClosureShim { + coroutine_closure_def_id, + target_kind, + }, + args, + }) + } else { + Some(Instance::new(coroutine_closure_def_id, args)) + } + } + _ => bug!( + "no built-in definition for `{trait_ref}::{}` for non-lending-closure type", + tcx.item_name(trait_item_id) + ), + } } else { Instance::try_resolve_item_for_coroutine(tcx, trait_item_id, trait_id, rcvr_args) } diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 2fc4bfd4aa3..f20ded355b1 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -328,6 +328,17 @@ fn layout_of_uncached<'tcx>( )? } + ty::CoroutineClosure(_, args) => { + let tys = args.as_coroutine_closure().upvar_tys(); + univariant( + &tys.iter() + .map(|ty| Ok(cx.layout_of(ty)?.layout)) + .try_collect::<IndexVec<_, _>>()?, + &ReprOptions::default(), + StructKind::AlwaysSized, + )? + } + ty::Tuple(tys) => { let kind = if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized }; diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs index 08e5476ae43..7b3d2ab22cf 100644 --- a/compiler/rustc_ty_utils/src/needs_drop.rs +++ b/compiler/rustc_ty_utils/src/needs_drop.rs @@ -172,6 +172,12 @@ where } } + ty::CoroutineClosure(_, args) => { + for upvar in args.as_coroutine_closure().upvar_tys() { + queue_type(self, upvar); + } + } + // Check for a `Drop` impl and whether this is a union or // `ManuallyDrop`. If it's a struct or enum without a `Drop` // impl then check whether the field types need `Drop`. diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs index 2158aacab03..60b1bbe8c2a 100644 --- a/compiler/rustc_ty_utils/src/ty.rs +++ b/compiler/rustc_ty_utils/src/ty.rs @@ -18,7 +18,9 @@ fn sized_constraint_for_ty<'tcx>( let result = match ty.kind() { Bool | Char | Int(..) | Uint(..) | Float(..) | RawPtr(..) | Ref(..) | FnDef(..) - | FnPtr(_) | Array(..) | Closure(..) | Coroutine(..) | Never => vec![], + | FnPtr(_) | Array(..) | Closure(..) | CoroutineClosure(..) | Coroutine(..) | Never => { + vec![] + } Str | Dynamic(..) | Slice(_) | Foreign(..) | Error(_) | CoroutineWitness(..) => { // these are never sized - return the target type diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs index 859000fb6cb..a4fe572067b 100644 --- a/compiler/rustc_type_ir/src/ty_kind.rs +++ b/compiler/rustc_type_ir/src/ty_kind.rs @@ -202,6 +202,13 @@ pub enum TyKind<I: Interner> { /// `ClosureArgs` for more details. Closure(I::DefId, I::GenericArgs), + /// The anonymous type of a closure. Used to represent the type of `async |a| a`. + /// + /// Coroutine-closure args contain both the - potentially substituted - generic + /// parameters of its parent and some synthetic parameters. See the documentation + /// for `CoroutineClosureArgs` for more details. + CoroutineClosure(I::DefId, I::GenericArgs), + /// The anonymous type of a coroutine. Used to represent the type of /// `|a| yield a`. /// @@ -317,16 +324,17 @@ const fn tykind_discriminant<I: Interner>(value: &TyKind<I>) -> usize { FnPtr(_) => 13, Dynamic(..) => 14, Closure(_, _) => 15, - Coroutine(_, _) => 16, - CoroutineWitness(_, _) => 17, - Never => 18, - Tuple(_) => 19, - Alias(_, _) => 20, - Param(_) => 21, - Bound(_, _) => 22, - Placeholder(_) => 23, - Infer(_) => 24, - Error(_) => 25, + CoroutineClosure(_, _) => 16, + Coroutine(_, _) => 17, + CoroutineWitness(_, _) => 18, + Never => 19, + Tuple(_) => 20, + Alias(_, _) => 21, + Param(_) => 22, + Bound(_, _) => 23, + Placeholder(_) => 24, + Infer(_) => 25, + Error(_) => 26, } } @@ -356,6 +364,7 @@ impl<I: Interner> PartialEq for TyKind<I> { a_p == b_p && a_r == b_r && a_repr == b_repr } (Closure(a_d, a_s), Closure(b_d, b_s)) => a_d == b_d && a_s == b_s, + (CoroutineClosure(a_d, a_s), CoroutineClosure(b_d, b_s)) => a_d == b_d && a_s == b_s, (Coroutine(a_d, a_s), Coroutine(b_d, b_s)) => a_d == b_d && a_s == b_s, (CoroutineWitness(a_d, a_s), CoroutineWitness(b_d, b_s)) => a_d == b_d && a_s == b_s, (Tuple(a_t), Tuple(b_t)) => a_t == b_t, @@ -430,6 +439,9 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> { } }, Closure(d, s) => f.debug_tuple("Closure").field(d).field(&this.wrap(s)).finish(), + CoroutineClosure(d, s) => { + f.debug_tuple("CoroutineClosure").field(d).field(&this.wrap(s)).finish() + } Coroutine(d, s) => f.debug_tuple("Coroutine").field(d).field(&this.wrap(s)).finish(), CoroutineWitness(d, s) => { f.debug_tuple("CoroutineWitness").field(d).field(&this.wrap(s)).finish() diff --git a/library/core/src/ops/async_function.rs b/library/core/src/ops/async_function.rs index 965873f163e..efbe9d164c3 100644 --- a/library/core/src/ops/async_function.rs +++ b/library/core/src/ops/async_function.rs @@ -106,3 +106,28 @@ mod impls { } } } + +mod internal_implementation_detail { + /// A helper trait that is used to enforce that the `ClosureKind` of a goal + /// is within the capabilities of a `CoroutineClosure`, and which allows us + /// to delay the projection of the tupled upvar types until after upvar + /// analysis is complete. + /// + /// The `Self` type is expected to be the `kind_ty` of the coroutine-closure, + /// and thus either `?0` or `i8`/`i16`/`i32` (see docs for `ClosureKind` + /// for an explanation of that). The `GoalKind` is also the same type, but + /// representing the kind of the trait that the closure is being called with. + #[cfg_attr(not(bootstrap), lang = "async_fn_kind_helper")] + trait AsyncFnKindHelper<GoalKind> { + // Projects a set of closure inputs (arguments), a region, and a set of upvars + // (by move and by ref) to the upvars that we expect the coroutine to have + // according to the `GoalKind` parameter above. + // + // The `Upvars` parameter should be the upvars of the parent coroutine-closure, + // and the `BorrowedUpvarsAsFnPtr` will be a function pointer that has the shape + // `for<'env> fn() -> (&'env T, ...)`. This allows us to represent the binder + // of the closure's self-capture, and these upvar types will be instantiated with + // the `'closure_env` region provided to the associated type. + type Upvars<'closure_env, Inputs, Upvars, BorrowedUpvarsAsFnPtr>; + } +} diff --git a/src/doc/unstable-book/src/library-features/async-fn-traits.md b/src/doc/unstable-book/src/library-features/async-fn-traits.md new file mode 100644 index 00000000000..e1c3f067e5b --- /dev/null +++ b/src/doc/unstable-book/src/library-features/async-fn-traits.md @@ -0,0 +1,13 @@ +# `async_fn_traits` + +See Also: [`fn_traits`](../library-features/fn-traits.md) + +---- + +The `async_fn_traits` feature allows for implementation of the [`AsyncFn*`] traits +for creating custom closure-like types that return futures. + +[`AsyncFn*`]: ../../std/ops/trait.AsyncFn.html + +The main difference to the `Fn*` family of traits is that `AsyncFn` can return a future +that borrows from itself (`FnOnce::Output` has no lifetime parameters, while `AsyncFn::CallFuture` does). diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index bd1d68e7074..3bac71dbc24 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -2286,6 +2286,7 @@ pub(crate) fn clean_middle_ty<'tcx>( } ty::Closure(..) => panic!("Closure"), + ty::CoroutineClosure(..) => panic!("CoroutineClosure"), ty::Coroutine(..) => panic!("Coroutine"), ty::Placeholder(..) => panic!("Placeholder"), ty::CoroutineWitness(..) => panic!("CoroutineWitness"), diff --git a/src/librustdoc/passes/collect_intra_doc_links.rs b/src/librustdoc/passes/collect_intra_doc_links.rs index a75d9bee304..fe02611b5d4 100644 --- a/src/librustdoc/passes/collect_intra_doc_links.rs +++ b/src/librustdoc/passes/collect_intra_doc_links.rs @@ -503,6 +503,7 @@ impl<'a, 'tcx> LinkCollector<'a, 'tcx> { } ty::Alias(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Dynamic(..) diff --git a/src/tools/clippy/clippy_lints/src/dereference.rs b/src/tools/clippy/clippy_lints/src/dereference.rs index 8ff54dfcfa0..194cf69ea7e 100644 --- a/src/tools/clippy/clippy_lints/src/dereference.rs +++ b/src/tools/clippy/clippy_lints/src/dereference.rs @@ -881,6 +881,7 @@ impl TyCoercionStability { | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Closure(..) + | ty::CoroutineClosure(..) | ty::Never | ty::Tuple(_) | ty::Alias(ty::Projection, _) => Self::Deref, diff --git a/src/tools/clippy/clippy_lints/src/utils/author.rs b/src/tools/clippy/clippy_lints/src/utils/author.rs index b26ebe5cee3..288df0fd663 100644 --- a/src/tools/clippy/clippy_lints/src/utils/author.rs +++ b/src/tools/clippy/clippy_lints/src/utils/author.rs @@ -490,6 +490,9 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { format!("ClosureKind::Coroutine(CoroutineKind::Coroutine(Movability::{movability:?})") }, }, + ClosureKind::CoroutineClosure(desugaring) => format!( + "ClosureKind::CoroutineClosure(CoroutineDesugaring::{desugaring:?})" + ), }; let ret_ty = match fn_decl.output { diff --git a/src/tools/clippy/tests/ui/author/blocks.stdout b/src/tools/clippy/tests/ui/author/blocks.stdout index 8c4d71e68f8..579f137f861 100644 --- a/src/tools/clippy/tests/ui/author/blocks.stdout +++ b/src/tools/clippy/tests/ui/author/blocks.stdout @@ -40,10 +40,10 @@ if let ExprKind::Block(block, None) = expr.kind { // report your lint here } -if let ExprKind::Closure { capture_clause: CaptureBy::Value { .. }, fn_decl: fn_decl, body: body_id, closure_kind: ClosureKind::Closure, .. } = expr.kind +if let ExprKind::Closure { capture_clause: CaptureBy::Value { .. }, fn_decl: fn_decl, body: body_id, closure_kind: ClosureKind::CoroutineClosure(CoroutineDesugaring::Async), .. } = expr.kind && let FnRetTy::DefaultReturn(_) = fn_decl.output && expr1 = &cx.tcx.hir().body(body_id).value - && let ExprKind::Closure { capture_clause: CaptureBy::Value { .. }, fn_decl: fn_decl1, body: body_id1, closure_kind: ClosureKind::Coroutine(CoroutineKind::Desugared(CoroutineDesugaring::Async, CoroutineSource::Closure)), .. } = expr1.kind + && let ExprKind::Closure { capture_clause: CaptureBy::Ref, fn_decl: fn_decl1, body: body_id1, closure_kind: ClosureKind::Coroutine(CoroutineKind::Desugared(CoroutineDesugaring::Async, CoroutineSource::Closure)), .. } = expr1.kind && let FnRetTy::DefaultReturn(_) = fn_decl1.output && expr2 = &cx.tcx.hir().body(body_id1).value && let ExprKind::Block(block, None) = expr2.kind diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.panic-abort.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.panic-abort.mir new file mode 100644 index 00000000000..1fae40c5f40 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.panic-abort.mir @@ -0,0 +1,47 @@ +// MIR for `main::{closure#0}::{closure#0}::{closure#0}` 0 coroutine_by_move + +fn main::{closure#0}::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}, _2: ResumeTy) -> () +yields () + { + debug _task_context => _2; + debug a => (_1.0: i32); + debug b => (_1.1: i32); + let mut _0: (); + let _3: i32; + scope 1 { + debug a => _3; + let _4: &i32; + scope 2 { + debug a => _4; + let _5: &i32; + scope 3 { + debug b => _5; + } + } + } + + bb0: { + StorageLive(_3); + _3 = (_1.0: i32); + FakeRead(ForLet(None), _3); + StorageLive(_4); + _4 = &_3; + FakeRead(ForLet(None), _4); + StorageLive(_5); + _5 = &(_1.1: i32); + FakeRead(ForLet(None), _5); + _0 = const (); + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb1, unwind: bb2]; + } + + bb1: { + return; + } + + bb2 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.panic-unwind.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.panic-unwind.mir new file mode 100644 index 00000000000..1fae40c5f40 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.panic-unwind.mir @@ -0,0 +1,47 @@ +// MIR for `main::{closure#0}::{closure#0}::{closure#0}` 0 coroutine_by_move + +fn main::{closure#0}::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}, _2: ResumeTy) -> () +yields () + { + debug _task_context => _2; + debug a => (_1.0: i32); + debug b => (_1.1: i32); + let mut _0: (); + let _3: i32; + scope 1 { + debug a => _3; + let _4: &i32; + scope 2 { + debug a => _4; + let _5: &i32; + scope 3 { + debug b => _5; + } + } + } + + bb0: { + StorageLive(_3); + _3 = (_1.0: i32); + FakeRead(ForLet(None), _3); + StorageLive(_4); + _4 = &_3; + FakeRead(ForLet(None), _4); + StorageLive(_5); + _5 = &(_1.1: i32); + FakeRead(ForLet(None), _5); + _0 = const (); + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb1, unwind: bb2]; + } + + bb1: { + return; + } + + bb2 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.panic-abort.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.panic-abort.mir new file mode 100644 index 00000000000..9886d6f68a4 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.panic-abort.mir @@ -0,0 +1,47 @@ +// MIR for `main::{closure#0}::{closure#0}::{closure#0}` 0 coroutine_by_mut + +fn main::{closure#0}::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}, _2: ResumeTy) -> () +yields () + { + debug _task_context => _2; + debug a => (_1.0: i32); + debug b => (*(_1.1: &i32)); + let mut _0: (); + let _3: i32; + scope 1 { + debug a => _3; + let _4: &i32; + scope 2 { + debug a => _4; + let _5: &i32; + scope 3 { + debug b => _5; + } + } + } + + bb0: { + StorageLive(_3); + _3 = (_1.0: i32); + FakeRead(ForLet(None), _3); + StorageLive(_4); + _4 = &_3; + FakeRead(ForLet(None), _4); + StorageLive(_5); + _5 = &(*(_1.1: &i32)); + FakeRead(ForLet(None), _5); + _0 = const (); + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb1, unwind: bb2]; + } + + bb1: { + return; + } + + bb2 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.panic-unwind.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.panic-unwind.mir new file mode 100644 index 00000000000..9886d6f68a4 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.panic-unwind.mir @@ -0,0 +1,47 @@ +// MIR for `main::{closure#0}::{closure#0}::{closure#0}` 0 coroutine_by_mut + +fn main::{closure#0}::{closure#0}::{closure#0}(_1: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}, _2: ResumeTy) -> () +yields () + { + debug _task_context => _2; + debug a => (_1.0: i32); + debug b => (*(_1.1: &i32)); + let mut _0: (); + let _3: i32; + scope 1 { + debug a => _3; + let _4: &i32; + scope 2 { + debug a => _4; + let _5: &i32; + scope 3 { + debug b => _5; + } + } + } + + bb0: { + StorageLive(_3); + _3 = (_1.0: i32); + FakeRead(ForLet(None), _3); + StorageLive(_4); + _4 = &_3; + FakeRead(ForLet(None), _4); + StorageLive(_5); + _5 = &(*(_1.1: &i32)); + FakeRead(ForLet(None), _5); + _0 = const (); + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb1, unwind: bb2]; + } + + bb1: { + return; + } + + bb2 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.panic-abort.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.panic-abort.mir new file mode 100644 index 00000000000..7df4eb49260 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.panic-abort.mir @@ -0,0 +1,10 @@ +// MIR for `main::{closure#0}::{closure#0}` 0 coroutine_closure_by_move + +fn main::{closure#0}::{closure#0}(_1: {coroutine-closure@$DIR/async_closure_shims.rs:39:33: 39:52}, _2: i32) -> {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10} { + let mut _0: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}; + + bb0: { + _0 = {coroutine@$DIR/async_closure_shims.rs:39:53: 42:10 (#0)} { a: move _2, b: move (_1.0: i32) }; + return; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.panic-unwind.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.panic-unwind.mir new file mode 100644 index 00000000000..7df4eb49260 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.panic-unwind.mir @@ -0,0 +1,10 @@ +// MIR for `main::{closure#0}::{closure#0}` 0 coroutine_closure_by_move + +fn main::{closure#0}::{closure#0}(_1: {coroutine-closure@$DIR/async_closure_shims.rs:39:33: 39:52}, _2: i32) -> {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10} { + let mut _0: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}; + + bb0: { + _0 = {coroutine@$DIR/async_closure_shims.rs:39:53: 42:10 (#0)} { a: move _2, b: move (_1.0: i32) }; + return; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.panic-abort.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.panic-abort.mir new file mode 100644 index 00000000000..517b8d0dd88 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.panic-abort.mir @@ -0,0 +1,16 @@ +// MIR for `main::{closure#0}::{closure#0}` 0 coroutine_closure_by_mut + +fn main::{closure#0}::{closure#0}(_1: &mut {coroutine-closure@$DIR/async_closure_shims.rs:39:33: 39:52}, _2: i32) -> {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10} { + debug a => _2; + debug b => ((*_1).0: i32); + let mut _0: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}; + let mut _3: &i32; + + bb0: { + StorageLive(_3); + _3 = &((*_1).0: i32); + _0 = {coroutine@$DIR/async_closure_shims.rs:39:53: 42:10 (#0)} { a: _2, b: move _3 }; + StorageDead(_3); + return; + } +} diff --git a/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.panic-unwind.mir b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.panic-unwind.mir new file mode 100644 index 00000000000..517b8d0dd88 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.panic-unwind.mir @@ -0,0 +1,16 @@ +// MIR for `main::{closure#0}::{closure#0}` 0 coroutine_closure_by_mut + +fn main::{closure#0}::{closure#0}(_1: &mut {coroutine-closure@$DIR/async_closure_shims.rs:39:33: 39:52}, _2: i32) -> {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10} { + debug a => _2; + debug b => ((*_1).0: i32); + let mut _0: {async closure body@$DIR/async_closure_shims.rs:39:53: 42:10}; + let mut _3: &i32; + + bb0: { + StorageLive(_3); + _3 = &((*_1).0: i32); + _0 = {coroutine@$DIR/async_closure_shims.rs:39:53: 42:10 (#0)} { a: _2, b: move _3 }; + StorageDead(_3); + return; + } +} diff --git a/tests/mir-opt/async_closure_shims.rs b/tests/mir-opt/async_closure_shims.rs new file mode 100644 index 00000000000..ef3bdaaa145 --- /dev/null +++ b/tests/mir-opt/async_closure_shims.rs @@ -0,0 +1,46 @@ +// edition:2021 +// skip-filecheck +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +#![feature(async_closure, noop_waker, async_fn_traits)] + +use std::future::Future; +use std::ops::{AsyncFnMut, AsyncFnOnce}; +use std::pin::pin; +use std::task::*; + +pub fn block_on<T>(fut: impl Future<Output = T>) -> T { + let mut fut = pin!(fut); + let ctx = &mut Context::from_waker(Waker::noop()); + + loop { + match fut.as_mut().poll(ctx) { + Poll::Pending => {} + Poll::Ready(t) => break t, + } + } +} + +async fn call_mut(f: &mut impl AsyncFnMut(i32)) { + f(0).await; +} + +async fn call_once(f: impl AsyncFnOnce(i32)) { + f(1).await; +} + +// EMIT_MIR async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_move.0.mir +// EMIT_MIR async_closure_shims.main-{closure#0}-{closure#0}.coroutine_closure_by_mut.0.mir +// EMIT_MIR async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_mut.0.mir +// EMIT_MIR async_closure_shims.main-{closure#0}-{closure#0}-{closure#0}.coroutine_by_move.0.mir +fn main() { + block_on(async { + let b = 2i32; + let mut async_closure = async move |a: i32| { + let a = &a; + let b = &b; + }; + call_mut(&mut async_closure).await; + call_once(async_closure).await; + }); +} diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir index 3c0d4008c90..9c8cf8763fd 100644 --- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir @@ -5,6 +5,7 @@ ty: Coroutine( DefId(0:4 ~ async_await[ccf8]::a::{closure#0}), [ + (), std::future::ResumeTy, (), (), @@ -22,6 +23,7 @@ ty: Coroutine( DefId(0:4 ~ async_await[ccf8]::a::{closure#0}), [ + (), std::future::ResumeTy, (), (), diff --git a/tests/ui-fulldeps/internal-lints/ty_tykind_usage.rs b/tests/ui-fulldeps/internal-lints/ty_tykind_usage.rs index ae7f341fe4e..cce223c77bb 100644 --- a/tests/ui-fulldeps/internal-lints/ty_tykind_usage.rs +++ b/tests/ui-fulldeps/internal-lints/ty_tykind_usage.rs @@ -29,6 +29,7 @@ fn main() { TyKind::FnPtr(..) => (), //~ ERROR usage of `ty::TyKind::<kind>` TyKind::Dynamic(..) => (), //~ ERROR usage of `ty::TyKind::<kind>` TyKind::Closure(..) => (), //~ ERROR usage of `ty::TyKind::<kind>` + TyKind::CoroutineClosure(..) => (), //~ ERROR usage of `ty::TyKind::<kind>` TyKind::Coroutine(..) => (), //~ ERROR usage of `ty::TyKind::<kind>` TyKind::CoroutineWitness(..) => (), //~ ERROR usage of `ty::TyKind::<kind>` TyKind::Never => (), //~ ERROR usage of `ty::TyKind::<kind>` diff --git a/tests/ui-fulldeps/internal-lints/ty_tykind_usage.stderr b/tests/ui-fulldeps/internal-lints/ty_tykind_usage.stderr index 45b7c26faad..2ff5aad95dd 100644 --- a/tests/ui-fulldeps/internal-lints/ty_tykind_usage.stderr +++ b/tests/ui-fulldeps/internal-lints/ty_tykind_usage.stderr @@ -109,71 +109,77 @@ LL | TyKind::Closure(..) => (), error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:32:9 | -LL | TyKind::Coroutine(..) => (), +LL | TyKind::CoroutineClosure(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:33:9 | -LL | TyKind::CoroutineWitness(..) => (), +LL | TyKind::Coroutine(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:34:9 | -LL | TyKind::Never => (), +LL | TyKind::CoroutineWitness(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:35:9 | -LL | TyKind::Tuple(..) => (), +LL | TyKind::Never => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:36:9 | -LL | TyKind::Alias(..) => (), +LL | TyKind::Tuple(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:37:9 | -LL | TyKind::Param(..) => (), +LL | TyKind::Alias(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:38:9 | -LL | TyKind::Bound(..) => (), +LL | TyKind::Param(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:39:9 | -LL | TyKind::Placeholder(..) => (), +LL | TyKind::Bound(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:40:9 | -LL | TyKind::Infer(..) => (), +LL | TyKind::Placeholder(..) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` --> $DIR/ty_tykind_usage.rs:41:9 | +LL | TyKind::Infer(..) => (), + | ^^^^^^ help: try using `ty::<kind>` directly: `ty` + +error: usage of `ty::TyKind::<kind>` + --> $DIR/ty_tykind_usage.rs:42:9 + | LL | TyKind::Error(_) => (), | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind::<kind>` - --> $DIR/ty_tykind_usage.rs:46:12 + --> $DIR/ty_tykind_usage.rs:47:12 | LL | if let TyKind::Int(int_ty) = kind {} | ^^^^^^ help: try using `ty::<kind>` directly: `ty` error: usage of `ty::TyKind` - --> $DIR/ty_tykind_usage.rs:48:24 + --> $DIR/ty_tykind_usage.rs:49:24 | LL | fn ty_kind(ty_bad: TyKind<'_>, ty_good: Ty<'_>) {} | ^^^^^^^^^^ @@ -181,7 +187,7 @@ LL | fn ty_kind(ty_bad: TyKind<'_>, ty_good: Ty<'_>) {} = help: try using `Ty` instead error: usage of `ty::TyKind` - --> $DIR/ty_tykind_usage.rs:50:37 + --> $DIR/ty_tykind_usage.rs:51:37 | LL | fn ir_ty_kind<I: Interner>(bad: IrTyKind<I>) -> IrTyKind<I> { | ^^^^^^^^^^^ @@ -189,7 +195,7 @@ LL | fn ir_ty_kind<I: Interner>(bad: IrTyKind<I>) -> IrTyKind<I> { = help: try using `Ty` instead error: usage of `ty::TyKind` - --> $DIR/ty_tykind_usage.rs:50:53 + --> $DIR/ty_tykind_usage.rs:51:53 | LL | fn ir_ty_kind<I: Interner>(bad: IrTyKind<I>) -> IrTyKind<I> { | ^^^^^^^^^^^ @@ -197,12 +203,12 @@ LL | fn ir_ty_kind<I: Interner>(bad: IrTyKind<I>) -> IrTyKind<I> { = help: try using `Ty` instead error: usage of `ty::TyKind::<kind>` - --> $DIR/ty_tykind_usage.rs:53:9 + --> $DIR/ty_tykind_usage.rs:54:9 | LL | IrTyKind::Bool | --------^^^^^^ | | | help: try using `ty::<kind>` directly: `ty` -error: aborting due to 32 previous errors +error: aborting due to 33 previous errors diff --git a/tests/ui/async-await/async-borrowck-escaping-closure-error.rs b/tests/ui/async-await/async-borrowck-escaping-closure-error.rs index f8ff9186842..2a3e382e118 100644 --- a/tests/ui/async-await/async-borrowck-escaping-closure-error.rs +++ b/tests/ui/async-await/async-borrowck-escaping-closure-error.rs @@ -1,10 +1,11 @@ // edition:2018 -// check-pass #![feature(async_closure)] fn foo() -> Box<dyn std::future::Future<Output = u32>> { let x = 0u32; Box::new((async || x)()) + //~^ ERROR cannot return value referencing local variable `x` + //~| ERROR cannot return value referencing temporary value } fn main() { diff --git a/tests/ui/async-await/async-borrowck-escaping-closure-error.stderr b/tests/ui/async-await/async-borrowck-escaping-closure-error.stderr new file mode 100644 index 00000000000..be67c78221a --- /dev/null +++ b/tests/ui/async-await/async-borrowck-escaping-closure-error.stderr @@ -0,0 +1,21 @@ +error[E0515]: cannot return value referencing local variable `x` + --> $DIR/async-borrowck-escaping-closure-error.rs:6:5 + | +LL | Box::new((async || x)()) + | ^^^^^^^^^------------^^^ + | | | + | | `x` is borrowed here + | returns a value referencing data owned by the current function + +error[E0515]: cannot return value referencing temporary value + --> $DIR/async-borrowck-escaping-closure-error.rs:6:5 + | +LL | Box::new((async || x)()) + | ^^^^^^^^^------------^^^ + | | | + | | temporary value created here + | returns a value referencing data owned by the current function + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0515`. diff --git a/tests/ui/async-await/async-closures/arg-mismatch.rs b/tests/ui/async-await/async-closures/arg-mismatch.rs new file mode 100644 index 00000000000..650e13677bc --- /dev/null +++ b/tests/ui/async-await/async-closures/arg-mismatch.rs @@ -0,0 +1,15 @@ +// aux-build:block-on.rs +// edition:2021 + +#![feature(async_closure)] + +extern crate block_on; + +fn main() { + block_on::block_on(async { + let c = async |x| {}; + c(1i32).await; + c(2usize).await; + //~^ ERROR mismatched types + }); +} diff --git a/tests/ui/async-await/async-closures/arg-mismatch.stderr b/tests/ui/async-await/async-closures/arg-mismatch.stderr new file mode 100644 index 00000000000..70853ae2815 --- /dev/null +++ b/tests/ui/async-await/async-closures/arg-mismatch.stderr @@ -0,0 +1,21 @@ +error[E0308]: mismatched types + --> $DIR/arg-mismatch.rs:12:11 + | +LL | c(2usize).await; + | - ^^^^^^ expected `i32`, found `usize` + | | + | arguments to this function are incorrect + | +note: closure parameter defined here + --> $DIR/arg-mismatch.rs:10:24 + | +LL | let c = async |x| {}; + | ^ +help: change the type of the numeric literal from `usize` to `i32` + | +LL | c(2i32).await; + | ~~~ + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0308`. diff --git a/tests/ui/async-await/async-closures/async-fn-mut-for-async-fn.rs b/tests/ui/async-await/async-closures/async-fn-mut-for-async-fn.rs new file mode 100644 index 00000000000..8d7dc6a276b --- /dev/null +++ b/tests/ui/async-await/async-closures/async-fn-mut-for-async-fn.rs @@ -0,0 +1,23 @@ +// aux-build:block-on.rs +// edition:2021 +// run-pass + +// FIXME(async_closures): When `fn_sig_for_fn_abi` is fixed, remove this. +// ignore-pass (test emits codegen-time warnings) + +#![feature(async_closure, async_fn_traits)] + +extern crate block_on; + +use std::ops::AsyncFnMut; + +fn main() { + block_on::block_on(async { + let x = async || {}; + + async fn needs_async_fn_mut(mut x: impl AsyncFnMut()) { + x().await; + } + needs_async_fn_mut(x).await; + }); +} diff --git a/tests/ui/async-await/async-closures/async-fn-once-for-async-fn.rs b/tests/ui/async-await/async-closures/async-fn-once-for-async-fn.rs new file mode 100644 index 00000000000..4afc43fe6bd --- /dev/null +++ b/tests/ui/async-await/async-closures/async-fn-once-for-async-fn.rs @@ -0,0 +1,23 @@ +// aux-build:block-on.rs +// edition:2021 +// run-pass + +// FIXME(async_closures): When `fn_sig_for_fn_abi` is fixed, remove this. +// ignore-pass (test emits codegen-time warnings) + +#![feature(async_closure, async_fn_traits)] + +extern crate block_on; + +use std::ops::AsyncFnOnce; + +fn main() { + block_on::block_on(async { + let x = async || {}; + + async fn needs_async_fn_once(x: impl AsyncFnOnce()) { + x().await; + } + needs_async_fn_once(x).await; + }); +} diff --git a/tests/ui/async-await/async-closures/auxiliary/block-on.rs b/tests/ui/async-await/async-closures/auxiliary/block-on.rs new file mode 100644 index 00000000000..3c27548b865 --- /dev/null +++ b/tests/ui/async-await/async-closures/auxiliary/block-on.rs @@ -0,0 +1,20 @@ +// edition: 2021 + +#![feature(async_closure, noop_waker, async_fn_traits)] + +use std::future::Future; +use std::pin::pin; +use std::task::*; + +pub fn block_on<T>(fut: impl Future<Output = T>) -> T { + let mut fut = pin!(fut); + // Poll loop, just to test the future... + let ctx = &mut Context::from_waker(Waker::noop()); + + loop { + match unsafe { fut.as_mut().poll(ctx) } { + Poll::Pending => {} + Poll::Ready(t) => break t, + } + } +} diff --git a/tests/ui/async-await/async-closures/await-inference-guidance.rs b/tests/ui/async-await/async-closures/await-inference-guidance.rs new file mode 100644 index 00000000000..3702915cbad --- /dev/null +++ b/tests/ui/async-await/async-closures/await-inference-guidance.rs @@ -0,0 +1,16 @@ +// aux-build:block-on.rs +// edition:2021 +// run-pass + +#![feature(async_closure)] + +extern crate block_on; + +fn main() { + block_on::block_on(async { + let x = async |x: &str| -> String { x.to_owned() }; + let mut s = x("hello, world").await; + s.truncate(4); + println!("{s}"); + }); +} diff --git a/tests/ui/async-await/async-closures/brand.rs b/tests/ui/async-await/async-closures/brand.rs new file mode 100644 index 00000000000..3bda7737bb4 --- /dev/null +++ b/tests/ui/async-await/async-closures/brand.rs @@ -0,0 +1,26 @@ +// aux-build:block-on.rs +// edition:2021 +// build-pass + +#![feature(async_closure, async_fn_traits)] + +extern crate block_on; + +use std::future::Future; +use std::marker::PhantomData; +use std::ops::AsyncFn; + +struct S; +struct B<'b>(PhantomData<&'b mut &'b mut ()>); + +impl S { + async fn q<F: AsyncFn(B<'_>)>(self, f: F) { + f(B(PhantomData)).await; + } +} + +fn main() { + block_on::block_on(async { + S.q(async |b: B<'_>| { println!("...") }).await; + }); +} diff --git a/tests/ui/async-await/async-closures/def-path.rs b/tests/ui/async-await/async-closures/def-path.rs index 2883a1715b0..87e99ddda64 100644 --- a/tests/ui/async-await/async-closures/def-path.rs +++ b/tests/ui/async-await/async-closures/def-path.rs @@ -8,7 +8,7 @@ fn main() { //~^ NOTE the expected `async` closure body let () = x(); //~^ ERROR mismatched types - //~| NOTE this expression has type `{static main::{closure#0}::{closure#0} upvar_tys= + //~| NOTE this expression has type `{static main::{closure#0}::{closure#0}< //~| NOTE expected `async` closure body, found `()` - //~| NOTE expected `async` closure body `{static main::{closure#0}::{closure#0} + //~| NOTE expected `async` closure body `{static main::{closure#0}::{closure#0}< } diff --git a/tests/ui/async-await/async-closures/def-path.stderr b/tests/ui/async-await/async-closures/def-path.stderr index 4b37e50aac4..dae45825f37 100644 --- a/tests/ui/async-await/async-closures/def-path.stderr +++ b/tests/ui/async-await/async-closures/def-path.stderr @@ -5,11 +5,11 @@ LL | let x = async || {}; | -- the expected `async` closure body LL | LL | let () = x(); - | ^^ --- this expression has type `{static main::{closure#0}::{closure#0} upvar_tys=?7t witness=?8t}` + | ^^ --- this expression has type `{static main::{closure#0}::{closure#0}<?7t> upvar_tys=?15t witness=?6t}` | | | expected `async` closure body, found `()` | - = note: expected `async` closure body `{static main::{closure#0}::{closure#0} upvar_tys=?7t witness=?8t}` + = note: expected `async` closure body `{static main::{closure#0}::{closure#0}<?7t> upvar_tys=?15t witness=?6t}` found unit type `()` error: aborting due to 1 previous error diff --git a/tests/ui/async-await/async-closures/drop.rs b/tests/ui/async-await/async-closures/drop.rs new file mode 100644 index 00000000000..1b7f2f8a600 --- /dev/null +++ b/tests/ui/async-await/async-closures/drop.rs @@ -0,0 +1,40 @@ +// aux-build:block-on.rs +// edition:2018 +// run-pass +// check-run-results + +#![feature(async_closure, async_fn_traits)] +#![allow(unused)] + +extern crate block_on; + +use std::ops::AsyncFnOnce; + +struct DropMe(i32); + +impl Drop for DropMe { + fn drop(&mut self) { + println!("{} was dropped", self.0); + } +} + +async fn call_once(f: impl AsyncFnOnce()) { + println!("before call"); + let fut = Box::pin(f()); + println!("after call"); + drop(fut); + println!("future dropped"); +} + +fn main() { + block_on::block_on(async { + let d = DropMe(42); + let async_closure = async move || { + let d = &d; + println!("called"); + }; + + call_once(async_closure).await; + println!("after"); + }); +} diff --git a/tests/ui/async-await/async-closures/drop.run.stdout b/tests/ui/async-await/async-closures/drop.run.stdout new file mode 100644 index 00000000000..ab233f491ba --- /dev/null +++ b/tests/ui/async-await/async-closures/drop.run.stdout @@ -0,0 +1,5 @@ +before call +after call +42 was dropped +future dropped +after diff --git a/tests/ui/async-await/async-closures/higher-ranked-return.rs b/tests/ui/async-await/async-closures/higher-ranked-return.rs new file mode 100644 index 00000000000..d98779c6ea3 --- /dev/null +++ b/tests/ui/async-await/async-closures/higher-ranked-return.rs @@ -0,0 +1,18 @@ +// aux-build:block-on.rs +// edition:2021 + +// known-bug: unknown +// Borrow checking doesn't like that higher-ranked output... + +#![feature(async_closure)] + +extern crate block_on; + +fn main() { + block_on::block_on(async { + let x = async move |x: &str| -> &str { + x + }; + let s = x("hello!").await; + }); +} diff --git a/tests/ui/async-await/async-closures/higher-ranked-return.stderr b/tests/ui/async-await/async-closures/higher-ranked-return.stderr new file mode 100644 index 00000000000..268631f67cd --- /dev/null +++ b/tests/ui/async-await/async-closures/higher-ranked-return.stderr @@ -0,0 +1,14 @@ +error: lifetime may not live long enough + --> $DIR/higher-ranked-return.rs:13:46 + | +LL | let x = async move |x: &str| -> &str { + | ________________________________-________----_^ + | | | | + | | | return type of async closure `{async closure body@$DIR/higher-ranked-return.rs:13:46: 15:10}` contains a lifetime `'2` + | | let's call the lifetime of this reference `'1` +LL | | x +LL | | }; + | |_________^ returning this value requires that `'1` must outlive `'2` + +error: aborting due to 1 previous error + diff --git a/tests/ui/async-await/async-closures/higher-ranked.rs b/tests/ui/async-await/async-closures/higher-ranked.rs index f0bdcf691ae..17b5116cceb 100644 --- a/tests/ui/async-await/async-closures/higher-ranked.rs +++ b/tests/ui/async-await/async-closures/higher-ranked.rs @@ -1,12 +1,16 @@ +// aux-build:block-on.rs // edition:2021 +// build-pass #![feature(async_closure)] +extern crate block_on; + fn main() { - let x = async move |x: &str| { - //~^ ERROR lifetime may not live long enough - // This error is proof that the `&str` type is higher-ranked. - // This won't work until async closures are fully impl'd. - println!("{x}"); - }; + block_on::block_on(async { + let x = async move |x: &str| { + println!("{x}"); + }; + x("hello!").await; + }); } diff --git a/tests/ui/async-await/async-closures/higher-ranked.stderr b/tests/ui/async-await/async-closures/higher-ranked.stderr deleted file mode 100644 index fb02a15b079..00000000000 --- a/tests/ui/async-await/async-closures/higher-ranked.stderr +++ /dev/null @@ -1,17 +0,0 @@ -error: lifetime may not live long enough - --> $DIR/higher-ranked.rs:6:34 - | -LL | let x = async move |x: &str| { - | ____________________________-___-_^ - | | | | - | | | return type of closure `{async closure body@$DIR/higher-ranked.rs:6:34: 11:6}` contains a lifetime `'2` - | | let's call the lifetime of this reference `'1` -LL | | -LL | | // This error is proof that the `&str` type is higher-ranked. -LL | | // This won't work until async closures are fully impl'd. -LL | | println!("{x}"); -LL | | }; - | |_____^ returning this value requires that `'1` must outlive `'2` - -error: aborting due to 1 previous error - diff --git a/tests/ui/async-await/async-closures/is-not-fn.rs b/tests/ui/async-await/async-closures/is-not-fn.rs new file mode 100644 index 00000000000..94c8e8563bd --- /dev/null +++ b/tests/ui/async-await/async-closures/is-not-fn.rs @@ -0,0 +1,12 @@ +// edition:2021 + +#![feature(async_closure)] + +fn main() { + fn needs_fn(x: impl FnOnce()) {} + needs_fn(async || {}); + //~^ ERROR expected a `FnOnce()` closure, found `{coroutine-closure@ + // FIXME(async_closures): This should explain in more detail how async fns don't + // implement the regular `Fn` traits. Or maybe we should just fix it and make them + // when there are no upvars or whatever. +} diff --git a/tests/ui/async-await/async-closures/is-not-fn.stderr b/tests/ui/async-await/async-closures/is-not-fn.stderr new file mode 100644 index 00000000000..12da4b1fc6f --- /dev/null +++ b/tests/ui/async-await/async-closures/is-not-fn.stderr @@ -0,0 +1,19 @@ +error[E0277]: expected a `FnOnce()` closure, found `{coroutine-closure@$DIR/is-not-fn.rs:7:14: 7:22}` + --> $DIR/is-not-fn.rs:7:14 + | +LL | needs_fn(async || {}); + | -------- ^^^^^^^^^^^ expected an `FnOnce()` closure, found `{coroutine-closure@$DIR/is-not-fn.rs:7:14: 7:22}` + | | + | required by a bound introduced by this call + | + = help: the trait `FnOnce<()>` is not implemented for `{coroutine-closure@$DIR/is-not-fn.rs:7:14: 7:22}` + = note: wrap the `{coroutine-closure@$DIR/is-not-fn.rs:7:14: 7:22}` in a closure with no arguments: `|| { /* code */ }` +note: required by a bound in `needs_fn` + --> $DIR/is-not-fn.rs:6:25 + | +LL | fn needs_fn(x: impl FnOnce()) {} + | ^^^^^^^^ required by this bound in `needs_fn` + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/async-await/async-closures/mangle.rs b/tests/ui/async-await/async-closures/mangle.rs new file mode 100644 index 00000000000..98065c3c711 --- /dev/null +++ b/tests/ui/async-await/async-closures/mangle.rs @@ -0,0 +1,36 @@ +// aux-build:block-on.rs +// edition:2021 +// build-pass +// revisions: v0 legacy +//[v0] compile-flags: -Csymbol-mangling-version=v0 +//[legacy] compile-flags: -Csymbol-mangling-version=legacy -Zunstable-options + +// FIXME(async_closures): When `fn_sig_for_fn_abi` is fixed, remove this. +// ignore-pass (test emits codegen-time warnings) + +#![feature(async_closure, noop_waker, async_fn_traits)] + +extern crate block_on; + +use std::future::Future; +use std::ops::{AsyncFnMut, AsyncFnOnce}; +use std::pin::pin; +use std::task::*; + +async fn call_mut(f: &mut impl AsyncFnMut()) { + f().await; +} + +async fn call_once(f: impl AsyncFnOnce()) { + f().await; +} + +fn main() { + block_on::block_on(async { + let mut async_closure = async move || { + println!("called"); + }; + call_mut(&mut async_closure).await; + call_once(async_closure).await; + }); +} diff --git a/tests/ui/async-await/async-closures/move-consuming-capture.rs b/tests/ui/async-await/async-closures/move-consuming-capture.rs new file mode 100644 index 00000000000..b8964c571f9 --- /dev/null +++ b/tests/ui/async-await/async-closures/move-consuming-capture.rs @@ -0,0 +1,20 @@ +// aux-build:block-on.rs +// edition:2021 + +#![feature(async_closure)] + +extern crate block_on; + +struct NoCopy; + +fn main() { + block_on::block_on(async { + let s = NoCopy; + let x = async move || { + drop(s); + }; + x().await; + x().await; + //~^ ERROR use of moved value: `x` + }); +} diff --git a/tests/ui/async-await/async-closures/move-consuming-capture.stderr b/tests/ui/async-await/async-closures/move-consuming-capture.stderr new file mode 100644 index 00000000000..2c2a0d1162d --- /dev/null +++ b/tests/ui/async-await/async-closures/move-consuming-capture.stderr @@ -0,0 +1,17 @@ +error[E0382]: use of moved value: `x` + --> $DIR/move-consuming-capture.rs:17:9 + | +LL | let x = async move || { + | - move occurs because `x` has type `{coroutine-closure@$DIR/move-consuming-capture.rs:13:17: 13:30}`, which does not implement the `Copy` trait +... +LL | x().await; + | --- `x` moved due to this method call +LL | x().await; + | ^ value used here after move + | +note: `async_call_once` takes ownership of the receiver `self`, which moves `x` + --> $SRC_DIR/core/src/ops/async_function.rs:LL:COL + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0382`. diff --git a/tests/ui/async-await/async-closures/move-is-async-fn.rs b/tests/ui/async-await/async-closures/move-is-async-fn.rs new file mode 100644 index 00000000000..943c0629541 --- /dev/null +++ b/tests/ui/async-await/async-closures/move-is-async-fn.rs @@ -0,0 +1,21 @@ +// aux-build:block-on.rs +// edition:2021 +// build-pass + +#![feature(async_closure)] + +extern crate block_on; + +fn main() { + block_on::block_on(async { + let s = String::from("hello, world"); + let c = async move || { + println!("{s}"); + }; + c().await; + c().await; + + fn is_static<T: 'static>(_: T) {} + is_static(c); + }); +} diff --git a/tests/ui/async-await/async-closures/mutate.rs b/tests/ui/async-await/async-closures/mutate.rs new file mode 100644 index 00000000000..cc1df5f034f --- /dev/null +++ b/tests/ui/async-await/async-closures/mutate.rs @@ -0,0 +1,19 @@ +// aux-build:block-on.rs +// edition:2021 +// run-pass + +#![feature(async_closure)] + +extern crate block_on; + +fn main() { + block_on::block_on(async { + let mut prefix = String::from("Hello"); + let mut c = async move |x: &str| { + prefix.push(','); + println!("{prefix} {x}!") + }; + c("world").await; + c("rust").await; + }); +} diff --git a/tests/ui/async-await/async-closures/not-lending.rs b/tests/ui/async-await/async-closures/not-lending.rs new file mode 100644 index 00000000000..90832e1a074 --- /dev/null +++ b/tests/ui/async-await/async-closures/not-lending.rs @@ -0,0 +1,21 @@ +// aux-build:block-on.rs +// edition:2021 + +#![feature(async_closure)] + +extern crate block_on; + +// Make sure that we can't make an async closure that evaluates to a self-borrow. +// i.e. that the generator may reference captures, but the future's return type can't. + +fn main() { + block_on::block_on(async { + let s = String::new(); + let x = async move || -> &String { &s }; + //~^ ERROR lifetime may not live long enough + + let s = String::new(); + let x = async move || { &s }; + //~^ ERROR lifetime may not live long enough + }); +} diff --git a/tests/ui/async-await/async-closures/not-lending.stderr b/tests/ui/async-await/async-closures/not-lending.stderr new file mode 100644 index 00000000000..1713e49b551 --- /dev/null +++ b/tests/ui/async-await/async-closures/not-lending.stderr @@ -0,0 +1,24 @@ +error: lifetime may not live long enough + --> $DIR/not-lending.rs:14:42 + | +LL | let x = async move || -> &String { &s }; + | ------------------------ ^^^^^^ returning this value requires that `'1` must outlive `'2` + | | | + | | return type of async closure `{async closure body@$DIR/not-lending.rs:14:42: 14:48}` contains a lifetime `'2` + | lifetime `'1` represents this closure's body + | + = note: closure implements `Fn`, so references to captured variables can't escape the closure + +error: lifetime may not live long enough + --> $DIR/not-lending.rs:18:31 + | +LL | let x = async move || { &s }; + | ------------- ^^^^^^ returning this value requires that `'1` must outlive `'2` + | | | + | | return type of async closure `{async closure body@$DIR/not-lending.rs:18:31: 18:37}` contains a lifetime `'2` + | lifetime `'1` represents this closure's body + | + = note: closure implements `Fn`, so references to captured variables can't escape the closure + +error: aborting due to 2 previous errors + diff --git a/tests/ui/async-await/async-closures/return-type-mismatch.rs b/tests/ui/async-await/async-closures/return-type-mismatch.rs new file mode 100644 index 00000000000..9ad6be0b6e6 --- /dev/null +++ b/tests/ui/async-await/async-closures/return-type-mismatch.rs @@ -0,0 +1,14 @@ +// aux-build:block-on.rs +// edition:2021 + +#![feature(async_closure)] + +extern crate block_on; + +fn main() { + block_on::block_on(async { + let x = async || -> i32 { 0 }; + let y: usize = x().await; + //~^ ERROR mismatched types + }); +} diff --git a/tests/ui/async-await/async-closures/return-type-mismatch.stderr b/tests/ui/async-await/async-closures/return-type-mismatch.stderr new file mode 100644 index 00000000000..53841f62777 --- /dev/null +++ b/tests/ui/async-await/async-closures/return-type-mismatch.stderr @@ -0,0 +1,14 @@ +error[E0308]: mismatched types + --> $DIR/return-type-mismatch.rs:11:24 + | +LL | let y: usize = x().await; + | ^^^^^^^^^ expected `usize`, found `i32` + | +help: you can convert an `i32` to a `usize` and panic if the converted value doesn't fit + | +LL | let y: usize = x().await.try_into().unwrap(); + | ++++++++++++++++++++ + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0308`. diff --git a/tests/ui/async-await/async-closures/wrong-fn-kind.rs b/tests/ui/async-await/async-closures/wrong-fn-kind.rs new file mode 100644 index 00000000000..24828832531 --- /dev/null +++ b/tests/ui/async-await/async-closures/wrong-fn-kind.rs @@ -0,0 +1,18 @@ +// edition:2021 + +// FIXME(async_closures): This needs a better error message! + +#![feature(async_closure, async_fn_traits)] + +use std::ops::AsyncFn; + +fn main() { + fn needs_async_fn(_: impl AsyncFn()) {} + + let mut x = 1; + needs_async_fn(async || { + //~^ ERROR i16: ops::async_function::internal_implementation_detail::AsyncFnKindHelper<i8> + // FIXME: Should say "closure is AsyncFnMut but it needs AsyncFn" or sth. + x += 1; + }); +} diff --git a/tests/ui/async-await/async-closures/wrong-fn-kind.stderr b/tests/ui/async-await/async-closures/wrong-fn-kind.stderr new file mode 100644 index 00000000000..ef95e6a211c --- /dev/null +++ b/tests/ui/async-await/async-closures/wrong-fn-kind.stderr @@ -0,0 +1,22 @@ +error[E0277]: the trait bound `i16: ops::async_function::internal_implementation_detail::AsyncFnKindHelper<i8>` is not satisfied + --> $DIR/wrong-fn-kind.rs:13:20 + | +LL | needs_async_fn(async || { + | _____--------------_^ + | | | + | | required by a bound introduced by this call +LL | | +LL | | // FIXME: Should say "closure is AsyncFnMut but it needs AsyncFn" or sth. +LL | | x += 1; +LL | | }); + | |_____^ the trait `ops::async_function::internal_implementation_detail::AsyncFnKindHelper<i8>` is not implemented for `i16` + | +note: required by a bound in `needs_async_fn` + --> $DIR/wrong-fn-kind.rs:10:31 + | +LL | fn needs_async_fn(_: impl AsyncFn()) {} + | ^^^^^^^^^ required by this bound in `needs_async_fn` + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs index 95683241aba..904d28fb0a7 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs @@ -12,6 +12,8 @@ pub async fn async_fn(x: &mut i32) -> &i32 { pub fn async_closure(x: &mut i32) -> impl Future<Output=&i32> { (async move || { + //~^ ERROR lifetime may not live long enough + //~| ERROR temporary value dropped while borrowed let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed y @@ -20,6 +22,8 @@ pub fn async_closure(x: &mut i32) -> impl Future<Output=&i32> { pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future<Output=&i32> { (async move || -> &i32 { + //~^ ERROR lifetime may not live long enough + //~| ERROR temporary value dropped while borrowed let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed y diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr index 628ba1a4818..bdf2820887c 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr @@ -11,7 +11,7 @@ LL | y | - returning this value requires that `*x` is borrowed for `'1` error[E0506]: cannot assign to `*x` because it is borrowed - --> $DIR/issue-74072-lifetime-name-annotations.rs:16:9 + --> $DIR/issue-74072-lifetime-name-annotations.rs:18:9 | LL | let y = &*x; | --- `*x` is borrowed here @@ -22,20 +22,92 @@ LL | y LL | })() | - return type of async closure is &'1 i32 +error: lifetime may not live long enough + --> $DIR/issue-74072-lifetime-name-annotations.rs:14:20 + | +LL | (async move || { + | ______-------------_^ + | | | | + | | | return type of async closure `{async closure body@$DIR/issue-74072-lifetime-name-annotations.rs:14:20: 20:6}` contains a lifetime `'2` + | | lifetime `'1` represents this closure's body +LL | | +LL | | +LL | | let y = &*x; +LL | | *x += 1; +LL | | y +LL | | })() + | |_____^ returning this value requires that `'1` must outlive `'2` + | + = note: closure implements `FnMut`, so references to captured variables can't escape the closure + +error[E0716]: temporary value dropped while borrowed + --> $DIR/issue-74072-lifetime-name-annotations.rs:14:5 + | +LL | pub fn async_closure(x: &mut i32) -> impl Future<Output=&i32> { + | - let's call the lifetime of this reference `'1` +LL | // (async move || { +LL | || +LL | || +LL | || let y = &*x; +LL | || *x += 1; +LL | || y +LL | || })() + | ||______^_- argument requires that borrow lasts for `'1` + | |_______| + | creates a temporary value which is freed while still in use +LL | } + | - temporary value is freed at the end of this statement + error[E0506]: cannot assign to `*x` because it is borrowed - --> $DIR/issue-74072-lifetime-name-annotations.rs:24:9 + --> $DIR/issue-74072-lifetime-name-annotations.rs:28:9 | -LL | (async move || -> &i32 { - | - let's call the lifetime of this reference `'1` LL | let y = &*x; | --- `*x` is borrowed here LL | *x += 1; | ^^^^^^^ `*x` is assigned to here but it was already borrowed LL | y | - returning this value requires that `*x` is borrowed for `'1` +LL | })() + | - return type of async closure is &'1 i32 + +error: lifetime may not live long enough + --> $DIR/issue-74072-lifetime-name-annotations.rs:24:28 + | +LL | (async move || -> &i32 { + | ______---------------------_^ + | | | | + | | | return type of async closure `{async closure body@$DIR/issue-74072-lifetime-name-annotations.rs:24:28: 30:6}` contains a lifetime `'2` + | | lifetime `'1` represents this closure's body +LL | | +LL | | +LL | | let y = &*x; +LL | | *x += 1; +LL | | y +LL | | })() + | |_____^ returning this value requires that `'1` must outlive `'2` + | + = note: closure implements `FnMut`, so references to captured variables can't escape the closure + +error[E0716]: temporary value dropped while borrowed + --> $DIR/issue-74072-lifetime-name-annotations.rs:24:5 + | +LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future<Output=&i32> { + | - let's call the lifetime of this reference `'1` +LL | // (async move || -> &i32 { +LL | || +LL | || +LL | || let y = &*x; +LL | || *x += 1; +LL | || y +LL | || })() + | ||______^_- argument requires that borrow lasts for `'1` + | |_______| + | creates a temporary value which is freed while still in use +LL | } + | - temporary value is freed at the end of this statement error[E0506]: cannot assign to `*x` because it is borrowed - --> $DIR/issue-74072-lifetime-name-annotations.rs:32:9 + --> $DIR/issue-74072-lifetime-name-annotations.rs:36:9 | LL | let y = &*x; | --- `*x` is borrowed here @@ -46,6 +118,7 @@ LL | y LL | } | - return type of async block is &'1 i32 -error: aborting due to 4 previous errors +error: aborting due to 8 previous errors -For more information about this error, try `rustc --explain E0506`. +Some errors have detailed explanations: E0506, E0716. +For more information about an error, try `rustc --explain E0506`. diff --git a/tests/ui/closures/binder/async-closure-with-binder.rs b/tests/ui/closures/binder/async-closure-with-binder.rs index 4fa599d37cb..69d30f369e9 100644 --- a/tests/ui/closures/binder/async-closure-with-binder.rs +++ b/tests/ui/closures/binder/async-closure-with-binder.rs @@ -1,8 +1,9 @@ // edition:2021 +// check-pass + #![feature(closure_lifetime_binder)] #![feature(async_closure)] + fn main() { - for<'a> async || (); - //~^ ERROR `for<...>` binders on `async` closures are not currently supported - //~^^ ERROR implicit types in closure signatures are forbidden when `for<...>` is present + for<'a> async || -> () {}; } diff --git a/tests/ui/closures/binder/async-closure-with-binder.stderr b/tests/ui/closures/binder/async-closure-with-binder.stderr deleted file mode 100644 index 1d4628b1a49..00000000000 --- a/tests/ui/closures/binder/async-closure-with-binder.stderr +++ /dev/null @@ -1,16 +0,0 @@ -error: `for<...>` binders on `async` closures are not currently supported - --> $DIR/async-closure-with-binder.rs:5:5 - | -LL | for<'a> async || (); - | ^^^^^^^ - -error: implicit types in closure signatures are forbidden when `for<...>` is present - --> $DIR/async-closure-with-binder.rs:5:5 - | -LL | for<'a> async || (); - | -------^^^^^^^^^ - | | - | `for<...>` is here - -error: aborting due to 2 previous errors - diff --git a/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.rs b/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.rs index a9d678c1e6a..66a432be357 100644 --- a/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.rs +++ b/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.rs @@ -9,7 +9,7 @@ impl Numberer { //~^ ERROR `async fn` is not permitted in Rust 2015 interval: Duration, //~^ ERROR cannot find type `Duration` in this scope - ) -> Numberer { //~WARN: changes to closure capture in Rust 2021 + ) -> Numberer { Numberer {} } } diff --git a/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.stderr b/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.stderr index 71e9e7602e8..60433e1c284 100644 --- a/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.stderr +++ b/tests/ui/span/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.stderr @@ -18,32 +18,7 @@ help: consider importing this struct LL + use std::time::Duration; | -warning: changes to closure capture in Rust 2021 will affect drop order - --> $DIR/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.rs:12:19 - | -LL | interval: Duration, - | -------- in Rust 2018, this causes the closure to capture `interval`, but in Rust 2021, it has no effect -LL | -LL | ) -> Numberer { - | _________________-_^ - | | | - | | in Rust 2018, `interval` is dropped here along with the closure, but in Rust 2021 `interval` is not part of the closure -LL | | Numberer {} -LL | | } - | |_____^ - | - = note: for more information, see <https://doc.rust-lang.org/nightly/edition-guide/rust-2021/disjoint-capture-in-closures.html> -note: the lint level is defined here - --> $DIR/drop-location-span-error-rust-2021-incompatible-closure-captures-96258.rs:1:9 - | -LL | #![warn(rust_2021_incompatible_closure_captures)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -help: add a dummy let to cause `interval` to be fully captured - | -LL | ) -> Numberer { let _ = &interval; - | ++++++++++++++++++ - -error: aborting due to 2 previous errors; 1 warning emitted +error: aborting due to 2 previous errors Some errors have detailed explanations: E0412, E0670. For more information about an error, try `rustc --explain E0412`. diff --git a/tests/ui/suggestions/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.stderr b/tests/ui/suggestions/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.stderr index 3065f83ea3d..dc4ec5d3ee2 100644 --- a/tests/ui/suggestions/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.stderr +++ b/tests/ui/suggestions/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.stderr @@ -18,25 +18,21 @@ help: use parentheses to call this function LL | bar(foo()); | ++ -error[E0277]: `{closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33}` is not a future +error[E0277]: `{coroutine-closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33}` is not a future --> $DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:12:9 | LL | bar(async_closure); - | --- ^^^^^^^^^^^^^ `{closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33}` is not a future + | --- ^^^^^^^^^^^^^ `{coroutine-closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33}` is not a future | | | required by a bound introduced by this call | - = help: the trait `Future` is not implemented for closure `{closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33}` - = note: {closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33} must be a future or must implement `IntoFuture` to be awaited + = help: the trait `Future` is not implemented for `{coroutine-closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33}` + = note: {coroutine-closure@$DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:11:25: 11:33} must be a future or must implement `IntoFuture` to be awaited note: required by a bound in `bar` --> $DIR/async-fn-ctor-passed-as-arg-where-it-should-have-been-called.rs:7:16 | LL | fn bar(f: impl Future<Output=()>) {} | ^^^^^^^^^^^^^^^^^ required by this bound in `bar` -help: use parentheses to call this closure - | -LL | bar(async_closure()); - | ++ error: aborting due to 2 previous errors diff --git a/tests/ui/symbol-names/basic.legacy.stderr b/tests/ui/symbol-names/basic.legacy.stderr index 61d27ec69f4..c1cbefac828 100644 --- a/tests/ui/symbol-names/basic.legacy.stderr +++ b/tests/ui/symbol-names/basic.legacy.stderr @@ -1,10 +1,10 @@ -error: symbol-name(_ZN5basic4main17h9308686d0228fa1dE) +error: symbol-name(_ZN5basic4main17h6fc0c8d27b1a289fE) --> $DIR/basic.rs:8:1 | LL | #[rustc_symbol_name] | ^^^^^^^^^^^^^^^^^^^^ -error: demangling(basic::main::h9308686d0228fa1d) +error: demangling(basic::main::h6fc0c8d27b1a289f) --> $DIR/basic.rs:8:1 | LL | #[rustc_symbol_name] diff --git a/tests/ui/symbol-names/issue-60925.legacy.stderr b/tests/ui/symbol-names/issue-60925.legacy.stderr index eb65f3b58ff..7dd68e6e3a8 100644 --- a/tests/ui/symbol-names/issue-60925.legacy.stderr +++ b/tests/ui/symbol-names/issue-60925.legacy.stderr @@ -1,10 +1,10 @@ -error: symbol-name(_ZN11issue_609253foo37Foo$LT$issue_60925..llv$u6d$..Foo$GT$3foo17h84ab5dafbd2a1508E) +error: symbol-name(_ZN11issue_609253foo37Foo$LT$issue_60925..llv$u6d$..Foo$GT$3foo17hab58a402db4ebf3aE) --> $DIR/issue-60925.rs:21:9 | LL | #[rustc_symbol_name] | ^^^^^^^^^^^^^^^^^^^^ -error: demangling(issue_60925::foo::Foo<issue_60925::llvm::Foo>::foo::h84ab5dafbd2a1508) +error: demangling(issue_60925::foo::Foo<issue_60925::llvm::Foo>::foo::hab58a402db4ebf3a) --> $DIR/issue-60925.rs:21:9 | LL | #[rustc_symbol_name] |