diff options
Diffstat (limited to 'compiler')
38 files changed, 610 insertions, 134 deletions
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 36ba1d81be6..d258c68b959 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -727,6 +727,12 @@ impl<'a, 'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R> } self.mutate_place(loc, (*destination, span), Deep, flow_state); } + TerminatorKind::TailCall { func, args, fn_span: _ } => { + self.consume_operand(loc, (func, span), flow_state); + for arg in args { + self.consume_operand(loc, (&arg.node, arg.span), flow_state); + } + } TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => { self.consume_operand(loc, (cond, span), flow_state); if let AssertKind::BoundsCheck { len, index } = &**msg { @@ -813,9 +819,8 @@ impl<'a, 'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R> TerminatorKind::UnwindResume | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::CoroutineDrop => { - // Returning from the function implicitly kills storage for all locals and statics. - // Often, the storage will already have been killed by an explicit // StorageDead, but we don't always emit those (notably on unwind paths), // so this "extra check" serves as a kind of backup. let borrow_set = self.borrow_set.clone(); diff --git a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs index 6979910a02d..30dfc4c21b0 100644 --- a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs +++ b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs @@ -125,6 +125,12 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> { } self.mutate_place(location, *destination, Deep); } + TerminatorKind::TailCall { func, args, .. } => { + self.consume_operand(location, func); + for arg in args { + self.consume_operand(location, &arg.node); + } + } TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => { self.consume_operand(location, cond); use rustc_middle::mir::AssertKind; diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index aa25e3adf28..8bba7ef4255 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1352,7 +1352,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } // FIXME: check the values } - TerminatorKind::Call { func, args, destination, call_source, target, .. } => { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => { + let call_source = match term.kind { + TerminatorKind::Call { call_source, .. } => call_source, + TerminatorKind::TailCall { .. } => CallSource::Normal, + _ => unreachable!(), + }; + self.check_operand(func, term_location); for arg in args { self.check_operand(&arg.node, term_location); @@ -1425,7 +1432,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { ); } - self.check_call_dest(body, term, &sig, *destination, *target, term_location); + if let TerminatorKind::Call { destination, target, .. } = term.kind { + self.check_call_dest(body, term, &sig, destination, target, term_location); + } // The ordinary liveness rules will ensure that all // regions in the type of the callee are live here. We @@ -1443,7 +1452,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { .add_location(region_vid, term_location); } - self.check_call_inputs(body, term, func, &sig, args, term_location, *call_source); + self.check_call_inputs(body, term, func, &sig, args, term_location, call_source); } TerminatorKind::Assert { cond, msg, .. } => { self.check_operand(cond, term_location); @@ -1675,6 +1684,11 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { span_mirbug!(self, block_data, "return on cleanup block") } } + TerminatorKind::TailCall { .. } => { + if is_cleanup { + span_mirbug!(self, block_data, "tailcall on cleanup block") + } + } TerminatorKind::CoroutineDrop { .. } => { if is_cleanup { span_mirbug!(self, block_data, "coroutine_drop in cleanup block") diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index c5b4277015a..5adbbb09ac8 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -491,6 +491,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { ) }); } + // FIXME(explicit_tail_calls): add support for tail calls to the cranelift backend, once cranelift supports tail calls + TerminatorKind::TailCall { fn_span, .. } => span_bug!( + *fn_span, + "tail calls are not yet supported in `rustc_codegen_cranelift` backend" + ), TerminatorKind::InlineAsm { template, operands, diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 9f7b95261d5..fd34ed88c0b 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -567,6 +567,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( { return None; } + TerminatorKind::TailCall { .. } => return None, TerminatorKind::Call { .. } => {} } } diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 0577ba32ffd..ac2b6ca4e95 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -281,6 +281,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::CoroutineDrop | TerminatorKind::Unreachable | TerminatorKind::SwitchInt { .. } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 4adb95f85d6..6a5525dc2b3 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1389,6 +1389,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_span, mergeable_succ(), ), + mir::TerminatorKind::TailCall { .. } => { + // FIXME(explicit_tail_calls): implement tail calls in ssa backend + span_bug!( + terminator.source_info.span, + "`TailCall` terminator is not yet supported by `rustc_codegen_ssa`" + ) + } mir::TerminatorKind::CoroutineDrop | mir::TerminatorKind::Yield { .. } => { bug!("coroutine ops in codegen") } diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index 0d8a17775dd..412effba32d 100644 --- a/compiler/rustc_const_eval/src/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -135,6 +135,8 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> { ccx: &'mir ConstCx<'mir, 'tcx>, tainted_by_errors: Option<ErrorGuaranteed>, ) -> ConstQualifs { + // FIXME(explicit_tail_calls): uhhhh I think we can return without return now, does it change anything + // Find the `Return` terminator if one exists. // // If no `Return` terminator exists, this MIR is divergent. Just return the conservative @@ -711,7 +713,14 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { self.super_terminator(terminator, location); match &terminator.kind { - TerminatorKind::Call { func, args, fn_span, call_source, .. } => { + TerminatorKind::Call { func, args, fn_span, .. } + | TerminatorKind::TailCall { func, args, fn_span, .. } => { + let call_source = match terminator.kind { + TerminatorKind::Call { call_source, .. } => call_source, + TerminatorKind::TailCall { .. } => CallSource::Normal, + _ => unreachable!(), + }; + let ConstCx { tcx, body, param_env, .. } = *self.ccx; let caller = self.def_id(); @@ -783,7 +792,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { callee, args: fn_args, span: *fn_span, - call_source: *call_source, + call_source, feature: Some(if tcx.features().const_trait_impl { sym::effects } else { @@ -830,7 +839,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { callee, args: fn_args, span: *fn_span, - call_source: *call_source, + call_source, feature: None, }); return; diff --git a/compiler/rustc_const_eval/src/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/check_consts/post_drop_elaboration.rs index f0763f1e490..f5e745454ab 100644 --- a/compiler/rustc_const_eval/src/check_consts/post_drop_elaboration.rs +++ b/compiler/rustc_const_eval/src/check_consts/post_drop_elaboration.rs @@ -108,6 +108,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> { mir::TerminatorKind::UnwindTerminate(_) | mir::TerminatorKind::Call { .. } + | mir::TerminatorKind::TailCall { .. } | mir::TerminatorKind::Assert { .. } | mir::TerminatorKind::FalseEdge { .. } | mir::TerminatorKind::FalseUnwind { .. } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 830c4bd3e26..6d3e5ea1031 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -26,8 +26,8 @@ use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayou use super::{ err_inval, throw_inval, throw_ub, throw_ub_custom, throw_unsup, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Memory, MemoryKind, - OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic, Projectable, Provenance, Scalar, - StackPopJump, + OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic, Projectable, Provenance, + ReturnAction, Scalar, }; use crate::errors; use crate::util; @@ -159,6 +159,19 @@ pub enum StackPopCleanup { Root { cleanup: bool }, } +/// Return type of [`InterpCx::pop_stack_frame`]. +pub struct StackPopInfo<'tcx, Prov: Provenance> { + /// Additional information about the action to be performed when returning from the popped + /// stack frame. + pub return_action: ReturnAction, + + /// [`return_to_block`](Frame::return_to_block) of the popped stack frame. + pub return_to_block: StackPopCleanup, + + /// [`return_place`](Frame::return_place) of the popped stack frame. + pub return_place: MPlaceTy<'tcx, Prov>, +} + /// State of a local variable including a memoized layout #[derive(Clone)] pub struct LocalState<'tcx, Prov: Provenance = CtfeProvenance> { @@ -803,14 +816,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { return_to_block: StackPopCleanup, ) -> InterpResult<'tcx> { trace!("body: {:#?}", body); + + // First push a stack frame so we have access to the local args + self.push_new_stack_frame(instance, body, return_to_block, return_place.clone())?; + + self.after_stack_frame_push(instance, body)?; + + Ok(()) + } + + /// Creates a new stack frame, initializes it and pushes it onto the stack. + /// A private helper for [`push_stack_frame`](InterpCx::push_stack_frame). + fn push_new_stack_frame( + &mut self, + instance: ty::Instance<'tcx>, + body: &'tcx mir::Body<'tcx>, + return_to_block: StackPopCleanup, + return_place: MPlaceTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx> { let dead_local = LocalState { value: LocalValue::Dead, layout: Cell::new(None) }; let locals = IndexVec::from_elem(dead_local, &body.local_decls); - // First push a stack frame so we have access to the local args let pre_frame = Frame { body, loc: Right(body.span), // Span used for errors caused during preamble. return_to_block, - return_place: return_place.clone(), + return_place, locals, instance, tracing_span: SpanGuard::new(), @@ -819,6 +849,15 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let frame = M::init_frame(self, pre_frame)?; self.stack_mut().push(frame); + Ok(()) + } + + /// A private helper for [`push_stack_frame`](InterpCx::push_stack_frame). + fn after_stack_frame_push( + &mut self, + instance: ty::Instance<'tcx>, + body: &'tcx mir::Body<'tcx>, + ) -> InterpResult<'tcx> { // Make sure all the constants required by this frame evaluate successfully (post-monomorphization check). for &const_ in &body.required_consts { let c = @@ -839,6 +878,61 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Ok(()) } + /// Pops a stack frame from the stack and returns some information about it. + /// + /// This also deallocates locals, if necessary. + /// + /// [`M::before_stack_pop`] should be called before calling this function. + /// [`M::after_stack_pop`] is called by this function automatically. + /// + /// [`M::before_stack_pop`]: Machine::before_stack_pop + /// [`M::after_stack_pop`]: Machine::after_stack_pop + pub fn pop_stack_frame( + &mut self, + unwinding: bool, + ) -> InterpResult<'tcx, StackPopInfo<'tcx, M::Provenance>> { + let cleanup = self.cleanup_current_frame_locals()?; + + let frame = + self.stack_mut().pop().expect("tried to pop a stack frame, but there were none"); + + let return_to_block = frame.return_to_block; + let return_place = frame.return_place.clone(); + + let return_action; + if cleanup { + return_action = M::after_stack_pop(self, frame, unwinding)?; + assert_ne!(return_action, ReturnAction::NoCleanup); + } else { + return_action = ReturnAction::NoCleanup; + }; + + Ok(StackPopInfo { return_action, return_to_block, return_place }) + } + + /// A private helper for [`pop_stack_frame`](InterpCx::pop_stack_frame). + /// Returns `true` if cleanup has been done, `false` otherwise. + fn cleanup_current_frame_locals(&mut self) -> InterpResult<'tcx, bool> { + // Cleanup: deallocate locals. + // Usually we want to clean up (deallocate locals), but in a few rare cases we don't. + // We do this while the frame is still on the stack, so errors point to the callee. + let return_to_block = self.frame().return_to_block; + let cleanup = match return_to_block { + StackPopCleanup::Goto { .. } => true, + StackPopCleanup::Root { cleanup, .. } => cleanup, + }; + + if cleanup { + // We need to take the locals out, since we need to mutate while iterating. + let locals = mem::take(&mut self.frame_mut().locals); + for local in &locals { + self.deallocate_local(local.value)?; + } + } + + Ok(cleanup) + } + /// Jump to the given block. #[inline] pub fn go_to_block(&mut self, target: mir::BasicBlock) { @@ -886,7 +980,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } /// Pops the current frame from the stack, deallocating the - /// memory for allocated locals. + /// memory for allocated locals, and jumps to an appropriate place. /// /// If `unwinding` is `false`, then we are performing a normal return /// from a function. In this case, we jump back into the frame of the caller, @@ -899,7 +993,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// The cleanup block ends with a special `Resume` terminator, which will /// cause us to continue unwinding. #[instrument(skip(self), level = "debug")] - pub(super) fn pop_stack_frame(&mut self, unwinding: bool) -> InterpResult<'tcx> { + pub(super) fn return_from_current_stack_frame( + &mut self, + unwinding: bool, + ) -> InterpResult<'tcx> { info!( "popping stack frame ({})", if unwinding { "during unwinding" } else { "returning from function" } @@ -947,45 +1044,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Ok(()) }; - // Cleanup: deallocate locals. - // Usually we want to clean up (deallocate locals), but in a few rare cases we don't. - // We do this while the frame is still on the stack, so errors point to the callee. - let return_to_block = self.frame().return_to_block; - let cleanup = match return_to_block { - StackPopCleanup::Goto { .. } => true, - StackPopCleanup::Root { cleanup, .. } => cleanup, - }; - if cleanup { - // We need to take the locals out, since we need to mutate while iterating. - let locals = mem::take(&mut self.frame_mut().locals); - for local in &locals { - self.deallocate_local(local.value)?; - } - } - // All right, now it is time to actually pop the frame. - // Note that its locals are gone already, but that's fine. - let frame = - self.stack_mut().pop().expect("tried to pop a stack frame, but there were none"); + let stack_pop_info = self.pop_stack_frame(unwinding)?; + // Report error from return value copy, if any. copy_ret_result?; - // If we are not doing cleanup, also skip everything else. - if !cleanup { - assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked"); - assert!(!unwinding, "tried to skip cleanup during unwinding"); - // Skip machine hook. - return Ok(()); - } - if M::after_stack_pop(self, frame, unwinding)? == StackPopJump::NoJump { - // The hook already did everything. - return Ok(()); + match stack_pop_info.return_action { + ReturnAction::Normal => {} + ReturnAction::NoJump => { + // The hook already did everything. + return Ok(()); + } + ReturnAction::NoCleanup => { + // If we are not doing cleanup, also skip everything else. + assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked"); + assert!(!unwinding, "tried to skip cleanup during unwinding"); + // Skip machine hook. + return Ok(()); + } } // Normal return, figure out where to jump. if unwinding { // Follow the unwind edge. - let unwind = match return_to_block { + let unwind = match stack_pop_info.return_to_block { StackPopCleanup::Goto { unwind, .. } => unwind, StackPopCleanup::Root { .. } => { panic!("encountered StackPopCleanup::Root when unwinding!") @@ -995,7 +1078,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.unwind_to_block(unwind) } else { // Follow the normal return edge. - match return_to_block { + match stack_pop_info.return_to_block { StackPopCleanup::Goto { ret, .. } => self.return_to_block(ret), StackPopCleanup::Root { .. } => { assert!( diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index e91ab7c1791..7f2e9ce06a5 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -23,10 +23,11 @@ use super::{ MemoryKind, Misalignment, OpTy, PlaceTy, Pointer, Provenance, }; -/// Data returned by Machine::stack_pop, -/// to provide further control over the popping of the stack frame +/// Data returned by [`Machine::after_stack_pop`], and consumed by +/// [`InterpCx::return_from_current_stack_frame`] to determine what actions should be done when +/// returning from a stack frame. #[derive(Eq, PartialEq, Debug, Copy, Clone)] -pub enum StackPopJump { +pub enum ReturnAction { /// Indicates that no special handling should be /// done - we'll either return normally or unwind /// based on the terminator for the function @@ -36,6 +37,9 @@ pub enum StackPopJump { /// Indicates that we should *not* jump to the return/unwind address, as the callback already /// took care of everything. NoJump, + + /// Returned by [`InterpCx::pop_stack_frame`] when no cleanup should be done. + NoCleanup, } /// Whether this kind of memory is allowed to leak @@ -522,10 +526,10 @@ pub trait Machine<'tcx>: Sized { _ecx: &mut InterpCx<'tcx, Self>, _frame: Frame<'tcx, Self::Provenance, Self::FrameExtra>, unwinding: bool, - ) -> InterpResult<'tcx, StackPopJump> { + ) -> InterpResult<'tcx, ReturnAction> { // By default, we do not support unwinding from panics assert!(!unwinding); - Ok(StackPopJump::Normal) + Ok(ReturnAction::Normal) } /// Called immediately after actual memory was allocated for a local diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs index 7d7b421f869..f703c6fbe3e 100644 --- a/compiler/rustc_const_eval/src/interpret/mod.rs +++ b/compiler/rustc_const_eval/src/interpret/mod.rs @@ -26,7 +26,7 @@ pub use self::intern::{ intern_const_alloc_for_constprop, intern_const_alloc_recursive, HasStaticRootDefId, InternKind, InternResult, }; -pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump}; +pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, ReturnAction}; pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind}; pub use self::operand::{ImmTy, Immediate, OpTy, Readable}; pub use self::place::{MPlaceTy, MemPlaceMeta, PlaceTy, Writeable}; diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 1baf62baa81..b3124dfdfbc 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -32,7 +32,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // We are unwinding and this fn has no cleanup code. // Just go on unwinding. trace!("unwinding: skipping frame"); - self.pop_stack_frame(/* unwinding */ true)?; + self.return_from_current_stack_frame(/* unwinding */ true)?; return Ok(true); }; let basic_block = &self.body().basic_blocks[loc.block]; diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index 68acddf63d8..25f6bd64055 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -4,9 +4,8 @@ use either::Either; use rustc_middle::ty::TyCtxt; use tracing::trace; -use rustc_middle::span_bug; use rustc_middle::{ - mir, + bug, mir, span_bug, ty::{ self, layout::{FnAbiOf, IntegerExt, LayoutOf, TyAndLayout}, @@ -26,7 +25,10 @@ use super::{ InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, Projectable, Provenance, Scalar, StackPopCleanup, }; -use crate::fluent_generated as fluent; +use crate::{ + fluent_generated as fluent, + interpret::{eval_context::StackPopInfo, ReturnAction}, +}; /// An argment passed to a function. #[derive(Clone, Debug)] @@ -47,6 +49,15 @@ impl<'tcx, Prov: Provenance> FnArg<'tcx, Prov> { } } +struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> { + callee: FnVal<'tcx, M::ExtraFnVal>, + args: Vec<FnArg<'tcx, M::Provenance>>, + fn_sig: ty::FnSig<'tcx>, + fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + /// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`]) + with_caller_location: bool, +} + impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// Make a copy of the given fn_arg. Any `InPlace` are degenerated to copies, no protection of the /// original memory occurs. @@ -84,7 +95,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { use rustc_middle::mir::TerminatorKind::*; match terminator.kind { Return => { - self.pop_stack_frame(/* unwinding */ false)? + self.return_from_current_stack_frame(/* unwinding */ false)? } Goto { target } => self.go_to_block(target), @@ -124,40 +135,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } => { let old_stack = self.frame_idx(); let old_loc = self.frame().loc; - let func = self.eval_operand(func, None)?; - let args = self.eval_fn_call_arguments(args)?; - - let fn_sig_binder = func.layout.ty.fn_sig(*self.tcx); - let fn_sig = - self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig_binder); - let extra_args = &args[fn_sig.inputs().len()..]; - let extra_args = - self.tcx.mk_type_list_from_iter(extra_args.iter().map(|arg| arg.layout().ty)); - - let (fn_val, fn_abi, with_caller_location) = match *func.layout.ty.kind() { - ty::FnPtr(_sig) => { - let fn_ptr = self.read_pointer(&func)?; - let fn_val = self.get_ptr_fn(fn_ptr)?; - (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false) - } - ty::FnDef(def_id, args) => { - let instance = self.resolve(def_id, args)?; - ( - FnVal::Instance(instance), - self.fn_abi_of_instance(instance, extra_args)?, - instance.def.requires_caller_location(*self.tcx), - ) - } - _ => span_bug!( - terminator.source_info.span, - "invalid callee of type {}", - func.layout.ty - ), - }; + + let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = + self.eval_callee_and_args(terminator, func, args)?; let destination = self.force_allocation(&self.eval_place(destination)?)?; self.eval_fn_call( - fn_val, + callee, (fn_sig.abi, fn_abi), &args, with_caller_location, @@ -172,6 +156,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } + TailCall { ref func, ref args, fn_span: _ } => { + let old_frame_idx = self.frame_idx(); + + let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = + self.eval_callee_and_args(terminator, func, args)?; + + self.eval_fn_tail_call(callee, (fn_sig.abi, fn_abi), &args, with_caller_location)?; + + if self.frame_idx() != old_frame_idx { + span_bug!( + terminator.source_info.span, + "evaluating this tail call pushed a new stack frame" + ); + } + } + Drop { place, target, unwind, replace: _ } => { let place = self.eval_place(place)?; let instance = Instance::resolve_drop_in_place(*self.tcx, place.layout.ty); @@ -209,7 +209,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { trace!("unwinding: resuming from cleanup"); // By definition, a Resume terminator means // that we're unwinding - self.pop_stack_frame(/* unwinding */ true)?; + self.return_from_current_stack_frame(/* unwinding */ true)?; return Ok(()); } @@ -514,6 +514,45 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Ok(()) } + /// Shared part of `Call` and `TailCall` implementation — finding and evaluating all the + /// necessary information about callee and arguments to make a call. + fn eval_callee_and_args( + &self, + terminator: &mir::Terminator<'tcx>, + func: &mir::Operand<'tcx>, + args: &[Spanned<mir::Operand<'tcx>>], + ) -> InterpResult<'tcx, EvaluatedCalleeAndArgs<'tcx, M>> { + let func = self.eval_operand(func, None)?; + let args = self.eval_fn_call_arguments(args)?; + + let fn_sig_binder = func.layout.ty.fn_sig(*self.tcx); + let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig_binder); + let extra_args = &args[fn_sig.inputs().len()..]; + let extra_args = + self.tcx.mk_type_list_from_iter(extra_args.iter().map(|arg| arg.layout().ty)); + + let (callee, fn_abi, with_caller_location) = match *func.layout.ty.kind() { + ty::FnPtr(_sig) => { + let fn_ptr = self.read_pointer(&func)?; + let fn_val = self.get_ptr_fn(fn_ptr)?; + (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false) + } + ty::FnDef(def_id, args) => { + let instance = self.resolve(def_id, args)?; + ( + FnVal::Instance(instance), + self.fn_abi_of_instance(instance, extra_args)?, + instance.def.requires_caller_location(*self.tcx), + ) + } + _ => { + span_bug!(terminator.source_info.span, "invalid callee of type {}", func.layout.ty) + } + }; + + Ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location }) + } + /// Call this function -- pushing the stack frame and initializing the arguments. /// /// `caller_fn_abi` is used to determine if all the arguments are passed the proper way. @@ -924,6 +963,49 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } + pub(crate) fn eval_fn_tail_call( + &mut self, + fn_val: FnVal<'tcx, M::ExtraFnVal>, + (caller_abi, caller_fn_abi): (Abi, &FnAbi<'tcx, Ty<'tcx>>), + args: &[FnArg<'tcx, M::Provenance>], + with_caller_location: bool, + ) -> InterpResult<'tcx> { + trace!("eval_fn_call: {:#?}", fn_val); + + // This is the "canonical" implementation of tails calls, + // a pop of the current stack frame, followed by a normal call + // which pushes a new stack frame, with the return address from + // the popped stack frame. + // + // Note that we are using `pop_stack_frame` and not `return_from_current_stack_frame`, + // as the latter "executes" the goto to the return block, but we don't want to, + // only the tail called function should return to the current return block. + M::before_stack_pop(self, self.frame())?; + + let StackPopInfo { return_action, return_to_block, return_place } = + self.pop_stack_frame(false)?; + + assert_eq!(return_action, ReturnAction::Normal); + + let StackPopCleanup::Goto { ret, unwind } = return_to_block else { + bug!("can't tailcall as root"); + }; + + // FIXME(explicit_tail_calls): + // we should check if both caller&callee can/n't unwind, + // see <https://github.com/rust-lang/rust/pull/113128#issuecomment-1614979803> + + self.eval_fn_call( + fn_val, + (caller_abi, caller_fn_abi), + args, + with_caller_location, + &return_place, + ret, + unwind, + ) + } + fn check_fn_target_features(&self, instance: ty::Instance<'tcx>) -> InterpResult<'tcx, ()> { // Calling functions with `#[target_feature]` is not unsafe on WASM, see #84988 let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index af19ed95193..82625ae3d47 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -854,6 +854,16 @@ impl<'tcx> TerminatorKind<'tcx> { } write!(fmt, ")") } + TailCall { func, args, .. } => { + write!(fmt, "tailcall {func:?}(")?; + for (index, arg) in args.iter().enumerate() { + if index > 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{:?}", arg)?; + } + write!(fmt, ")") + } Assert { cond, expected, msg, .. } => { write!(fmt, "assert(")?; if !expected { @@ -921,7 +931,12 @@ impl<'tcx> TerminatorKind<'tcx> { pub fn fmt_successor_labels(&self) -> Vec<Cow<'static, str>> { use self::TerminatorKind::*; match *self { - Return | UnwindResume | UnwindTerminate(_) | Unreachable | CoroutineDrop => vec![], + Return + | TailCall { .. } + | UnwindResume + | UnwindTerminate(_) + | Unreachable + | CoroutineDrop => vec![], Goto { .. } => vec!["".into()], SwitchInt { ref targets, .. } => targets .values diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 2c2884f1897..620fa962d79 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -744,6 +744,36 @@ pub enum TerminatorKind<'tcx> { fn_span: Span, }, + /// Tail call. + /// + /// Roughly speaking this is a chimera of [`Call`] and [`Return`], with some caveats. + /// Semantically tail calls consists of two actions: + /// - pop of the current stack frame + /// - a call to the `func`, with the return address of the **current** caller + /// - so that a `return` inside `func` returns to the caller of the caller + /// of the function that is currently being executed + /// + /// Note that in difference with [`Call`] this is missing + /// - `destination` (because it's always the return place) + /// - `target` (because it's always taken from the current stack frame) + /// - `unwind` (because it's always taken from the current stack frame) + /// + /// [`Call`]: TerminatorKind::Call + /// [`Return`]: TerminatorKind::Return + TailCall { + /// The function that’s being called. + func: Operand<'tcx>, + /// Arguments the function is called with. + /// These are owned by the callee, which is free to modify them. + /// This allows the memory occupied by "by-value" arguments to be + /// reused across function calls without duplicating the contents. + args: Box<[Spanned<Operand<'tcx>>]>, + // FIXME(explicit_tail_calls): should we have the span for `become`? is this span accurate? do we need it? + /// This `Span` is the span of the function, without the dot and receiver + /// (e.g. `foo(a, b)` in `x.foo(a, b)` + fn_span: Span, + }, + /// Evaluates the operand, which must have type `bool`. If it is not equal to `expected`, /// initiates a panic. Initiating a panic corresponds to a `Call` terminator with some /// unspecified constant as the function to call, all the operands stored in the `AssertMessage` @@ -870,6 +900,7 @@ impl TerminatorKind<'_> { TerminatorKind::Unreachable => "Unreachable", TerminatorKind::Drop { .. } => "Drop", TerminatorKind::Call { .. } => "Call", + TerminatorKind::TailCall { .. } => "TailCall", TerminatorKind::Assert { .. } => "Assert", TerminatorKind::Yield { .. } => "Yield", TerminatorKind::CoroutineDrop => "CoroutineDrop", diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs index ed592612358..5b035d9579d 100644 --- a/compiler/rustc_middle/src/mir/terminator.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -439,6 +439,7 @@ mod helper { | CoroutineDrop | Return | Unreachable + | TailCall { .. } | Call { target: None, unwind: _, .. } => (&[]).into_iter().copied().chain(None), InlineAsm { ref targets, unwind: UnwindAction::Cleanup(u), .. } => { targets.iter().copied().chain(Some(u)) @@ -479,6 +480,7 @@ mod helper { | CoroutineDrop | Return | Unreachable + | TailCall { .. } | Call { target: None, unwind: _, .. } => (&mut []).into_iter().chain(None), InlineAsm { ref mut targets, unwind: UnwindAction::Cleanup(ref mut u), .. } => { targets.iter_mut().chain(Some(u)) @@ -501,6 +503,7 @@ impl<'tcx> TerminatorKind<'tcx> { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::CoroutineDrop | TerminatorKind::Yield { .. } @@ -521,6 +524,7 @@ impl<'tcx> TerminatorKind<'tcx> { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::CoroutineDrop | TerminatorKind::Yield { .. } @@ -606,9 +610,12 @@ impl<'tcx> TerminatorKind<'tcx> { pub fn edges(&self) -> TerminatorEdges<'_, 'tcx> { use TerminatorKind::*; match *self { - Return | UnwindResume | UnwindTerminate(_) | CoroutineDrop | Unreachable => { - TerminatorEdges::None - } + Return + | TailCall { .. } + | UnwindResume + | UnwindTerminate(_) + | CoroutineDrop + | Unreachable => TerminatorEdges::None, Goto { target } => TerminatorEdges::Single(target), diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 7628a1ed2fe..0d3c419748b 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -540,6 +540,17 @@ macro_rules! make_mir_visitor { ); } + TerminatorKind::TailCall { + func, + args, + fn_span: _, + } => { + self.visit_operand(func, location); + for arg in args { + self.visit_operand(&$($mutability)? arg.node, location); + } + }, + TerminatorKind::Assert { cond, expected: _, diff --git a/compiler/rustc_mir_build/src/build/expr/stmt.rs b/compiler/rustc_mir_build/src/build/expr/stmt.rs index 2bdeb579a02..88b76c46c90 100644 --- a/compiler/rustc_mir_build/src/build/expr/stmt.rs +++ b/compiler/rustc_mir_build/src/build/expr/stmt.rs @@ -2,7 +2,9 @@ use crate::build::scope::BreakableTarget; use crate::build::{BlockAnd, BlockAndExtension, BlockFrame, Builder}; use rustc_middle::middle::region; use rustc_middle::mir::*; +use rustc_middle::span_bug; use rustc_middle::thir::*; +use rustc_span::source_map::Spanned; use tracing::debug; impl<'a, 'tcx> Builder<'a, 'tcx> { @@ -91,9 +93,42 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ExprKind::Return { value } => { this.break_scope(block, value, BreakableTarget::Return, source_info) } - // FIXME(explicit_tail_calls): properly lower tail calls here ExprKind::Become { value } => { - this.break_scope(block, Some(value), BreakableTarget::Return, source_info) + let v = &this.thir[value]; + let ExprKind::Scope { value, lint_level, region_scope } = v.kind else { + span_bug!(v.span, "`thir_check_tail_calls` should have disallowed this {v:?}") + }; + + let v = &this.thir[value]; + let ExprKind::Call { ref args, fun, fn_span, .. } = v.kind else { + span_bug!(v.span, "`thir_check_tail_calls` should have disallowed this {v:?}") + }; + + this.in_scope((region_scope, source_info), lint_level, |this| { + let fun = unpack!(block = this.as_local_operand(block, fun)); + let args: Box<[_]> = args + .into_iter() + .copied() + .map(|arg| Spanned { + node: unpack!(block = this.as_local_call_operand(block, arg)), + span: this.thir.exprs[arg].span, + }) + .collect(); + + this.record_operands_moved(&args); + + debug!("expr_into_dest: fn_span={:?}", fn_span); + + unpack!(block = this.break_for_tail_call(block, &args, source_info)); + + this.cfg.terminate( + block, + source_info, + TerminatorKind::TailCall { func: fun, args, fn_span }, + ); + + this.cfg.start_new_block().unit() + }) } _ => { assert!( diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs index 5b6de39bb2e..948301e2ece 100644 --- a/compiler/rustc_mir_build/src/build/scope.rs +++ b/compiler/rustc_mir_build/src/build/scope.rs @@ -745,6 +745,91 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.cfg.terminate(block, source_info, TerminatorKind::UnwindResume); } + /// Sets up the drops for explict tail calls. + /// + /// Unlike other kinds of early exits, tail calls do not go through the drop tree. + /// Instead, all scheduled drops are immediately added to the CFG. + pub(crate) fn break_for_tail_call( + &mut self, + mut block: BasicBlock, + args: &[Spanned<Operand<'tcx>>], + source_info: SourceInfo, + ) -> BlockAnd<()> { + let arg_drops: Vec<_> = args + .iter() + .rev() + .filter_map(|arg| match &arg.node { + Operand::Copy(_) => bug!("copy op in tail call args"), + Operand::Move(place) => { + let local = + place.as_local().unwrap_or_else(|| bug!("projection in tail call args")); + + Some(DropData { source_info, local, kind: DropKind::Value }) + } + Operand::Constant(_) => None, + }) + .collect(); + + let mut unwind_to = self.diverge_cleanup_target( + self.scopes.scopes.iter().rev().nth(1).unwrap().region_scope, + DUMMY_SP, + ); + let unwind_drops = &mut self.scopes.unwind_drops; + + // the innermost scope contains only the destructors for the tail call arguments + // we only want to drop these in case of a panic, so we skip it + for scope in self.scopes.scopes[1..].iter().rev().skip(1) { + // FIXME(explicit_tail_calls) code duplication with `build_scope_drops` + for drop_data in scope.drops.iter().rev() { + let source_info = drop_data.source_info; + let local = drop_data.local; + + match drop_data.kind { + DropKind::Value => { + // `unwind_to` should drop the value that we're about to + // schedule. If dropping this value panics, then we continue + // with the *next* value on the unwind path. + debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local); + debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind); + unwind_to = unwind_drops.drops[unwind_to].next; + + let mut unwind_entry_point = unwind_to; + + // the tail call arguments must be dropped if any of these drops panic + for drop in arg_drops.iter().copied() { + unwind_entry_point = unwind_drops.add_drop(drop, unwind_entry_point); + } + + unwind_drops.add_entry_point(block, unwind_entry_point); + + let next = self.cfg.start_new_block(); + self.cfg.terminate( + block, + source_info, + TerminatorKind::Drop { + place: local.into(), + target: next, + unwind: UnwindAction::Continue, + replace: false, + }, + ); + block = next; + } + DropKind::Storage => { + // Only temps and vars need their storage dead. + assert!(local.index() > self.arg_count); + self.cfg.push( + block, + Statement { source_info, kind: StatementKind::StorageDead(local) }, + ); + } + } + } + } + + block.unit() + } + fn leave_top_scope(&mut self, block: BasicBlock) -> BasicBlock { // If we are emitting a `drop` statement, we need to have the cached // diverge cleanup pads ready in case that drop panics. @@ -1523,6 +1608,7 @@ impl<'tcx> DropTreeBuilder<'tcx> for Unwind { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::Yield { .. } | TerminatorKind::CoroutineDrop diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs index 1c7aa9f9ed0..263e777d03a 100644 --- a/compiler/rustc_mir_build/src/lints.rs +++ b/compiler/rustc_mir_build/src/lints.rs @@ -217,12 +217,28 @@ impl<'mir, 'tcx, C: TerminatorClassifier<'tcx>> TriColorVisitor<BasicBlocks<'tcx | TerminatorKind::FalseUnwind { .. } | TerminatorKind::Goto { .. } | TerminatorKind::SwitchInt { .. } => ControlFlow::Continue(()), + + // Note that tail call terminator technically returns to the caller, + // but for purposes of this lint it makes sense to count it as possibly recursive, + // since it's still a call. + // + // If this'll be repurposed for something else, this might need to be changed. + TerminatorKind::TailCall { .. } => ControlFlow::Continue(()), } } fn node_settled(&mut self, bb: BasicBlock) -> ControlFlow<Self::BreakVal> { // When we examine a node for the last time, remember it if it is a recursive call. let terminator = self.body[bb].terminator(); + + // FIXME(explicit_tail_calls): highlight tail calls as "recursive call site" + // + // We don't want to lint functions that recurse only through tail calls + // (such as `fn g() { become () }`), so just adding `| TailCall { ... }` + // here won't work. + // + // But at the same time we would like to highlight both calls in a function like + // `fn f() { if false { become f() } else { f() } }`, so we need to figure something out. if self.classifier.is_recursive_terminator(self.tcx, self.body, terminator) { self.reachable_recursive_calls.push(terminator.source_info.span); } diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs index 706bb796349..574da949b0e 100644 --- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs +++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs @@ -145,6 +145,7 @@ where | TerminatorKind::InlineAsm { .. } | TerminatorKind::UnwindResume | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::SwitchInt { .. } | TerminatorKind::Unreachable | TerminatorKind::Yield { .. } => {} diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs index 29169c31263..f850a710277 100644 --- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs @@ -288,6 +288,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> { | TerminatorKind::Goto { .. } | TerminatorKind::UnwindResume | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::SwitchInt { .. } | TerminatorKind::Unreachable => {} } @@ -325,6 +326,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> { | TerminatorKind::Goto { .. } | TerminatorKind::UnwindResume | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::SwitchInt { .. } | TerminatorKind::Unreachable => {} } diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index 1fb77bef3d4..7b39db821d8 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -489,6 +489,12 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> { self.gather_init(destination.as_ref(), InitKind::NonPanicPathOnly); } } + TerminatorKind::TailCall { ref func, ref args, .. } => { + self.gather_operand(func); + for arg in args { + self.gather_operand(&arg.node); + } + } TerminatorKind::InlineAsm { template: _, ref operands, diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 7c1ff6fda53..1582c2e8a90 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -269,6 +269,9 @@ pub trait ValueAnalysis<'tcx> { TerminatorKind::SwitchInt { discr, targets } => { return self.handle_switch_int(discr, targets, state); } + TerminatorKind::TailCall { .. } => { + // FIXME(explicit_tail_calls): determine if we need to do something here (probably not) + } TerminatorKind::Goto { .. } | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 9d44001f915..fa5da89e8ba 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -1367,6 +1367,10 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool { | TerminatorKind::Call { .. } | TerminatorKind::InlineAsm { .. } | TerminatorKind::Assert { .. } => return true, + + TerminatorKind::TailCall { .. } => { + unreachable!("tail calls can't be present in generators") + } } } @@ -1916,6 +1920,7 @@ impl<'tcx> Visitor<'tcx> for EnsureCoroutineFieldAssignmentsNeverAlias<'_> { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::Drop { .. } | TerminatorKind::Assert { .. } diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 3333bebff3a..a1c1422912e 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -40,7 +40,9 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { fn is_call_like(bbd: &BasicBlockData<'_>) -> bool { use TerminatorKind::*; match bbd.terminator().kind { - Call { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => true, + Call { .. } | TailCall { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => { + true + } Goto { .. } | SwitchInt { .. } @@ -137,6 +139,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { self.penalty += LANDINGPAD_PENALTY; } } + TerminatorKind::TailCall { .. } => { + self.penalty += CALL_PENALTY; + } TerminatorKind::SwitchInt { discr, targets } => { if discr.constant().is_some() { // Not only will this become a `Goto`, but likely other diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 360dccb240d..83fb9ff9743 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -358,9 +358,12 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera } // These terminators have no coverage-relevant successors. - CoroutineDrop | Return | Unreachable | UnwindResume | UnwindTerminate(_) => { - CoverageSuccessors::NotChainable(&[]) - } + CoroutineDrop + | Return + | TailCall { .. } + | Unreachable + | UnwindResume + | UnwindTerminate(_) => CoverageSuccessors::NotChainable(&[]), } } diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 2ca166929ee..a0f8f580b1d 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -193,7 +193,8 @@ fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { | TerminatorKind::Goto { .. } => None, // Call `func` operand can have a more specific span when part of a chain of calls - | TerminatorKind::Call { ref func, .. } => { + TerminatorKind::Call { ref func, .. } + | TerminatorKind::TailCall { ref func, .. } => { let mut span = terminator.source_info.span; if let mir::Operand::Constant(box constant) = func { if constant.span.lo() > span.lo() { diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index b1016c0867c..ab73a8af317 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -628,6 +628,12 @@ impl WriteInfo { self.add_operand(&arg.node); } } + TerminatorKind::TailCall { func, args, .. } => { + self.add_operand(func); + for arg in args { + self.add_operand(&arg.node); + } + } TerminatorKind::InlineAsm { operands, .. } => { for asm_operand in operands { match asm_operand { diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 5075e072754..fe73715480f 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -383,6 +383,8 @@ impl<'tcx> Inliner<'tcx> { ) -> Option<CallSite<'tcx>> { // Only consider direct calls to functions let terminator = bb_data.terminator(); + + // FIXME(explicit_tail_calls): figure out if we can inline tail calls if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { let func_ty = func.ty(caller_body, self.tcx); if let ty::FnDef(def_id, args) = *func_ty.kind() { @@ -550,6 +552,9 @@ impl<'tcx> Inliner<'tcx> { // inline-asm is detected. LLVM will still possibly do an inline later on // if the no-attribute function ends up with the same instruction set anyway. return Err("Cannot move inline-asm across instruction sets"); + } else if let TerminatorKind::TailCall { .. } = term.kind { + // FIXME(explicit_tail_calls): figure out how exactly functions containing tail calls can be inlined (and if they even should) + return Err("can't inline functions with tail calls"); } else { work_list.extend(term.successors()) } @@ -1038,6 +1043,10 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { *target = self.map_block(*target); *unwind = self.map_unwind(*unwind); } + TerminatorKind::TailCall { .. } => { + // check_mir_body forbids tail calls + unreachable!() + } TerminatorKind::Call { ref mut target, ref mut unwind, .. } => { if let Some(ref mut tgt) = *target { *tgt = self.map_block(*tgt); diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 85a61f95ca3..97ec0cb39de 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -596,6 +596,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"), // Disallowed during optimizations. diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index 8d6c00bbedb..82ad8879d17 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -799,6 +799,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::Drop { .. } | TerminatorKind::Yield { .. } diff --git a/compiler/rustc_mir_transform/src/mentioned_items.rs b/compiler/rustc_mir_transform/src/mentioned_items.rs index db2bb60bdac..d928d7cf764 100644 --- a/compiler/rustc_mir_transform/src/mentioned_items.rs +++ b/compiler/rustc_mir_transform/src/mentioned_items.rs @@ -38,7 +38,7 @@ impl<'tcx> Visitor<'tcx> for MentionedItemsVisitor<'_, 'tcx> { self.super_terminator(terminator, location); let span = || self.body.source_info(location).span; match &terminator.kind { - mir::TerminatorKind::Call { func, .. } => { + mir::TerminatorKind::Call { func, .. } | mir::TerminatorKind::TailCall { func, .. } => { let callee_ty = func.ty(self.body, self.tcx); self.mentioned_items .push(Spanned { node: MentionedItem::Fn(callee_ty), span: span() }); diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index fb52bfa468a..1df5737e859 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -75,6 +75,7 @@ impl RemoveNoopLandingPads { | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Unreachable | TerminatorKind::Call { .. } + | TerminatorKind::TailCall { .. } | TerminatorKind::Assert { .. } | TerminatorKind::Drop { .. } | TerminatorKind::InlineAsm { .. } => false, diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 2018a8fe667..ab5c25c4937 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -400,40 +400,44 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { self.check_edge(location, *target, EdgeKind::Normal); self.check_unwind_edge(location, *unwind); } - TerminatorKind::Call { args, destination, target, unwind, .. } => { - if let Some(target) = target { - self.check_edge(location, *target, EdgeKind::Normal); - } - self.check_unwind_edge(location, *unwind); + TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => { + // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks + if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind { + if let Some(target) = target { + self.check_edge(location, target, EdgeKind::Normal); + } + self.check_unwind_edge(location, unwind); + + // The code generation assumes that there are no critical call edges. The assumption + // is used to simplify inserting code that should be executed along the return edge + // from the call. FIXME(tmiasko): Since this is a strictly code generation concern, + // the code generation should be responsible for handling it. + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) + && self.is_critical_call_edge(target, unwind) + { + self.fail( + location, + format!( + "encountered critical edge in `Call` terminator {:?}", + terminator.kind, + ), + ); + } - // The code generation assumes that there are no critical call edges. The assumption - // is used to simplify inserting code that should be executed along the return edge - // from the call. FIXME(tmiasko): Since this is a strictly code generation concern, - // the code generation should be responsible for handling it. - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) - && self.is_critical_call_edge(*target, *unwind) - { - self.fail( - location, - format!( - "encountered critical edge in `Call` terminator {:?}", - terminator.kind, - ), - ); + // The call destination place and Operand::Move place used as an argument might be + // passed by a reference to the callee. Consequently they cannot be packed. + if is_within_packed(self.tcx, &self.body.local_decls, destination).is_some() { + // This is bad! The callee will expect the memory to be aligned. + self.fail( + location, + format!( + "encountered packed place in `Call` terminator destination: {:?}", + terminator.kind, + ), + ); + } } - // The call destination place and Operand::Move place used as an argument might be - // passed by a reference to the callee. Consequently they cannot be packed. - if is_within_packed(self.tcx, &self.body.local_decls, *destination).is_some() { - // This is bad! The callee will expect the memory to be aligned. - self.fail( - location, - format!( - "encountered packed place in `Call` terminator destination: {:?}", - terminator.kind, - ), - ); - } for arg in args { if let Operand::Move(place) = &arg.node { if is_within_packed(self.tcx, &self.body.local_decls, *place).is_some() { @@ -1498,15 +1502,22 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } } - TerminatorKind::Call { func, .. } => { + TerminatorKind::Call { func, .. } | TerminatorKind::TailCall { func, .. } => { let func_ty = func.ty(&self.body.local_decls, self.tcx); match func_ty.kind() { ty::FnPtr(..) | ty::FnDef(..) => {} _ => self.fail( location, - format!("encountered non-callable type {func_ty} in `Call` terminator"), + format!( + "encountered non-callable type {func_ty} in `{}` terminator", + terminator.kind.name() + ), ), } + + if let TerminatorKind::TailCall { .. } = terminator.kind { + // FIXME(explicit_tail_calls): implement tail-call specific checks here (such as signature matching, forbidding closures, etc) + } } TerminatorKind::Assert { cond, .. } => { let cond_ty = cond.ty(&self.body.local_decls, self.tcx); diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 005a22f642f..bfd505c0672 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -755,7 +755,8 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> { }; match terminator.kind { - mir::TerminatorKind::Call { ref func, ref args, ref fn_span, .. } => { + mir::TerminatorKind::Call { ref func, ref args, ref fn_span, .. } + | mir::TerminatorKind::TailCall { ref func, ref args, ref fn_span } => { let callee_ty = func.ty(self.body, tcx); // *Before* monomorphizing, record that we already handled this mention. self.used_mentioned_items.insert(MentionedItem::Fn(callee_ty)); diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs index 9d5a14b5145..8bfaa112c2f 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs @@ -644,6 +644,7 @@ impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> { target: target.map(|t| t.as_usize()), unwind: unwind.stable(tables), }, + mir::TerminatorKind::TailCall { func: _, args: _, fn_span: _ } => todo!(), mir::TerminatorKind::Assert { cond, expected, msg, target, unwind } => { TerminatorKind::Assert { cond: cond.stable(tables), |