interpret: refactor function call handling to be better-abstracted

1 files changed, 0 insertions, 1073 deletions

diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
deleted file mode 100644
index 47d0e22b527..00000000000
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ /dev/null
@@ -1,1073 +0,0 @@
-use std::borrow::Cow;
-
-use either::Either;
-use rustc_middle::ty::layout::{FnAbiOf, IntegerExt, LayoutOf, TyAndLayout};
-use rustc_middle::ty::{self, AdtDef, Instance, Ty};
-use rustc_middle::{bug, mir, span_bug};
-use rustc_span::source_map::Spanned;
-use rustc_span::sym;
-use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
-use rustc_target::abi::{self, FieldIdx, Integer};
-use rustc_target::spec::abi::Abi;
-use tracing::trace;
-
-use super::{
-    throw_ub, throw_ub_custom, throw_unsup_format, CtfeProvenance, FnVal, ImmTy, InterpCx,
-    InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, Projectable, Provenance, Scalar,
-    StackPopCleanup,
-};
-use crate::fluent_generated as fluent;
-use crate::interpret::eval_context::StackPopInfo;
-use crate::interpret::ReturnAction;
-
-/// An argment passed to a function.
-#[derive(Clone, Debug)]
-pub enum FnArg<'tcx, Prov: Provenance = CtfeProvenance> {
-    /// Pass a copy of the given operand.
-    Copy(OpTy<'tcx, Prov>),
-    /// Allow for the argument to be passed in-place: destroy the value originally stored at that place and
-    /// make the place inaccessible for the duration of the function call.
-    InPlace(MPlaceTy<'tcx, Prov>),
-}
-
-impl<'tcx, Prov: Provenance> FnArg<'tcx, Prov> {
-    pub fn layout(&self) -> &TyAndLayout<'tcx> {
-        match self {
-            FnArg::Copy(op) => &op.layout,
-            FnArg::InPlace(mplace) => &mplace.layout,
-        }
-    }
-}
-
-struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> {
-    callee: FnVal<'tcx, M::ExtraFnVal>,
-    args: Vec<FnArg<'tcx, M::Provenance>>,
-    fn_sig: ty::FnSig<'tcx>,
-    fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
-    /// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`])
-    with_caller_location: bool,
-}
-
-impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
-    /// Make a copy of the given fn_arg. Any `InPlace` are degenerated to copies, no protection of the
-    /// original memory occurs.
-    pub fn copy_fn_arg(&self, arg: &FnArg<'tcx, M::Provenance>) -> OpTy<'tcx, M::Provenance> {
-        match arg {
-            FnArg::Copy(op) => op.clone(),
-            FnArg::InPlace(mplace) => mplace.clone().into(),
-        }
-    }
-
-    /// Make a copy of the given fn_args. Any `InPlace` are degenerated to copies, no protection of the
-    /// original memory occurs.
-    pub fn copy_fn_args(
-        &self,
-        args: &[FnArg<'tcx, M::Provenance>],
-    ) -> Vec<OpTy<'tcx, M::Provenance>> {
-        args.iter().map(|fn_arg| self.copy_fn_arg(fn_arg)).collect()
-    }
-
-    pub fn fn_arg_field(
-        &self,
-        arg: &FnArg<'tcx, M::Provenance>,
-        field: usize,
-    ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
-        Ok(match arg {
-            FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?),
-            FnArg::InPlace(mplace) => FnArg::InPlace(self.project_field(mplace, field)?),
-        })
-    }
-
-    pub(super) fn eval_terminator(
-        &mut self,
-        terminator: &mir::Terminator<'tcx>,
-    ) -> InterpResult<'tcx> {
-        use rustc_middle::mir::TerminatorKind::*;
-        match terminator.kind {
-            Return => {
-                self.return_from_current_stack_frame(/* unwinding */ false)?
-            }
-
-            Goto { target } => self.go_to_block(target),
-
-            SwitchInt { ref discr, ref targets } => {
-                let discr = self.read_immediate(&self.eval_operand(discr, None)?)?;
-                trace!("SwitchInt({:?})", *discr);
-
-                // Branch to the `otherwise` case by default, if no match is found.
-                let mut target_block = targets.otherwise();
-
-                for (const_int, target) in targets.iter() {
-                    // Compare using MIR BinOp::Eq, to also support pointer values.
-                    // (Avoiding `self.binary_op` as that does some redundant layout computation.)
-                    let res = self.binary_op(
-                        mir::BinOp::Eq,
-                        &discr,
-                        &ImmTy::from_uint(const_int, discr.layout),
-                    )?;
-                    if res.to_scalar().to_bool()? {
-                        target_block = target;
-                        break;
-                    }
-                }
-
-                self.go_to_block(target_block);
-            }
-
-            Call {
-                ref func,
-                ref args,
-                destination,
-                target,
-                unwind,
-                call_source: _,
-                fn_span: _,
-            } => {
-                let old_stack = self.frame_idx();
-                let old_loc = self.frame().loc;
-
-                let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
-                    self.eval_callee_and_args(terminator, func, args)?;
-
-                let destination = self.force_allocation(&self.eval_place(destination)?)?;
-                self.eval_fn_call(
-                    callee,
-                    (fn_sig.abi, fn_abi),
-                    &args,
-                    with_caller_location,
-                    &destination,
-                    target,
-                    if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
-                )?;
-                // Sanity-check that `eval_fn_call` either pushed a new frame or
-                // did a jump to another block.
-                if self.frame_idx() == old_stack && self.frame().loc == old_loc {
-                    span_bug!(terminator.source_info.span, "evaluating this call made no progress");
-                }
-            }
-
-            TailCall { ref func, ref args, fn_span: _ } => {
-                let old_frame_idx = self.frame_idx();
-
-                let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
-                    self.eval_callee_and_args(terminator, func, args)?;
-
-                self.eval_fn_tail_call(callee, (fn_sig.abi, fn_abi), &args, with_caller_location)?;
-
-                if self.frame_idx() != old_frame_idx {
-                    span_bug!(
-                        terminator.source_info.span,
-                        "evaluating this tail call pushed a new stack frame"
-                    );
-                }
-            }
-
-            Drop { place, target, unwind, replace: _ } => {
-                let place = self.eval_place(place)?;
-                let instance = Instance::resolve_drop_in_place(*self.tcx, place.layout.ty);
-                if let ty::InstanceKind::DropGlue(_, None) = instance.def {
-                    // This is the branch we enter if and only if the dropped type has no drop glue
-                    // whatsoever. This can happen as a result of monomorphizing a drop of a
-                    // generic. In order to make sure that generic and non-generic code behaves
-                    // roughly the same (and in keeping with Mir semantics) we do nothing here.
-                    self.go_to_block(target);
-                    return Ok(());
-                }
-                trace!("TerminatorKind::drop: {:?}, type {}", place, place.layout.ty);
-                self.drop_in_place(&place, instance, target, unwind)?;
-            }
-
-            Assert { ref cond, expected, ref msg, target, unwind } => {
-                let ignored =
-                    M::ignore_optional_overflow_checks(self) && msg.is_optional_overflow_check();
-                let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?;
-                if ignored || expected == cond_val {
-                    self.go_to_block(target);
-                } else {
-                    M::assert_panic(self, msg, unwind)?;
-                }
-            }
-
-            UnwindTerminate(reason) => {
-                M::unwind_terminate(self, reason)?;
-            }
-
-            // When we encounter Resume, we've finished unwinding
-            // cleanup for the current stack frame. We pop it in order
-            // to continue unwinding the next frame
-            UnwindResume => {
-                trace!("unwinding: resuming from cleanup");
-                // By definition, a Resume terminator means
-                // that we're unwinding
-                self.return_from_current_stack_frame(/* unwinding */ true)?;
-                return Ok(());
-            }
-
-            // It is UB to ever encounter this.
-            Unreachable => throw_ub!(Unreachable),
-
-            // These should never occur for MIR we actually run.
-            FalseEdge { .. } | FalseUnwind { .. } | Yield { .. } | CoroutineDrop => span_bug!(
-                terminator.source_info.span,
-                "{:#?} should have been eliminated by MIR pass",
-                terminator.kind
-            ),
-
-            InlineAsm { template, ref operands, options, ref targets, .. } => {
-                M::eval_inline_asm(self, template, operands, options, targets)?;
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Evaluate the arguments of a function call
-    pub(super) fn eval_fn_call_arguments(
-        &self,
-        ops: &[Spanned<mir::Operand<'tcx>>],
-    ) -> InterpResult<'tcx, Vec<FnArg<'tcx, M::Provenance>>> {
-        ops.iter()
-            .map(|op| {
-                let arg = match &op.node {
-                    mir::Operand::Copy(_) | mir::Operand::Constant(_) => {
-                        // Make a regular copy.
-                        let op = self.eval_operand(&op.node, None)?;
-                        FnArg::Copy(op)
-                    }
-                    mir::Operand::Move(place) => {
-                        // If this place lives in memory, preserve its location.
-                        // We call `place_to_op` which will be an `MPlaceTy` whenever there exists
-                        // an mplace for this place. (This is in contrast to `PlaceTy::as_mplace_or_local`
-                        // which can return a local even if that has an mplace.)
-                        let place = self.eval_place(*place)?;
-                        let op = self.place_to_op(&place)?;
-
-                        match op.as_mplace_or_imm() {
-                            Either::Left(mplace) => FnArg::InPlace(mplace),
-                            Either::Right(_imm) => {
-                                // This argument doesn't live in memory, so there's no place
-                                // to make inaccessible during the call.
-                                // We rely on there not being any stray `PlaceTy` that would let the
-                                // caller directly access this local!
-                                // This is also crucial for tail calls, where we want the `FnArg` to
-                                // stay valid when the old stack frame gets popped.
-                                FnArg::Copy(op)
-                            }
-                        }
-                    }
-                };
-
-                Ok(arg)
-            })
-            .collect()
-    }
-
-    /// Find the wrapped inner type of a transparent wrapper.
-    /// Must not be called on 1-ZST (as they don't have a uniquely defined "wrapped field").
-    ///
-    /// We work with `TyAndLayout` here since that makes it much easier to iterate over all fields.
-    fn unfold_transparent(
-        &self,
-        layout: TyAndLayout<'tcx>,
-        may_unfold: impl Fn(AdtDef<'tcx>) -> bool,
-    ) -> TyAndLayout<'tcx> {
-        match layout.ty.kind() {
-            ty::Adt(adt_def, _) if adt_def.repr().transparent() && may_unfold(*adt_def) => {
-                assert!(!adt_def.is_enum());
-                // Find the non-1-ZST field, and recurse.
-                let (_, field) = layout.non_1zst_field(self).unwrap();
-                self.unfold_transparent(field, may_unfold)
-            }
-            // Not a transparent type, no further unfolding.
-            _ => layout,
-        }
-    }
-
-    /// Unwrap types that are guaranteed a null-pointer-optimization
-    fn unfold_npo(&self, layout: TyAndLayout<'tcx>) -> InterpResult<'tcx, TyAndLayout<'tcx>> {
-        // Check if this is `Option` wrapping some type or if this is `Result` wrapping a 1-ZST and
-        // another type.
-        let ty::Adt(def, args) = layout.ty.kind() else {
-            // Not an ADT, so definitely no NPO.
-            return Ok(layout);
-        };
-        let inner = if self.tcx.is_diagnostic_item(sym::Option, def.did()) {
-            // The wrapped type is the only arg.
-            self.layout_of(args[0].as_type().unwrap())?
-        } else if self.tcx.is_diagnostic_item(sym::Result, def.did()) {
-            // We want to extract which (if any) of the args is not a 1-ZST.
-            let lhs = self.layout_of(args[0].as_type().unwrap())?;
-            let rhs = self.layout_of(args[1].as_type().unwrap())?;
-            if lhs.is_1zst() {
-                rhs
-            } else if rhs.is_1zst() {
-                lhs
-            } else {
-                return Ok(layout); // no NPO
-            }
-        } else {
-            return Ok(layout); // no NPO
-        };
-
-        // Check if the inner type is one of the NPO-guaranteed ones.
-        // For that we first unpeel transparent *structs* (but not unions).
-        let is_npo = |def: AdtDef<'tcx>| {
-            self.tcx.has_attr(def.did(), sym::rustc_nonnull_optimization_guaranteed)
-        };
-        let inner = self.unfold_transparent(inner, /* may_unfold */ |def| {
-            // Stop at NPO tpyes so that we don't miss that attribute in the check below!
-            def.is_struct() && !is_npo(def)
-        });
-        Ok(match inner.ty.kind() {
-            ty::Ref(..) | ty::FnPtr(..) => {
-                // Option<&T> behaves like &T, and same for fn()
-                inner
-            }
-            ty::Adt(def, _) if is_npo(*def) => {
-                // Once we found a `nonnull_optimization_guaranteed` type, further strip off
-                // newtype structs from it to find the underlying ABI type.
-                self.unfold_transparent(inner, /* may_unfold */ |def| def.is_struct())
-            }
-            _ => {
-                // Everything else we do not unfold.
-                layout
-            }
-        })
-    }
-
-    /// Check if these two layouts look like they are fn-ABI-compatible.
-    /// (We also compare the `PassMode`, so this doesn't have to check everything. But it turns out
-    /// that only checking the `PassMode` is insufficient.)
-    fn layout_compat(
-        &self,
-        caller: TyAndLayout<'tcx>,
-        callee: TyAndLayout<'tcx>,
-    ) -> InterpResult<'tcx, bool> {
-        // Fast path: equal types are definitely compatible.
-        if caller.ty == callee.ty {
-            return Ok(true);
-        }
-        // 1-ZST are compatible with all 1-ZST (and with nothing else).
-        if caller.is_1zst() || callee.is_1zst() {
-            return Ok(caller.is_1zst() && callee.is_1zst());
-        }
-        // Unfold newtypes and NPO optimizations.
-        let unfold = |layout: TyAndLayout<'tcx>| {
-            self.unfold_npo(self.unfold_transparent(layout, /* may_unfold */ |_def| true))
-        };
-        let caller = unfold(caller)?;
-        let callee = unfold(callee)?;
-        // Now see if these inner types are compatible.
-
-        // Compatible pointer types. For thin pointers, we have to accept even non-`repr(transparent)`
-        // things as compatible due to `DispatchFromDyn`. For instance, `Rc<i32>` and `*mut i32`
-        // must be compatible. So we just accept everything with Pointer ABI as compatible,
-        // even if this will accept some code that is not stably guaranteed to work.
-        // This also handles function pointers.
-        let thin_pointer = |layout: TyAndLayout<'tcx>| match layout.abi {
-            abi::Abi::Scalar(s) => match s.primitive() {
-                abi::Primitive::Pointer(addr_space) => Some(addr_space),
-                _ => None,
-            },
-            _ => None,
-        };
-        if let (Some(caller), Some(callee)) = (thin_pointer(caller), thin_pointer(callee)) {
-            return Ok(caller == callee);
-        }
-        // For wide pointers we have to get the pointee type.
-        let pointee_ty = |ty: Ty<'tcx>| -> InterpResult<'tcx, Option<Ty<'tcx>>> {
-            // We cannot use `builtin_deref` here since we need to reject `Box<T, MyAlloc>`.
-            Ok(Some(match ty.kind() {
-                ty::Ref(_, ty, _) => *ty,
-                ty::RawPtr(ty, _) => *ty,
-                // We only accept `Box` with the default allocator.
-                _ if ty.is_box_global(*self.tcx) => ty.boxed_ty(),
-                _ => return Ok(None),
-            }))
-        };
-        if let (Some(caller), Some(callee)) = (pointee_ty(caller.ty)?, pointee_ty(callee.ty)?) {
-            // This is okay if they have the same metadata type.
-            let meta_ty = |ty: Ty<'tcx>| {
-                // Even if `ty` is normalized, the search for the unsized tail will project
-                // to fields, which can yield non-normalized types. So we need to provide a
-                // normalization function.
-                let normalize = |ty| self.tcx.normalize_erasing_regions(self.param_env, ty);
-                ty.ptr_metadata_ty(*self.tcx, normalize)
-            };
-            return Ok(meta_ty(caller) == meta_ty(callee));
-        }
-
-        // Compatible integer types (in particular, usize vs ptr-sized-u32/u64).
-        // `char` counts as `u32.`
-        let int_ty = |ty: Ty<'tcx>| {
-            Some(match ty.kind() {
-                ty::Int(ity) => (Integer::from_int_ty(&self.tcx, *ity), /* signed */ true),
-                ty::Uint(uty) => (Integer::from_uint_ty(&self.tcx, *uty), /* signed */ false),
-                ty::Char => (Integer::I32, /* signed */ false),
-                _ => return None,
-            })
-        };
-        if let (Some(caller), Some(callee)) = (int_ty(caller.ty), int_ty(callee.ty)) {
-            // This is okay if they are the same integer type.
-            return Ok(caller == callee);
-        }
-
-        // Fall back to exact equality.
-        // FIXME: We are missing the rules for "repr(C) wrapping compatible types".
-        Ok(caller == callee)
-    }
-
-    fn check_argument_compat(
-        &self,
-        caller_abi: &ArgAbi<'tcx, Ty<'tcx>>,
-        callee_abi: &ArgAbi<'tcx, Ty<'tcx>>,
-    ) -> InterpResult<'tcx, bool> {
-        // We do not want to accept things as ABI-compatible that just "happen to be" compatible on the current target,
-        // so we implement a type-based check that reflects the guaranteed rules for ABI compatibility.
-        if self.layout_compat(caller_abi.layout, callee_abi.layout)? {
-            // Ensure that our checks imply actual ABI compatibility for this concrete call.
-            assert!(caller_abi.eq_abi(callee_abi));
-            return Ok(true);
-        } else {
-            trace!(
-                "check_argument_compat: incompatible ABIs:\ncaller: {:?}\ncallee: {:?}",
-                caller_abi, callee_abi
-            );
-            return Ok(false);
-        }
-    }
-
-    /// Initialize a single callee argument, checking the types for compatibility.
-    fn pass_argument<'x, 'y>(
-        &mut self,
-        caller_args: &mut impl Iterator<
-            Item = (&'x FnArg<'tcx, M::Provenance>, &'y ArgAbi<'tcx, Ty<'tcx>>),
-        >,
-        callee_abi: &ArgAbi<'tcx, Ty<'tcx>>,
-        callee_arg: &mir::Place<'tcx>,
-        callee_ty: Ty<'tcx>,
-        already_live: bool,
-    ) -> InterpResult<'tcx>
-    where
-        'tcx: 'x,
-        'tcx: 'y,
-    {
-        assert_eq!(callee_ty, callee_abi.layout.ty);
-        if matches!(callee_abi.mode, PassMode::Ignore) {
-            // This one is skipped. Still must be made live though!
-            if !already_live {
-                self.storage_live(callee_arg.as_local().unwrap())?;
-            }
-            return Ok(());
-        }
-        // Find next caller arg.
-        let Some((caller_arg, caller_abi)) = caller_args.next() else {
-            throw_ub_custom!(fluent::const_eval_not_enough_caller_args);
-        };
-        assert_eq!(caller_arg.layout().layout, caller_abi.layout.layout);
-        // Sadly we cannot assert that `caller_arg.layout().ty` and `caller_abi.layout.ty` are
-        // equal; in closures the types sometimes differ. We just hope that `caller_abi` is the
-        // right type to print to the user.
-
-        // Check compatibility
-        if !self.check_argument_compat(caller_abi, callee_abi)? {
-            throw_ub!(AbiMismatchArgument {
-                caller_ty: caller_abi.layout.ty,
-                callee_ty: callee_abi.layout.ty
-            });
-        }
-        // We work with a copy of the argument for now; if this is in-place argument passing, we
-        // will later protect the source it comes from. This means the callee cannot observe if we
-        // did in-place of by-copy argument passing, except for pointer equality tests.
-        let caller_arg_copy = self.copy_fn_arg(caller_arg);
-        if !already_live {
-            let local = callee_arg.as_local().unwrap();
-            let meta = caller_arg_copy.meta();
-            // `check_argument_compat` ensures that if metadata is needed, both have the same type,
-            // so we know they will use the metadata the same way.
-            assert!(!meta.has_meta() || caller_arg_copy.layout.ty == callee_ty);
-
-            self.storage_live_dyn(local, meta)?;
-        }
-        // Now we can finally actually evaluate the callee place.
-        let callee_arg = self.eval_place(*callee_arg)?;
-        // We allow some transmutes here.
-        // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This
-        // is true for all `copy_op`, but there are a lot of special cases for argument passing
-        // specifically.)
-        self.copy_op_allow_transmute(&caller_arg_copy, &callee_arg)?;
-        // If this was an in-place pass, protect the place it comes from for the duration of the call.
-        if let FnArg::InPlace(mplace) = caller_arg {
-            M::protect_in_place_function_argument(self, mplace)?;
-        }
-        Ok(())
-    }
-
-    /// Shared part of `Call` and `TailCall` implementation — finding and evaluating all the
-    /// necessary information about callee and arguments to make a call.
-    fn eval_callee_and_args(
-        &self,
-        terminator: &mir::Terminator<'tcx>,
-        func: &mir::Operand<'tcx>,
-        args: &[Spanned<mir::Operand<'tcx>>],
-    ) -> InterpResult<'tcx, EvaluatedCalleeAndArgs<'tcx, M>> {
-        let func = self.eval_operand(func, None)?;
-        let args = self.eval_fn_call_arguments(args)?;
-
-        let fn_sig_binder = func.layout.ty.fn_sig(*self.tcx);
-        let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig_binder);
-        let extra_args = &args[fn_sig.inputs().len()..];
-        let extra_args =
-            self.tcx.mk_type_list_from_iter(extra_args.iter().map(|arg| arg.layout().ty));
-
-        let (callee, fn_abi, with_caller_location) = match *func.layout.ty.kind() {
-            ty::FnPtr(_sig) => {
-                let fn_ptr = self.read_pointer(&func)?;
-                let fn_val = self.get_ptr_fn(fn_ptr)?;
-                (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false)
-            }
-            ty::FnDef(def_id, args) => {
-                let instance = self.resolve(def_id, args)?;
-                (
-                    FnVal::Instance(instance),
-                    self.fn_abi_of_instance(instance, extra_args)?,
-                    instance.def.requires_caller_location(*self.tcx),
-                )
-            }
-            _ => {
-                span_bug!(terminator.source_info.span, "invalid callee of type {}", func.layout.ty)
-            }
-        };
-
-        Ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location })
-    }
-
-    /// Call this function -- pushing the stack frame and initializing the arguments.
-    ///
-    /// `caller_fn_abi` is used to determine if all the arguments are passed the proper way.
-    /// However, we also need `caller_abi` to determine if we need to do untupling of arguments.
-    ///
-    /// `with_caller_location` indicates whether the caller passed a caller location. Miri
-    /// implements caller locations without argument passing, but to match `FnAbi` we need to know
-    /// when those arguments are present.
-    pub(crate) fn eval_fn_call(
-        &mut self,
-        fn_val: FnVal<'tcx, M::ExtraFnVal>,
-        (caller_abi, caller_fn_abi): (Abi, &FnAbi<'tcx, Ty<'tcx>>),
-        args: &[FnArg<'tcx, M::Provenance>],
-        with_caller_location: bool,
-        destination: &MPlaceTy<'tcx, M::Provenance>,
-        target: Option<mir::BasicBlock>,
-        mut unwind: mir::UnwindAction,
-    ) -> InterpResult<'tcx> {
-        trace!("eval_fn_call: {:#?}", fn_val);
-
-        let instance = match fn_val {
-            FnVal::Instance(instance) => instance,
-            FnVal::Other(extra) => {
-                return M::call_extra_fn(
-                    self,
-                    extra,
-                    caller_abi,
-                    args,
-                    destination,
-                    target,
-                    unwind,
-                );
-            }
-        };
-
-        match instance.def {
-            ty::InstanceKind::Intrinsic(def_id) => {
-                assert!(self.tcx.intrinsic(def_id).is_some());
-                // FIXME: Should `InPlace` arguments be reset to uninit?
-                if let Some(fallback) = M::call_intrinsic(
-                    self,
-                    instance,
-                    &self.copy_fn_args(args),
-                    destination,
-                    target,
-                    unwind,
-                )? {
-                    assert!(!self.tcx.intrinsic(fallback.def_id()).unwrap().must_be_overridden);
-                    assert!(matches!(fallback.def, ty::InstanceKind::Item(_)));
-                    return self.eval_fn_call(
-                        FnVal::Instance(fallback),
-                        (caller_abi, caller_fn_abi),
-                        args,
-                        with_caller_location,
-                        destination,
-                        target,
-                        unwind,
-                    );
-                } else {
-                    Ok(())
-                }
-            }
-            ty::InstanceKind::VTableShim(..)
-            | ty::InstanceKind::ReifyShim(..)
-            | ty::InstanceKind::ClosureOnceShim { .. }
-            | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | ty::InstanceKind::CoroutineKindShim { .. }
-            | ty::InstanceKind::FnPtrShim(..)
-            | ty::InstanceKind::DropGlue(..)
-            | ty::InstanceKind::CloneShim(..)
-            | ty::InstanceKind::FnPtrAddrShim(..)
-            | ty::InstanceKind::ThreadLocalShim(..)
-            | ty::InstanceKind::AsyncDropGlueCtorShim(..)
-            | ty::InstanceKind::Item(_) => {
-                // We need MIR for this fn
-                let Some((body, instance)) = M::find_mir_or_eval_fn(
-                    self,
-                    instance,
-                    caller_abi,
-                    args,
-                    destination,
-                    target,
-                    unwind,
-                )?
-                else {
-                    return Ok(());
-                };
-
-                // Compute callee information using the `instance` returned by
-                // `find_mir_or_eval_fn`.
-                // FIXME: for variadic support, do we have to somehow determine callee's extra_args?
-                let callee_fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
-
-                if callee_fn_abi.c_variadic || caller_fn_abi.c_variadic {
-                    throw_unsup_format!("calling a c-variadic function is not supported");
-                }
-
-                if M::enforce_abi(self) {
-                    if caller_fn_abi.conv != callee_fn_abi.conv {
-                        throw_ub_custom!(
-                            fluent::const_eval_incompatible_calling_conventions,
-                            callee_conv = format!("{:?}", callee_fn_abi.conv),
-                            caller_conv = format!("{:?}", caller_fn_abi.conv),
-                        )
-                    }
-                }
-
-                // Check that all target features required by the callee (i.e., from
-                // the attribute `#[target_feature(enable = ...)]`) are enabled at
-                // compile time.
-                self.check_fn_target_features(instance)?;
-
-                if !callee_fn_abi.can_unwind {
-                    // The callee cannot unwind, so force the `Unreachable` unwind handling.
-                    unwind = mir::UnwindAction::Unreachable;
-                }
-
-                self.push_stack_frame(
-                    instance,
-                    body,
-                    destination,
-                    StackPopCleanup::Goto { ret: target, unwind },
-                )?;
-
-                // If an error is raised here, pop the frame again to get an accurate backtrace.
-                // To this end, we wrap it all in a `try` block.
-                let res: InterpResult<'tcx> = try {
-                    trace!(
-                        "caller ABI: {:?}, args: {:#?}",
-                        caller_abi,
-                        args.iter()
-                            .map(|arg| (
-                                arg.layout().ty,
-                                match arg {
-                                    FnArg::Copy(op) => format!("copy({op:?})"),
-                                    FnArg::InPlace(mplace) => format!("in-place({mplace:?})"),
-                                }
-                            ))
-                            .collect::<Vec<_>>()
-                    );
-                    trace!(
-                        "spread_arg: {:?}, locals: {:#?}",
-                        body.spread_arg,
-                        body.args_iter()
-                            .map(|local| (
-                                local,
-                                self.layout_of_local(self.frame(), local, None).unwrap().ty,
-                            ))
-                            .collect::<Vec<_>>()
-                    );
-
-                    // In principle, we have two iterators: Where the arguments come from, and where
-                    // they go to.
-
-                    // For where they come from: If the ABI is RustCall, we untuple the
-                    // last incoming argument. These two iterators do not have the same type,
-                    // so to keep the code paths uniform we accept an allocation
-                    // (for RustCall ABI only).
-                    let caller_args: Cow<'_, [FnArg<'tcx, M::Provenance>]> =
-                        if caller_abi == Abi::RustCall && !args.is_empty() {
-                            // Untuple
-                            let (untuple_arg, args) = args.split_last().unwrap();
-                            trace!("eval_fn_call: Will pass last argument by untupling");
-                            Cow::from(
-                                args.iter()
-                                    .map(|a| Ok(a.clone()))
-                                    .chain(
-                                        (0..untuple_arg.layout().fields.count())
-                                            .map(|i| self.fn_arg_field(untuple_arg, i)),
-                                    )
-                                    .collect::<InterpResult<'_, Vec<_>>>()?,
-                            )
-                        } else {
-                            // Plain arg passing
-                            Cow::from(args)
-                        };
-                    // If `with_caller_location` is set we pretend there is an extra argument (that
-                    // we will not pass).
-                    assert_eq!(
-                        caller_args.len() + if with_caller_location { 1 } else { 0 },
-                        caller_fn_abi.args.len(),
-                        "mismatch between caller ABI and caller arguments",
-                    );
-                    let mut caller_args = caller_args
-                        .iter()
-                        .zip(caller_fn_abi.args.iter())
-                        .filter(|arg_and_abi| !matches!(arg_and_abi.1.mode, PassMode::Ignore));
-
-                    // Now we have to spread them out across the callee's locals,
-                    // taking into account the `spread_arg`. If we could write
-                    // this is a single iterator (that handles `spread_arg`), then
-                    // `pass_argument` would be the loop body. It takes care to
-                    // not advance `caller_iter` for ignored arguments.
-                    let mut callee_args_abis = callee_fn_abi.args.iter();
-                    for local in body.args_iter() {
-                        // Construct the destination place for this argument. At this point all
-                        // locals are still dead, so we cannot construct a `PlaceTy`.
-                        let dest = mir::Place::from(local);
-                        // `layout_of_local` does more than just the instantiation we need to get the
-                        // type, but the result gets cached so this avoids calling the instantiation
-                        // query *again* the next time this local is accessed.
-                        let ty = self.layout_of_local(self.frame(), local, None)?.ty;
-                        if Some(local) == body.spread_arg {
-                            // Make the local live once, then fill in the value field by field.
-                            self.storage_live(local)?;
-                            // Must be a tuple
-                            let ty::Tuple(fields) = ty.kind() else {
-                                span_bug!(self.cur_span(), "non-tuple type for `spread_arg`: {ty}")
-                            };
-                            for (i, field_ty) in fields.iter().enumerate() {
-                                let dest = dest.project_deeper(
-                                    &[mir::ProjectionElem::Field(
-                                        FieldIdx::from_usize(i),
-                                        field_ty,
-                                    )],
-                                    *self.tcx,
-                                );
-                                let callee_abi = callee_args_abis.next().unwrap();
-                                self.pass_argument(
-                                    &mut caller_args,
-                                    callee_abi,
-                                    &dest,
-                                    field_ty,
-                                    /* already_live */ true,
-                                )?;
-                            }
-                        } else {
-                            // Normal argument. Cannot mark it as live yet, it might be unsized!
-                            let callee_abi = callee_args_abis.next().unwrap();
-                            self.pass_argument(
-                                &mut caller_args,
-                                callee_abi,
-                                &dest,
-                                ty,
-                                /* already_live */ false,
-                            )?;
-                        }
-                    }
-                    // If the callee needs a caller location, pretend we consume one more argument from the ABI.
-                    if instance.def.requires_caller_location(*self.tcx) {
-                        callee_args_abis.next().unwrap();
-                    }
-                    // Now we should have no more caller args or callee arg ABIs
-                    assert!(
-                        callee_args_abis.next().is_none(),
-                        "mismatch between callee ABI and callee body arguments"
-                    );
-                    if caller_args.next().is_some() {
-                        throw_ub_custom!(fluent::const_eval_too_many_caller_args);
-                    }
-                    // Don't forget to check the return type!
-                    if !self.check_argument_compat(&caller_fn_abi.ret, &callee_fn_abi.ret)? {
-                        throw_ub!(AbiMismatchReturn {
-                            caller_ty: caller_fn_abi.ret.layout.ty,
-                            callee_ty: callee_fn_abi.ret.layout.ty
-                        });
-                    }
-
-                    // Protect return place for in-place return value passing.
-                    M::protect_in_place_function_argument(self, &destination)?;
-
-                    // Don't forget to mark "initially live" locals as live.
-                    self.storage_live_for_always_live_locals()?;
-                };
-                match res {
-                    Err(err) => {
-                        self.stack_mut().pop();
-                        Err(err)
-                    }
-                    Ok(()) => Ok(()),
-                }
-            }
-            // `InstanceKind::Virtual` does not have callable MIR. Calls to `Virtual` instances must be
-            // codegen'd / interpreted as virtual calls through the vtable.
-            ty::InstanceKind::Virtual(def_id, idx) => {
-                let mut args = args.to_vec();
-                // We have to implement all "object safe receivers". So we have to go search for a
-                // pointer or `dyn Trait` type, but it could be wrapped in newtypes. So recursively
-                // unwrap those newtypes until we are there.
-                // An `InPlace` does nothing here, we keep the original receiver intact. We can't
-                // really pass the argument in-place anyway, and we are constructing a new
-                // `Immediate` receiver.
-                let mut receiver = self.copy_fn_arg(&args[0]);
-                let receiver_place = loop {
-                    match receiver.layout.ty.kind() {
-                        ty::Ref(..) | ty::RawPtr(..) => {
-                            // We do *not* use `deref_pointer` here: we don't want to conceptually
-                            // create a place that must be dereferenceable, since the receiver might
-                            // be a raw pointer and (for `*const dyn Trait`) we don't need to
-                            // actually access memory to resolve this method.
-                            // Also see <https://github.com/rust-lang/miri/issues/2786>.
-                            let val = self.read_immediate(&receiver)?;
-                            break self.ref_to_mplace(&val)?;
-                        }
-                        ty::Dynamic(.., ty::Dyn) => break receiver.assert_mem_place(), // no immediate unsized values
-                        ty::Dynamic(.., ty::DynStar) => {
-                            // Not clear how to handle this, so far we assume the receiver is always a pointer.
-                            span_bug!(
-                                self.cur_span(),
-                                "by-value calls on a `dyn*`... are those a thing?"
-                            );
-                        }
-                        _ => {
-                            // Not there yet, search for the only non-ZST field.
-                            // (The rules for `DispatchFromDyn` ensure there's exactly one such field.)
-                            let (idx, _) = receiver.layout.non_1zst_field(self).expect(
-                                "not exactly one non-1-ZST field in a `DispatchFromDyn` type",
-                            );
-                            receiver = self.project_field(&receiver, idx)?;
-                        }
-                    }
-                };
-
-                // Obtain the underlying trait we are working on, and the adjusted receiver argument.
-                let (trait_, dyn_ty, adjusted_recv) = if let ty::Dynamic(data, _, ty::DynStar) =
-                    receiver_place.layout.ty.kind()
-                {
-                    let recv = self.unpack_dyn_star(&receiver_place, data)?;
-
-                    (data.principal(), recv.layout.ty, recv.ptr())
-                } else {
-                    // Doesn't have to be a `dyn Trait`, but the unsized tail must be `dyn Trait`.
-                    // (For that reason we also cannot use `unpack_dyn_trait`.)
-                    let receiver_tail = self
-                        .tcx
-                        .struct_tail_erasing_lifetimes(receiver_place.layout.ty, self.param_env);
-                    let ty::Dynamic(receiver_trait, _, ty::Dyn) = receiver_tail.kind() else {
-                        span_bug!(
-                            self.cur_span(),
-                            "dynamic call on non-`dyn` type {}",
-                            receiver_tail
-                        )
-                    };
-                    assert!(receiver_place.layout.is_unsized());
-
-                    // Get the required information from the vtable.
-                    let vptr = receiver_place.meta().unwrap_meta().to_pointer(self)?;
-                    let dyn_ty = self.get_ptr_vtable_ty(vptr, Some(receiver_trait))?;
-
-                    // It might be surprising that we use a pointer as the receiver even if this
-                    // is a by-val case; this works because by-val passing of an unsized `dyn
-                    // Trait` to a function is actually desugared to a pointer.
-                    (receiver_trait.principal(), dyn_ty, receiver_place.ptr())
-                };
-
-                // Now determine the actual method to call. Usually we use the easy way of just
-                // looking up the method at index `idx`.
-                let vtable_entries = self.vtable_entries(trait_, dyn_ty);
-                let Some(ty::VtblEntry::Method(fn_inst)) = vtable_entries.get(idx).copied() else {
-                    // FIXME(fee1-dead) these could be variants of the UB info enum instead of this
-                    throw_ub_custom!(fluent::const_eval_dyn_call_not_a_method);
-                };
-                trace!("Virtual call dispatches to {fn_inst:#?}");
-                // We can also do the lookup based on `def_id` and `dyn_ty`, and check that that
-                // produces the same result.
-                if cfg!(debug_assertions) {
-                    let tcx = *self.tcx;
-
-                    let trait_def_id = tcx.trait_of_item(def_id).unwrap();
-                    let virtual_trait_ref =
-                        ty::TraitRef::from_method(tcx, trait_def_id, instance.args);
-                    let existential_trait_ref =
-                        ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref);
-                    let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty);
-
-                    let concrete_method = Instance::expect_resolve_for_vtable(
-                        tcx,
-                        self.param_env,
-                        def_id,
-                        instance.args.rebase_onto(tcx, trait_def_id, concrete_trait_ref.args),
-                        self.cur_span(),
-                    );
-                    assert_eq!(fn_inst, concrete_method);
-                }
-
-                // Adjust receiver argument. Layout can be any (thin) ptr.
-                let receiver_ty = Ty::new_mut_ptr(self.tcx.tcx, dyn_ty);
-                args[0] = FnArg::Copy(
-                    ImmTy::from_immediate(
-                        Scalar::from_maybe_pointer(adjusted_recv, self).into(),
-                        self.layout_of(receiver_ty)?,
-                    )
-                    .into(),
-                );
-                trace!("Patched receiver operand to {:#?}", args[0]);
-                // Need to also adjust the type in the ABI. Strangely, the layout there is actually
-                // already fine! Just the type is bogus. This is due to what `force_thin_self_ptr`
-                // does in `fn_abi_new_uncached`; supposedly, codegen relies on having the bogus
-                // type, so we just patch this up locally.
-                let mut caller_fn_abi = caller_fn_abi.clone();
-                caller_fn_abi.args[0].layout.ty = receiver_ty;
-
-                // recurse with concrete function
-                self.eval_fn_call(
-                    FnVal::Instance(fn_inst),
-                    (caller_abi, &caller_fn_abi),
-                    &args,
-                    with_caller_location,
-                    destination,
-                    target,
-                    unwind,
-                )
-            }
-        }
-    }
-
-    pub(crate) fn eval_fn_tail_call(
-        &mut self,
-        fn_val: FnVal<'tcx, M::ExtraFnVal>,
-        (caller_abi, caller_fn_abi): (Abi, &FnAbi<'tcx, Ty<'tcx>>),
-        args: &[FnArg<'tcx, M::Provenance>],
-        with_caller_location: bool,
-    ) -> InterpResult<'tcx> {
-        trace!("eval_fn_call: {:#?}", fn_val);
-
-        // This is the "canonical" implementation of tails calls,
-        // a pop of the current stack frame, followed by a normal call
-        // which pushes a new stack frame, with the return address from
-        // the popped stack frame.
-        //
-        // Note that we are using `pop_stack_frame` and not `return_from_current_stack_frame`,
-        // as the latter "executes" the goto to the return block, but we don't want to,
-        // only the tail called function should return to the current return block.
-        M::before_stack_pop(self, self.frame())?;
-
-        let StackPopInfo { return_action, return_to_block, return_place } =
-            self.pop_stack_frame(false)?;
-
-        assert_eq!(return_action, ReturnAction::Normal);
-
-        let StackPopCleanup::Goto { ret, unwind } = return_to_block else {
-            bug!("can't tailcall as root");
-        };
-
-        // FIXME(explicit_tail_calls):
-        //   we should check if both caller&callee can/n't unwind,
-        //   see <https://github.com/rust-lang/rust/pull/113128#issuecomment-1614979803>
-
-        self.eval_fn_call(
-            fn_val,
-            (caller_abi, caller_fn_abi),
-            args,
-            with_caller_location,
-            &return_place,
-            ret,
-            unwind,
-        )
-    }
-
-    fn check_fn_target_features(&self, instance: ty::Instance<'tcx>) -> InterpResult<'tcx, ()> {
-        // Calling functions with `#[target_feature]` is not unsafe on WASM, see #84988
-        let attrs = self.tcx.codegen_fn_attrs(instance.def_id());
-        if !self.tcx.sess.target.is_like_wasm
-            && attrs
-                .target_features
-                .iter()
-                .any(|feature| !self.tcx.sess.target_features.contains(feature))
-        {
-            throw_ub_custom!(
-                fluent::const_eval_unavailable_target_features_for_fn,
-                unavailable_feats = attrs
-                    .target_features
-                    .iter()
-                    .filter(|&feature| !self.tcx.sess.target_features.contains(feature))
-                    .fold(String::new(), |mut s, feature| {
-                        if !s.is_empty() {
-                            s.push_str(", ");
-                        }
-                        s.push_str(feature.as_str());
-                        s
-                    }),
-            );
-        }
-        Ok(())
-    }
-
-    fn drop_in_place(
-        &mut self,
-        place: &PlaceTy<'tcx, M::Provenance>,
-        instance: ty::Instance<'tcx>,
-        target: mir::BasicBlock,
-        unwind: mir::UnwindAction,
-    ) -> InterpResult<'tcx> {
-        trace!("drop_in_place: {:?},\n  instance={:?}", place, instance);
-        // We take the address of the object. This may well be unaligned, which is fine
-        // for us here. However, unaligned accesses will probably make the actual drop
-        // implementation fail -- a problem shared by rustc.
-        let place = self.force_allocation(place)?;
-
-        // We behave a bit different from codegen here.
-        // Codegen creates an `InstanceKind::Virtual` with index 0 (the slot of the drop method) and
-        // then dispatches that to the normal call machinery. However, our call machinery currently
-        // only supports calling `VtblEntry::Method`; it would choke on a `MetadataDropInPlace`. So
-        // instead we do the virtual call stuff ourselves. It's easier here than in `eval_fn_call`
-        // since we can just get a place of the underlying type and use `mplace_to_ref`.
-        let place = match place.layout.ty.kind() {
-            ty::Dynamic(data, _, ty::Dyn) => {
-                // Dropping a trait object. Need to find actual drop fn.
-                self.unpack_dyn_trait(&place, data)?
-            }
-            ty::Dynamic(data, _, ty::DynStar) => {
-                // Dropping a `dyn*`. Need to find actual drop fn.
-                self.unpack_dyn_star(&place, data)?
-            }
-            _ => {
-                debug_assert_eq!(
-                    instance,
-                    ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty)
-                );
-                place
-            }
-        };
-        let instance = ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty);
-        let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
-
-        let arg = self.mplace_to_ref(&place)?;
-        let ret = MPlaceTy::fake_alloc_zst(self.layout_of(self.tcx.types.unit)?);
-
-        self.eval_fn_call(
-            FnVal::Instance(instance),
-            (Abi::Rust, fn_abi),
-            &[FnArg::Copy(arg.into())],
-            false,
-            &ret.into(),
-            Some(target),
-            unwind,
-        )
-    }
-}