diff options
Diffstat (limited to 'compiler/rustc_const_eval/src/interpret')
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/call.rs | 61 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/cast.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/discriminant.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/intrinsics.rs | 152 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/machine.rs | 26 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/memory.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/operand.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/projection.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/stack.rs | 62 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/step.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/traits.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/util.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/validity.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/visitor.rs | 21 |
14 files changed, 260 insertions, 133 deletions
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index 405208e94f4..37677f9e048 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -62,7 +62,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { pub(super) fn fn_arg_field( &self, arg: &FnArg<'tcx, M::Provenance>, - field: usize, + field: FieldIdx, ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { interp_ok(match arg { FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?), @@ -339,7 +339,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { caller_fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[FnArg<'tcx, M::Provenance>], with_caller_location: bool, - destination: &MPlaceTy<'tcx, M::Provenance>, + destination: &PlaceTy<'tcx, M::Provenance>, mut stack_pop: StackPopCleanup, ) -> InterpResult<'tcx> { // Compute callee information. @@ -487,7 +487,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } // Protect return place for in-place return value passing. - M::protect_in_place_function_argument(self, &destination)?; + // We only need to protect anything if this is actually an in-memory place. + if let Left(mplace) = destination.as_mplace_or_local() { + M::protect_in_place_function_argument(self, &mplace)?; + } // Don't forget to mark "initially live" locals as live. self.storage_live_for_always_live_locals()?; @@ -512,7 +515,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { (caller_abi, caller_fn_abi): (ExternAbi, &FnAbi<'tcx, Ty<'tcx>>), args: &[FnArg<'tcx, M::Provenance>], with_caller_location: bool, - destination: &MPlaceTy<'tcx, M::Provenance>, + destination: &PlaceTy<'tcx, M::Provenance>, target: Option<mir::BasicBlock>, unwind: mir::UnwindAction, ) -> InterpResult<'tcx> { @@ -597,10 +600,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Cow::from( args.iter() .map(|a| interp_ok(a.clone())) - .chain( - (0..untuple_arg.layout().fields.count()) - .map(|i| self.fn_arg_field(untuple_arg, i)), - ) + .chain((0..untuple_arg.layout().fields.count()).map(|i| { + self.fn_arg_field(untuple_arg, FieldIdx::from_usize(i)) + })) .collect::<InterpResult<'_, Vec<_>>>()?, ) } else { @@ -776,10 +778,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Note that we are using `pop_stack_frame_raw` and not `return_from_current_stack_frame`, // as the latter "executes" the goto to the return block, but we don't want to, // only the tail called function should return to the current return block. - M::before_stack_pop(self, self.frame())?; - - let StackPopInfo { return_action, return_to_block, return_place } = - self.pop_stack_frame_raw(false)?; + let StackPopInfo { return_action, return_to_block, return_place } = self + .pop_stack_frame_raw(false, |_this, _return_place| { + // This function's return value is just discarded, the tail-callee will fill in the return place instead. + interp_ok(()) + })?; assert_eq!(return_action, ReturnAction::Normal); @@ -850,7 +853,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { (ExternAbi::Rust, fn_abi), &[FnArg::Copy(arg.into())], false, - &ret, + &ret.into(), Some(target), unwind, ) @@ -891,28 +894,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { throw_ub_custom!(fluent::const_eval_unwind_past_top); } - M::before_stack_pop(self, self.frame())?; - - // Copy return value. Must of course happen *before* we deallocate the locals. - // Must be *after* `before_stack_pop` as otherwise the return place might still be protected. - let copy_ret_result = if !unwinding { - let op = self - .local_to_op(mir::RETURN_PLACE, None) - .expect("return place should always be live"); - let dest = self.frame().return_place.clone(); - let res = self.copy_op_allow_transmute(&op, &dest); - trace!("return value: {:?}", self.dump_place(&dest.into())); - // We delay actually short-circuiting on this error until *after* the stack frame is - // popped, since we want this error to be attributed to the caller, whose type defines - // this transmute. - res - } else { + // Get out the return value. Must happen *before* the frame is popped as we have to get the + // local's value out. + let return_op = + self.local_to_op(mir::RETURN_PLACE, None).expect("return place should always be live"); + // Do the actual pop + copy. + let stack_pop_info = self.pop_stack_frame_raw(unwinding, |this, return_place| { + this.copy_op_allow_transmute(&return_op, return_place)?; + trace!("return value: {:?}", this.dump_place(return_place)); interp_ok(()) - }; - - // All right, now it is time to actually pop the frame. - // An error here takes precedence over the copy error. - let (stack_pop_info, ()) = self.pop_stack_frame_raw(unwinding).and(copy_ret_result)?; + })?; match stack_pop_info.return_action { ReturnAction::Normal => {} @@ -924,7 +915,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // If we are not doing cleanup, also skip everything else. assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked"); assert!(!unwinding, "tried to skip cleanup during unwinding"); - // Skip machine hook. + // Don't jump anywhere. return interp_ok(()); } } diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index 643a5805019..9e15f4572d7 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -1,6 +1,6 @@ use std::assert_matches::assert_matches; -use rustc_abi::Integer; +use rustc_abi::{FieldIdx, Integer}; use rustc_apfloat::ieee::{Double, Half, Quad, Single}; use rustc_apfloat::{Float, FloatConvert}; use rustc_middle::mir::CastKind; @@ -484,6 +484,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let mut found_cast_field = false; for i in 0..src.layout.fields.count() { let cast_ty_field = cast_ty.field(self, i); + let i = FieldIdx::from_usize(i); let src_field = self.project_field(src, i)?; let dst_field = self.project_field(dest, i)?; if src_field.layout.is_1zst() && cast_ty_field.is_1zst() { diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs index 2f0b1cb6d1e..6c4b000e16b 100644 --- a/compiler/rustc_const_eval/src/interpret/discriminant.rs +++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs @@ -1,6 +1,6 @@ //! Functions for reading and writing discriminants of multi-variant layouts (enums and coroutines). -use rustc_abi::{self as abi, TagEncoding, VariantIdx, Variants}; +use rustc_abi::{self as abi, FieldIdx, TagEncoding, VariantIdx, Variants}; use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout}; use rustc_middle::ty::{self, CoroutineArgsExt, ScalarInt, Ty}; use rustc_middle::{mir, span_bug}; @@ -231,7 +231,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { &self, layout: TyAndLayout<'tcx>, variant_index: VariantIdx, - ) -> InterpResult<'tcx, Option<(ScalarInt, usize)>> { + ) -> InterpResult<'tcx, Option<(ScalarInt, FieldIdx)>> { // Layout computation excludes uninhabited variants from consideration. // Therefore, there's no way to represent those variants in the given layout. // Essentially, uninhabited variants do not have a tag that corresponds to their diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 090b2a692cf..ab27182c211 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -17,16 +17,16 @@ use tracing::trace; use super::memory::MemoryKind; use super::util::ensure_monomorphic_enough; use super::{ - Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult, - MPlaceTy, Machine, OpTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval, - err_ub_custom, err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format, + Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult, Machine, + OpTy, PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval, err_ub_custom, + err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format, }; use crate::fluent_generated as fluent; /// Directly returns an `Allocation` containing an absolute path representation of the given type. pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> { let path = crate::util::type_name(tcx, ty); - let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes()); + let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ()); tcx.mk_const_alloc(alloc) } @@ -50,13 +50,6 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>( ensure_monomorphic_enough(tcx, tp_ty)?; ConstValue::from_bool(tp_ty.needs_drop(tcx, typing_env)) } - sym::pref_align_of => { - // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough. - let layout = tcx - .layout_of(typing_env.as_query_input(tp_ty)) - .map_err(|e| err_inval!(Layout(*e)))?; - ConstValue::from_target_usize(layout.align.pref.bytes(), &tcx) - } sym::type_id => { ensure_monomorphic_enough(tcx, tp_ty)?; ConstValue::from_u128(tcx.type_id_hash(tp_ty).as_u128()) @@ -112,7 +105,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { &mut self, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ret: Option<mir::BasicBlock>, ) -> InterpResult<'tcx, bool> { let instance_args = instance.args; @@ -144,14 +137,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_scalar(Scalar::from_target_usize(result, self), dest)?; } - sym::pref_align_of - | sym::needs_drop - | sym::type_id - | sym::type_name - | sym::variant_count => { + sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => { let gid = GlobalId { instance, promoted: None }; let ty = match intrinsic_name { - sym::pref_align_of | sym::variant_count => self.tcx.types.usize, + sym::variant_count => self.tcx.types.usize, sym::needs_drop => self.tcx.types.bool, sym::type_id => self.tcx.types.u128, sym::type_name => Ty::new_static_str(self.tcx.tcx), @@ -518,6 +507,103 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?, sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?, + sym::floorf16 => self.float_round_intrinsic::<Half>( + args, + dest, + rustc_apfloat::Round::TowardNegative, + )?, + sym::floorf32 => self.float_round_intrinsic::<Single>( + args, + dest, + rustc_apfloat::Round::TowardNegative, + )?, + sym::floorf64 => self.float_round_intrinsic::<Double>( + args, + dest, + rustc_apfloat::Round::TowardNegative, + )?, + sym::floorf128 => self.float_round_intrinsic::<Quad>( + args, + dest, + rustc_apfloat::Round::TowardNegative, + )?, + + sym::ceilf16 => self.float_round_intrinsic::<Half>( + args, + dest, + rustc_apfloat::Round::TowardPositive, + )?, + sym::ceilf32 => self.float_round_intrinsic::<Single>( + args, + dest, + rustc_apfloat::Round::TowardPositive, + )?, + sym::ceilf64 => self.float_round_intrinsic::<Double>( + args, + dest, + rustc_apfloat::Round::TowardPositive, + )?, + sym::ceilf128 => self.float_round_intrinsic::<Quad>( + args, + dest, + rustc_apfloat::Round::TowardPositive, + )?, + + sym::truncf16 => { + self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)? + } + sym::truncf32 => { + self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)? + } + sym::truncf64 => { + self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)? + } + sym::truncf128 => { + self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)? + } + + sym::roundf16 => self.float_round_intrinsic::<Half>( + args, + dest, + rustc_apfloat::Round::NearestTiesToAway, + )?, + sym::roundf32 => self.float_round_intrinsic::<Single>( + args, + dest, + rustc_apfloat::Round::NearestTiesToAway, + )?, + sym::roundf64 => self.float_round_intrinsic::<Double>( + args, + dest, + rustc_apfloat::Round::NearestTiesToAway, + )?, + sym::roundf128 => self.float_round_intrinsic::<Quad>( + args, + dest, + rustc_apfloat::Round::NearestTiesToAway, + )?, + + sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>( + args, + dest, + rustc_apfloat::Round::NearestTiesToEven, + )?, + sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>( + args, + dest, + rustc_apfloat::Round::NearestTiesToEven, + )?, + sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>( + args, + dest, + rustc_apfloat::Round::NearestTiesToEven, + )?, + sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>( + args, + dest, + rustc_apfloat::Round::NearestTiesToEven, + )?, + // Unsupported intrinsic: skip the return_to_block below. _ => return interp_ok(false), } @@ -587,7 +673,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { &mut self, a: &ImmTy<'tcx, M::Provenance>, b: &ImmTy<'tcx, M::Provenance>, - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { assert_eq!(a.layout.ty, b.layout.ty); assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..)); @@ -801,7 +887,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_min_intrinsic<F>( &mut self, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ()> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, @@ -822,7 +908,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_max_intrinsic<F>( &mut self, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ()> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, @@ -843,7 +929,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_minimum_intrinsic<F>( &mut self, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ()> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, @@ -859,7 +945,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_maximum_intrinsic<F>( &mut self, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ()> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, @@ -875,7 +961,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_copysign_intrinsic<F>( &mut self, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ()> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, @@ -890,7 +976,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_abs_intrinsic<F>( &mut self, args: &[OpTy<'tcx, M::Provenance>], - dest: &MPlaceTy<'tcx, M::Provenance>, + dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, ()> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, @@ -900,4 +986,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_scalar(x.abs(), dest)?; interp_ok(()) } + + fn float_round_intrinsic<F>( + &mut self, + args: &[OpTy<'tcx, M::Provenance>], + dest: &PlaceTy<'tcx, M::Provenance>, + mode: rustc_apfloat::Round, + ) -> InterpResult<'tcx, ()> + where + F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>, + { + let x: F = self.read_scalar(&args[0])?.to_float()?; + let res = x.round_to_integral(mode).value; + let res = self.adjust_nan(res, &[x]); + self.write_scalar(res, dest)?; + interp_ok(()) + } } diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index a1386b4e1be..b9e022c9604 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -147,6 +147,12 @@ pub trait Machine<'tcx>: Sized { /// already been checked before. const ALL_CONSTS_ARE_PRECHECKED: bool = true; + /// Determines whether rustc_const_eval functions that make use of the [Machine] should make + /// tracing calls (to the `tracing` library). By default this is `false`, meaning the tracing + /// calls will supposedly be optimized out. This flag is set to `true` inside Miri, to allow + /// tracing the interpretation steps, among other things. + const TRACING_ENABLED: bool = false; + /// Whether memory accesses should be alignment-checked. fn enforce_alignment(ecx: &InterpCx<'tcx, Self>) -> bool; @@ -202,7 +208,7 @@ pub trait Machine<'tcx>: Sized { instance: ty::Instance<'tcx>, abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[FnArg<'tcx, Self::Provenance>], - destination: &MPlaceTy<'tcx, Self::Provenance>, + destination: &PlaceTy<'tcx, Self::Provenance>, target: Option<mir::BasicBlock>, unwind: mir::UnwindAction, ) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>>; @@ -214,7 +220,7 @@ pub trait Machine<'tcx>: Sized { fn_val: Self::ExtraFnVal, abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[FnArg<'tcx, Self::Provenance>], - destination: &MPlaceTy<'tcx, Self::Provenance>, + destination: &PlaceTy<'tcx, Self::Provenance>, target: Option<mir::BasicBlock>, unwind: mir::UnwindAction, ) -> InterpResult<'tcx>; @@ -228,7 +234,7 @@ pub trait Machine<'tcx>: Sized { ecx: &mut InterpCx<'tcx, Self>, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx, Self::Provenance>], - destination: &MPlaceTy<'tcx, Self::Provenance>, + destination: &PlaceTy<'tcx, Self::Provenance>, target: Option<mir::BasicBlock>, unwind: mir::UnwindAction, ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>>; @@ -530,11 +536,9 @@ pub trait Machine<'tcx>: Sized { interp_ok(()) } - /// Called just before the return value is copied to the caller-provided return place. - fn before_stack_pop( - _ecx: &InterpCx<'tcx, Self>, - _frame: &Frame<'tcx, Self::Provenance, Self::FrameExtra>, - ) -> InterpResult<'tcx> { + /// Called just before the frame is removed from the stack (followed by return value copy and + /// local cleanup). + fn before_stack_pop(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> { interp_ok(()) } @@ -622,6 +626,10 @@ pub trait Machine<'tcx>: Sized { // Default to no caching. Cow::Owned(compute_range()) } + + /// Compute the value passed to the constructors of the `AllocBytes` type for + /// abstract machine allocations. + fn get_default_alloc_params(&self) -> <Self::Bytes as AllocBytes>::AllocParams; } /// A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines @@ -669,7 +677,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) { fn_val: !, _abi: &FnAbi<$tcx, Ty<$tcx>>, _args: &[FnArg<$tcx>], - _destination: &MPlaceTy<$tcx, Self::Provenance>, + _destination: &PlaceTy<$tcx, Self::Provenance>, _target: Option<mir::BasicBlock>, _unwind: mir::UnwindAction, ) -> InterpResult<$tcx> { diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 43bf48a9b96..99a4bc1b7d6 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -233,10 +233,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { kind: MemoryKind<M::MemoryKind>, init: AllocInit, ) -> InterpResult<'tcx, Pointer<M::Provenance>> { + let params = self.machine.get_default_alloc_params(); let alloc = if M::PANIC_ON_ALLOC_FAIL { - Allocation::new(size, align, init) + Allocation::new(size, align, init, params) } else { - Allocation::try_new(size, align, init)? + Allocation::try_new(size, align, init, params)? }; self.insert_allocation(alloc, kind) } @@ -248,7 +249,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { kind: MemoryKind<M::MemoryKind>, mutability: Mutability, ) -> InterpResult<'tcx, Pointer<M::Provenance>> { - let alloc = Allocation::from_bytes(bytes, align, mutability); + let params = self.machine.get_default_alloc_params(); + let alloc = Allocation::from_bytes(bytes, align, mutability, params); self.insert_allocation(alloc, kind) } diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 36da9037e43..77667ba823a 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -12,6 +12,7 @@ use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter}; use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt}; use rustc_middle::{bug, mir, span_bug, ty}; +use rustc_span::DUMMY_SP; use tracing::trace; use super::{ @@ -307,10 +308,10 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { #[inline] pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self { // Can use any typing env, since `Ordering` is always monomorphic. - let ty = tcx.ty_ordering_enum(None); + let ty = tcx.ty_ordering_enum(DUMMY_SP); let layout = tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)).unwrap(); - Self::from_scalar(Scalar::from_i8(c as i8), layout) + Self::from_scalar(Scalar::Int(c.into()), layout) } pub fn from_pair(a: Self, b: Self, cx: &(impl HasTypingEnv<'tcx> + HasTyCtxt<'tcx>)) -> Self { diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index 8ecb3e13d5c..ad47a19a14d 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -10,7 +10,7 @@ use std::marker::PhantomData; use std::ops::Range; -use rustc_abi::{self as abi, Size, VariantIdx}; +use rustc_abi::{self as abi, FieldIdx, Size, VariantIdx}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::{bug, mir, span_bug, ty}; @@ -144,22 +144,22 @@ where /// always possible without allocating, so it can take `&self`. Also return the field's layout. /// This supports both struct and array fields, but not slices! /// - /// This also works for arrays, but then the `usize` index type is restricting. - /// For indexing into arrays, use `mplace_index`. + /// This also works for arrays, but then the `FieldIdx` index type is restricting. + /// For indexing into arrays, use [`Self::project_index`]. pub fn project_field<P: Projectable<'tcx, M::Provenance>>( &self, base: &P, - field: usize, + field: FieldIdx, ) -> InterpResult<'tcx, P> { // Slices nominally have length 0, so they will panic somewhere in `fields.offset`. debug_assert!( !matches!(base.layout().ty.kind(), ty::Slice(..)), "`field` projection called on a slice -- call `index` projection instead" ); - let offset = base.layout().fields.offset(field); + let offset = base.layout().fields.offset(field.as_usize()); // Computing the layout does normalization, so we get a normalized type out of this // even if the field type is non-normalized (possible e.g. via associated types). - let field_layout = base.layout().field(self, field); + let field_layout = base.layout().field(self, field.as_usize()); // Offset may need adjustment for unsized fields. let (meta, offset) = if field_layout.is_unsized() { @@ -244,7 +244,7 @@ where } _ => span_bug!( self.cur_span(), - "`mplace_index` called on non-array type {:?}", + "`project_index` called on non-array type {:?}", base.layout().ty ), }; @@ -260,7 +260,7 @@ where ) -> InterpResult<'tcx, (P, u64)> { assert!(base.layout().ty.ty_adt_def().unwrap().repr().simd()); // SIMD types must be newtypes around arrays, so all we have to do is project to their only field. - let array = self.project_field(base, 0)?; + let array = self.project_field(base, FieldIdx::ZERO)?; let len = array.len(self)?; interp_ok((array, len)) } @@ -384,7 +384,7 @@ where UnwrapUnsafeBinder(target) => base.transmute(self.layout_of(target)?, self)?, // We don't want anything happening here, this is here as a dummy. Subtype(_) => base.transmute(base.layout(), self)?, - Field(field, _) => self.project_field(base, field.index())?, + Field(field, _) => self.project_field(base, field)?, Downcast(_, variant) => self.project_downcast(base, variant)?, Deref => self.deref_pointer(&base.to_op(self)?)?.into(), Index(local) => { diff --git a/compiler/rustc_const_eval/src/interpret/stack.rs b/compiler/rustc_const_eval/src/interpret/stack.rs index d7b03776bc4..2a2d1bb2754 100644 --- a/compiler/rustc_const_eval/src/interpret/stack.rs +++ b/compiler/rustc_const_eval/src/interpret/stack.rs @@ -15,9 +15,9 @@ use rustc_span::Span; use tracing::{info_span, instrument, trace}; use super::{ - AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, - MemPlaceMeta, MemoryKind, Operand, Pointer, Provenance, ReturnAction, Scalar, - from_known_layout, interp_ok, throw_ub, throw_unsup, + AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, Machine, MemPlace, MemPlaceMeta, + MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout, + interp_ok, throw_ub, throw_unsup, }; use crate::errors; @@ -76,8 +76,10 @@ pub struct Frame<'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> { return_to_block: StackPopCleanup, /// The location where the result of the current stack frame should be written to, - /// and its layout in the caller. - pub return_place: MPlaceTy<'tcx, Prov>, + /// and its layout in the caller. This place is to be interpreted relative to the + /// *caller's* stack frame. We use a `PlaceTy` instead of an `MPlaceTy` since this + /// avoids having to move *all* return places into Miri's memory. + pub return_place: PlaceTy<'tcx, Prov>, /// The list of locals for this stack frame, stored in order as /// `[return_ptr, arguments..., variables..., temporaries...]`. @@ -129,7 +131,7 @@ pub struct StackPopInfo<'tcx, Prov: Provenance> { pub return_to_block: StackPopCleanup, /// [`return_place`](Frame::return_place) of the popped stack frame. - pub return_place: MPlaceTy<'tcx, Prov>, + pub return_place: PlaceTy<'tcx, Prov>, } /// State of a local variable including a memoized layout @@ -353,7 +355,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { &mut self, instance: ty::Instance<'tcx>, body: &'tcx mir::Body<'tcx>, - return_place: &MPlaceTy<'tcx, M::Provenance>, + return_place: &PlaceTy<'tcx, M::Provenance>, return_to_block: StackPopCleanup, ) -> InterpResult<'tcx> { trace!("body: {:#?}", body); @@ -404,9 +406,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// it. /// /// This also deallocates locals, if necessary. + /// `copy_ret_val` gets called after the frame has been taken from the stack but before the locals have been deallocated. /// - /// [`M::before_stack_pop`] should be called before calling this function. - /// [`M::after_stack_pop`] is called by this function automatically. + /// [`M::before_stack_pop`] and [`M::after_stack_pop`] are called by this function + /// automatically. /// /// The high-level version of this is `return_from_current_stack_frame`. /// @@ -415,47 +418,44 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { pub(super) fn pop_stack_frame_raw( &mut self, unwinding: bool, + copy_ret_val: impl FnOnce(&mut Self, &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx>, ) -> InterpResult<'tcx, StackPopInfo<'tcx, M::Provenance>> { - let cleanup = self.cleanup_current_frame_locals()?; - + M::before_stack_pop(self)?; let frame = self.stack_mut().pop().expect("tried to pop a stack frame, but there were none"); + // Copy return value (unless we are unwinding). + if !unwinding { + copy_ret_val(self, &frame.return_place)?; + } + let return_to_block = frame.return_to_block; let return_place = frame.return_place.clone(); - let return_action; - if cleanup { - return_action = M::after_stack_pop(self, frame, unwinding)?; - assert_ne!(return_action, ReturnAction::NoCleanup); - } else { - return_action = ReturnAction::NoCleanup; - }; - - interp_ok(StackPopInfo { return_action, return_to_block, return_place }) - } - - /// A private helper for [`pop_stack_frame_raw`](InterpCx::pop_stack_frame_raw). - /// Returns `true` if cleanup has been done, `false` otherwise. - fn cleanup_current_frame_locals(&mut self) -> InterpResult<'tcx, bool> { // Cleanup: deallocate locals. // Usually we want to clean up (deallocate locals), but in a few rare cases we don't. // We do this while the frame is still on the stack, so errors point to the callee. - let return_to_block = self.frame().return_to_block; let cleanup = match return_to_block { StackPopCleanup::Goto { .. } => true, StackPopCleanup::Root { cleanup, .. } => cleanup, }; - if cleanup { + let return_action = if cleanup { // We need to take the locals out, since we need to mutate while iterating. - let locals = mem::take(&mut self.frame_mut().locals); - for local in &locals { + for local in &frame.locals { self.deallocate_local(local.value)?; } - } - interp_ok(cleanup) + // Call the machine hook, which determines the next steps. + let return_action = M::after_stack_pop(self, frame, unwinding)?; + assert_ne!(return_action, ReturnAction::NoCleanup); + return_action + } else { + // We also skip the machine hook when there's no cleanup. This not a real "pop" anyway. + ReturnAction::NoCleanup + }; + + interp_ok(StackPopInfo { return_action, return_to_block, return_place }) } /// In the current stack frame, mark all locals as live that are not arguments and don't have diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 363ceee1970..833fcc38817 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -333,7 +333,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } for (field_index, operand) in operands.iter_enumerated() { let field_index = active_field_index.unwrap_or(field_index); - let field_dest = self.project_field(&variant_dest, field_index.as_usize())?; + let field_dest = self.project_field(&variant_dest, field_index)?; let op = self.eval_operand(operand, Some(field_dest.layout))?; self.copy_op(&op, &field_dest)?; } @@ -506,7 +506,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = self.eval_callee_and_args(terminator, func, args)?; - let destination = self.force_allocation(&self.eval_place(destination)?)?; + let destination = self.eval_place(destination)?; self.init_fn_call( callee, (fn_sig.abi, fn_abi), diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs index a5029eea5a7..7249ef23bf6 100644 --- a/compiler/rustc_const_eval/src/interpret/traits.rs +++ b/compiler/rustc_const_eval/src/interpret/traits.rs @@ -1,4 +1,4 @@ -use rustc_abi::{Align, Size}; +use rustc_abi::{Align, FieldIdx, Size}; use rustc_middle::mir::interpret::{InterpResult, Pointer}; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, ExistentialPredicateStableCmpExt, Ty, TyCtxt, VtblEntry}; @@ -137,8 +137,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { matches!(val.layout().ty.kind(), ty::Dynamic(_, _, ty::DynStar)), "`unpack_dyn_star` only makes sense on `dyn*` types" ); - let data = self.project_field(val, 0)?; - let vtable = self.project_field(val, 1)?; + let data = self.project_field(val, FieldIdx::ZERO)?; + let vtable = self.project_field(val, FieldIdx::ONE)?; let vtable = self.read_pointer(&vtable.to_op(self)?)?; let ty = self.get_ptr_vtable_ty(vtable, Some(expected_trait))?; // `data` is already the right thing but has the wrong type. So we transmute it. diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs index ba579e25f03..83a17092619 100644 --- a/compiler/rustc_const_eval/src/interpret/util.rs +++ b/compiler/rustc_const_eval/src/interpret/util.rs @@ -38,10 +38,29 @@ pub(crate) fn create_static_alloc<'tcx>( static_def_id: LocalDefId, layout: TyAndLayout<'tcx>, ) -> InterpResult<'tcx, MPlaceTy<'tcx>> { - let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit)?; + let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit, ())?; let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id.into()); assert_eq!(ecx.machine.static_root_ids, None); ecx.machine.static_root_ids = Some((alloc_id, static_def_id)); assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none()); interp_ok(ecx.ptr_to_mplace(Pointer::from(alloc_id).into(), layout)) } + +/// This struct is needed to enforce `#[must_use]` on [tracing::span::EnteredSpan] +/// while wrapping them in an `Option`. +#[must_use] +pub enum MaybeEnteredSpan { + Some(tracing::span::EnteredSpan), + None, +} + +#[macro_export] +macro_rules! enter_trace_span { + ($machine:ident, $($tt:tt)*) => { + if $machine::TRACING_ENABLED { + $crate::interpret::tracing_utils::MaybeEnteredSpan::Some(tracing::info_span!($($tt)*).entered()) + } else { + $crate::interpret::tracing_utils::MaybeEnteredSpan::None + } + } +} diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 8f39afa642a..7d76d925ef2 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -294,7 +294,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { // First, check if we are projecting to a variant. match layout.variants { Variants::Multiple { tag_field, .. } => { - if tag_field == field { + if tag_field.as_usize() == field { return match layout.ty.kind() { ty::Adt(def, ..) if def.is_enum() => PathElem::EnumTag, ty::Coroutine(..) => PathElem::CoroutineTag, diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index 3647c109a6e..5aea91233bd 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -112,8 +112,10 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { // So we transmute it to a raw pointer. let raw_ptr_ty = Ty::new_mut_ptr(*self.ecx().tcx, self.ecx().tcx.types.unit); let raw_ptr_ty = self.ecx().layout_of(raw_ptr_ty)?; - let vtable_field = - self.ecx().project_field(v, 1)?.transmute(raw_ptr_ty, self.ecx())?; + let vtable_field = self + .ecx() + .project_field(v, FieldIdx::ONE)? + .transmute(raw_ptr_ty, self.ecx())?; self.visit_field(v, 1, &vtable_field)?; // Then unpack the first field, and continue. @@ -140,14 +142,16 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { // `Box` has two fields: the pointer we care about, and the allocator. assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields"); - let (unique_ptr, alloc) = - (self.ecx().project_field(v, 0)?, self.ecx().project_field(v, 1)?); + let (unique_ptr, alloc) = ( + self.ecx().project_field(v, FieldIdx::ZERO)?, + self.ecx().project_field(v, FieldIdx::ONE)?, + ); // Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`... // (which means another 2 fields, the second of which is a `PhantomData`) assert_eq!(unique_ptr.layout().fields.count(), 2); let (nonnull_ptr, phantom) = ( - self.ecx().project_field(&unique_ptr, 0)?, - self.ecx().project_field(&unique_ptr, 1)?, + self.ecx().project_field(&unique_ptr, FieldIdx::ZERO)?, + self.ecx().project_field(&unique_ptr, FieldIdx::ONE)?, ); assert!( phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()), @@ -156,7 +160,7 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { ); // ... that contains a `NonNull`... (gladly, only a single field here) assert_eq!(nonnull_ptr.layout().fields.count(), 1); - let raw_ptr = self.ecx().project_field(&nonnull_ptr, 0)?; // the actual raw ptr + let raw_ptr = self.ecx().project_field(&nonnull_ptr, FieldIdx::ZERO)?; // the actual raw ptr // ... whose only field finally is a raw ptr we can dereference. self.visit_box(ty, &raw_ptr)?; @@ -188,9 +192,8 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { } FieldsShape::Arbitrary { memory_index, .. } => { for idx in Self::aggregate_field_iter(memory_index) { - let idx = idx.as_usize(); let field = self.ecx().project_field(v, idx)?; - self.visit_field(v, idx, &field)?; + self.visit_field(v, idx.as_usize(), &field)?; } } FieldsShape::Array { .. } => { |