diff options
Diffstat (limited to 'compiler/rustc_const_eval')
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/call.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/eval_context.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/operand.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/place.rs | 13 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/stack.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/step.rs | 65 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/validity.rs | 4 |
7 files changed, 86 insertions, 27 deletions
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index 64cb934ac8d..4cb88d44e1b 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -27,8 +27,9 @@ use crate::{enter_trace_span, fluent_generated as fluent}; pub enum FnArg<'tcx, Prov: Provenance = CtfeProvenance> { /// Pass a copy of the given operand. Copy(OpTy<'tcx, Prov>), - /// Allow for the argument to be passed in-place: destroy the value originally stored at that place and - /// make the place inaccessible for the duration of the function call. + /// Allow for the argument to be passed in-place: destroy the value originally stored at that + /// place and make the place inaccessible for the duration of the function call. This *must* be + /// an in-memory place so that we can do the proper alias checks. InPlace(MPlaceTy<'tcx, Prov>), } @@ -379,6 +380,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } + // *Before* pushing the new frame, determine whether the return destination is in memory. + // Need to use `place_to_op` to be *sure* we get the mplace if there is one. + let destination_mplace = self.place_to_op(destination)?.as_mplace_or_imm().left(); + + // Push the "raw" frame -- this leaves locals uninitialized. self.push_stack_frame_raw(instance, body, destination, cont)?; // If an error is raised here, pop the frame again to get an accurate backtrace. @@ -496,7 +502,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Protect return place for in-place return value passing. // We only need to protect anything if this is actually an in-memory place. - if let Left(mplace) = destination.as_mplace_or_local() { + if let Some(mplace) = destination_mplace { M::protect_in_place_function_argument(self, &mplace)?; } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index a8a1ac1c980..9681d89ce35 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -325,8 +325,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let _trace = enter_trace_span!( M, "instantiate_from_frame_and_normalize_erasing_regions", - "{}", - frame.instance + %frame.instance ); frame .instance @@ -583,6 +582,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { span: Span, layout: Option<TyAndLayout<'tcx>>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let _trace = enter_trace_span!(M, const_eval::eval_mir_constant, ?val); let const_val = val.eval(*self.tcx, self.typing_env, span).map_err(|err| { if M::ALL_CONSTS_ARE_PRECHECKED { match err { diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 53a440b646b..560b0e1ae4e 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -175,6 +175,16 @@ impl<Prov: Provenance> Immediate<Prov> { } interp_ok(()) } + + pub fn has_provenance(&self) -> bool { + match self { + Immediate::Scalar(scalar) => matches!(scalar, Scalar::Ptr { .. }), + Immediate::ScalarPair(s1, s2) => { + matches!(s1, Scalar::Ptr { .. }) || matches!(s2, Scalar::Ptr { .. }) + } + Immediate::Uninit => false, + } + } } // ScalarPair needs a type to interpret, so we often have an immediate and a type together diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 3255ffa54aa..a86fdf80f60 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -234,6 +234,12 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> { } /// A place is either an mplace or some local. + /// + /// Note that the return value can be different even for logically identical places! + /// Specifically, if a local is stored in-memory, this may return `Local` or `MPlaceTy` + /// depending on how the place was constructed. In other words, seeing `Local` here does *not* + /// imply that this place does not point to memory. Every caller must therefore always handle + /// both cases. #[inline(always)] pub fn as_mplace_or_local( &self, @@ -759,6 +765,13 @@ where &mut self, dest: &impl Writeable<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { + // If this is an efficiently represented local variable without provenance, skip the + // `as_mplace_or_mutable_local` that would otherwise force this local into memory. + if let Right(imm) = dest.to_op(self)?.as_mplace_or_imm() { + if !imm.has_provenance() { + return interp_ok(()); + } + } match self.as_mplace_or_mutable_local(&dest.to_place())? { Right((local_val, _local_layout, local)) => { local_val.clear_provenance()?; diff --git a/compiler/rustc_const_eval/src/interpret/stack.rs b/compiler/rustc_const_eval/src/interpret/stack.rs index 73cc87508ef..7cabfd96121 100644 --- a/compiler/rustc_const_eval/src/interpret/stack.rs +++ b/compiler/rustc_const_eval/src/interpret/stack.rs @@ -20,7 +20,7 @@ use super::{ MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout, interp_ok, throw_ub, throw_unsup, }; -use crate::errors; +use crate::{enter_trace_span, errors}; // The Phantomdata exists to prevent this type from being `Send`. If it were sent across a thread // boundary and dropped in the other thread, it would exit the span in the other thread. @@ -386,6 +386,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Make sure all the constants required by this frame evaluate successfully (post-monomorphization check). for &const_ in body.required_consts() { + // We can't use `eval_mir_constant` here as that assumes that all required consts have + // already been checked, so we need a separate tracing call. + let _trace = enter_trace_span!(M, const_eval::required_consts, ?const_.const_); let c = self.instantiate_from_current_frame_and_normalize_erasing_regions(const_.const_)?; c.eval(*self.tcx, self.typing_env, const_.span).map_err(|err| { diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index f1995b3f132..084d45cf2cb 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -4,6 +4,7 @@ use either::Either; use rustc_abi::{FIRST_VARIANT, FieldIdx}; +use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexSlice; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, mir, span_bug}; @@ -389,8 +390,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// Evaluate the arguments of a function call fn eval_fn_call_argument( - &self, + &mut self, op: &mir::Operand<'tcx>, + move_definitely_disjoint: bool, ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { interp_ok(match op { mir::Operand::Copy(_) | mir::Operand::Constant(_) => { @@ -399,24 +401,19 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { FnArg::Copy(op) } mir::Operand::Move(place) => { - // If this place lives in memory, preserve its location. - // We call `place_to_op` which will be an `MPlaceTy` whenever there exists - // an mplace for this place. (This is in contrast to `PlaceTy::as_mplace_or_local` - // which can return a local even if that has an mplace.) let place = self.eval_place(*place)?; - let op = self.place_to_op(&place)?; - - match op.as_mplace_or_imm() { - Either::Left(mplace) => FnArg::InPlace(mplace), - Either::Right(_imm) => { - // This argument doesn't live in memory, so there's no place - // to make inaccessible during the call. - // We rely on there not being any stray `PlaceTy` that would let the - // caller directly access this local! - // This is also crucial for tail calls, where we want the `FnArg` to - // stay valid when the old stack frame gets popped. - FnArg::Copy(op) + if move_definitely_disjoint { + // We still have to ensure that no *other* pointers are used to access this place, + // so *if* it is in memory then we have to treat it as `InPlace`. + // Use `place_to_op` to guarantee that we notice it being in memory. + let op = self.place_to_op(&place)?; + match op.as_mplace_or_imm() { + Either::Left(mplace) => FnArg::InPlace(mplace), + Either::Right(_imm) => FnArg::Copy(op), } + } else { + // We have to force this into memory to detect aliasing among `Move` arguments. + FnArg::InPlace(self.force_allocation(&place)?) } } }) @@ -425,18 +422,46 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// Shared part of `Call` and `TailCall` implementation — finding and evaluating all the /// necessary information about callee and arguments to make a call. fn eval_callee_and_args( - &self, + &mut self, terminator: &mir::Terminator<'tcx>, func: &mir::Operand<'tcx>, args: &[Spanned<mir::Operand<'tcx>>], ) -> InterpResult<'tcx, EvaluatedCalleeAndArgs<'tcx, M>> { let func = self.eval_operand(func, None)?; + + // Evaluating function call arguments. The tricky part here is dealing with `Move` + // arguments: we have to ensure no two such arguments alias. This would be most easily done + // by just forcing them all into memory and then doing the usual in-place argument + // protection, but then we'd force *a lot* of arguments into memory. So we do some syntactic + // pre-processing here where if all `move` arguments are syntactically distinct local + // variables (and none is indirect), we can skip the in-memory forcing. + let move_definitely_disjoint = 'move_definitely_disjoint: { + let mut previous_locals = FxHashSet::<mir::Local>::default(); + for arg in args { + let mir::Operand::Move(place) = arg.node else { + continue; // we can skip non-`Move` arguments. + }; + if place.is_indirect_first_projection() { + // An indirect `Move` argument could alias with anything else... + break 'move_definitely_disjoint false; + } + if !previous_locals.insert(place.local) { + // This local is the base for two arguments! They might overlap. + break 'move_definitely_disjoint false; + } + } + // We found no violation so they are all definitely disjoint. + true + }; let args = args .iter() - .map(|arg| self.eval_fn_call_argument(&arg.node)) + .map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint)) .collect::<InterpResult<'tcx, Vec<_>>>()?; - let fn_sig_binder = func.layout.ty.fn_sig(*self.tcx); + let fn_sig_binder = { + let _trace = enter_trace_span!(M, "fn_sig", ty = ?func.layout.ty.kind()); + func.layout.ty.fn_sig(*self.tcx) + }; let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.typing_env, fn_sig_binder); let extra_args = &args[fn_sig.inputs().len()..]; let extra_args = diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 5e8bee65706..02e3d90f4af 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -1418,7 +1418,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let _trace = enter_trace_span!( M, "validate_operand", - "recursive={recursive}, reset_provenance_and_padding={reset_provenance_and_padding}, val={val:?}" + recursive, + reset_provenance_and_padding, + ?val, ); // Note that we *could* actually be in CTFE here with `-Zextra-const-ub-checks`, but it's |