diff options
Diffstat (limited to 'compiler/rustc_const_eval/src')
29 files changed, 859 insertions, 590 deletions
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index 57af0ff0714..9dc34260de7 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -42,6 +42,7 @@ fn eval_body_using_ecx<'mir, 'tcx>( | DefKind::Static | DefKind::ConstParam | DefKind::AnonConst + | DefKind::InlineConst | DefKind::AssocConst ), "Unexpected DefKind: {:?}", @@ -61,7 +62,7 @@ fn eval_body_using_ecx<'mir, 'tcx>( cid.instance, body, Some(&ret.into()), - StackPopCleanup::None { cleanup: false }, + StackPopCleanup::Root { cleanup: false }, )?; // The main interpreter loop. @@ -214,6 +215,7 @@ pub fn eval_to_const_value_raw_provider<'tcx>( tcx: TyCtxt<'tcx>, key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>, ) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> { + assert!(key.param_env.is_const()); // see comment in eval_to_allocation_raw_provider for what we're doing here if key.param_env.reveal() == Reveal::All { let mut key = key; @@ -248,6 +250,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>( tcx: TyCtxt<'tcx>, key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>, ) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> { + assert!(key.param_env.is_const()); // Because the constant is computed twice (once per value of `Reveal`), we are at risk of // reporting the same error twice here. To resolve this, we check whether we can evaluate the // constant in the more restrictive `Reveal::UserFacing`, which most likely already was diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs index 821b048eb9b..05fbbf45d7c 100644 --- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs @@ -1,5 +1,5 @@ use rustc_hir as hir; -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_span::symbol::Symbol; @@ -15,7 +15,8 @@ pub fn is_unstable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Symbol> { } } -pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, hir_id: hir::HirId) -> bool { +pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { + let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); let parent_id = tcx.hir().get_parent_node(hir_id); matches!( tcx.hir().get(parent_id), @@ -29,15 +30,15 @@ pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, hir_id: hir::HirId) -> bool { /// Checks whether the function has a `const` modifier or, in case it is an intrinsic, whether /// said intrinsic has a `rustc_const_{un,}stable` attribute. fn is_const_fn_raw(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local()); - - let node = tcx.hir().get(hir_id); + let def_id = def_id.expect_local(); + let node = tcx.hir().get_by_def_id(def_id); if let hir::Node::ForeignItem(hir::ForeignItem { kind: hir::ForeignItemKind::Fn(..), .. }) = node { // Intrinsics use `rustc_const_{un,}stable` attributes to indicate constness. All other // foreign items cannot be evaluated at compile-time. + let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = tcx.hir().get_foreign_abi(hir_id) { tcx.lookup_const_stability(def_id).is_some() } else { @@ -50,7 +51,7 @@ fn is_const_fn_raw(tcx: TyCtxt<'_>, def_id: DefId) -> bool { // If the function itself is not annotated with `const`, it may still be a `const fn` // if it resides in a const trait impl. - is_parent_const_impl_raw(tcx, hir_id) + is_parent_const_impl_raw(tcx, def_id) } else { matches!(node, hir::Node::Ctor(_)) } diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index dacd8f7c12c..89717b75f12 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -260,7 +260,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, args: &[OpTy<'tcx>], _ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>, _unwind: StackPopUnwind, // unwinding is not supported in consts - ) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>> { + ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> { debug!("find_mir_or_eval_fn: {:?}", instance); // Only check non-glue functions @@ -279,11 +279,21 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, if let Some(new_instance) = ecx.hook_special_const_fn(instance, args)? { // We call another const fn instead. - return Self::find_mir_or_eval_fn(ecx, new_instance, _abi, args, _ret, _unwind); + // However, we return the *original* instance to make backtraces work out + // (and we hope this does not confuse the FnAbi checks too much). + return Ok(Self::find_mir_or_eval_fn( + ecx, + new_instance, + _abi, + args, + _ret, + _unwind, + )? + .map(|(body, _instance)| (body, instance))); } } // This is a const fn. Call it. - Ok(Some(ecx.load_mir(instance.def, None)?)) + Ok(Some((ecx.load_mir(instance.def, None)?, instance))) } fn call_intrinsic( @@ -337,6 +347,33 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, )?; ecx.write_pointer(ptr, dest)?; } + sym::const_deallocate => { + let ptr = ecx.read_pointer(&args[0])?; + let size = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?; + let align = ecx.read_scalar(&args[2])?.to_machine_usize(ecx)?; + + let size = Size::from_bytes(size); + let align = match Align::from_bytes(align) { + Ok(a) => a, + Err(err) => throw_ub_format!("align has to be a power of 2, {}", err), + }; + + // If an allocation is created in an another const, + // we don't deallocate it. + let (alloc_id, _, _) = ecx.memory.ptr_get_alloc(ptr)?; + let is_allocated_in_another_const = matches!( + ecx.tcx.get_global_alloc(alloc_id), + Some(interpret::GlobalAlloc::Memory(_)) + ); + + if !is_allocated_in_another_const { + ecx.memory.deallocate( + ptr, + Some((size, align)), + interpret::MemoryKind::Machine(MemoryKind::Heap), + )?; + } + } _ => { return Err(ConstEvalErrKind::NeedsRfc(format!( "calling intrinsic `{}`", @@ -388,13 +425,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, Err(ConstEvalErrKind::NeedsRfc("pointer arithmetic or comparison".to_string()).into()) } - fn box_alloc( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _dest: &PlaceTy<'tcx>, - ) -> InterpResult<'tcx> { - Err(ConstEvalErrKind::NeedsRfc("heap allocations via `box` keyword".to_string()).into()) - } - fn before_terminator(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> { // The step limit has already been hit in a previous call to `before_terminator`. if ecx.machine.steps_remaining == 0 { @@ -423,14 +453,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, } #[inline(always)] - fn stack( + fn stack<'a>( ecx: &'a InterpCx<'mir, 'tcx, Self>, ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { &ecx.machine.stack } #[inline(always)] - fn stack_mut( + fn stack_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { &mut ecx.machine.stack diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs index a334165df4c..91b17d1ac1e 100644 --- a/compiler/rustc_const_eval/src/const_eval/mod.rs +++ b/compiler/rustc_const_eval/src/const_eval/mod.rs @@ -25,9 +25,9 @@ pub use fn_queries::*; pub use machine::*; pub(crate) fn const_caller_location( - tcx: TyCtxt<'tcx>, + tcx: TyCtxt<'_>, (file, line, col): (Symbol, u32, u32), -) -> ConstValue<'tcx> { +) -> ConstValue<'_> { trace!("const_caller_location: {}:{}:{}", file, line, col); let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), false); diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index cf084faade8..0a8112da2ab 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -7,7 +7,11 @@ use rustc_hir::{self as hir, def_id::DefId, definitions::DefPathData}; use rustc_index::vec::IndexVec; use rustc_macros::HashStable; use rustc_middle::mir; -use rustc_middle::ty::layout::{self, LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; +use rustc_middle::mir::interpret::{InterpError, InvalidProgramInfo}; +use rustc_middle::ty::layout::{ + self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOf, LayoutOfHelpers, + TyAndLayout, +}; use rustc_middle::ty::{ self, query::TyCtxtAt, subst::SubstsRef, ParamEnv, Ty, TyCtxt, TypeFoldable, }; @@ -15,7 +19,7 @@ use rustc_mir_dataflow::storage::AlwaysLiveLocals; use rustc_query_system::ich::StableHashingContext; use rustc_session::Limit; use rustc_span::{Pos, Span}; -use rustc_target::abi::{Align, HasDataLayout, Size, TargetDataLayout}; +use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout}; use super::{ AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, @@ -152,11 +156,11 @@ pub enum StackPopCleanup { /// `ret` stores the block we jump to on a normal return, while `unwind` /// stores the block used for cleanup during unwinding. Goto { ret: Option<mir::BasicBlock>, unwind: StackPopUnwind }, - /// Just do nothing: Used by Main and for the `box_alloc` hook in miri. + /// The root frame of the stack: nowhere else to jump to. /// `cleanup` says whether locals are deallocated. Static computation /// wants them leaked to intern what they need (and just throw away /// the entire `ecx` when it is done). - None { cleanup: bool }, + Root { cleanup: bool }, } /// State of a local variable including a memoized layout @@ -332,6 +336,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> LayoutOfHelpers<'tcx> for InterpC } } +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> FnAbiOfHelpers<'tcx> for InterpCx<'mir, 'tcx, M> { + type FnAbiOfResult = InterpResult<'tcx, &'tcx FnAbi<'tcx, Ty<'tcx>>>; + + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + _span: Span, + _fn_abi_request: FnAbiRequest<'tcx>, + ) -> InterpErrorInfo<'tcx> { + match err { + FnAbiError::Layout(err) => err_inval!(Layout(err)).into(), + FnAbiError::AdjustForForeignAbi(err) => { + err_inval!(FnAbiAdjustForForeignAbi(err)).into() + } + } + } +} + /// Test if it is valid for a MIR assignment to assign `src`-typed place to `dest`-typed value. /// This test should be symmetric, as it is primarily about layout compatibility. pub(super) fn mir_assign_valid_types<'tcx>( @@ -508,7 +530,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub(super) fn subst_from_current_frame_and_normalize_erasing_regions<T: TypeFoldable<'tcx>>( &self, value: T, - ) -> T { + ) -> Result<T, InterpError<'tcx>> { self.subst_from_frame_and_normalize_erasing_regions(self.frame(), value) } @@ -518,8 +540,18 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { &self, frame: &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>, value: T, - ) -> T { - frame.instance.subst_mir_and_normalize_erasing_regions(*self.tcx, self.param_env, value) + ) -> Result<T, InterpError<'tcx>> { + frame + .instance + .try_subst_mir_and_normalize_erasing_regions(*self.tcx, self.param_env, value) + .or_else(|e| { + self.tcx.sess.delay_span_bug( + self.cur_span(), + format!("failed to normalize {}", e.get_type_for_failure()).as_str(), + ); + + Err(InterpError::InvalidProgram(InvalidProgramInfo::TooGeneric)) + }) } /// The `substs` are assumed to already be in our interpreter "universe" (param_env). @@ -554,7 +586,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let layout = from_known_layout(self.tcx, self.param_env, layout, || { let local_ty = frame.body.local_decls[local].ty; let local_ty = - self.subst_from_frame_and_normalize_erasing_regions(frame, local_ty); + self.subst_from_frame_and_normalize_erasing_regions(frame, local_ty)?; self.layout_of(local_ty) })?; if let Some(state) = frame.locals.get(local) { @@ -605,19 +637,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { match self.size_and_align_of(metadata, &field)? { Some(size_and_align) => size_and_align, None => { - // A field with extern type. If this field is at offset 0, we behave - // like the underlying extern type. - // FIXME: Once we have made decisions for how to handle size and alignment - // of `extern type`, this should be adapted. It is just a temporary hack - // to get some code to work that probably ought to work. - if sized_size == Size::ZERO { - return Ok(None); - } else { - span_bug!( - self.cur_span(), - "Fields cannot be extern types, unless they are at offset 0" - ) - } + // A field with an extern type. We don't know the actual dynamic size + // or the alignment. + return Ok(None); } }; @@ -702,7 +724,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { for const_ in &body.required_consts { let span = const_.span; let const_ = - self.subst_from_current_frame_and_normalize_erasing_regions(const_.literal); + self.subst_from_current_frame_and_normalize_erasing_regions(const_.literal)?; self.mir_const_to_op(&const_, None).map_err(|err| { // If there was an error, set the span of the current frame to this constant. // Avoiding doing this when evaluation succeeds. @@ -827,7 +849,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // because this is CTFE and the final value will be thoroughly validated anyway. let cleanup = match return_to_block { StackPopCleanup::Goto { .. } => true, - StackPopCleanup::None { cleanup, .. } => cleanup, + StackPopCleanup::Root { cleanup, .. } => cleanup, }; if !cleanup { @@ -852,8 +874,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Follow the unwind edge. let unwind = match return_to_block { StackPopCleanup::Goto { unwind, .. } => unwind, - StackPopCleanup::None { .. } => { - panic!("Encountered StackPopCleanup::None when unwinding!") + StackPopCleanup::Root { .. } => { + panic!("encountered StackPopCleanup::Root when unwinding!") } }; self.unwind_to_block(unwind) @@ -861,7 +883,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Follow the normal return edge. match return_to_block { StackPopCleanup::Goto { ret, .. } => self.return_to_block(ret), - StackPopCleanup::None { .. } => Ok(()), + StackPopCleanup::Root { .. } => { + assert!( + self.stack().is_empty(), + "only the topmost frame can have StackPopCleanup::Root" + ); + Ok(()) + } } } } @@ -918,12 +946,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } else { self.param_env }; + let param_env = param_env.with_const(); let val = self.tcx.eval_to_allocation_raw(param_env.and(gid))?; self.raw_const_to_mplace(val) } #[must_use] - pub fn dump_place(&'a self, place: Place<M::PointerTag>) -> PlacePrinter<'a, 'mir, 'tcx, M> { + pub fn dump_place(&self, place: Place<M::PointerTag>) -> PlacePrinter<'_, 'mir, 'tcx, M> { PlacePrinter { ecx: self, place } } diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index 84e79408397..a1dd587c17a 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -292,14 +292,15 @@ pub enum InternKind { /// Any errors here would anyway be turned into `const_err` lints, whereas validation failures /// are hard errors. #[tracing::instrument(level = "debug", skip(ecx))] -pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>>( +pub fn intern_const_alloc_recursive< + 'mir, + 'tcx: 'mir, + M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>, +>( ecx: &mut InterpCx<'mir, 'tcx, M>, intern_kind: InternKind, ret: &MPlaceTy<'tcx>, -) -> Result<(), ErrorReported> -where - 'tcx: 'mir, -{ +) -> Result<(), ErrorReported> { let tcx = ecx.tcx; let base_intern_mode = match intern_kind { InternKind::Static(mutbl) => InternMode::Static(mutbl), diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 698742fe98c..d6f856a6f0a 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -140,7 +140,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::min_align_of_val | sym::size_of_val => { // Avoid `deref_operand` -- this is not a deref, the ptr does not have to be - // dereferencable! + // dereferenceable! let place = self.ref_to_mplace(&self.read_immediate(&args[0])?)?; let (size, align) = self .size_and_align_of_mplace(&place)? @@ -322,6 +322,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::copy => { self.copy_intrinsic(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?; } + sym::write_bytes => { + self.write_bytes_intrinsic(&args[0], &args[1], &args[2])?; + } sym::offset => { let ptr = self.read_pointer(&args[0])?; let offset_count = self.read_scalar(&args[1])?.to_machine_isize(self)?; @@ -394,10 +397,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::transmute => { self.copy_op_transmute(&args[0], dest)?; } - sym::assert_inhabited => { + sym::assert_inhabited | sym::assert_zero_valid | sym::assert_uninit_valid => { let ty = instance.substs.type_at(0); let layout = self.layout_of(ty)?; + // For *all* intrinsics we first check `is_uninhabited` to give a more specific + // error message. if layout.abi.is_uninhabited() { // The run-time intrinsic panics just to get a good backtrace; here we abort // since there is no problem showing a backtrace even for aborts. @@ -409,52 +414,59 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ), )?; } + if intrinsic_name == sym::assert_zero_valid + && !layout.might_permit_raw_init(self, /*zero:*/ true) + { + M::abort( + self, + format!( + "aborted execution: attempted to zero-initialize type `{}`, which is invalid", + ty + ), + )?; + } + if intrinsic_name == sym::assert_uninit_valid + && !layout.might_permit_raw_init(self, /*zero:*/ false) + { + M::abort( + self, + format!( + "aborted execution: attempted to leave type `{}` uninitialized, which is invalid", + ty + ), + )?; + } } sym::simd_insert => { let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); let elem = &args[2]; - let input = &args[0]; - let (len, e_ty) = input.layout.ty.simd_size_and_type(*self.tcx); + let (input, input_len) = self.operand_to_simd(&args[0])?; + let (dest, dest_len) = self.place_to_simd(dest)?; + assert_eq!(input_len, dest_len, "Return vector length must match input length"); assert!( - index < len, - "Index `{}` must be in bounds of vector type `{}`: `[0, {})`", + index < dest_len, + "Index `{}` must be in bounds of vector with length {}`", index, - e_ty, - len - ); - assert_eq!( - input.layout, dest.layout, - "Return type `{}` must match vector type `{}`", - dest.layout.ty, input.layout.ty - ); - assert_eq!( - elem.layout.ty, e_ty, - "Scalar element type `{}` must match vector element type `{}`", - elem.layout.ty, e_ty + dest_len ); - for i in 0..len { - let place = self.place_index(dest, i)?; - let value = if i == index { *elem } else { self.operand_index(input, i)? }; - self.copy_op(&value, &place)?; + for i in 0..dest_len { + let place = self.mplace_index(&dest, i)?; + let value = + if i == index { *elem } else { self.mplace_index(&input, i)?.into() }; + self.copy_op(&value, &place.into())?; } } sym::simd_extract => { let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); - let (len, e_ty) = args[0].layout.ty.simd_size_and_type(*self.tcx); + let (input, input_len) = self.operand_to_simd(&args[0])?; assert!( - index < len, - "index `{}` is out-of-bounds of vector type `{}` with length `{}`", + index < input_len, + "index `{}` must be in bounds of vector with length `{}`", index, - e_ty, - len + input_len ); - assert_eq!( - e_ty, dest.layout.ty, - "Return type `{}` must match vector element type `{}`", - dest.layout.ty, e_ty - ); - self.copy_op(&self.operand_index(&args[0], index)?, dest)?; + self.copy_op(&self.mplace_index(&input, index)?.into(), dest)?; } sym::likely | sym::unlikely | sym::black_box => { // These just return their argument @@ -558,6 +570,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.memory.copy(src, align, dst, align, size, nonoverlapping) } + pub(crate) fn write_bytes_intrinsic( + &mut self, + dst: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>, + byte: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>, + count: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>, + ) -> InterpResult<'tcx> { + let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap().ty)?; + + let dst = self.read_pointer(&dst)?; + let byte = self.read_scalar(&byte)?.to_u8()?; + let count = self.read_scalar(&count)?.to_machine_usize(self)?; + + let len = layout + .size + .checked_mul(count, self) + .ok_or_else(|| err_ub_format!("overflow computing total size of `write_bytes`"))?; + + let bytes = std::iter::repeat(byte).take(len.bytes_usize()); + self.memory.write_bytes(dst, bytes) + } + pub(crate) fn raw_eq_intrinsic( &mut self, lhs: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::PointerTag>, diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs index b5e97ec8fe0..058903dcdee 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs @@ -82,7 +82,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ) -> MPlaceTy<'tcx, M::PointerTag> { let loc_details = &self.tcx.sess.opts.debugging_opts.location_detail; let file = if loc_details.file { - self.allocate_str(&filename.as_str(), MemoryKind::CallerLocation, Mutability::Not) + self.allocate_str(filename.as_str(), MemoryKind::CallerLocation, Mutability::Not) } else { // FIXME: This creates a new allocation each time. It might be preferable to // perform this allocation only once, and re-use the `MPlaceTy`. diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs index 5b4a5ac3577..ca000f93eb6 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/type_name.rs @@ -88,7 +88,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { } fn path_crate(mut self, cnum: CrateNum) -> Result<Self::Path, Self::Error> { - self.path.push_str(&self.tcx.crate_name(cnum).as_str()); + self.path.push_str(self.tcx.crate_name(cnum).as_str()); Ok(self) } @@ -148,7 +148,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { } } -impl PrettyPrinter<'tcx> for AbsolutePathPrinter<'tcx> { +impl<'tcx> PrettyPrinter<'tcx> for AbsolutePathPrinter<'tcx> { fn region_should_not_be_omitted(&self, _region: ty::Region<'_>) -> bool { false } diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 323e102b872..23ec3875cbc 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -131,6 +131,10 @@ pub trait Machine<'mir, 'tcx>: Sized { /// Whether to enforce the validity invariant fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; + /// Whether to enforce validity (e.g., initialization and not having ptr provenance) + /// of integers and floats. + fn enforce_number_validity(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool; + /// Whether function calls should be [ABI](Abi)-checked. fn enforce_abi(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { true @@ -163,7 +167,7 @@ pub trait Machine<'mir, 'tcx>: Sized { args: &[OpTy<'tcx, Self::PointerTag>], ret: Option<(&PlaceTy<'tcx, Self::PointerTag>, mir::BasicBlock)>, unwind: StackPopUnwind, - ) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>>; + ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>>; /// Execute `fn_val`. It is the hook's responsibility to advance the instruction /// pointer as appropriate. @@ -208,12 +212,6 @@ pub trait Machine<'mir, 'tcx>: Sized { right: &ImmTy<'tcx, Self::PointerTag>, ) -> InterpResult<'tcx, (Scalar<Self::PointerTag>, bool, Ty<'tcx>)>; - /// Heap allocations via the `box` keyword. - fn box_alloc( - ecx: &mut InterpCx<'mir, 'tcx, Self>, - dest: &PlaceTy<'tcx, Self::PointerTag>, - ) -> InterpResult<'tcx>; - /// Called to read the specified `local` from the `frame`. /// Since reading a ZST is not actually accessing memory or locals, this is never invoked /// for ZST reads. @@ -370,12 +368,12 @@ pub trait Machine<'mir, 'tcx>: Sized { ) -> InterpResult<'tcx, Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>>; /// Borrow the current thread's stack. - fn stack( + fn stack<'a>( ecx: &'a InterpCx<'mir, 'tcx, Self>, ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>]; /// Mutably borrow the current thread's stack. - fn stack_mut( + fn stack_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>>; @@ -427,6 +425,11 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) { } #[inline(always)] + fn enforce_number_validity(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool { + true + } + + #[inline(always)] fn call_extra_fn( _ecx: &mut InterpCx<$mir, $tcx, Self>, fn_val: !, diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index b8b6ff93753..4aa3c83cc02 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -1057,20 +1057,19 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { Some(dest_ptr) => dest_ptr, }; + // This checks relocation edges on the src, which needs to happen before + // `prepare_relocation_copy`. + let src_bytes = src_alloc + .get_bytes_with_uninit_and_ptr(&tcx, src_range) + .map_err(|e| e.to_interp_error(src_alloc_id))? + .as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation // first copy the relocations to a temporary buffer, because // `get_bytes_mut` will clear the relocations, which is correct, // since we don't want to keep any relocations at the target. - // (`get_bytes_with_uninit_and_ptr` below checks that there are no - // relocations overlapping the edges; those would not be handled correctly). let relocations = src_alloc.prepare_relocation_copy(self, src_range, dest_offset, num_copies); // Prepare a copy of the initialization mask. let compressed = src_alloc.compress_uninit_range(src_range); - // This checks relocation edges on the src. - let src_bytes = src_alloc - .get_bytes_with_uninit_and_ptr(&tcx, src_range) - .map_err(|e| e.to_interp_error(src_alloc_id))? - .as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation // Destination alloc preparations and access hooks. let (dest_alloc, extra) = self.get_raw_mut(dest_alloc_id)?; diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index b6682b13ed2..e9c94c0cc43 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -106,7 +106,7 @@ pub struct ImmTy<'tcx, Tag: Provenance = AllocId> { #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] rustc_data_structures::static_assert_size!(ImmTy<'_>, 72); -impl<Tag: Provenance> std::fmt::Display for ImmTy<'tcx, Tag> { +impl<Tag: Provenance> std::fmt::Display for ImmTy<'_, Tag> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { /// Helper function for printing a scalar to a FmtPrinter fn p<'a, 'tcx, F: std::fmt::Write, Tag: Provenance>( @@ -437,6 +437,18 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }) } + /// Converts a repr(simd) operand into an operand where `place_index` accesses the SIMD elements. + /// Also returns the number of elements. + pub fn operand_to_simd( + &self, + base: &OpTy<'tcx, M::PointerTag>, + ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, u64)> { + // Basically we just transmute this place into an array following simd_size_and_type. + // This only works in memory, but repr(simd) types should never be immediates anyway. + assert!(base.layout.ty.is_simd()); + self.mplace_to_simd(&base.assert_mem_place()) + } + /// Read from a local. Will not actually access the local if reading from a ZST. /// Will not access memory, instead an indirect `Operand` is returned. /// @@ -500,7 +512,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.param_env, self.layout_of(self.subst_from_current_frame_and_normalize_erasing_regions( place.ty(&self.frame().body.local_decls, *self.tcx).ty - ))?, + )?)?, op.layout, )); Ok(op) @@ -522,7 +534,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Constant(ref constant) => { let val = - self.subst_from_current_frame_and_normalize_erasing_regions(constant.literal); + self.subst_from_current_frame_and_normalize_erasing_regions(constant.literal)?; // This can still fail: // * During ConstProp, with `TooGeneric` or since the `requried_consts` were not all // checked yet. @@ -556,7 +568,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ty::ConstKind::Param(_) | ty::ConstKind::Bound(..) => throw_inval!(TooGeneric), ty::ConstKind::Error(_) => throw_inval!(AlreadyReported(ErrorReported)), ty::ConstKind::Unevaluated(uv) => { - let instance = self.resolve(uv.def, uv.substs(*self.tcx))?; + let instance = self.resolve(uv.def, uv.substs)?; Ok(self.eval_to_allocation(GlobalId { instance, promoted: uv.promoted })?.into()) } ty::ConstKind::Infer(..) | ty::ConstKind::Placeholder(..) => { diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index ac000b1bb56..48c90e1881a 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -130,7 +130,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let signed = left_layout.abi.is_signed(); let size = u128::from(left_layout.size.bits()); let overflow = r >= size; - let r = r % size; // mask to type size + // The shift offset is implicitly masked to the type size, to make sure this operation + // is always defined. This is the one MIR operator that does *not* directly map to a + // single LLVM operation. See + // <https://github.com/rust-lang/rust/blob/a3b9405ae7bb6ab4e8103b414e75c44598a10fd2/compiler/rustc_codegen_ssa/src/common.rs#L131-L158> + // for the corresponding truncation in our codegen backends. + let r = r % size; let r = u32::try_from(r).unwrap(); // we masked so this will always fit let result = if signed { let l = self.sign_extend(l, left_layout) as i128; @@ -323,9 +328,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.binary_int_op(bin_op, l, left.layout, r, right.layout) } _ if left.layout.ty.is_any_ptr() => { - // The RHS type must be the same *or an integer type* (for `Offset`). + // The RHS type must be a `pointer` *or an integer type* (for `Offset`). + // (Even when both sides are pointers, their type might differ, see issue #91636) assert!( - right.layout.ty == left.layout.ty || right.layout.ty.is_integral(), + right.layout.ty.is_any_ptr() || right.layout.ty.is_integral(), "Unexpected types for BinOp: {:?} {:?} {:?}", left.layout.ty, bin_op, diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index d425b84bdaf..818b95b7fc4 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -150,7 +150,7 @@ impl<Tag: Provenance> MemPlace<Tag> { } #[inline] - pub fn offset( + pub fn offset<'tcx>( self, offset: Size, meta: MemPlaceMeta<Tag>, @@ -200,7 +200,7 @@ impl<'tcx, Tag: Provenance> MPlaceTy<'tcx, Tag> { } } else { // Go through the layout. There are lots of types that support a length, - // e.g., SIMD types. + // e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!) match self.layout.fields { FieldsShape::Array { count, .. } => Ok(count), _ => bug!("len not supported on sized type {:?}", self.layout.ty), @@ -327,7 +327,7 @@ where self.memory.get_mut(place.ptr, size, place.align) } - /// Check if this mplace is dereferencable and sufficiently aligned. + /// Check if this mplace is dereferenceable and sufficiently aligned. fn check_mplace_access( &self, mplace: MPlaceTy<'tcx, M::PointerTag>, @@ -362,21 +362,15 @@ where // Re-use parent metadata to determine dynamic field layout. // With custom DSTS, this *will* execute user-defined code, but the same // happens at run-time so that's okay. - let align = match self.size_and_align_of(&base.meta, &field_layout)? { - Some((_, align)) => align, - None if offset == Size::ZERO => { - // An extern type at offset 0, we fall back to its static alignment. - // FIXME: Once we have made decisions for how to handle size and alignment - // of `extern type`, this should be adapted. It is just a temporary hack - // to get some code to work that probably ought to work. - field_layout.align.abi + match self.size_and_align_of(&base.meta, &field_layout)? { + Some((_, align)) => (base.meta, offset.align_to(align)), + None => { + // For unsized types with an extern type tail we perform no adjustments. + // NOTE: keep this in sync with `PlaceRef::project_field` in the codegen backend. + assert!(matches!(base.meta, MemPlaceMeta::None)); + (base.meta, offset) } - None => span_bug!( - self.cur_span(), - "cannot compute offset for extern type field at non-0 offset" - ), - }; - (base.meta, offset.align_to(align)) + } } else { // base.meta could be present; we might be accessing a sized field of an unsized // struct. @@ -420,7 +414,7 @@ where // Iterates over all fields of an array. Much more efficient than doing the // same by repeatedly calling `mplace_array`. - pub(super) fn mplace_array_fields( + pub(super) fn mplace_array_fields<'a>( &self, base: &'a MPlaceTy<'tcx, Tag>, ) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, MPlaceTy<'tcx, Tag>>> + 'a> @@ -533,6 +527,22 @@ where }) } + /// Converts a repr(simd) place into a place where `place_index` accesses the SIMD elements. + /// Also returns the number of elements. + pub fn mplace_to_simd( + &self, + base: &MPlaceTy<'tcx, M::PointerTag>, + ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, u64)> { + // Basically we just transmute this place into an array following simd_size_and_type. + // (Transmuting is okay since this is an in-memory place. We also double-check the size + // stays the same.) + let (len, e_ty) = base.layout.ty.simd_size_and_type(*self.tcx); + let array = self.tcx.mk_array(e_ty, len); + let layout = self.layout_of(array)?; + assert_eq!(layout.size, base.layout.size); + Ok((MPlaceTy { layout, ..*base }, len)) + } + /// Gets the place of a field inside the place, and also the field's type. /// Just a convenience function, but used quite a bit. /// This is the only projection that might have a side-effect: We cannot project @@ -594,6 +604,16 @@ where }) } + /// Converts a repr(simd) place into a place where `place_index` accesses the SIMD elements. + /// Also returns the number of elements. + pub fn place_to_simd( + &mut self, + base: &PlaceTy<'tcx, M::PointerTag>, + ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, u64)> { + let mplace = self.force_allocation(base)?; + self.mplace_to_simd(&mplace) + } + /// Computes a place. You should only use this if you intend to write into this /// place; for reading, a more efficient alternative is `eval_place_for_read`. pub fn eval_place( @@ -617,7 +637,7 @@ where self.param_env, self.layout_of(self.subst_from_current_frame_and_normalize_erasing_regions( place.ty(&self.frame().body.local_decls, *self.tcx).ty - ))?, + )?)?, place_ty.layout, )); Ok(place_ty) diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index e6037d561de..57ba9b40992 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -140,8 +140,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Defined to do nothing. These are added by optimization passes, to avoid changing the // size of MIR constantly. Nop => {} - - LlvmInlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"), } self.stack_mut()[frame_idx].loc.as_mut().unwrap().statement_index += 1; @@ -199,9 +197,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Aggregate(ref kind, ref operands) => { // active_field_index is for union initialization. let (dest, active_field_index) = match **kind { - mir::AggregateKind::Adt(adt_def, variant_index, _, _, active_field_index) => { + mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => { self.write_discriminant(variant_index, &dest)?; - if adt_def.is_enum() { + if self.tcx.adt_def(adt_did).is_enum() { assert!(active_field_index.is_none()); (self.place_downcast(&dest, variant_index)?, None) } else { @@ -242,11 +240,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let elem_size = first.layout.size; let first_ptr = first.ptr; let rest_ptr = first_ptr.offset(elem_size, self)?; + // For the alignment of `rest_ptr`, we crucially do *not* use `first.align` as + // that place might be more aligned than its type mandates (a `u8` array could + // be 4-aligned if it sits at the right spot in a struct). Instead we use + // `first.layout.align`, i.e., the alignment given by the type. self.memory.copy_repeatedly( first_ptr, first.align, rest_ptr, - first.align, + first.layout.align.abi, elem_size, length - 1, /*nonoverlapping:*/ true, @@ -267,12 +269,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.write_immediate(place.to_ref(self), &dest)?; } - NullaryOp(mir::NullOp::Box, _) => { - M::box_alloc(self, &dest)?; - } - NullaryOp(null_op, ty) => { - let ty = self.subst_from_current_frame_and_normalize_erasing_regions(ty); + let ty = self.subst_from_current_frame_and_normalize_erasing_regions(ty)?; let layout = self.layout_of(ty)?; if layout.is_unsized() { // FIXME: This should be a span_bug (#80742) @@ -285,7 +283,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let val = match null_op { mir::NullOp::SizeOf => layout.size.bytes(), mir::NullOp::AlignOf => layout.align.abi.bytes(), - mir::NullOp::Box => unreachable!(), }; self.write_scalar(Scalar::from_machine_usize(val, self), &dest)?; } @@ -298,7 +295,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Cast(cast_kind, ref operand, cast_ty) => { let src = self.eval_operand(operand, None)?; - let cast_ty = self.subst_from_current_frame_and_normalize_erasing_regions(cast_ty); + let cast_ty = + self.subst_from_current_frame_and_normalize_erasing_regions(cast_ty)?; self.cast(&src, cast_kind, cast_ty, &dest)?; } diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index 00208574c55..f3910c9765d 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -1,14 +1,14 @@ use std::borrow::Cow; use std::convert::TryFrom; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::layout::{self, LayoutOf as _, TyAndLayout}; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::Instance; use rustc_middle::{ mir, ty::{self, Ty}, }; use rustc_target::abi; +use rustc_target::abi::call::{ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode}; use rustc_target::spec::abi::Abi; use super::{ @@ -17,10 +17,6 @@ use super::{ }; impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { - fn fn_can_unwind(&self, attrs: CodegenFnAttrFlags, abi: Abi) -> bool { - layout::fn_can_unwind(*self.tcx, attrs, abi) - } - pub(super) fn eval_terminator( &mut self, terminator: &mir::Terminator<'tcx>, @@ -64,25 +60,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let old_stack = self.frame_idx(); let old_loc = self.frame().loc; let func = self.eval_operand(func, None)?; - let (fn_val, abi, caller_can_unwind) = match *func.layout.ty.kind() { - ty::FnPtr(sig) => { - let caller_abi = sig.abi(); + let args = self.eval_operands(args)?; + + let fn_sig_binder = func.layout.ty.fn_sig(*self.tcx); + let fn_sig = + self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig_binder); + let extra_args = &args[fn_sig.inputs().len()..]; + let extra_args = self.tcx.mk_type_list(extra_args.iter().map(|arg| arg.layout.ty)); + + let (fn_val, fn_abi, with_caller_location) = match *func.layout.ty.kind() { + ty::FnPtr(_sig) => { let fn_ptr = self.read_pointer(&func)?; let fn_val = self.memory.get_fn(fn_ptr)?; - ( - fn_val, - caller_abi, - self.fn_can_unwind(CodegenFnAttrFlags::empty(), caller_abi), - ) + (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false) } ty::FnDef(def_id, substs) => { - let sig = func.layout.ty.fn_sig(*self.tcx); + let instance = + self.resolve(ty::WithOptConstParam::unknown(def_id), substs)?; ( - FnVal::Instance( - self.resolve(ty::WithOptConstParam::unknown(def_id), substs)?, - ), - sig.abi(), - self.fn_can_unwind(self.tcx.codegen_fn_attrs(def_id).flags, sig.abi()), + FnVal::Instance(instance), + self.fn_abi_of_instance(instance, extra_args)?, + instance.def.requires_caller_location(*self.tcx), ) } _ => span_bug!( @@ -91,7 +89,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { func.layout.ty ), }; - let args = self.eval_operands(args)?; + let dest_place; let ret = match destination { Some((dest, ret)) => { @@ -102,10 +100,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }; self.eval_fn_call( fn_val, - abi, - &args[..], + (fn_sig.abi, fn_abi), + &args, + with_caller_location, ret, - match (cleanup, caller_can_unwind) { + match (cleanup, fn_abi.can_unwind) { (Some(cleanup), true) => StackPopUnwind::Cleanup(*cleanup), (None, true) => StackPopUnwind::Skip, (_, false) => StackPopUnwind::NotAllowed, @@ -174,68 +173,128 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } fn check_argument_compat( - rust_abi: bool, - caller: TyAndLayout<'tcx>, - callee: TyAndLayout<'tcx>, + caller_abi: &ArgAbi<'tcx, Ty<'tcx>>, + callee_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> bool { - if caller.ty == callee.ty { - // No question + // Heuristic for type comparison. + let layout_compat = || { + if caller_abi.layout.ty == callee_abi.layout.ty { + // No question + return true; + } + // Compare layout + match (caller_abi.layout.abi, callee_abi.layout.abi) { + // Different valid ranges are okay (once we enforce validity, + // that will take care to make it UB to leave the range, just + // like for transmute). + (abi::Abi::Scalar(caller), abi::Abi::Scalar(callee)) => { + caller.value == callee.value + } + ( + abi::Abi::ScalarPair(caller1, caller2), + abi::Abi::ScalarPair(callee1, callee2), + ) => caller1.value == callee1.value && caller2.value == callee2.value, + // Be conservative + _ => false, + } + }; + // Padding must be fully equal. + let pad_compat = || caller_abi.pad == callee_abi.pad; + // When comparing the PassMode, we have to be smart about comparing the attributes. + let arg_attr_compat = |a1: ArgAttributes, a2: ArgAttributes| { + // There's only one regular attribute that matters for the call ABI: InReg. + // Everything else is things like noalias, dereferencable, nonnull, ... + // (This also applies to pointee_size, pointee_align.) + if a1.regular.contains(ArgAttribute::InReg) != a2.regular.contains(ArgAttribute::InReg) + { + return false; + } + // We also compare the sign extension mode -- this could let the callee make assumptions + // about bits that conceptually were not even passed. + if a1.arg_ext != a2.arg_ext { + return false; + } return true; - } - if !rust_abi { - // Don't risk anything - return false; - } - // Compare layout - match (caller.abi, callee.abi) { - // Different valid ranges are okay (once we enforce validity, - // that will take care to make it UB to leave the range, just - // like for transmute). - (abi::Abi::Scalar(caller), abi::Abi::Scalar(callee)) => caller.value == callee.value, - (abi::Abi::ScalarPair(caller1, caller2), abi::Abi::ScalarPair(callee1, callee2)) => { - caller1.value == callee1.value && caller2.value == callee2.value + }; + let mode_compat = || match (caller_abi.mode, callee_abi.mode) { + (PassMode::Ignore, PassMode::Ignore) => true, + (PassMode::Direct(a1), PassMode::Direct(a2)) => arg_attr_compat(a1, a2), + (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => { + arg_attr_compat(a1, a2) && arg_attr_compat(b1, b2) } - // Be conservative + (PassMode::Cast(c1), PassMode::Cast(c2)) => c1 == c2, + ( + PassMode::Indirect { attrs: a1, extra_attrs: None, on_stack: s1 }, + PassMode::Indirect { attrs: a2, extra_attrs: None, on_stack: s2 }, + ) => arg_attr_compat(a1, a2) && s1 == s2, + ( + PassMode::Indirect { attrs: a1, extra_attrs: Some(e1), on_stack: s1 }, + PassMode::Indirect { attrs: a2, extra_attrs: Some(e2), on_stack: s2 }, + ) => arg_attr_compat(a1, a2) && arg_attr_compat(e1, e2) && s1 == s2, _ => false, + }; + + if layout_compat() && pad_compat() && mode_compat() { + return true; } + trace!( + "check_argument_compat: incompatible ABIs:\ncaller: {:?}\ncallee: {:?}", + caller_abi, + callee_abi + ); + return false; } - /// Pass a single argument, checking the types for compatibility. - fn pass_argument( + /// Initialize a single callee argument, checking the types for compatibility. + fn pass_argument<'x, 'y>( &mut self, - rust_abi: bool, - caller_arg: &mut impl Iterator<Item = OpTy<'tcx, M::PointerTag>>, + caller_args: &mut impl Iterator< + Item = (&'x OpTy<'tcx, M::PointerTag>, &'y ArgAbi<'tcx, Ty<'tcx>>), + >, + callee_abi: &ArgAbi<'tcx, Ty<'tcx>>, callee_arg: &PlaceTy<'tcx, M::PointerTag>, - ) -> InterpResult<'tcx> { - if rust_abi && callee_arg.layout.is_zst() { - // Nothing to do. - trace!("Skipping callee ZST"); + ) -> InterpResult<'tcx> + where + 'tcx: 'x, + 'tcx: 'y, + { + if matches!(callee_abi.mode, PassMode::Ignore) { + // This one is skipped. return Ok(()); } - let caller_arg = caller_arg.next().ok_or_else(|| { + // Find next caller arg. + let (caller_arg, caller_abi) = caller_args.next().ok_or_else(|| { err_ub_format!("calling a function with fewer arguments than it requires") })?; - if rust_abi { - assert!(!caller_arg.layout.is_zst(), "ZSTs must have been already filtered out"); - } // Now, check - if !Self::check_argument_compat(rust_abi, caller_arg.layout, callee_arg.layout) { + if !Self::check_argument_compat(caller_abi, callee_abi) { throw_ub_format!( "calling a function with argument of type {:?} passing data of type {:?}", callee_arg.layout.ty, caller_arg.layout.ty ) } - // We allow some transmutes here + // We allow some transmutes here. + // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This + // is true for all `copy_op`, but there are a lot of special cases for argument passing + // specifically.) self.copy_op_transmute(&caller_arg, callee_arg) } /// Call this function -- pushing the stack frame and initializing the arguments. + /// + /// `caller_fn_abi` is used to determine if all the arguments are passed the proper way. + /// However, we also need `caller_abi` to determine if we need to do untupling of arguments. + /// + /// `with_caller_location` indicates whether the caller passed a caller location. Miri + /// implements caller locations without argument passing, but to match `FnAbi` we need to know + /// when those arguments are present. pub(crate) fn eval_fn_call( &mut self, fn_val: FnVal<'tcx, M::ExtraFnVal>, - caller_abi: Abi, + (caller_abi, caller_fn_abi): (Abi, &FnAbi<'tcx, Ty<'tcx>>), args: &[OpTy<'tcx, M::PointerTag>], + with_caller_location: bool, ret: Option<(&PlaceTy<'tcx, M::PointerTag>, mir::BasicBlock)>, mut unwind: StackPopUnwind, ) -> InterpResult<'tcx> { @@ -248,39 +307,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } }; - let get_abi = |this: &Self, instance_ty: Ty<'tcx>| match instance_ty.kind() { - ty::FnDef(..) => instance_ty.fn_sig(*this.tcx).abi(), - ty::Closure(..) => Abi::RustCall, - ty::Generator(..) => Abi::Rust, - _ => span_bug!(this.cur_span(), "unexpected callee ty: {:?}", instance_ty), - }; - - // ABI check - let check_abi = |callee_abi: Abi| -> InterpResult<'tcx> { - let normalize_abi = |abi| match abi { - Abi::Rust | Abi::RustCall | Abi::RustIntrinsic | Abi::PlatformIntrinsic => - // These are all the same ABI, really. - { - Abi::Rust - } - abi => abi, - }; - if normalize_abi(caller_abi) != normalize_abi(callee_abi) { - throw_ub_format!( - "calling a function with ABI {} using caller ABI {}", - callee_abi.name(), - caller_abi.name() - ) - } - Ok(()) - }; - match instance.def { ty::InstanceDef::Intrinsic(..) => { - if M::enforce_abi(self) { - check_abi(get_abi(self, instance.ty(*self.tcx, self.param_env)))?; - } assert!(caller_abi == Abi::RustIntrinsic || caller_abi == Abi::PlatformIntrinsic); + // caller_fn_abi is not relevant here, we interpret the arguments directly for each intrinsic. M::call_intrinsic(self, instance, args, ret, unwind) } ty::InstanceDef::VtableShim(..) @@ -291,26 +321,37 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { | ty::InstanceDef::CloneShim(..) | ty::InstanceDef::Item(_) => { // We need MIR for this fn - let body = + let (body, instance) = match M::find_mir_or_eval_fn(self, instance, caller_abi, args, ret, unwind)? { Some(body) => body, None => return Ok(()), }; - // Check against the ABI of the MIR body we are calling (not the ABI of `instance`; - // these can differ when `find_mir_or_eval_fn` does something clever like resolve - // exported symbol names). - let callee_def_id = body.source.def_id(); - let callee_abi = get_abi(self, self.tcx.type_of(callee_def_id)); + // Compute callee information using the `instance` returned by + // `find_mir_or_eval_fn`. + // FIXME: for variadic support, do we have to somehow determine calle's extra_args? + let callee_fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; + + if callee_fn_abi.c_variadic != caller_fn_abi.c_variadic { + throw_ub_format!( + "calling a c-variadic function via a non-variadic call site, or vice versa" + ); + } + if callee_fn_abi.c_variadic { + throw_unsup_format!("calling a c-variadic function is not supported"); + } if M::enforce_abi(self) { - check_abi(callee_abi)?; + if caller_fn_abi.conv != callee_fn_abi.conv { + throw_ub_format!( + "calling a function with calling convention {:?} using calling convention {:?}", + callee_fn_abi.conv, + caller_fn_abi.conv + ) + } } - if !matches!(unwind, StackPopUnwind::NotAllowed) - && !self - .fn_can_unwind(self.tcx.codegen_fn_attrs(callee_def_id).flags, callee_abi) - { + if !matches!(unwind, StackPopUnwind::NotAllowed) && !callee_fn_abi.can_unwind { // The callee cannot unwind. unwind = StackPopUnwind::NotAllowed; } @@ -343,12 +384,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { .collect::<Vec<_>>() ); - // Figure out how to pass which arguments. - // The Rust ABI is special: ZST get skipped. - let rust_abi = matches!(caller_abi, Abi::Rust | Abi::RustCall); - - // We have two iterators: Where the arguments come from, - // and where they go to. + // In principle, we have two iterators: Where the arguments come from, and where + // they go to. // For where they come from: If the ABI is RustCall, we untuple the // last incoming argument. These two iterators do not have the same type, @@ -373,53 +410,59 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Plain arg passing Cow::from(args) }; - // Skip ZSTs - let mut caller_iter = - caller_args.iter().filter(|op| !rust_abi || !op.layout.is_zst()).copied(); + // If `with_caller_location` is set we pretend there is an extra argument (that + // we will not pass). + assert_eq!( + caller_args.len() + if with_caller_location { 1 } else { 0 }, + caller_fn_abi.args.len(), + "mismatch between caller ABI and caller arguments", + ); + let mut caller_args = caller_args + .iter() + .zip(caller_fn_abi.args.iter()) + .filter(|arg_and_abi| !matches!(arg_and_abi.1.mode, PassMode::Ignore)); // Now we have to spread them out across the callee's locals, // taking into account the `spread_arg`. If we could write // this is a single iterator (that handles `spread_arg`), then // `pass_argument` would be the loop body. It takes care to // not advance `caller_iter` for ZSTs. + let mut callee_args_abis = callee_fn_abi.args.iter(); for local in body.args_iter() { let dest = self.eval_place(mir::Place::from(local))?; if Some(local) == body.spread_arg { // Must be a tuple for i in 0..dest.layout.fields.count() { let dest = self.place_field(&dest, i)?; - self.pass_argument(rust_abi, &mut caller_iter, &dest)?; + let callee_abi = callee_args_abis.next().unwrap(); + self.pass_argument(&mut caller_args, callee_abi, &dest)?; } } else { // Normal argument - self.pass_argument(rust_abi, &mut caller_iter, &dest)?; + let callee_abi = callee_args_abis.next().unwrap(); + self.pass_argument(&mut caller_args, callee_abi, &dest)?; } } - // Now we should have no more caller args - if caller_iter.next().is_some() { + // If the callee needs a caller location, pretend we consume one more argument from the ABI. + if instance.def.requires_caller_location(*self.tcx) { + callee_args_abis.next().unwrap(); + } + // Now we should have no more caller args or callee arg ABIs + assert!( + callee_args_abis.next().is_none(), + "mismatch between callee ABI and callee body arguments" + ); + if caller_args.next().is_some() { throw_ub_format!("calling a function with more arguments than it expected") } // Don't forget to check the return type! - if let Some((caller_ret, _)) = ret { - let callee_ret = self.eval_place(mir::Place::return_place())?; - if !Self::check_argument_compat( - rust_abi, - caller_ret.layout, - callee_ret.layout, - ) { - throw_ub_format!( - "calling a function with return type {:?} passing \ - return place of type {:?}", - callee_ret.layout.ty, - caller_ret.layout.ty - ) - } - } else { - let local = mir::RETURN_PLACE; - let callee_layout = self.layout_of_local(self.frame(), local, None)?; - if !callee_layout.abi.is_uninhabited() { - throw_ub_format!("calling a returning function without a return place") - } + if !Self::check_argument_compat(&caller_fn_abi.ret, &callee_fn_abi.ret) { + throw_ub_format!( + "calling a function with return type {:?} passing \ + return place of type {:?}", + callee_fn_abi.ret.layout.ty, + caller_fn_abi.ret.layout.ty, + ) } }; match res { @@ -464,7 +507,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { )); trace!("Patched self operand to {:#?}", args[0]); // recurse with concrete function - self.eval_fn_call(fn_val, caller_abi, &args, ret, unwind) + self.eval_fn_call( + fn_val, + (caller_abi, caller_fn_abi), + &args, + with_caller_location, + ret, + unwind, + ) } } } @@ -489,6 +539,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } _ => (instance, place), }; + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; let arg = ImmTy::from_immediate( place.to_ref(self), @@ -500,8 +551,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.eval_fn_call( FnVal::Instance(instance), - Abi::Rust, + (Abi::Rust, fn_abi), &[arg.into()], + false, Some((&dest.into(), target)), match unwind { Some(cleanup) => StackPopUnwind::Cleanup(cleanup), diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs index a16388d5de2..6a3378a3896 100644 --- a/compiler/rustc_const_eval/src/interpret/util.rs +++ b/compiler/rustc_const_eval/src/interpret/util.rs @@ -3,13 +3,17 @@ use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitor}; use std::convert::TryInto; use std::ops::ControlFlow; -/// Returns `true` if a used generic parameter requires substitution. +/// Checks whether a type contains generic parameters which require substitution. +/// +/// In case it does, returns a `TooGeneric` const eval error. Note that due to polymorphization +/// types may be "concrete enough" even though they still contain generic parameters in +/// case these parameters are unused. crate fn ensure_monomorphic_enough<'tcx, T>(tcx: TyCtxt<'tcx>, ty: T) -> InterpResult<'tcx> where T: TypeFoldable<'tcx>, { debug!("ensure_monomorphic_enough: ty={:?}", ty); - if !ty.potentially_needs_subst() { + if !ty.needs_subst() { return Ok(()); } @@ -21,12 +25,8 @@ where impl<'tcx> TypeVisitor<'tcx> for UsedParamsNeedSubstVisitor<'tcx> { type BreakTy = FoundParam; - fn tcx_for_anon_const_substs(&self) -> Option<TyCtxt<'tcx>> { - Some(self.tcx) - } - fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> { - if !ty.potentially_needs_subst() { + if !ty.needs_subst() { return ControlFlow::CONTINUE; } @@ -44,7 +44,7 @@ where let is_used = unused_params.contains(index).map_or(true, |unused| !unused); // Only recurse when generic parameters in fns, closures and generators // are used and require substitution. - match (is_used, subst.definitely_needs_subst(self.tcx)) { + match (is_used, subst.needs_subst()) { // Just in case there are closures or generators within this subst, // recurse. (true, true) => return subst.super_visit_with(self), diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index fc69770bf6a..9dc7930fc51 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -14,6 +14,7 @@ use rustc_middle::mir::interpret::InterpError; use rustc_middle::ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_span::symbol::{sym, Symbol}; +use rustc_span::DUMMY_SP; use rustc_target::abi::{Abi, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange}; use std::hash::Hash; @@ -266,14 +267,14 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' match layout.variants { Variants::Single { index } => { // Inside a variant - PathElem::Field(def.variants[index].fields[field].ident.name) + PathElem::Field(def.variants[index].fields[field].name) } Variants::Multiple { .. } => bug!("we handled variants above"), } } // other ADTs - ty::Adt(def, _) => PathElem::Field(def.non_enum_variant().fields[field].ident.name), + ty::Adt(def, _) => PathElem::Field(def.non_enum_variant().fields[field].name), // arrays/slices ty::Array(..) | ty::Slice(..) => PathElem::ArrayElem(field), @@ -520,7 +521,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let value = self.read_scalar(value)?; // NOTE: Keep this in sync with the array optimization for int/float // types below! - if self.ctfe_mode.is_some() { + if M::enforce_number_validity(self.ecx) { // Integers/floats in CTFE: Must be scalar bits, pointers are dangerous let is_bits = value.check_init().map_or(false, |v| v.try_to_int().is_ok()); if !is_bits { @@ -528,9 +529,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' { "{}", value } expected { "initialized plain (non-pointer) bytes" } ) } - } else { - // At run-time, for now, we accept *anything* for these types, including - // uninit. We should fix that, but let's start low. } Ok(true) } @@ -728,7 +726,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> new_op: &OpTy<'tcx, M::PointerTag>, ) -> InterpResult<'tcx> { let name = match old_op.layout.ty.kind() { - ty::Adt(adt, _) => PathElem::Variant(adt.variants[variant_id].ident.name), + ty::Adt(adt, _) => PathElem::Variant(adt.variants[variant_id].name), // Generators also have variants ty::Generator(..) => PathElem::GeneratorState(variant_id), _ => bug!("Unexpected type with variant: {:?}", old_op.layout.ty), @@ -739,9 +737,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> #[inline(always)] fn visit_union( &mut self, - _op: &OpTy<'tcx, M::PointerTag>, + op: &OpTy<'tcx, M::PointerTag>, _fields: NonZeroUsize, ) -> InterpResult<'tcx> { + // Special check preventing `UnsafeCell` inside unions in the inner part of constants. + if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { inner: true, .. })) { + if !op.layout.ty.is_freeze(self.ecx.tcx.at(DUMMY_SP), self.ecx.param_env) { + throw_validation_failure!(self.path, { "`UnsafeCell` in a `const`" }); + } + } Ok(()) } @@ -855,9 +859,10 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> } }; + let allow_uninit_and_ptr = !M::enforce_number_validity(self.ecx); match alloc.check_bytes( alloc_range(Size::ZERO, size), - /*allow_uninit_and_ptr*/ self.ctfe_mode.is_none(), + allow_uninit_and_ptr, ) { // In the happy case, we needn't check anything else. Ok(()) => {} diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs index f308e764e86..92854af55bb 100644 --- a/compiler/rustc_const_eval/src/lib.rs +++ b/compiler/rustc_const_eval/src/lib.rs @@ -11,8 +11,6 @@ Rust MIR: a lowered representation of Rust. #![feature(crate_visibility_modifier)] #![feature(decl_macro)] #![feature(exact_size_is_empty)] -#![feature(in_band_lifetimes)] -#![feature(iter_zip)] #![feature(let_else)] #![feature(map_try_insert)] #![feature(min_specialization)] diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs index 3785c170f6b..6799514a449 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs @@ -1,8 +1,8 @@ //! The `Visitor` responsible for actually checking a `mir::Body` for invalid operations. use rustc_errors::{Applicability, Diagnostic, ErrorReported}; +use rustc_hir as hir; use rustc_hir::def_id::DefId; -use rustc_hir::{self as hir, HirId, LangItem}; use rustc_index::bit_set::BitSet; use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::traits::{ImplSource, Obligation, ObligationCause}; @@ -14,8 +14,7 @@ use rustc_middle::ty::{self, adjustment::PointerCast, Instance, InstanceDef, Ty, use rustc_middle::ty::{Binder, TraitPredicate, TraitRef}; use rustc_mir_dataflow::{self, Analysis}; use rustc_span::{sym, Span, Symbol}; -use rustc_trait_selection::traits::error_reporting::InferCtxtExt; -use rustc_trait_selection::traits::{self, SelectionContext, TraitEngine}; +use rustc_trait_selection::traits::SelectionContext; use std::mem; use std::ops::Deref; @@ -36,7 +35,7 @@ pub struct Qualifs<'mir, 'tcx> { needs_non_const_drop: Option<QualifResults<'mir, 'tcx, NeedsNonConstDrop>>, } -impl Qualifs<'mir, 'tcx> { +impl<'mir, 'tcx> Qualifs<'mir, 'tcx> { /// Returns `true` if `local` is `NeedsDrop` at the given `Location`. /// /// Only updates the cursor if absolutely necessary @@ -186,7 +185,7 @@ pub struct Checker<'mir, 'tcx> { secondary_errors: Vec<Diagnostic>, } -impl Deref for Checker<'mir, 'tcx> { +impl<'mir, 'tcx> Deref for Checker<'mir, 'tcx> { type Target = ConstCx<'mir, 'tcx>; fn deref(&self) -> &Self::Target { @@ -194,7 +193,7 @@ impl Deref for Checker<'mir, 'tcx> { } } -impl Checker<'mir, 'tcx> { +impl<'mir, 'tcx> Checker<'mir, 'tcx> { pub fn new(ccx: &'mir ConstCx<'mir, 'tcx>) -> Self { Checker { span: ccx.body.span, @@ -222,8 +221,7 @@ impl Checker<'mir, 'tcx> { // Prevent const trait methods from being annotated as `stable`. // FIXME: Do this as part of stability checking. if self.is_const_stable_const_fn() { - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - if crate::const_eval::is_parent_const_impl_raw(tcx, hir_id) { + if crate::const_eval::is_parent_const_impl_raw(tcx, def_id) { self.ccx .tcx .sess @@ -255,16 +253,6 @@ impl Checker<'mir, 'tcx> { self.visit_body(&body); } - // Ensure that the end result is `Sync` in a non-thread local `static`. - let should_check_for_sync = self.const_kind() - == hir::ConstContext::Static(hir::Mutability::Not) - && !tcx.is_thread_local_static(def_id.to_def_id()); - - if should_check_for_sync { - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - check_return_ty_is_sync(tcx, &body, hir_id); - } - // If we got through const-checking without emitting any "primary" errors, emit any // "secondary" errors if they occurred. let secondary_errors = mem::take(&mut self.secondary_errors); @@ -284,7 +272,7 @@ impl Checker<'mir, 'tcx> { struct StorageDeads { locals: BitSet<Local>, } - impl Visitor<'tcx> for StorageDeads { + impl<'tcx> Visitor<'tcx> for StorageDeads { fn visit_statement(&mut self, stmt: &Statement<'tcx>, _: Location) { if let StatementKind::StorageDead(l) = stmt.kind { self.locals.insert(l); @@ -359,7 +347,7 @@ impl Checker<'mir, 'tcx> { fn check_local_or_return_ty(&mut self, ty: Ty<'tcx>, local: Local) { let kind = self.body.local_kind(local); - for ty in ty.walk(self.tcx) { + for ty in ty.walk() { let ty = match ty.unpack() { GenericArgKind::Type(ty) => ty, @@ -471,7 +459,7 @@ impl Checker<'mir, 'tcx> { } } -impl Visitor<'tcx> for Checker<'mir, 'tcx> { +impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { fn visit_basic_block_data(&mut self, bb: BasicBlock, block: &BasicBlockData<'tcx>) { trace!("visit_basic_block_data: bb={:?} is_cleanup={:?}", bb, block.is_cleanup); @@ -643,7 +631,6 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { } Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _) => {} - Rvalue::NullaryOp(NullOp::Box, _) => self.check_op(ops::HeapAllocation), Rvalue::ShallowInitBox(_, _) => {} Rvalue::UnaryOp(_, ref operand) => { @@ -722,7 +709,7 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { match elem { ProjectionElem::Deref => { let base_ty = Place::ty_from(place_local, proj_base, self.body, self.tcx).ty; - if let ty::RawPtr(_) = base_ty.kind() { + if base_ty.is_unsafe_ptr() { if proj_base.is_empty() { let decl = &self.body.local_decls[place_local]; if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info { @@ -731,7 +718,13 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { return; } } - self.check_op(ops::RawPtrDeref); + + // `*const T` is stable, `*mut T` is not + if !base_ty.is_mutable_ptr() { + return; + } + + self.check_op(ops::RawMutPtrDeref); } if context.is_mutating_use() { @@ -758,10 +751,6 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { self.super_statement(statement, location); match statement.kind { - StatementKind::LlvmInlineAsm { .. } => { - self.check_op(ops::InlineAsm); - } - StatementKind::Assign(..) | StatementKind::SetDiscriminant { .. } | StatementKind::FakeRead(..) @@ -806,7 +795,7 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { if let Some(trait_id) = tcx.trait_of_item(callee) { trace!("attempting to call a trait method"); if !self.tcx.features().const_trait_impl { - self.check_op(ops::FnCallNonConst); + self.check_op(ops::FnCallNonConst(Some((callee, substs)))); return; } @@ -816,14 +805,13 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { param_env, Binder::dummy(TraitPredicate { trait_ref, - constness: ty::BoundConstness::ConstIfConst, + constness: ty::BoundConstness::NotConst, polarity: ty::ImplPolarity::Positive, }), ); let implsrc = tcx.infer_ctxt().enter(|infcx| { - let mut selcx = - SelectionContext::with_constness(&infcx, hir::Constness::Const); + let mut selcx = SelectionContext::new(&infcx); selcx.select(&obligation) }); @@ -836,6 +824,10 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { return; } Ok(Some(ImplSource::UserDefined(data))) => { + if let hir::Constness::NotConst = tcx.impl_constness(data.impl_def_id) { + self.check_op(ops::FnCallNonConst(None)); + return; + } let callee_name = tcx.item_name(callee); if let Some(&did) = tcx .associated_item_def_ids(data.impl_def_id) @@ -862,7 +854,7 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { } if !nonconst_call_permission { - self.check_op(ops::FnCallNonConst); + self.check_op(ops::FnCallNonConst(None)); return; } } @@ -931,7 +923,7 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { } if !nonconst_call_permission { - self.check_op(ops::FnCallNonConst); + self.check_op(ops::FnCallNonConst(None)); return; } } @@ -998,11 +990,12 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { } let mut err_span = self.span; + let ty_of_dropped_place = dropped_place.ty(self.body, self.tcx).ty; - let ty_needs_non_const_drop = qualifs::NeedsNonConstDrop::in_any_value_of_ty( - self.ccx, - dropped_place.ty(self.body, self.tcx).ty, - ); + let ty_needs_non_const_drop = + qualifs::NeedsNonConstDrop::in_any_value_of_ty(self.ccx, ty_of_dropped_place); + + debug!(?ty_of_dropped_place, ?ty_needs_non_const_drop); if !ty_needs_non_const_drop { return; @@ -1047,20 +1040,7 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> { } } -fn check_return_ty_is_sync(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, hir_id: HirId) { - let ty = body.return_ty(); - tcx.infer_ctxt().enter(|infcx| { - let cause = traits::ObligationCause::new(body.span, hir_id, traits::SharedStatic); - let mut fulfillment_cx = traits::FulfillmentContext::new(); - let sync_def_id = tcx.require_lang_item(LangItem::Sync, Some(body.span)); - fulfillment_cx.register_bound(&infcx, ty::ParamEnv::empty(), ty, sync_def_id, cause); - if let Err(err) = fulfillment_cx.select_all_or_error(&infcx) { - infcx.report_fulfillment_errors(&err, None, false); - } - }); -} - -fn place_as_reborrow( +fn place_as_reborrow<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, place: Place<'tcx>, diff --git a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs index dc44409d500..b026bb2bad6 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs @@ -28,7 +28,7 @@ pub struct ConstCx<'mir, 'tcx> { pub const_kind: Option<hir::ConstContext>, } -impl ConstCx<'mir, 'tcx> { +impl<'mir, 'tcx> ConstCx<'mir, 'tcx> { pub fn new(tcx: TyCtxt<'tcx>, body: &'mir mir::Body<'tcx>) -> Self { let def_id = body.source.def_id().expect_local(); let param_env = tcx.param_env(def_id); @@ -72,11 +72,7 @@ impl ConstCx<'mir, 'tcx> { } } -pub fn rustc_allow_const_fn_unstable( - tcx: TyCtxt<'tcx>, - def_id: DefId, - feature_gate: Symbol, -) -> bool { +pub fn rustc_allow_const_fn_unstable(tcx: TyCtxt<'_>, def_id: DefId, feature_gate: Symbol) -> bool { let attrs = tcx.get_attrs(def_id); attr::rustc_allow_const_fn_unstable(&tcx.sess, attrs).any(|name| name == feature_gate) } @@ -89,7 +85,7 @@ pub fn rustc_allow_const_fn_unstable( // functions can be called in a const-context by users of the stable compiler. "const-stable" // functions are subject to more stringent restrictions than "const-unstable" functions: They // cannot use unstable features and can only call other "const-stable" functions. -pub fn is_const_stable_const_fn(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool { +pub fn is_const_stable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> bool { use attr::{ConstStability, Stability, StabilityLevel}; // A default body marked const is not const-stable because const diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs index 230d023efb9..24c4a4915e5 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs @@ -1,12 +1,14 @@ //! Concrete error types for all operations which may be invalid in a certain const context. -use rustc_errors::{struct_span_err, DiagnosticBuilder}; +use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def_id::DefId; -use rustc_middle::mir; +use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::{mir, ty::AssocKind}; use rustc_session::parse::feature_err; use rustc_span::symbol::sym; -use rustc_span::{Span, Symbol}; +use rustc_span::{symbol::Ident, Span, Symbol}; +use rustc_span::{BytePos, Pos}; use super::ConstCx; @@ -37,7 +39,7 @@ pub trait NonConstOp: std::fmt::Debug { DiagnosticImportance::Primary } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx>; + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx>; } #[derive(Debug)] @@ -51,7 +53,7 @@ impl NonConstOp for FloatingPointOp { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_fn_floating_point_arithmetic, @@ -65,24 +67,78 @@ impl NonConstOp for FloatingPointOp { #[derive(Debug)] pub struct FnCallIndirect; impl NonConstOp for FnCallIndirect { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { ccx.tcx.sess.struct_span_err(span, "function pointers are not allowed in const fn") } } /// A function call where the callee is not marked as `const`. #[derive(Debug)] -pub struct FnCallNonConst; -impl NonConstOp for FnCallNonConst { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { - struct_span_err!( +pub struct FnCallNonConst<'tcx>(pub Option<(DefId, SubstsRef<'tcx>)>); +impl<'a> NonConstOp for FnCallNonConst<'a> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + let mut err = struct_span_err!( ccx.tcx.sess, span, E0015, "calls in {}s are limited to constant functions, \ tuple structs and tuple variants", ccx.const_kind(), - ) + ); + + if let FnCallNonConst(Some((callee, substs))) = *self { + if let Some(trait_def_id) = ccx.tcx.lang_items().eq_trait() { + if let Some(eq_item) = ccx.tcx.associated_items(trait_def_id).find_by_name_and_kind( + ccx.tcx, + Ident::with_dummy_span(sym::eq), + AssocKind::Fn, + trait_def_id, + ) { + if callee == eq_item.def_id && substs.len() == 2 { + match (substs[0].unpack(), substs[1].unpack()) { + (GenericArgKind::Type(self_ty), GenericArgKind::Type(rhs_ty)) + if self_ty == rhs_ty + && self_ty.is_ref() + && self_ty.peel_refs().is_primitive() => + { + let mut num_refs = 0; + let mut tmp_ty = self_ty; + while let rustc_middle::ty::Ref(_, inner_ty, _) = tmp_ty.kind() { + num_refs += 1; + tmp_ty = inner_ty; + } + let deref = "*".repeat(num_refs); + + if let Ok(call_str) = + ccx.tcx.sess.source_map().span_to_snippet(span) + { + if let Some(eq_idx) = call_str.find("==") { + if let Some(rhs_idx) = call_str[(eq_idx + 2)..] + .find(|c: char| !c.is_whitespace()) + { + let rhs_pos = span.lo() + + BytePos::from_usize(eq_idx + 2 + rhs_idx); + let rhs_span = span.with_lo(rhs_pos).with_hi(rhs_pos); + err.multipart_suggestion( + "consider dereferencing here", + vec![ + (span.shrink_to_lo(), deref.clone()), + (rhs_span, deref), + ], + Applicability::MachineApplicable, + ); + } + } + } + } + _ => {} + } + } + } + } + } + + err } } @@ -93,7 +149,7 @@ impl NonConstOp for FnCallNonConst { pub struct FnCallUnstable(pub DefId, pub Option<Symbol>); impl NonConstOp for FnCallUnstable { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let FnCallUnstable(def_id, feature) = *self; let mut err = ccx.tcx.sess.struct_span_err( @@ -127,7 +183,7 @@ impl NonConstOp for FnPtrCast { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_fn_fn_ptr_basics, @@ -148,7 +204,7 @@ impl NonConstOp for Generator { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let msg = format!("{}s are not allowed in {}s", self.0, ccx.const_kind()); if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 { feature_err(&ccx.tcx.sess.parse_sess, sym::const_async_blocks, span, &msg) @@ -161,7 +217,7 @@ impl NonConstOp for Generator { #[derive(Debug)] pub struct HeapAllocation; impl NonConstOp for HeapAllocation { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let mut err = struct_span_err!( ccx.tcx.sess, span, @@ -185,7 +241,7 @@ impl NonConstOp for HeapAllocation { #[derive(Debug)] pub struct InlineAsm; impl NonConstOp for InlineAsm { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { struct_span_err!( ccx.tcx.sess, span, @@ -201,7 +257,7 @@ pub struct LiveDrop { pub dropped_at: Option<Span>, } impl NonConstOp for LiveDrop { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let mut err = struct_span_err!( ccx.tcx.sess, span, @@ -229,7 +285,7 @@ impl NonConstOp for TransientCellBorrow { // not additionally emit a feature gate error if activating the feature gate won't work. DiagnosticImportance::Secondary } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_refs_to_cell, @@ -245,7 +301,7 @@ impl NonConstOp for TransientCellBorrow { /// it in the future for static items. pub struct CellBorrow; impl NonConstOp for CellBorrow { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let mut err = struct_span_err!( ccx.tcx.sess, span, @@ -292,7 +348,7 @@ impl NonConstOp for MutBorrow { DiagnosticImportance::Secondary } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let raw = match self.0 { hir::BorrowKind::Raw => "raw ", hir::BorrowKind::Ref => "", @@ -331,7 +387,7 @@ impl NonConstOp for TransientMutBorrow { Status::Unstable(sym::const_mut_refs) } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let raw = match self.0 { hir::BorrowKind::Raw => "raw ", hir::BorrowKind::Ref => "", @@ -358,7 +414,7 @@ impl NonConstOp for MutDeref { DiagnosticImportance::Secondary } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_mut_refs, @@ -372,7 +428,7 @@ impl NonConstOp for MutDeref { #[derive(Debug)] pub struct PanicNonStr; impl NonConstOp for PanicNonStr { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { ccx.tcx.sess.struct_span_err( span, "argument to `panic!()` in a const context must have type `&str`", @@ -386,7 +442,7 @@ impl NonConstOp for PanicNonStr { #[derive(Debug)] pub struct RawPtrComparison; impl NonConstOp for RawPtrComparison { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let mut err = ccx .tcx .sess @@ -400,18 +456,18 @@ impl NonConstOp for RawPtrComparison { } #[derive(Debug)] -pub struct RawPtrDeref; -impl NonConstOp for RawPtrDeref { +pub struct RawMutPtrDeref; +impl NonConstOp for RawMutPtrDeref { fn status_in_item(&self, _: &ConstCx<'_, '_>) -> Status { - Status::Unstable(sym::const_raw_ptr_deref) + Status::Unstable(sym::const_mut_refs) } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, - sym::const_raw_ptr_deref, + sym::const_mut_refs, span, - &format!("dereferencing raw pointers in {}s is unstable", ccx.const_kind(),), + &format!("dereferencing raw mutable pointers in {}s is unstable", ccx.const_kind(),), ) } } @@ -422,7 +478,7 @@ impl NonConstOp for RawPtrDeref { #[derive(Debug)] pub struct RawPtrToIntCast; impl NonConstOp for RawPtrToIntCast { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let mut err = ccx .tcx .sess @@ -447,7 +503,7 @@ impl NonConstOp for StaticAccess { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { let mut err = struct_span_err!( ccx.tcx.sess, span, @@ -473,7 +529,7 @@ impl NonConstOp for StaticAccess { #[derive(Debug)] pub struct ThreadLocalAccess; impl NonConstOp for ThreadLocalAccess { - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { struct_span_err!( ccx.tcx.sess, span, @@ -504,7 +560,11 @@ pub mod ty { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>( + &self, + ccx: &ConstCx<'_, 'tcx>, + span: Span, + ) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_mut_refs, @@ -534,7 +594,11 @@ pub mod ty { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>( + &self, + ccx: &ConstCx<'_, 'tcx>, + span: Span, + ) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_fn_fn_ptr_basics, @@ -551,7 +615,11 @@ pub mod ty { Status::Unstable(sym::const_impl_trait) } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>( + &self, + ccx: &ConstCx<'_, 'tcx>, + span: Span, + ) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_impl_trait, @@ -581,7 +649,11 @@ pub mod ty { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>( + &self, + ccx: &ConstCx<'_, 'tcx>, + span: Span, + ) -> DiagnosticBuilder<'tcx> { let mut err = feature_err( &ccx.tcx.sess.parse_sess, sym::const_fn_trait_bound, @@ -620,7 +692,11 @@ pub mod ty { } } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>( + &self, + ccx: &ConstCx<'_, 'tcx>, + span: Span, + ) -> DiagnosticBuilder<'tcx> { let mut err = feature_err( &ccx.tcx.sess.parse_sess, sym::const_fn_trait_bound, @@ -647,7 +723,11 @@ pub mod ty { Status::Unstable(sym::const_trait_bound_opt_out) } - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + fn build_error<'tcx>( + &self, + ccx: &ConstCx<'_, 'tcx>, + span: Span, + ) -> DiagnosticBuilder<'tcx> { feature_err( &ccx.tcx.sess.parse_sess, sym::const_trait_bound_opt_out, diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs index 7a2be3c3bad..4e210f66353 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs @@ -23,7 +23,7 @@ pub fn checking_enabled(ccx: &ConstCx<'_, '_>) -> bool { /// /// This is separate from the rest of the const checking logic because it must run after drop /// elaboration. -pub fn check_live_drops(tcx: TyCtxt<'tcx>, body: &mir::Body<'tcx>) { +pub fn check_live_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mir::Body<'tcx>) { let def_id = body.source.def_id().expect_local(); let const_kind = tcx.hir().body_const_context(def_id); if const_kind.is_none() { @@ -50,7 +50,7 @@ struct CheckLiveDrops<'mir, 'tcx> { } // So we can access `body` and `tcx`. -impl std::ops::Deref for CheckLiveDrops<'mir, 'tcx> { +impl<'mir, 'tcx> std::ops::Deref for CheckLiveDrops<'mir, 'tcx> { type Target = ConstCx<'mir, 'tcx>; fn deref(&self) -> &Self::Target { @@ -58,13 +58,13 @@ impl std::ops::Deref for CheckLiveDrops<'mir, 'tcx> { } } -impl CheckLiveDrops<'mir, 'tcx> { +impl CheckLiveDrops<'_, '_> { fn check_live_drop(&self, span: Span) { ops::LiveDrop { dropped_at: None }.build_error(self.ccx, span).emit(); } } -impl Visitor<'tcx> for CheckLiveDrops<'mir, 'tcx> { +impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> { fn visit_basic_block_data(&mut self, bb: BasicBlock, block: &mir::BasicBlockData<'tcx>) { trace!("visit_basic_block_data: bb={:?} is_cleanup={:?}", bb, block.is_cleanup); @@ -80,7 +80,8 @@ impl Visitor<'tcx> for CheckLiveDrops<'mir, 'tcx> { trace!("visit_terminator: terminator={:?} location={:?}", terminator, location); match &terminator.kind { - mir::TerminatorKind::Drop { place: dropped_place, .. } => { + mir::TerminatorKind::Drop { place: dropped_place, .. } + | mir::TerminatorKind::DropAndReplace { place: dropped_place, .. } => { let dropped_ty = dropped_place.ty(self.body, self.tcx).ty; if !NeedsNonConstDrop::in_any_value_of_ty(self.ccx, dropped_ty) { // Instead of throwing a bug, we just return here. This is because we have to @@ -104,11 +105,6 @@ impl Visitor<'tcx> for CheckLiveDrops<'mir, 'tcx> { } } - mir::TerminatorKind::DropAndReplace { .. } => span_bug!( - terminator.source_info.span, - "`DropAndReplace` should be removed by drop elaboration", - ), - mir::TerminatorKind::Abort | mir::TerminatorKind::Call { .. } | mir::TerminatorKind::Assert { .. } diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs index abc5a3c6a52..91610b15eb9 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs @@ -3,18 +3,18 @@ //! See the `Qualif` trait for more info. use rustc_errors::ErrorReported; -use rustc_hir as hir; use rustc_infer::infer::TyCtxtInferExt; +use rustc_infer::traits::TraitEngine; use rustc_middle::mir::*; use rustc_middle::ty::{self, subst::SubstsRef, AdtDef, Ty}; use rustc_span::DUMMY_SP; use rustc_trait_selection::traits::{ - self, ImplSource, Obligation, ObligationCause, SelectionContext, + self, FulfillmentContext, ImplSource, Obligation, ObligationCause, SelectionContext, }; use super::ConstCx; -pub fn in_any_value_of_ty( +pub fn in_any_value_of_ty<'tcx>( cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>, error_occured: Option<ErrorReported>, @@ -59,7 +59,7 @@ pub trait Qualif { /// from a call to another function. /// /// It also determines the `Qualif`s for primitive types. - fn in_any_value_of_ty(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool; + fn in_any_value_of_ty<'tcx>(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool; /// Returns `true` if this `Qualif` is inherent to the given struct or enum. /// @@ -69,7 +69,7 @@ pub trait Qualif { /// with a custom `Drop` impl is inherently `NeedsDrop`. /// /// Returning `true` for `in_adt_inherently` but `false` for `in_any_value_of_ty` is unsound. - fn in_adt_inherently( + fn in_adt_inherently<'tcx>( cx: &ConstCx<'_, 'tcx>, adt: &'tcx AdtDef, substs: SubstsRef<'tcx>, @@ -90,11 +90,15 @@ impl Qualif for HasMutInterior { qualifs.has_mut_interior } - fn in_any_value_of_ty(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { + fn in_any_value_of_ty<'tcx>(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { !ty.is_freeze(cx.tcx.at(DUMMY_SP), cx.param_env) } - fn in_adt_inherently(cx: &ConstCx<'_, 'tcx>, adt: &'tcx AdtDef, _: SubstsRef<'tcx>) -> bool { + fn in_adt_inherently<'tcx>( + cx: &ConstCx<'_, 'tcx>, + adt: &'tcx AdtDef, + _: SubstsRef<'tcx>, + ) -> bool { // Exactly one type, `UnsafeCell`, has the `HasMutInterior` qualif inherently. // It arises structurally for all other types. Some(adt.did) == cx.tcx.lang_items().unsafe_cell_type() @@ -116,11 +120,15 @@ impl Qualif for NeedsDrop { qualifs.needs_drop } - fn in_any_value_of_ty(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { + fn in_any_value_of_ty<'tcx>(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { ty.needs_drop(cx.tcx, cx.param_env) } - fn in_adt_inherently(cx: &ConstCx<'_, 'tcx>, adt: &'tcx AdtDef, _: SubstsRef<'tcx>) -> bool { + fn in_adt_inherently<'tcx>( + cx: &ConstCx<'_, 'tcx>, + adt: &'tcx AdtDef, + _: SubstsRef<'tcx>, + ) -> bool { adt.has_dtor(cx.tcx) } } @@ -138,15 +146,10 @@ impl Qualif for NeedsNonConstDrop { qualifs.needs_non_const_drop } - fn in_any_value_of_ty(cx: &ConstCx<'_, 'tcx>, mut ty: Ty<'tcx>) -> bool { - // Avoid selecting for simple cases. - match ty::util::needs_drop_components(ty, &cx.tcx.data_layout).as_deref() { - Ok([]) => return false, - Err(ty::util::AlwaysRequiresDrop) => return true, - // If we've got a single component, select with that - // to increase the chance that we hit the selection cache. - Ok([t]) => ty = t, - Ok([..]) => {} + fn in_any_value_of_ty<'tcx>(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { + // Avoid selecting for simple cases, such as builtin types. + if ty::util::is_trivially_const_drop(ty) { + return false; } let Some(drop_trait) = cx.tcx.lang_items().drop_trait() else { @@ -154,31 +157,57 @@ impl Qualif for NeedsNonConstDrop { // without having the lang item present. return false; }; - let trait_ref = - ty::TraitRef { def_id: drop_trait, substs: cx.tcx.mk_substs_trait(ty, &[]) }; + let obligation = Obligation::new( ObligationCause::dummy(), cx.param_env, ty::Binder::dummy(ty::TraitPredicate { - trait_ref, + trait_ref: ty::TraitRef { + def_id: drop_trait, + substs: cx.tcx.mk_substs_trait(ty, &[]), + }, constness: ty::BoundConstness::ConstIfConst, polarity: ty::ImplPolarity::Positive, }), ); - let implsrc = cx.tcx.infer_ctxt().enter(|infcx| { - let mut selcx = SelectionContext::with_constness(&infcx, hir::Constness::Const); - selcx.select(&obligation) - }); - !matches!( - implsrc, - Ok(Some( + cx.tcx.infer_ctxt().enter(|infcx| { + let mut selcx = SelectionContext::new(&infcx); + let Some(impl_src) = selcx.select(&obligation).ok().flatten() else { + // If we couldn't select a const drop candidate, then it's bad + return true; + }; + + if !matches!( + impl_src, ImplSource::ConstDrop(_) | ImplSource::Param(_, ty::BoundConstness::ConstIfConst) - )) - ) + ) { + // If our const drop candidate is not ConstDrop or implied by the param env, + // then it's bad + return true; + } + + if impl_src.borrow_nested_obligations().is_empty() { + return false; + } + + // If we successfully found one, then select all of the predicates + // implied by our const drop impl. + let mut fcx = FulfillmentContext::new(); + for nested in impl_src.nested_obligations() { + fcx.register_predicate_obligation(&infcx, nested); + } + + // If we had any errors, then it's bad + !fcx.select_all_or_error(&infcx).is_empty() + }) } - fn in_adt_inherently(cx: &ConstCx<'_, 'tcx>, adt: &'tcx AdtDef, _: SubstsRef<'tcx>) -> bool { + fn in_adt_inherently<'tcx>( + cx: &ConstCx<'_, 'tcx>, + adt: &'tcx AdtDef, + _: SubstsRef<'tcx>, + ) -> bool { adt.has_non_const_dtor(cx.tcx) } } @@ -193,17 +222,16 @@ impl Qualif for CustomEq { qualifs.custom_eq } - fn in_any_value_of_ty(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { + fn in_any_value_of_ty<'tcx>(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool { // If *any* component of a composite data type does not implement `Structural{Partial,}Eq`, // we know that at least some values of that type are not structural-match. I say "some" // because that component may be part of an enum variant (e.g., // `Option::<NonStructuralMatchTy>::Some`), in which case some values of this type may be // structural-match (`Option::None`). - let id = cx.tcx.hir().local_def_id_to_hir_id(cx.def_id()); - traits::search_for_structural_match_violation(id, cx.body.span, cx.tcx, ty).is_some() + traits::search_for_structural_match_violation(cx.body.span, cx.tcx, ty).is_some() } - fn in_adt_inherently( + fn in_adt_inherently<'tcx>( cx: &ConstCx<'_, 'tcx>, adt: &'tcx AdtDef, substs: SubstsRef<'tcx>, @@ -216,7 +244,11 @@ impl Qualif for CustomEq { // FIXME: Use `mir::visit::Visitor` for the `in_*` functions if/when it supports early return. /// Returns `true` if this `Rvalue` contains qualif `Q`. -pub fn in_rvalue<Q, F>(cx: &ConstCx<'_, 'tcx>, in_local: &mut F, rvalue: &Rvalue<'tcx>) -> bool +pub fn in_rvalue<'tcx, Q, F>( + cx: &ConstCx<'_, 'tcx>, + in_local: &mut F, + rvalue: &Rvalue<'tcx>, +) -> bool where Q: Qualif, F: FnMut(Local) -> bool, @@ -255,7 +287,8 @@ where Rvalue::Aggregate(kind, operands) => { // Return early if we know that the struct or enum being constructed is always // qualified. - if let AggregateKind::Adt(def, _, substs, ..) = **kind { + if let AggregateKind::Adt(adt_did, _, substs, ..) = **kind { + let def = cx.tcx.adt_def(adt_did); if Q::in_adt_inherently(cx, def, substs) { return true; } @@ -271,7 +304,7 @@ where } /// Returns `true` if this `Place` contains qualif `Q`. -pub fn in_place<Q, F>(cx: &ConstCx<'_, 'tcx>, in_local: &mut F, place: PlaceRef<'tcx>) -> bool +pub fn in_place<'tcx, Q, F>(cx: &ConstCx<'_, 'tcx>, in_local: &mut F, place: PlaceRef<'tcx>) -> bool where Q: Qualif, F: FnMut(Local) -> bool, @@ -303,7 +336,11 @@ where } /// Returns `true` if this `Operand` contains qualif `Q`. -pub fn in_operand<Q, F>(cx: &ConstCx<'_, 'tcx>, in_local: &mut F, operand: &Operand<'tcx>) -> bool +pub fn in_operand<'tcx, Q, F>( + cx: &ConstCx<'_, 'tcx>, + in_local: &mut F, + operand: &Operand<'tcx>, +) -> bool where Q: Qualif, F: FnMut(Local) -> bool, @@ -318,7 +355,7 @@ where // Check the qualifs of the value of `const` items. if let Some(ct) = constant.literal.const_for_ty() { - if let ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs_: _, promoted }) = ct.val { + if let ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs: _, promoted }) = ct.val { // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible // only for `NeedsNonConstDrop` with precise drop checking. This is the only const // check performed after the promotion. Verify that with an assertion. diff --git a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs b/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs index fcce829eba4..fd7febc17a3 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs @@ -7,6 +7,7 @@ use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::{self, BasicBlock, Local, Location, Statement, StatementKind}; use rustc_mir_dataflow::fmt::DebugWithContext; use rustc_mir_dataflow::JoinSemiLattice; +use rustc_mir_dataflow::{Analysis, AnalysisDomain, CallReturnPlaces}; use rustc_span::DUMMY_SP; use std::fmt; @@ -27,7 +28,7 @@ struct TransferFunction<'a, 'mir, 'tcx, Q> { _qualif: PhantomData<Q>, } -impl<Q> TransferFunction<'a, 'mir, 'tcx, Q> +impl<'a, 'mir, 'tcx, Q> TransferFunction<'a, 'mir, 'tcx, Q> where Q: Qualif, { @@ -80,18 +81,18 @@ where fn apply_call_return_effect( &mut self, _block: BasicBlock, - _func: &mir::Operand<'tcx>, - _args: &[mir::Operand<'tcx>], - return_place: mir::Place<'tcx>, + return_places: CallReturnPlaces<'_, 'tcx>, ) { - // We cannot reason about another function's internals, so use conservative type-based - // qualification for the result of a function call. - let return_ty = return_place.ty(self.ccx.body, self.ccx.tcx).ty; - let qualif = Q::in_any_value_of_ty(self.ccx, return_ty); + return_places.for_each(|place| { + // We cannot reason about another function's internals, so use conservative type-based + // qualification for the result of a function call. + let return_ty = place.ty(self.ccx.body, self.ccx.tcx).ty; + let qualif = Q::in_any_value_of_ty(self.ccx, return_ty); - if !return_place.is_indirect() { - self.assign_qualif_direct(&return_place, qualif); - } + if !place.is_indirect() { + self.assign_qualif_direct(&place, qualif); + } + }); } fn address_of_allows_mutation(&self, _mt: mir::Mutability, _place: mir::Place<'tcx>) -> bool { @@ -126,7 +127,7 @@ where } } -impl<Q> Visitor<'tcx> for TransferFunction<'_, '_, 'tcx, Q> +impl<'tcx, Q> Visitor<'tcx> for TransferFunction<'_, '_, 'tcx, Q> where Q: Qualif, { @@ -264,7 +265,7 @@ where } } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub(super) struct State { /// Describes whether a local contains qualif. pub qualif: BitSet<Local>, @@ -273,6 +274,19 @@ pub(super) struct State { pub borrow: BitSet<Local>, } +impl Clone for State { + fn clone(&self) -> Self { + State { qualif: self.qualif.clone(), borrow: self.borrow.clone() } + } + + // Data flow engine when possible uses `clone_from` for domain values. + // Providing an implementation will avoid some intermediate memory allocations. + fn clone_from(&mut self, other: &Self) { + self.qualif.clone_from(&other.qualif); + self.borrow.clone_from(&other.borrow); + } +} + impl State { #[inline] pub(super) fn contains(&self, local: Local) -> bool { @@ -316,7 +330,7 @@ impl JoinSemiLattice for State { } } -impl<Q> rustc_mir_dataflow::AnalysisDomain<'tcx> for FlowSensitiveAnalysis<'_, '_, 'tcx, Q> +impl<'tcx, Q> AnalysisDomain<'tcx> for FlowSensitiveAnalysis<'_, '_, 'tcx, Q> where Q: Qualif, { @@ -336,7 +350,7 @@ where } } -impl<Q> rustc_mir_dataflow::Analysis<'tcx> for FlowSensitiveAnalysis<'_, '_, 'tcx, Q> +impl<'tcx, Q> Analysis<'tcx> for FlowSensitiveAnalysis<'_, '_, 'tcx, Q> where Q: Qualif, { @@ -362,10 +376,8 @@ where &self, state: &mut Self::Domain, block: BasicBlock, - func: &mir::Operand<'tcx>, - args: &[mir::Operand<'tcx>], - return_place: mir::Place<'tcx>, + return_places: CallReturnPlaces<'_, 'tcx>, ) { - self.transfer_function(state).apply_call_return_effect(block, func, args, return_place) + self.transfer_function(state).apply_call_return_effect(block, return_places) } } diff --git a/compiler/rustc_const_eval/src/transform/mod.rs b/compiler/rustc_const_eval/src/transform/mod.rs index 38c28f34934..a2928bdf51b 100644 --- a/compiler/rustc_const_eval/src/transform/mod.rs +++ b/compiler/rustc_const_eval/src/transform/mod.rs @@ -1,5 +1,3 @@ pub mod check_consts; pub mod promote_consts; pub mod validate; - -pub use rustc_middle::mir::MirPass; diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs index 7bf378601e0..ac282a5ecc8 100644 --- a/compiler/rustc_const_eval/src/transform/promote_consts.rs +++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs @@ -27,7 +27,6 @@ use std::cell::Cell; use std::{cmp, iter, mem}; use crate::transform::check_consts::{qualifs, ConstCx}; -use crate::transform::MirPass; /// A `MirPass` for promotion. /// @@ -42,6 +41,10 @@ pub struct PromoteTemps<'tcx> { } impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> { + fn phase_change(&self) -> Option<MirPhase> { + Some(MirPhase::ConstPromotion) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // There's not really any point in promoting errorful MIR. // @@ -93,17 +96,8 @@ impl TempState { /// returned value in a promoted MIR, unless it's a subset /// of a larger candidate. #[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum Candidate { - /// Borrow of a constant temporary, candidate for lifetime extension. - Ref(Location), -} - -impl Candidate { - fn source_info(&self, body: &Body<'_>) -> SourceInfo { - match self { - Candidate::Ref(location) => *body.source_info(*location), - } - } +pub struct Candidate { + location: Location, } struct Collector<'a, 'tcx> { @@ -167,15 +161,15 @@ impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> { match *rvalue { Rvalue::Ref(..) => { - self.candidates.push(Candidate::Ref(location)); + self.candidates.push(Candidate { location }); } _ => {} } } } -pub fn collect_temps_and_candidates( - ccx: &ConstCx<'mir, 'tcx>, +pub fn collect_temps_and_candidates<'tcx>( + ccx: &ConstCx<'_, 'tcx>, rpo: &mut ReversePostorder<'_, 'tcx>, ) -> (IndexVec<Local, TempState>, Vec<Candidate>) { let mut collector = Collector { @@ -197,7 +191,7 @@ struct Validator<'a, 'tcx> { temps: &'a IndexVec<Local, TempState>, } -impl std::ops::Deref for Validator<'a, 'tcx> { +impl<'a, 'tcx> std::ops::Deref for Validator<'a, 'tcx> { type Target = ConstCx<'a, 'tcx>; fn deref(&self) -> &Self::Target { @@ -209,36 +203,33 @@ struct Unpromotable; impl<'tcx> Validator<'_, 'tcx> { fn validate_candidate(&self, candidate: Candidate) -> Result<(), Unpromotable> { - match candidate { - Candidate::Ref(loc) => { - let statement = &self.body[loc.block].statements[loc.statement_index]; - match &statement.kind { - StatementKind::Assign(box (_, Rvalue::Ref(_, kind, place))) => { - // We can only promote interior borrows of promotable temps (non-temps - // don't get promoted anyway). - self.validate_local(place.local)?; - - // The reference operation itself must be promotable. - // (Needs to come after `validate_local` to avoid ICEs.) - self.validate_ref(*kind, place)?; - - // We do not check all the projections (they do not get promoted anyway), - // but we do stay away from promoting anything involving a dereference. - if place.projection.contains(&ProjectionElem::Deref) { - return Err(Unpromotable); - } + let loc = candidate.location; + let statement = &self.body[loc.block].statements[loc.statement_index]; + match &statement.kind { + StatementKind::Assign(box (_, Rvalue::Ref(_, kind, place))) => { + // We can only promote interior borrows of promotable temps (non-temps + // don't get promoted anyway). + self.validate_local(place.local)?; + + // The reference operation itself must be promotable. + // (Needs to come after `validate_local` to avoid ICEs.) + self.validate_ref(*kind, place)?; - // We cannot promote things that need dropping, since the promoted value - // would not get dropped. - if self.qualif_local::<qualifs::NeedsDrop>(place.local) { - return Err(Unpromotable); - } + // We do not check all the projections (they do not get promoted anyway), + // but we do stay away from promoting anything involving a dereference. + if place.projection.contains(&ProjectionElem::Deref) { + return Err(Unpromotable); + } - Ok(()) - } - _ => bug!(), + // We cannot promote things that need dropping, since the promoted value + // would not get dropped. + if self.qualif_local::<qualifs::NeedsDrop>(place.local) { + return Err(Unpromotable); } + + Ok(()) } + _ => bug!(), } } @@ -517,7 +508,6 @@ impl<'tcx> Validator<'_, 'tcx> { } Rvalue::NullaryOp(op, _) => match op { - NullOp::Box => return Err(Unpromotable), NullOp::SizeOf => {} NullOp::AlignOf => {} }, @@ -853,17 +843,13 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { ty, val: ty::ConstKind::Unevaluated(ty::Unevaluated { def, - substs_: Some(InternalSubsts::for_item( - tcx, - def.did, - |param, _| { - if let ty::GenericParamDefKind::Lifetime = param.kind { - tcx.lifetimes.re_erased.into() - } else { - tcx.mk_param_from_def(param) - } - }, - )), + substs: InternalSubsts::for_item(tcx, def.did, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), promoted: Some(promoted_id), }), }) @@ -871,58 +857,55 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { })) }; let (blocks, local_decls) = self.source.basic_blocks_and_local_decls_mut(); - match candidate { - Candidate::Ref(loc) => { - let statement = &mut blocks[loc.block].statements[loc.statement_index]; - match statement.kind { - StatementKind::Assign(box ( - _, - Rvalue::Ref(ref mut region, borrow_kind, ref mut place), - )) => { - // Use the underlying local for this (necessarily interior) borrow. - let ty = local_decls.local_decls()[place.local].ty; - let span = statement.source_info.span; - - let ref_ty = tcx.mk_ref( - tcx.lifetimes.re_erased, - ty::TypeAndMut { ty, mutbl: borrow_kind.to_mutbl_lossy() }, - ); - - *region = tcx.lifetimes.re_erased; - - let mut projection = vec![PlaceElem::Deref]; - projection.extend(place.projection); - place.projection = tcx.intern_place_elems(&projection); - - // Create a temp to hold the promoted reference. - // This is because `*r` requires `r` to be a local, - // otherwise we would use the `promoted` directly. - let mut promoted_ref = LocalDecl::new(ref_ty, span); - promoted_ref.source_info = statement.source_info; - let promoted_ref = local_decls.push(promoted_ref); - assert_eq!(self.temps.push(TempState::Unpromotable), promoted_ref); - - let promoted_ref_statement = Statement { - source_info: statement.source_info, - kind: StatementKind::Assign(Box::new(( - Place::from(promoted_ref), - Rvalue::Use(promoted_operand(ref_ty, span)), - ))), - }; - self.extra_statements.push((loc, promoted_ref_statement)); - - Rvalue::Ref( - tcx.lifetimes.re_erased, - borrow_kind, - Place { - local: mem::replace(&mut place.local, promoted_ref), - projection: List::empty(), - }, - ) - } - _ => bug!(), - } + let loc = candidate.location; + let statement = &mut blocks[loc.block].statements[loc.statement_index]; + match statement.kind { + StatementKind::Assign(box ( + _, + Rvalue::Ref(ref mut region, borrow_kind, ref mut place), + )) => { + // Use the underlying local for this (necessarily interior) borrow. + let ty = local_decls.local_decls()[place.local].ty; + let span = statement.source_info.span; + + let ref_ty = tcx.mk_ref( + tcx.lifetimes.re_erased, + ty::TypeAndMut { ty, mutbl: borrow_kind.to_mutbl_lossy() }, + ); + + *region = tcx.lifetimes.re_erased; + + let mut projection = vec![PlaceElem::Deref]; + projection.extend(place.projection); + place.projection = tcx.intern_place_elems(&projection); + + // Create a temp to hold the promoted reference. + // This is because `*r` requires `r` to be a local, + // otherwise we would use the `promoted` directly. + let mut promoted_ref = LocalDecl::new(ref_ty, span); + promoted_ref.source_info = statement.source_info; + let promoted_ref = local_decls.push(promoted_ref); + assert_eq!(self.temps.push(TempState::Unpromotable), promoted_ref); + + let promoted_ref_statement = Statement { + source_info: statement.source_info, + kind: StatementKind::Assign(Box::new(( + Place::from(promoted_ref), + Rvalue::Use(promoted_operand(ref_ty, span)), + ))), + }; + self.extra_statements.push((loc, promoted_ref_statement)); + + Rvalue::Ref( + tcx.lifetimes.re_erased, + borrow_kind, + Place { + local: mem::replace(&mut place.local, promoted_ref), + projection: List::empty(), + }, + ) } + _ => bug!(), } }; @@ -964,17 +947,13 @@ pub fn promote_candidates<'tcx>( let mut extra_statements = vec![]; for candidate in candidates.into_iter().rev() { - match candidate { - Candidate::Ref(Location { block, statement_index }) => { - if let StatementKind::Assign(box (place, _)) = - &body[block].statements[statement_index].kind - { - if let Some(local) = place.as_local() { - if temps[local] == TempState::PromotedOut { - // Already promoted. - continue; - } - } + let Location { block, statement_index } = candidate.location; + if let StatementKind::Assign(box (place, _)) = &body[block].statements[statement_index].kind + { + if let Some(local) = place.as_local() { + if temps[local] == TempState::PromotedOut { + // Already promoted. + continue; } } } @@ -982,11 +961,10 @@ pub fn promote_candidates<'tcx>( // Declare return place local so that `mir::Body::new` doesn't complain. let initial_locals = iter::once(LocalDecl::new(tcx.types.never, body.span)).collect(); - let mut scope = body.source_scopes[candidate.source_info(body).scope].clone(); + let mut scope = body.source_scopes[body.source_info(candidate.location).scope].clone(); scope.parent_scope = None; let promoted = Body::new( - tcx, body.source, // `promoted` gets filled in below IndexVec::new(), IndexVec::from_elem_n(scope, 1), diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index b09b2227f3e..22ef0b2dda5 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -1,14 +1,13 @@ //! Validates the MIR to ensure that invariants are upheld. -use super::MirPass; use rustc_index::bit_set::BitSet; use rustc_infer::infer::TyCtxtInferExt; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::traversal; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::{ - AggregateKind, BasicBlock, Body, BorrowKind, Local, Location, MirPhase, Operand, PlaceElem, - PlaceRef, ProjectionElem, Rvalue, SourceScope, Statement, StatementKind, Terminator, + AggregateKind, BasicBlock, Body, BorrowKind, Local, Location, MirPass, MirPhase, Operand, + PlaceElem, PlaceRef, ProjectionElem, Rvalue, SourceScope, Statement, StatementKind, Terminator, TerminatorKind, START_BLOCK, }; use rustc_middle::ty::fold::BottomUpFolder; @@ -67,7 +66,7 @@ impl<'tcx> MirPass<'tcx> for Validator { /// /// The point of this function is to approximate "equal up to subtyping". However, /// the approximation is incorrect as variance is ignored. -pub fn equal_up_to_regions( +pub fn equal_up_to_regions<'tcx>( tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, src: Ty<'tcx>, @@ -353,7 +352,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { StatementKind::SetDiscriminant { .. } | StatementKind::StorageLive(..) | StatementKind::StorageDead(..) - | StatementKind::LlvmInlineAsm(..) | StatementKind::Retag(_, _) | StatementKind::Coverage(_) | StatementKind::Nop => {} @@ -496,10 +494,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.check_edge(location, *unwind, EdgeKind::Unwind); } } - TerminatorKind::InlineAsm { destination, .. } => { + TerminatorKind::InlineAsm { destination, cleanup, .. } => { if let Some(destination) = destination { self.check_edge(location, *destination, EdgeKind::Normal); } + if let Some(cleanup) = cleanup { + self.check_edge(location, *cleanup, EdgeKind::Unwind); + } } // Nothing to validate for these. TerminatorKind::Resume diff --git a/compiler/rustc_const_eval/src/util/aggregate.rs b/compiler/rustc_const_eval/src/util/aggregate.rs index 4bc0357cab8..e5f5e7072d5 100644 --- a/compiler/rustc_const_eval/src/util/aggregate.rs +++ b/compiler/rustc_const_eval/src/util/aggregate.rs @@ -22,7 +22,8 @@ pub fn expand_aggregate<'tcx>( ) -> impl Iterator<Item = Statement<'tcx>> + TrustedLen { let mut set_discriminant = None; let active_field_index = match kind { - AggregateKind::Adt(adt_def, variant_index, _, _, active_field_index) => { + AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => { + let adt_def = tcx.adt_def(adt_did); if adt_def.is_enum() { set_discriminant = Some(Statement { kind: StatementKind::SetDiscriminant { place: Box::new(lhs), variant_index }, |