diff options
Diffstat (limited to 'compiler/rustc_const_eval/src')
39 files changed, 398 insertions, 1758 deletions
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index f9786acfc6c..5fbf5b41109 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -8,10 +8,11 @@ use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::traits::ObligationCause; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; +use rustc_middle::span_bug; use rustc_middle::ty::{self, adjustment::PointerCoercion, Ty, TyCtxt}; use rustc_middle::ty::{Instance, InstanceDef, TypeVisitableExt}; use rustc_mir_dataflow::Analysis; -use rustc_span::{sym, Span, Symbol}; +use rustc_span::{sym, Span, Symbol, DUMMY_SP}; use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _; use rustc_trait_selection::traits::{self, ObligationCauseCode, ObligationCtxt}; use rustc_type_ir::visit::{TypeSuperVisitable, TypeVisitor}; @@ -19,6 +20,8 @@ use rustc_type_ir::visit::{TypeSuperVisitable, TypeVisitor}; use std::mem; use std::ops::Deref; +use tracing::{debug, instrument, trace}; + use super::ops::{self, NonConstOp, Status}; use super::qualifs::{self, HasMutInterior, NeedsDrop, NeedsNonConstDrop}; use super::resolver::FlowSensitiveAnalysis; @@ -579,7 +582,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { } } - Rvalue::BinaryOp(op, box (lhs, rhs)) | Rvalue::CheckedBinaryOp(op, box (lhs, rhs)) => { + Rvalue::BinaryOp(op, box (lhs, rhs)) => { let lhs_ty = lhs.ty(self.body, self.tcx); let rhs_ty = rhs.ty(self.body, self.tcx); @@ -738,7 +741,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { let cause = ObligationCause::new( terminator.source_info.span, self.body.source.def_id().expect_local(), - ObligationCauseCode::ItemObligation(callee), + ObligationCauseCode::WhereClause(callee, DUMMY_SP), ); let normalized_predicates = ocx.normalize(&cause, param_env, predicates); ocx.register_obligations(traits::predicates_for_generics( diff --git a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs b/compiler/rustc_const_eval/src/check_consts/mod.rs index 12e7ec15e32..308b90cd470 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs +++ b/compiler/rustc_const_eval/src/check_consts/mod.rs @@ -8,6 +8,7 @@ use rustc_attr as attr; use rustc_errors::DiagCtxt; use rustc_hir as hir; use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::ty::{self, PolyFnSig, TyCtxt}; use rustc_span::Symbol; diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/check_consts/ops.rs index dda8f3ed87d..90b622cae65 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs +++ b/compiler/rustc_const_eval/src/check_consts/ops.rs @@ -8,7 +8,8 @@ use rustc_hir::def_id::DefId; use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::traits::{ImplSource, Obligation, ObligationCause}; use rustc_middle::mir::{self, CallSource}; -use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::span_bug; +use rustc_middle::ty::print::{with_no_trimmed_paths, PrintTraitRefExt as _}; use rustc_middle::ty::{ self, suggest_constraining_type_param, Closure, FnDef, FnPtr, GenericArgKind, GenericArgsRef, Param, TraitRef, Ty, @@ -18,6 +19,7 @@ use rustc_session::parse::feature_err; use rustc_span::symbol::sym; use rustc_span::{BytePos, Pos, Span, Symbol}; use rustc_trait_selection::traits::SelectionContext; +use tracing::debug; use super::ConstCx; use crate::errors; @@ -307,7 +309,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> { } if let ConstContext::Static(_) = ccx.const_kind() { - err.note("consider wrapping this expression in `Lazy::new(|| ...)` from the `once_cell` crate: https://crates.io/crates/once_cell"); + err.note("consider wrapping this expression in `std::sync::LazyLock::new(|| ...)`"); } err diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/check_consts/post_drop_elaboration.rs index 5cd13783c23..f0763f1e490 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs +++ b/compiler/rustc_const_eval/src/check_consts/post_drop_elaboration.rs @@ -2,6 +2,7 @@ use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::{self, BasicBlock, Location}; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_span::{symbol::sym, Span}; +use tracing::trace; use super::check::Qualifs; use super::ops::{self, NonConstOp}; diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/check_consts/qualifs.rs index 1847847d9d2..5949444e599 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs +++ b/compiler/rustc_const_eval/src/check_consts/qualifs.rs @@ -5,6 +5,7 @@ use rustc_errors::ErrorGuaranteed; use rustc_hir::LangItem; use rustc_infer::infer::TyCtxtInferExt; +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::mir::*; use rustc_middle::traits::BuiltinImplSource; @@ -12,6 +13,7 @@ use rustc_middle::ty::{self, AdtDef, GenericArgsRef, Ty}; use rustc_trait_selection::traits::{ ImplSource, Obligation, ObligationCause, ObligationCtxt, SelectionContext, }; +use tracing::{instrument, trace}; use super::ConstCx; @@ -260,7 +262,7 @@ where | Rvalue::Cast(_, operand, _) | Rvalue::ShallowInitBox(operand, _) => in_operand::<Q, _>(cx, in_local, operand), - Rvalue::BinaryOp(_, box (lhs, rhs)) | Rvalue::CheckedBinaryOp(_, box (lhs, rhs)) => { + Rvalue::BinaryOp(_, box (lhs, rhs)) => { in_operand::<Q, _>(cx, in_local, lhs) || in_operand::<Q, _>(cx, in_local, rhs) } diff --git a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs b/compiler/rustc_const_eval/src/check_consts/resolver.rs index 5ae3ffaaec2..011341472b4 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs +++ b/compiler/rustc_const_eval/src/check_consts/resolver.rs @@ -200,7 +200,6 @@ where | mir::Rvalue::Repeat(..) | mir::Rvalue::Len(..) | mir::Rvalue::BinaryOp(..) - | mir::Rvalue::CheckedBinaryOp(..) | mir::Rvalue::NullaryOp(..) | mir::Rvalue::UnaryOp(..) | mir::Rvalue::Discriminant(..) diff --git a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs index 7b6828c6e18..530a05a1ed8 100644 --- a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs @@ -1,9 +1,12 @@ -use crate::interpret::{self, HasStaticRootDefId, ImmTy, Immediate, InterpCx, PointerArithmetic}; +use crate::interpret::{ + self, throw_machine_stop, HasStaticRootDefId, ImmTy, Immediate, InterpCx, PointerArithmetic, +}; use rustc_middle::mir::interpret::{AllocId, ConstAllocation, InterpResult}; use rustc_middle::mir::*; use rustc_middle::query::TyCtxtAt; use rustc_middle::ty; use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::{bug, span_bug}; use rustc_span::def_id::DefId; /// Macro for machine-specific `InterpError` without allocation. @@ -105,7 +108,7 @@ impl<'mir, 'tcx: 'mir> interpret::Machine<'mir, 'tcx> for DummyMachine { _destination: &interpret::MPlaceTy<'tcx, Self::Provenance>, _target: Option<BasicBlock>, _unwind: UnwindAction, - ) -> interpret::InterpResult<'tcx> { + ) -> interpret::InterpResult<'tcx, Option<ty::Instance<'tcx>>> { unimplemented!() } @@ -122,7 +125,7 @@ impl<'mir, 'tcx: 'mir> interpret::Machine<'mir, 'tcx> for DummyMachine { bin_op: BinOp, left: &interpret::ImmTy<'tcx, Self::Provenance>, right: &interpret::ImmTy<'tcx, Self::Provenance>, - ) -> interpret::InterpResult<'tcx, (ImmTy<'tcx, Self::Provenance>, bool)> { + ) -> interpret::InterpResult<'tcx, ImmTy<'tcx, Self::Provenance>> { use rustc_middle::mir::BinOp::*; Ok(match bin_op { Eq | Ne | Lt | Le | Gt | Ge => { @@ -151,7 +154,7 @@ impl<'mir, 'tcx: 'mir> interpret::Machine<'mir, 'tcx> for DummyMachine { Ge => left >= right, _ => bug!(), }; - (ImmTy::from_bool(res, *ecx.tcx), false) + ImmTy::from_bool(res, *ecx.tcx) } // Some more operations are possible with atomics. diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs index 763344207c4..650669ac690 100644 --- a/compiler/rustc_const_eval/src/const_eval/error.rs +++ b/compiler/rustc_const_eval/src/const_eval/error.rs @@ -2,15 +2,16 @@ use std::mem; use rustc_errors::{DiagArgName, DiagArgValue, DiagMessage, Diagnostic, IntoDiagArg}; use rustc_hir::CRATE_HIR_ID; -use rustc_middle::mir::interpret::Provenance; +use rustc_middle::mir::interpret::{Provenance, ReportedErrorInfo}; use rustc_middle::mir::AssertKind; use rustc_middle::query::TyCtxtAt; use rustc_middle::ty::TyCtxt; use rustc_middle::ty::{layout::LayoutError, ConstInt}; -use rustc_span::{Span, Symbol, DUMMY_SP}; +use rustc_span::{Span, Symbol}; use super::CompileTimeInterpreter; use crate::errors::{self, FrameNote, ReportErrorExt}; +use crate::interpret::{err_inval, err_machine_stop}; use crate::interpret::{ErrorHandled, Frame, InterpError, InterpErrorInfo, MachineStopType}; /// The CTFE machine has some custom error kinds. @@ -121,7 +122,7 @@ where pub(super) fn report<'tcx, C, F, E>( tcx: TyCtxt<'tcx>, error: InterpError<'tcx>, - span: Option<Span>, + span: Span, get_span_and_frames: C, mk: F, ) -> ErrorHandled @@ -135,16 +136,16 @@ where // Don't emit a new diagnostic for these errors, they are already reported elsewhere or // should remain silent. err_inval!(Layout(LayoutError::Unknown(_))) | err_inval!(TooGeneric) => { - ErrorHandled::TooGeneric(span.unwrap_or(DUMMY_SP)) + ErrorHandled::TooGeneric(span) } - err_inval!(AlreadyReported(guar)) => ErrorHandled::Reported(guar, span.unwrap_or(DUMMY_SP)), + err_inval!(AlreadyReported(guar)) => ErrorHandled::Reported(guar, span), err_inval!(Layout(LayoutError::ReferencesError(guar))) => { - ErrorHandled::Reported(guar.into(), span.unwrap_or(DUMMY_SP)) + ErrorHandled::Reported(ReportedErrorInfo::tainted_by_errors(guar), span) } // Report remaining errors. _ => { let (our_span, frames) = get_span_and_frames(); - let span = span.unwrap_or(our_span); + let span = span.substitute_dummy(our_span); let err = mk(span, frames); let mut err = tcx.dcx().create_err(err); diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index 40afd9f162f..966951e7d9a 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -1,8 +1,10 @@ use std::sync::atomic::Ordering::Relaxed; use either::{Left, Right}; +use tracing::{debug, instrument, trace}; use rustc_hir::def::DefKind; +use rustc_middle::bug; use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo}; use rustc_middle::mir::{self, ConstAlloc, ConstValue}; use rustc_middle::query::TyCtxtAt; @@ -12,7 +14,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::lint; use rustc_span::def_id::LocalDefId; -use rustc_span::Span; +use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::{self, Abi}; use super::{CanAccessMutGlobal, CompileTimeEvalContext, CompileTimeInterpreter}; @@ -24,7 +26,7 @@ use crate::interpret::{ InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup, }; -use crate::interpret::{eval_nullary_intrinsic, InternResult}; +use crate::interpret::{eval_nullary_intrinsic, throw_exhaust, InternResult}; use crate::CTRL_C_RECEIVED; // Returns a pointer to where the result lives @@ -222,7 +224,7 @@ pub(super) fn op_to_const<'tcx>( // This codepath solely exists for `valtree_to_const_value` to not need to generate // a `ConstValue::Indirect` for wide references, so it is tightly restricted to just // that case. - let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap().ty; // `false` = no raw ptrs + let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); // `false` = no raw ptrs debug_assert!( matches!( ecx.tcx.struct_tail_without_normalization(pointee_ty).kind(), @@ -298,7 +300,7 @@ pub fn eval_to_const_value_raw_provider<'tcx>( super::report( tcx, error.into_kind(), - Some(span), + span, || (span, vec![]), |span, _| errors::NullaryIntrinsicError { span }, ) @@ -406,7 +408,7 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>( super::report( *ecx.tcx, error, - None, + DUMMY_SP, || super::get_span_and_frames(ecx.tcx, ecx.stack()), |span, frames| ConstEvalError { span, error_kind: kind, instance, frame_notes: frames }, ) @@ -461,7 +463,7 @@ fn report_validation_error<'mir, 'tcx>( crate::const_eval::report( *ecx.tcx, error, - None, + DUMMY_SP, || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()), move |span, frames| errors::ValidationFailure { span, ub_note, frames, raw_bytes }, ) diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs index ddad6683afb..8c66888d100 100644 --- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs @@ -81,8 +81,8 @@ fn is_promotable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> bool { if cfg!(debug_assertions) && stab.promotable { let sig = tcx.fn_sig(def_id); assert_eq!( - sig.skip_binder().unsafety(), - hir::Unsafety::Normal, + sig.skip_binder().safety(), + hir::Safety::Safe, "don't mark const unsafe fns as promotable", // https://github.com/rust-lang/rust/pull/53851#issuecomment-418760682 ); diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index dd835279df3..b8b4426c2ec 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -10,6 +10,7 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_hir::def_id::LocalDefId; use rustc_hir::LangItem; +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::mir::AssertMessage; use rustc_middle::query::TyCtxtAt; @@ -20,12 +21,14 @@ use rustc_span::symbol::{sym, Symbol}; use rustc_span::Span; use rustc_target::abi::{Align, Size}; use rustc_target::spec::abi::Abi as CallAbi; +use tracing::debug; use crate::errors::{LongRunning, LongRunningWarn}; use crate::fluent_generated as fluent; use crate::interpret::{ - self, compile_time_machine, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, FnVal, - Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, Scalar, + self, compile_time_machine, err_ub, throw_exhaust, throw_inval, throw_ub_custom, throw_unsup, + throw_unsup_format, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, FnVal, Frame, + GlobalAlloc, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, Scalar, }; use super::error::*; @@ -459,17 +462,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, dest: &MPlaceTy<'tcx, Self::Provenance>, target: Option<mir::BasicBlock>, _unwind: mir::UnwindAction, - ) -> InterpResult<'tcx> { + ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> { // Shared intrinsics. if ecx.emulate_intrinsic(instance, args, dest, target)? { - return Ok(()); + return Ok(None); } let intrinsic_name = ecx.tcx.item_name(instance.def_id()); // CTFE-specific intrinsics. - let Some(ret) = target else { - throw_unsup_format!("intrinsic `{intrinsic_name}` is not supported at compile-time"); - }; match intrinsic_name { sym::ptr_guaranteed_cmp => { let a = ecx.read_scalar(&args[0])?; @@ -536,14 +536,22 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, // not the optimization stage.) sym::is_val_statically_known => ecx.write_scalar(Scalar::from_bool(false), dest)?, _ => { - throw_unsup_format!( - "intrinsic `{intrinsic_name}` is not supported at compile-time" - ); + // We haven't handled the intrinsic, let's see if we can use a fallback body. + if ecx.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden { + throw_unsup_format!( + "intrinsic `{intrinsic_name}` is not supported at compile-time" + ); + } + return Ok(Some(ty::Instance { + def: ty::InstanceDef::Item(instance.def_id()), + args: instance.args, + })); } } - ecx.go_to_block(ret); - Ok(()) + // Intrinsic is done, jump to next block. + ecx.return_to_block(target)?; + Ok(None) } fn assert_panic( @@ -582,7 +590,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, _bin_op: mir::BinOp, _left: &ImmTy<'tcx>, _right: &ImmTy<'tcx>, - ) -> InterpResult<'tcx, (ImmTy<'tcx>, bool)> { + ) -> InterpResult<'tcx, ImmTy<'tcx>> { throw_unsup_format!("pointer arithmetic or comparison is not supported at compile-time"); } @@ -752,11 +760,21 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, ecx: &InterpCx<'mir, 'tcx, Self>, alloc_id: AllocId, ) -> InterpResult<'tcx> { + // Check if this is the currently evaluated static. if Some(alloc_id) == ecx.machine.static_root_ids.map(|(id, _)| id) { - Err(ConstEvalErrKind::RecursiveStatic.into()) - } else { - Ok(()) + return Err(ConstEvalErrKind::RecursiveStatic.into()); + } + // If this is another static, make sure we fire off the query to detect cycles. + // But only do that when checks for static recursion are enabled. + if ecx.machine.static_root_ids.is_some() { + if let Some(GlobalAlloc::Static(def_id)) = ecx.tcx.try_get_global_alloc(alloc_id) { + if ecx.tcx.is_foreign_item(def_id) { + throw_unsup!(ExternStatic(def_id)); + } + ecx.ctfe_query(|tcx| tcx.eval_static_initializer(def_id))?; + } } + Ok(()) } } diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs index 8efc67bcb0c..4ae4816e33a 100644 --- a/compiler/rustc_const_eval/src/const_eval/mod.rs +++ b/compiler/rustc_const_eval/src/const_eval/mod.rs @@ -1,10 +1,12 @@ // Not in interpret to make sure we do not use private implementation details +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::mir::interpret::InterpErrorInfo; use rustc_middle::query::{Key, TyCtxtAt}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_target::abi::VariantIdx; +use tracing::instrument; use crate::interpret::{format_interp_error, InterpCx}; diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index dcfce4e35e0..b906bfa775f 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -1,10 +1,12 @@ use rustc_data_structures::stack::ensure_sufficient_stack; +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId}; use rustc_middle::ty::layout::{LayoutCx, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; use rustc_span::DUMMY_SP; use rustc_target::abi::{Abi, VariantIdx}; +use tracing::{debug, instrument, trace}; use super::eval_queries::{mk_eval_cx_to_read_const_val, op_to_const}; use super::machine::CompileTimeEvalContext; diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs index 90d4f1168e4..e5ea4c3442e 100644 --- a/compiler/rustc_const_eval/src/errors.rs +++ b/compiler/rustc_const_eval/src/errors.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use either::Either; use rustc_errors::{ codes::*, Diag, DiagArgValue, DiagCtxt, DiagMessage, Diagnostic, EmissionGuarantee, Level, }; @@ -481,6 +482,8 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> { DivisionOverflow => const_eval_division_overflow, RemainderOverflow => const_eval_remainder_overflow, PointerArithOverflow => const_eval_pointer_arithmetic_overflow, + ArithOverflow { .. } => const_eval_overflow_arith, + ShiftOverflow { .. } => const_eval_overflow_shift, InvalidMeta(InvalidMetaKind::SliceTooBig) => const_eval_invalid_meta_slice, InvalidMeta(InvalidMetaKind::TooBig) => const_eval_invalid_meta, UnterminatedCString(_) => const_eval_unterminated_c_string, @@ -539,6 +542,19 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> { | UninhabitedEnumVariantWritten(_) | UninhabitedEnumVariantRead(_) => {} + ArithOverflow { intrinsic } => { + diag.arg("intrinsic", intrinsic); + } + ShiftOverflow { intrinsic, shift_amount } => { + diag.arg("intrinsic", intrinsic); + diag.arg( + "shift_amount", + match shift_amount { + Either::Left(v) => v.to_string(), + Either::Right(v) => v.to_string(), + }, + ); + } BoundsCheckFailed { len, index } => { diag.arg("len", len); diag.arg("index", index); diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index 76e59ea9055..ecfb25e32cd 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -7,11 +7,14 @@ use rustc_middle::mir::CastKind; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::{IntegerExt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, FloatTy, Ty}; +use rustc_middle::{bug, span_bug}; use rustc_target::abi::Integer; use rustc_type_ir::TyKind::*; +use tracing::trace; use super::{ - util::ensure_monomorphic_enough, FnVal, ImmTy, Immediate, InterpCx, Machine, OpTy, PlaceTy, + err_inval, throw_ub, throw_ub_custom, util::ensure_monomorphic_enough, FnVal, ImmTy, Immediate, + InterpCx, Machine, OpTy, PlaceTy, }; use crate::fluent_generated as fluent; diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs index caacc6f57d3..0790f06e6e1 100644 --- a/compiler/rustc_const_eval/src/interpret/discriminant.rs +++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs @@ -1,12 +1,16 @@ //! Functions for reading and writing discriminants of multi-variant layouts (enums and coroutines). use rustc_middle::mir; +use rustc_middle::span_bug; use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt}; use rustc_middle::ty::{self, ScalarInt, Ty}; use rustc_target::abi::{self, TagEncoding}; use rustc_target::abi::{VariantIdx, Variants}; +use tracing::{instrument, trace}; -use super::{ImmTy, InterpCx, InterpResult, Machine, Readable, Scalar, Writeable}; +use super::{ + err_ub, throw_ub, ImmTy, InterpCx, InterpResult, Machine, Readable, Scalar, Writeable, +}; impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Writes the discriminant of the given variant. @@ -169,7 +173,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let tag_val = ImmTy::from_uint(tag_bits, tag_layout); let niche_start_val = ImmTy::from_uint(niche_start, tag_layout); let variant_index_relative_val = - self.wrapping_binary_op(mir::BinOp::Sub, &tag_val, &niche_start_val)?; + self.binary_op(mir::BinOp::Sub, &tag_val, &niche_start_val)?; let variant_index_relative = variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size); // Check if this is in the range that indicates an actual discriminant. @@ -289,11 +293,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let variant_index_relative_val = ImmTy::from_uint(variant_index_relative, tag_layout); let tag = self - .wrapping_binary_op( - mir::BinOp::Add, - &variant_index_relative_val, - &niche_start_val, - )? + .binary_op(mir::BinOp::Add, &variant_index_relative_val, &niche_start_val)? .to_scalar() .assert_int(); Ok(Some((tag, tag_field))) diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 126d64329f8..d82623fa009 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -2,6 +2,7 @@ use std::cell::Cell; use std::{fmt, mem}; use either::{Either, Left, Right}; +use tracing::{debug, info, info_span, instrument, trace}; use hir::CRATE_HIR_ID; use rustc_errors::DiagCtxt; @@ -17,15 +18,17 @@ use rustc_middle::ty::layout::{ TyAndLayout, }; use rustc_middle::ty::{self, GenericArgsRef, ParamEnv, Ty, TyCtxt, TypeFoldable, Variance}; +use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::storage::always_storage_live_locals; use rustc_session::Limit; use rustc_span::Span; use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout}; use super::{ - GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, - Memory, MemoryKind, OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic, Projectable, - Provenance, Scalar, StackPopJump, + err_inval, throw_inval, throw_ub, throw_ub_custom, throw_unsup, GlobalId, Immediate, + InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Memory, MemoryKind, + OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic, Projectable, Provenance, Scalar, + StackPopJump, }; use crate::errors; use crate::util; @@ -1179,9 +1182,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { M::eval_mir_constant(self, *val, span, layout, |ecx, val, span, layout| { let const_val = val.eval(*ecx.tcx, ecx.param_env, span).map_err(|err| { - if M::ALL_CONSTS_ARE_PRECHECKED && !matches!(err, ErrorHandled::TooGeneric(..)) { - // Looks like the const is not captued by `required_consts`, that's bad. - bug!("interpret const eval failure of {val:?} which is not in required_consts"); + if M::ALL_CONSTS_ARE_PRECHECKED { + match err { + ErrorHandled::TooGeneric(..) => {}, + ErrorHandled::Reported(reported, span) => { + if reported.is_tainted_by_errors() { + // const-eval will return "tainted" errors if e.g. the layout cannot + // be computed as the type references non-existing names. + // See <https://github.com/rust-lang/rust/issues/124348>. + } else { + // Looks like the const is not captued by `required_consts`, that's bad. + span_bug!(span, "interpret const eval failure of {val:?} which is not in required_consts"); + } + } + } } err.emit_note(*ecx.tcx); err diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index d4168273f29..c34b658fb37 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -23,8 +23,9 @@ use rustc_middle::query::TyCtxtAt; use rustc_middle::ty::layout::TyAndLayout; use rustc_span::def_id::LocalDefId; use rustc_span::sym; +use tracing::{instrument, trace}; -use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy}; +use super::{err_ub, AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy}; use crate::const_eval; use crate::errors::NestedStaticInThreadLocal; diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 4d37c3c22cd..3a8f8dc4cb9 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -8,21 +8,19 @@ use rustc_middle::ty::layout::{LayoutOf as _, ValidityRequirement}; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_middle::{ - mir::{ - self, - interpret::{ - Allocation, ConstAllocation, GlobalId, InterpResult, PointerArithmetic, Scalar, - }, - BinOp, ConstValue, NonDivergingIntrinsic, - }, + bug, + mir::{self, BinOp, ConstValue, NonDivergingIntrinsic}, ty::layout::TyAndLayout, }; use rustc_span::symbol::{sym, Symbol}; use rustc_target::abi::Size; +use tracing::trace; use super::{ - memory::MemoryKind, util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, - MPlaceTy, Machine, OpTy, Pointer, + err_inval, err_ub_custom, err_unsup_format, memory::MemoryKind, throw_inval, throw_ub_custom, + throw_ub_format, util::ensure_monomorphic_enough, Allocation, CheckInAllocMsg, ConstAllocation, + GlobalId, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, Pointer, PointerArithmetic, + Scalar, }; use crate::fluent_generated as fluent; @@ -113,10 +111,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ) -> InterpResult<'tcx, bool> { let instance_args = instance.args; let intrinsic_name = self.tcx.item_name(instance.def_id()); - let Some(ret) = ret else { - // We don't support any intrinsic without return place. - return Ok(false); - }; match intrinsic_name { sym::caller_location => { @@ -253,14 +247,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { match (self.ptr_try_get_alloc_id(a), self.ptr_try_get_alloc_id(b)) { (Err(a), Err(b)) => { // Neither pointer points to an allocation. - // If these are inequal or null, this *will* fail the deref check below. + // This is okay only if they are the same. + if a != b { + // We'd catch this below in the "dereferenceable" check, but + // show a nicer error for this particular case. + throw_ub_custom!( + fluent::const_eval_offset_from_different_integers, + name = intrinsic_name, + ); + } + // This will always return 0. (a, b) } (Err(_), _) | (_, Err(_)) => { // We managed to find a valid allocation for one pointer, but not the other. // That means they are definitely not pointing to the same allocation. throw_ub_custom!( - fluent::const_eval_different_allocations, + fluent::const_eval_offset_from_different_allocations, name = intrinsic_name, ); } @@ -268,7 +271,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Found allocation for both. They must be into the same allocation. if a_alloc_id != b_alloc_id { throw_ub_custom!( - fluent::const_eval_different_allocations, + fluent::const_eval_offset_from_different_allocations, name = intrinsic_name, ); } @@ -284,13 +287,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let (val, overflowed) = { let a_offset = ImmTy::from_uint(a_offset, usize_layout); let b_offset = ImmTy::from_uint(b_offset, usize_layout); - self.overflowing_binary_op(BinOp::Sub, &a_offset, &b_offset)? + self.binary_op(BinOp::SubWithOverflow, &a_offset, &b_offset)? + .to_scalar_pair() }; - if overflowed { + if overflowed.to_bool()? { // a < b if intrinsic_name == sym::ptr_offset_from_unsigned { throw_ub_custom!( - fluent::const_eval_unsigned_offset_from_overflow, + fluent::const_eval_offset_from_unsigned_overflow, a_offset = a_offset, b_offset = b_offset, ); @@ -298,7 +302,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // The signed form of the intrinsic allows this. If we interpret the // difference as isize, we'll get the proper signed difference. If that // seems *positive*, they were more than isize::MAX apart. - let dist = val.to_scalar().to_target_isize(self)?; + let dist = val.to_target_isize(self)?; if dist >= 0 { throw_ub_custom!( fluent::const_eval_offset_from_underflow, @@ -308,7 +312,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { dist } else { // b >= a - let dist = val.to_scalar().to_target_isize(self)?; + let dist = val.to_target_isize(self)?; // If converting to isize produced a *negative* result, we had an overflow // because they were more than isize::MAX apart. if dist < 0 { @@ -376,7 +380,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }; M::panic_nounwind(self, &msg)?; - // Skip the `go_to_block` at the end. + // Skip the `return_to_block` at the end (we panicked, we do not return). return Ok(true); } } @@ -414,7 +418,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } self.copy_op(&self.project_index(&input, index)?, dest)?; } - sym::likely | sym::unlikely | sym::black_box => { + sym::black_box => { // These just return their argument self.copy_op(&args[0], dest)?; } @@ -437,11 +441,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?; } + // Unsupported intrinsic: skip the return_to_block below. _ => return Ok(false), } trace!("{:?}", self.dump_place(&dest.clone().into())); - self.go_to_block(ret); + self.return_to_block(ret)?; Ok(true) } @@ -513,9 +518,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Performs an exact division, resulting in undefined behavior where // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`. // First, check x % y != 0 (or if that computation overflows). - let (res, overflow) = self.overflowing_binary_op(BinOp::Rem, a, b)?; - assert!(!overflow); // All overflow is UB, so this should never return on overflow. - if res.to_scalar().assert_bits(a.layout.size) != 0 { + let rem = self.binary_op(BinOp::Rem, a, b)?; + if rem.to_scalar().assert_bits(a.layout.size) != 0 { throw_ub_custom!( fluent::const_eval_exact_div_has_remainder, a = format!("{a}"), @@ -523,7 +527,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ) } // `Rem` says this is all right, so we can let `Div` do its job. - self.binop_ignore_overflow(BinOp::Div, a, b, &dest.clone().into()) + let res = self.binary_op(BinOp::Div, a, b)?; + self.write_immediate(*res, dest) } pub fn saturating_arith( @@ -536,8 +541,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { assert!(matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..))); assert!(matches!(mir_op, BinOp::Add | BinOp::Sub)); - let (val, overflowed) = self.overflowing_binary_op(mir_op, l, r)?; - Ok(if overflowed { + let (val, overflowed) = + self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair(); + Ok(if overflowed.to_bool()? { let size = l.layout.size; let num_bits = size.bits(); if l.layout.abi.is_signed() { @@ -568,7 +574,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } } else { - val.to_scalar() + val }) } @@ -605,7 +611,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { nonoverlapping: bool, ) -> InterpResult<'tcx> { let count = self.read_target_usize(count)?; - let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap().ty)?; + let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap())?; let (size, align) = (layout.size, layout.align.abi); // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max), // but no actual allocation can be big enough for the difference to be noticeable. @@ -649,7 +655,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { byte: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, count: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, ) -> InterpResult<'tcx> { - let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap().ty)?; + let layout = self.layout_of(dst.layout.ty.builtin_deref(true).unwrap())?; let dst = self.read_pointer(dst)?; let byte = self.read_scalar(byte)?.to_u8()?; @@ -688,7 +694,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { lhs: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, rhs: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, ) -> InterpResult<'tcx, Scalar<M::Provenance>> { - let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap().ty)?; + let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap())?; assert!(layout.is_sized()); let get_bytes = |this: &InterpCx<'mir, 'tcx, M>, diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 8bc569bed54..72a16dbe4d6 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -18,9 +18,9 @@ use rustc_target::abi::{Align, Size}; use rustc_target::spec::abi::Abi as CallAbi; use super::{ - AllocBytes, AllocId, AllocKind, AllocRange, Allocation, ConstAllocation, CtfeProvenance, FnArg, - Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, MemoryKind, Misalignment, OpTy, PlaceTy, - Pointer, Provenance, + throw_unsup, throw_unsup_format, AllocBytes, AllocId, AllocKind, AllocRange, Allocation, + ConstAllocation, CtfeProvenance, FnArg, Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, + MemoryKind, Misalignment, OpTy, PlaceTy, Pointer, Provenance, }; /// Data returned by Machine::stack_pop, @@ -216,6 +216,9 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized { /// Directly process an intrinsic without pushing a stack frame. It is the hook's /// responsibility to advance the instruction pointer as appropriate. + /// + /// Returns `None` if the intrinsic was fully handled. + /// Otherwise, returns an `Instance` of the function that implements the intrinsic. fn call_intrinsic( ecx: &mut InterpCx<'mir, 'tcx, Self>, instance: ty::Instance<'tcx>, @@ -223,7 +226,7 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized { destination: &MPlaceTy<'tcx, Self::Provenance>, target: Option<mir::BasicBlock>, unwind: mir::UnwindAction, - ) -> InterpResult<'tcx>; + ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>>; /// Called to evaluate `Assert` MIR terminators that trigger a panic. fn assert_panic( @@ -249,7 +252,7 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized { bin_op: mir::BinOp, left: &ImmTy<'tcx, Self::Provenance>, right: &ImmTy<'tcx, Self::Provenance>, - ) -> InterpResult<'tcx, (ImmTy<'tcx, Self::Provenance>, bool)>; + ) -> InterpResult<'tcx, ImmTy<'tcx, Self::Provenance>>; /// Generate the NaN returned by a float operation, given the list of inputs. /// (This is all inputs, not just NaN inputs!) diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 594e3b3212f..23273e69444 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -16,16 +16,20 @@ use std::ptr; use rustc_ast::Mutability; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_hir::def::DefKind; +use rustc_middle::bug; use rustc_middle::mir::display_allocation; use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt}; use rustc_target::abi::{Align, HasDataLayout, Size}; +use tracing::{debug, instrument, trace}; + use crate::fluent_generated as fluent; use super::{ - alloc_range, AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckAlignMsg, - CheckInAllocMsg, CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, - Misalignment, Pointer, PointerArithmetic, Provenance, Scalar, + alloc_range, err_ub, err_ub_custom, throw_ub, throw_ub_custom, throw_unsup, throw_unsup_format, + AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckAlignMsg, CheckInAllocMsg, + CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Misalignment, Pointer, + PointerArithmetic, Provenance, Scalar, }; #[derive(Debug, PartialEq, Copy, Clone)] @@ -411,6 +415,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// to the allocation it points to. Supports both shared and mutable references, as the actual /// checking is offloaded to a helper closure. /// + /// `alloc_size` will only get called for non-zero-sized accesses. + /// /// Returns `None` if and only if the size is 0. fn check_and_deref_ptr<T>( &self, @@ -423,18 +429,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { M::ProvenanceExtra, ) -> InterpResult<'tcx, (Size, Align, T)>, ) -> InterpResult<'tcx, Option<T>> { + // Everything is okay with size 0. + if size.bytes() == 0 { + return Ok(None); + } + Ok(match self.ptr_try_get_alloc_id(ptr) { Err(addr) => { - // We couldn't get a proper allocation. This is only okay if the access size is 0, - // and the address is not null. - if size.bytes() > 0 || addr == 0 { - throw_ub!(DanglingIntPointer(addr, msg)); - } - None + // We couldn't get a proper allocation. + throw_ub!(DanglingIntPointer(addr, msg)); } Ok((alloc_id, offset, prov)) => { let (alloc_size, _alloc_align, ret_val) = alloc_size(alloc_id, offset, prov)?; - // Test bounds. This also ensures non-null. + // Test bounds. // It is sufficient to check this for the end pointer. Also check for overflow! if offset.checked_add(size, &self.tcx).map_or(true, |end| end > alloc_size) { throw_ub!(PointerOutOfBounds { @@ -445,14 +452,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { msg, }) } - // Ensure we never consider the null pointer dereferenceable. - if M::Provenance::OFFSET_IS_ADDR { - assert_ne!(ptr.addr(), Size::ZERO); - } - // We can still be zero-sized in this branch, in which case we have to - // return `None`. - if size.bytes() == 0 { None } else { Some(ret_val) } + Some(ret_val) } }) } @@ -639,16 +640,18 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { size, CheckInAllocMsg::MemoryAccessTest, |alloc_id, offset, prov| { - if !self.memory.validation_in_progress.get() { - // We want to call the hook on *all* accesses that involve an AllocId, - // including zero-sized accesses. That means we have to do it here - // rather than below in the `Some` branch. - M::before_alloc_read(self, alloc_id)?; - } let alloc = self.get_alloc_raw(alloc_id)?; Ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc))) }, )?; + // We want to call the hook on *all* accesses that involve an AllocId, including zero-sized + // accesses. That means we cannot rely on the closure above or the `Some` branch below. We + // do this after `check_and_deref_ptr` to ensure some basic sanity has already been checked. + if !self.memory.validation_in_progress.get() { + if let Ok((alloc_id, ..)) = self.ptr_try_get_alloc_id(ptr) { + M::before_alloc_read(self, alloc_id)?; + } + } if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc { let range = alloc_range(offset, size); diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 718c91b2f76..8f01b872010 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -4,19 +4,21 @@ use std::assert_matches::assert_matches; use either::{Either, Left, Right}; +use tracing::trace; use rustc_hir::def::Namespace; use rustc_middle::mir::interpret::ScalarSizeMismatch; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{HasParamEnv, HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter}; use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; use rustc_middle::{mir, ty}; use rustc_target::abi::{self, Abi, HasDataLayout, Size}; use super::{ - alloc_range, from_known_layout, mir_assign_valid_types, CtfeProvenance, InterpCx, InterpResult, - MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer, Projectable, - Provenance, Scalar, + alloc_range, err_ub, from_known_layout, mir_assign_valid_types, throw_ub, CtfeProvenance, + InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, + Pointer, Projectable, Provenance, Scalar, }; /// An `Immediate` represents a single immediate self-contained Rust value. @@ -248,6 +250,15 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { Self::from_scalar(Scalar::from_i8(c as i8), layout) } + pub fn from_pair(a: Self, b: Self, tcx: TyCtxt<'tcx>) -> Self { + let layout = tcx + .layout_of( + ty::ParamEnv::reveal_all().and(Ty::new_tup(tcx, &[a.layout.ty, b.layout.ty])), + ) + .unwrap(); + Self::from_scalar_pair(a.to_scalar(), b.to_scalar(), layout) + } + /// Return the immediate as a `ScalarInt`. Ensures that it has the size that the layout of the /// immediate indicates. #[inline] @@ -269,6 +280,17 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral()) } + #[inline] + #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) + pub fn to_pair(self, cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>)) -> (Self, Self) { + let layout = self.layout; + let (val0, val1) = self.to_scalar_pair(); + ( + ImmTy::from_scalar(val0, layout.field(cx, 0)), + ImmTy::from_scalar(val1, layout.field(cx, 1)), + ) + } + /// Compute the "sub-immediate" that is located within the `base` at the given offset with the /// given layout. // Not called `offset` to avoid confusion with the trait method. diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index 9af755e40de..62641b868e6 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -1,77 +1,23 @@ +use either::Either; + use rustc_apfloat::{Float, FloatConvert}; use rustc_middle::mir; use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::{self, FloatTy, ScalarInt, Ty}; +use rustc_middle::ty::{self, FloatTy, ScalarInt}; +use rustc_middle::{bug, span_bug}; use rustc_span::symbol::sym; -use rustc_target::abi::Abi; - -use super::{ImmTy, Immediate, InterpCx, Machine, PlaceTy}; +use tracing::trace; -use crate::fluent_generated as fluent; - -impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { - /// Applies the binary operation `op` to the two operands and writes a tuple of the result - /// and a boolean signifying the potential overflow to the destination. - pub fn binop_with_overflow( - &mut self, - op: mir::BinOp, - left: &ImmTy<'tcx, M::Provenance>, - right: &ImmTy<'tcx, M::Provenance>, - dest: &PlaceTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx> { - let (val, overflowed) = self.overflowing_binary_op(op, left, right)?; - debug_assert_eq!( - Ty::new_tup(self.tcx.tcx, &[val.layout.ty, self.tcx.types.bool]), - dest.layout.ty, - "type mismatch for result of {op:?}", - ); - // Write the result to `dest`. - if let Abi::ScalarPair(..) = dest.layout.abi { - // We can use the optimized path and avoid `place_field` (which might do - // `force_allocation`). - let pair = Immediate::ScalarPair(val.to_scalar(), Scalar::from_bool(overflowed)); - self.write_immediate(pair, dest)?; - } else { - assert!(self.tcx.sess.opts.unstable_opts.randomize_layout); - // With randomized layout, `(int, bool)` might cease to be a `ScalarPair`, so we have to - // do a component-wise write here. This code path is slower than the above because - // `place_field` will have to `force_allocate` locals here. - let val_field = self.project_field(dest, 0)?; - self.write_scalar(val.to_scalar(), &val_field)?; - let overflowed_field = self.project_field(dest, 1)?; - self.write_scalar(Scalar::from_bool(overflowed), &overflowed_field)?; - } - Ok(()) - } - - /// Applies the binary operation `op` to the arguments and writes the result to the - /// destination. - pub fn binop_ignore_overflow( - &mut self, - op: mir::BinOp, - left: &ImmTy<'tcx, M::Provenance>, - right: &ImmTy<'tcx, M::Provenance>, - dest: &PlaceTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx> { - let val = self.wrapping_binary_op(op, left, right)?; - assert_eq!(val.layout.ty, dest.layout.ty, "type mismatch for result of {op:?}"); - self.write_immediate(*val, dest) - } -} +use super::{err_ub, throw_ub, ImmTy, InterpCx, Machine}; impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { - fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> (ImmTy<'tcx, M::Provenance>, bool) { + fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> ImmTy<'tcx, M::Provenance> { let res = Ord::cmp(&lhs, &rhs); - return (ImmTy::from_ordering(res, *self.tcx), false); + return ImmTy::from_ordering(res, *self.tcx); } - fn binary_char_op( - &self, - bin_op: mir::BinOp, - l: char, - r: char, - ) -> (ImmTy<'tcx, M::Provenance>, bool) { + fn binary_char_op(&self, bin_op: mir::BinOp, l: char, r: char) -> ImmTy<'tcx, M::Provenance> { use rustc_middle::mir::BinOp::*; if bin_op == Cmp { @@ -87,15 +33,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Ge => l >= r, _ => span_bug!(self.cur_span(), "Invalid operation on char: {:?}", bin_op), }; - (ImmTy::from_bool(res, *self.tcx), false) + ImmTy::from_bool(res, *self.tcx) } - fn binary_bool_op( - &self, - bin_op: mir::BinOp, - l: bool, - r: bool, - ) -> (ImmTy<'tcx, M::Provenance>, bool) { + fn binary_bool_op(&self, bin_op: mir::BinOp, l: bool, r: bool) -> ImmTy<'tcx, M::Provenance> { use rustc_middle::mir::BinOp::*; let res = match bin_op { @@ -110,7 +51,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { BitXor => l ^ r, _ => span_bug!(self.cur_span(), "Invalid operation on bool: {:?}", bin_op), }; - (ImmTy::from_bool(res, *self.tcx), false) + ImmTy::from_bool(res, *self.tcx) } fn binary_float_op<F: Float + FloatConvert<F> + Into<Scalar<M::Provenance>>>( @@ -119,14 +60,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { layout: TyAndLayout<'tcx>, l: F, r: F, - ) -> (ImmTy<'tcx, M::Provenance>, bool) { + ) -> ImmTy<'tcx, M::Provenance> { use rustc_middle::mir::BinOp::*; // Performs appropriate non-deterministic adjustments of NaN results. let adjust_nan = |f: F| -> F { if f.is_nan() { M::generate_nan(self, &[l, r]) } else { f } }; - let val = match bin_op { + match bin_op { Eq => ImmTy::from_bool(l == r, *self.tcx), Ne => ImmTy::from_bool(l != r, *self.tcx), Lt => ImmTy::from_bool(l < r, *self.tcx), @@ -139,8 +80,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Div => ImmTy::from_scalar(adjust_nan((l / r).value).into(), layout), Rem => ImmTy::from_scalar(adjust_nan((l % r).value).into(), layout), _ => span_bug!(self.cur_span(), "invalid float op: `{:?}`", bin_op), - }; - (val, false) + } } fn binary_int_op( @@ -148,7 +88,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { bin_op: mir::BinOp, left: &ImmTy<'tcx, M::Provenance>, right: &ImmTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (ImmTy<'tcx, M::Provenance>, bool)> { + ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { use rustc_middle::mir::BinOp::*; // This checks the size, so that we can just assert it below. @@ -168,25 +108,27 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ShrUnchecked => Some(sym::unchecked_shr), _ => None, }; + let with_overflow = bin_op.is_overflowing(); // Shift ops can have an RHS with a different numeric type. if matches!(bin_op, Shl | ShlUnchecked | Shr | ShrUnchecked) { let size = left.layout.size.bits(); - // The shift offset is implicitly masked to the type size. (This is the one MIR operator - // that does *not* directly map to a single LLVM operation.) Compute how much we - // actually shift and whether there was an overflow due to shifting too much. + // Compute the equivalent shift modulo `size` that is in the range `0..size`. (This is + // the one MIR operator that does *not* directly map to a single LLVM operation.) let (shift_amount, overflow) = if right.layout.abi.is_signed() { let shift_amount = r_signed(); let overflow = shift_amount < 0 || shift_amount >= i128::from(size); // Deliberately wrapping `as` casts: shift_amount *can* be negative, but the result // of the `as` will be equal modulo `size` (since it is a power of two). let masked_amount = (shift_amount as u128) % u128::from(size); - assert_eq!(overflow, shift_amount != (masked_amount as i128)); + assert_eq!(overflow, shift_amount != i128::try_from(masked_amount).unwrap()); (masked_amount, overflow) } else { let shift_amount = r_unsigned(); + let overflow = shift_amount >= u128::from(size); let masked_amount = shift_amount % u128::from(size); - (masked_amount, shift_amount != masked_amount) + assert_eq!(overflow, shift_amount != masked_amount); + (masked_amount, overflow) }; let shift_amount = u32::try_from(shift_amount).unwrap(); // we masked so this will always fit // Compute the shifted result. @@ -208,19 +150,18 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ScalarInt::truncate_from_uint(result, left.layout.size).0 }; - if overflow && let Some(intrinsic_name) = throw_ub_on_overflow { - throw_ub_custom!( - fluent::const_eval_overflow_shift, - val = if right.layout.abi.is_signed() { - r_signed().to_string() + if overflow && let Some(intrinsic) = throw_ub_on_overflow { + throw_ub!(ShiftOverflow { + intrinsic, + shift_amount: if right.layout.abi.is_signed() { + Either::Right(r_signed()) } else { - r_unsigned().to_string() - }, - name = intrinsic_name - ); + Either::Left(r_unsigned()) + } + }); } - return Ok((ImmTy::from_scalar_int(result, left.layout), overflow)); + return Ok(ImmTy::from_scalar_int(result, left.layout)); } // For the remaining ops, the types must be the same on both sides @@ -245,7 +186,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => None, }; if let Some(op) = op { - return Ok((ImmTy::from_bool(op(&l_signed(), &r_signed()), *self.tcx), false)); + return Ok(ImmTy::from_bool(op(&l_signed(), &r_signed()), *self.tcx)); } if bin_op == Cmp { return Ok(self.three_way_compare(l_signed(), r_signed())); @@ -255,9 +196,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Rem if r.is_null() => throw_ub!(RemainderByZero), Div => Some(i128::overflowing_div), Rem => Some(i128::overflowing_rem), - Add | AddUnchecked => Some(i128::overflowing_add), - Sub | SubUnchecked => Some(i128::overflowing_sub), - Mul | MulUnchecked => Some(i128::overflowing_mul), + Add | AddUnchecked | AddWithOverflow => Some(i128::overflowing_add), + Sub | SubUnchecked | SubWithOverflow => Some(i128::overflowing_sub), + Mul | MulUnchecked | MulWithOverflow => Some(i128::overflowing_mul), _ => None, }; if let Some(op) = op { @@ -281,10 +222,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // If that truncation loses any information, we have an overflow. let (result, lossy) = ScalarInt::truncate_from_int(result, left.layout.size); let overflow = oflo || lossy; - if overflow && let Some(intrinsic_name) = throw_ub_on_overflow { - throw_ub_custom!(fluent::const_eval_overflow, name = intrinsic_name); + if overflow && let Some(intrinsic) = throw_ub_on_overflow { + throw_ub!(ArithOverflow { intrinsic }); } - return Ok((ImmTy::from_scalar_int(result, left.layout), overflow)); + let res = ImmTy::from_scalar_int(result, left.layout); + return Ok(if with_overflow { + let overflow = ImmTy::from_bool(overflow, *self.tcx); + ImmTy::from_pair(res, overflow, *self.tcx) + } else { + res + }); } } // From here on it's okay to treat everything as unsigned. @@ -295,7 +242,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { return Ok(self.three_way_compare(l, r)); } - let val = match bin_op { + Ok(match bin_op { Eq => ImmTy::from_bool(l == r, *self.tcx), Ne => ImmTy::from_bool(l != r, *self.tcx), @@ -308,40 +255,42 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { BitAnd => ImmTy::from_uint(l & r, left.layout), BitXor => ImmTy::from_uint(l ^ r, left.layout), - Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Rem | Div => { + _ => { assert!(!left.layout.abi.is_signed()); let op: fn(u128, u128) -> (u128, bool) = match bin_op { - Add | AddUnchecked => u128::overflowing_add, - Sub | SubUnchecked => u128::overflowing_sub, - Mul | MulUnchecked => u128::overflowing_mul, + Add | AddUnchecked | AddWithOverflow => u128::overflowing_add, + Sub | SubUnchecked | SubWithOverflow => u128::overflowing_sub, + Mul | MulUnchecked | MulWithOverflow => u128::overflowing_mul, Div if r == 0 => throw_ub!(DivisionByZero), Rem if r == 0 => throw_ub!(RemainderByZero), Div => u128::overflowing_div, Rem => u128::overflowing_rem, - _ => bug!(), + _ => span_bug!( + self.cur_span(), + "invalid binary op {:?}: {:?}, {:?} (both {})", + bin_op, + left, + right, + right.layout.ty, + ), }; let (result, oflo) = op(l, r); // Truncate to target type. // If that truncation loses any information, we have an overflow. let (result, lossy) = ScalarInt::truncate_from_uint(result, left.layout.size); let overflow = oflo || lossy; - if overflow && let Some(intrinsic_name) = throw_ub_on_overflow { - throw_ub_custom!(fluent::const_eval_overflow, name = intrinsic_name); + if overflow && let Some(intrinsic) = throw_ub_on_overflow { + throw_ub!(ArithOverflow { intrinsic }); + } + let res = ImmTy::from_scalar_int(result, left.layout); + if with_overflow { + let overflow = ImmTy::from_bool(overflow, *self.tcx); + ImmTy::from_pair(res, overflow, *self.tcx) + } else { + res } - return Ok((ImmTy::from_scalar_int(result, left.layout), overflow)); } - - _ => span_bug!( - self.cur_span(), - "invalid binary op {:?}: {:?}, {:?} (both {})", - bin_op, - left, - right, - right.layout.ty, - ), - }; - - Ok((val, false)) + }) } fn binary_ptr_op( @@ -349,7 +298,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { bin_op: mir::BinOp, left: &ImmTy<'tcx, M::Provenance>, right: &ImmTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (ImmTy<'tcx, M::Provenance>, bool)> { + ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { use rustc_middle::mir::BinOp::*; match bin_op { @@ -357,7 +306,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Offset => { let ptr = left.to_scalar().to_pointer(self)?; let offset_count = right.to_scalar().to_target_isize(self)?; - let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty; + let pointee_ty = left.layout.ty.builtin_deref(true).unwrap(); // We cannot overflow i64 as a type's size must be <= isize::MAX. let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap(); @@ -368,10 +317,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { offset_count.checked_mul(pointee_size).ok_or(err_ub!(PointerArithOverflow))?; let offset_ptr = self.ptr_offset_inbounds(ptr, offset_bytes)?; - Ok(( - ImmTy::from_scalar(Scalar::from_maybe_pointer(offset_ptr, self), left.layout), - false, - )) + Ok(ImmTy::from_scalar(Scalar::from_maybe_pointer(offset_ptr, self), left.layout)) } // Fall back to machine hook so Miri can support more pointer ops. @@ -379,13 +325,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - /// Returns the result of the specified operation, and whether it overflowed. - pub fn overflowing_binary_op( + /// Returns the result of the specified operation. + /// + /// Whether this produces a scalar or a pair depends on the specific `bin_op`. + pub fn binary_op( &self, bin_op: mir::BinOp, left: &ImmTy<'tcx, M::Provenance>, right: &ImmTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (ImmTy<'tcx, M::Provenance>, bool)> { + ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { trace!( "Running binary op {:?}: {:?} ({}), {:?} ({})", bin_op, @@ -457,24 +405,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - #[inline] - pub fn wrapping_binary_op( - &self, - bin_op: mir::BinOp, - left: &ImmTy<'tcx, M::Provenance>, - right: &ImmTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { - let (val, _overflow) = self.overflowing_binary_op(bin_op, left, right)?; - Ok(val) - } - /// Returns the result of the specified operation, whether it overflowed, and /// the result type. - pub fn overflowing_unary_op( + pub fn unary_op( &self, un_op: mir::UnOp, val: &ImmTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (ImmTy<'tcx, M::Provenance>, bool)> { + ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { use rustc_middle::mir::UnOp::*; let layout = val.layout; @@ -488,7 +425,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Not => !val, _ => span_bug!(self.cur_span(), "Invalid bool op {:?}", un_op), }; - Ok((ImmTy::from_bool(res, *self.tcx), false)) + Ok(ImmTy::from_bool(res, *self.tcx)) } ty::Float(fty) => { // No NaN adjustment here, `-` is a bitwise operation! @@ -497,37 +434,25 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { (Neg, FloatTy::F64) => Scalar::from_f64(-val.to_f64()?), _ => span_bug!(self.cur_span(), "Invalid float op {:?}", un_op), }; - Ok((ImmTy::from_scalar(res, layout), false)) + Ok(ImmTy::from_scalar(res, layout)) } _ => { assert!(layout.ty.is_integral()); let val = val.to_bits(layout.size)?; - let (res, overflow) = match un_op { - Not => (self.truncate(!val, layout), false), // bitwise negation, then truncate + let res = match un_op { + Not => self.truncate(!val, layout), // bitwise negation, then truncate Neg => { // arithmetic negation assert!(layout.abi.is_signed()); let val = self.sign_extend(val, layout) as i128; - let (res, overflow) = val.overflowing_neg(); + let res = val.wrapping_neg(); let res = res as u128; // Truncate to target type. - // If that truncation loses any information, we have an overflow. - let truncated = self.truncate(res, layout); - (truncated, overflow || self.sign_extend(truncated, layout) != res) + self.truncate(res, layout) } }; - Ok((ImmTy::from_uint(res, layout), overflow)) + Ok(ImmTy::from_uint(res, layout)) } } } - - #[inline] - pub fn wrapping_unary_op( - &self, - un_op: mir::UnOp, - val: &ImmTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { - let (val, _overflow) = self.overflowing_unary_op(un_op, val)?; - Ok(val) - } } diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index e5241f1ba19..588db0f547f 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -5,18 +5,21 @@ use std::assert_matches::assert_matches; use either::{Either, Left, Right}; +use tracing::{instrument, trace}; use rustc_ast::Mutability; use rustc_middle::mir; use rustc_middle::ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Ty; +use rustc_middle::{bug, span_bug}; use rustc_target::abi::{Abi, Align, HasDataLayout, Size}; use super::{ - alloc_range, mir_assign_valid_types, AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance, - ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy, - Operand, Pointer, PointerArithmetic, Projectable, Provenance, Readable, Scalar, + alloc_range, mir_assign_valid_types, throw_ub, AllocRef, AllocRefMut, CheckAlignMsg, + CtfeProvenance, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, Misalignment, + OffsetMode, OpTy, Operand, Pointer, PointerArithmetic, Projectable, Provenance, Readable, + Scalar, }; #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] @@ -415,7 +418,7 @@ where val: &ImmTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { let pointee_type = - val.layout.ty.builtin_deref(true).expect("`ref_to_mplace` called on non-ptr type").ty; + val.layout.ty.builtin_deref(true).expect("`ref_to_mplace` called on non-ptr type"); let layout = self.layout_of(pointee_type)?; let (ptr, meta) = val.to_scalar_and_meta(); diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index 5ff78f7b8c9..cfc4d701aa7 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -14,10 +14,16 @@ use rustc_middle::mir; use rustc_middle::ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Ty; +use rustc_middle::{bug, span_bug}; use rustc_target::abi::Size; use rustc_target::abi::{self, VariantIdx}; -use super::{InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, Provenance, Scalar}; +use tracing::{debug, instrument}; + +use super::{ + throw_ub, throw_unsup_format, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, + Provenance, Scalar, +}; /// Describes the constraints placed on offset-projections. #[derive(Copy, Clone, Debug)] diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index b29034e991e..58210fe3971 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -3,10 +3,12 @@ //! The main entry point is the `step` method. use either::Either; +use tracing::{info, instrument, trace}; use rustc_index::IndexSlice; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::{bug, span_bug}; use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; use super::{ @@ -166,23 +168,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let left = self.read_immediate(&self.eval_operand(left, layout)?)?; let layout = util::binop_right_homogeneous(bin_op).then_some(left.layout); let right = self.read_immediate(&self.eval_operand(right, layout)?)?; - self.binop_ignore_overflow(bin_op, &left, &right, &dest)?; - } - - CheckedBinaryOp(bin_op, box (ref left, ref right)) => { - // Due to the extra boolean in the result, we can never reuse the `dest.layout`. - let left = self.read_immediate(&self.eval_operand(left, None)?)?; - let layout = util::binop_right_homogeneous(bin_op).then_some(left.layout); - let right = self.read_immediate(&self.eval_operand(right, layout)?)?; - self.binop_with_overflow(bin_op, &left, &right, &dest)?; + let result = self.binary_op(bin_op, &left, &right)?; + assert_eq!(result.layout, dest.layout, "layout mismatch for result of {bin_op:?}"); + self.write_immediate(*result, &dest)?; } UnaryOp(un_op, ref operand) => { // The operand always has the same type as the result. let val = self.read_immediate(&self.eval_operand(operand, Some(dest.layout))?)?; - let val = self.wrapping_unary_op(un_op, &val)?; - assert_eq!(val.layout, dest.layout, "layout mismatch for result of {un_op:?}"); - self.write_immediate(*val, &dest)?; + let result = self.unary_op(un_op, &val)?; + assert_eq!(result.layout, dest.layout, "layout mismatch for result of {un_op:?}"); + self.write_immediate(*result, &dest)?; } Aggregate(box ref kind, ref operands) => { diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index 9c31532a9ce..360cefb5a1f 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -1,7 +1,9 @@ use std::borrow::Cow; use either::Either; +use tracing::trace; +use rustc_middle::span_bug; use rustc_middle::{ mir, ty::{ @@ -19,8 +21,9 @@ use rustc_target::abi::{ use rustc_target::spec::abi::Abi; use super::{ - CtfeProvenance, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, - Projectable, Provenance, Scalar, StackPopCleanup, + throw_ub, throw_ub_custom, throw_unsup_format, CtfeProvenance, FnVal, ImmTy, InterpCx, + InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, Projectable, Provenance, Scalar, + StackPopCleanup, }; use crate::fluent_generated as fluent; @@ -95,7 +98,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { for (const_int, target) in targets.iter() { // Compare using MIR BinOp::Eq, to also support pointer values. // (Avoiding `self.binary_op` as that does some redundant layout computation.) - let res = self.wrapping_binary_op( + let res = self.binary_op( mir::BinOp::Eq, &discr, &ImmTy::from_uint(const_int, discr.layout), @@ -169,10 +172,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } Drop { place, target, unwind, replace: _ } => { - let frame = self.frame(); - let ty = place.ty(&frame.body.local_decls, *self.tcx).ty; - let ty = self.instantiate_from_frame_and_normalize_erasing_regions(frame, ty)?; - let instance = Instance::resolve_drop_in_place(*self.tcx, ty); + let place = self.eval_place(place)?; + let instance = Instance::resolve_drop_in_place(*self.tcx, place.layout.ty); if let ty::InstanceDef::DropGlue(_, None) = instance.def { // This is the branch we enter if and only if the dropped type has no drop glue // whatsoever. This can happen as a result of monomorphizing a drop of a @@ -181,8 +182,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.go_to_block(target); return Ok(()); } - let place = self.eval_place(place)?; - trace!("TerminatorKind::drop: {:?}, type {}", place, ty); + trace!("TerminatorKind::drop: {:?}, type {}", place, place.layout.ty); self.drop_in_place(&place, instance, target, unwind)?; } @@ -539,14 +539,28 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ty::InstanceDef::Intrinsic(def_id) => { assert!(self.tcx.intrinsic(def_id).is_some()); // FIXME: Should `InPlace` arguments be reset to uninit? - M::call_intrinsic( + if let Some(fallback) = M::call_intrinsic( self, instance, &self.copy_fn_args(args), destination, target, unwind, - ) + )? { + assert!(!self.tcx.intrinsic(fallback.def_id()).unwrap().must_be_overridden); + assert!(matches!(fallback.def, ty::InstanceDef::Item(_))); + return self.eval_fn_call( + FnVal::Instance(fallback), + (caller_abi, caller_fn_abi), + args, + with_caller_location, + destination, + target, + unwind, + ); + } else { + Ok(()) + } } ty::InstanceDef::VTableShim(..) | ty::InstanceDef::ReifyShim(..) @@ -938,6 +952,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // implementation fail -- a problem shared by rustc. let place = self.force_allocation(place)?; + // We behave a bit different from codegen here. + // Codegen creates an `InstanceDef::Virtual` with index 0 (the slot of the drop method) and + // then dispatches that to the normal call machinery. However, our call machinery currently + // only supports calling `VtblEntry::Method`; it would choke on a `MetadataDropInPlace`. So + // instead we do the virtual call stuff ourselves. It's easier here than in `eval_fn_call` + // since we can just get a place of the underlying type and use `mplace_to_ref`. let place = match place.layout.ty.kind() { ty::Dynamic(data, _, ty::Dyn) => { // Dropping a trait object. Need to find actual drop fn. diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs index b603ef0d27a..8b590459405 100644 --- a/compiler/rustc_const_eval/src/interpret/traits.rs +++ b/compiler/rustc_const_eval/src/interpret/traits.rs @@ -2,6 +2,7 @@ use rustc_middle::mir::interpret::{InterpResult, Pointer}; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_target::abi::{Align, Size}; +use tracing::trace; use super::util::ensure_monomorphic_enough; use super::{InterpCx, Machine}; diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs index c83ef14c03f..f0f9df73d94 100644 --- a/compiler/rustc_const_eval/src/interpret/util.rs +++ b/compiler/rustc_const_eval/src/interpret/util.rs @@ -1,5 +1,4 @@ use crate::const_eval::{CompileTimeEvalContext, CompileTimeInterpreter, InterpretationResult}; -use crate::interpret::{MemPlaceMeta, MemoryKind}; use rustc_hir::def_id::LocalDefId; use rustc_middle::mir; use rustc_middle::mir::interpret::{Allocation, InterpResult, Pointer}; @@ -8,8 +7,9 @@ use rustc_middle::ty::{ self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, }; use std::ops::ControlFlow; +use tracing::debug; -use super::{InterpCx, MPlaceTy}; +use super::{throw_inval, InterpCx, MPlaceTy, MemPlaceMeta, MemoryKind}; /// Checks whether a type contains generic parameters which must be instantiated. /// diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 14566719ccd..cf6027a312f 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -8,11 +8,13 @@ use std::fmt::Write; use std::num::NonZero; use either::{Left, Right}; +use tracing::trace; use hir::def::DefKind; use rustc_ast::Mutability; use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; +use rustc_middle::bug; use rustc_middle::mir::interpret::{ ExpectedKind, InterpError, InvalidMetaKind, Misalignment, PointerKind, Provenance, ValidationErrorInfo, ValidationErrorKind, ValidationErrorKind::*, @@ -27,9 +29,9 @@ use rustc_target::abi::{ use std::hash::Hash; use super::{ - format_interp_error, machine::AllocMap, AllocId, CheckInAllocMsg, GlobalAlloc, ImmTy, - Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, Pointer, Projectable, - Scalar, ValueVisitor, + err_ub, format_interp_error, machine::AllocMap, throw_ub, AllocId, CheckInAllocMsg, + GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, + Pointer, Projectable, Scalar, ValueVisitor, }; // for the validation errors @@ -433,6 +435,11 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' found_bytes: has.bytes() }, ); + // Make sure this is non-null. We checked dereferenceability above, but if `size` is zero + // that does not imply non-null. + if self.ecx.scalar_may_be_null(Scalar::from_maybe_pointer(place.ptr(), self.ecx))? { + throw_validation_failure!(self.path, NullPtr { ptr_kind }) + } // Do not allow pointers to uninhabited types. if place.layout.abi.is_uninhabited() { let ty = place.layout.ty; @@ -455,8 +462,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // `!` is a ZST and we want to validate it. if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr()) { let mut skip_recursive_check = false; - let alloc_actual_mutbl = mutability(self.ecx, alloc_id); - if let GlobalAlloc::Static(did) = self.ecx.tcx.global_alloc(alloc_id) { + if let Some(GlobalAlloc::Static(did)) = self.ecx.tcx.try_get_global_alloc(alloc_id) + { let DefKind::Static { nested, .. } = self.ecx.tcx.def_kind(did) else { bug!() }; // Special handling for pointers to statics (irrespective of their type). assert!(!self.ecx.tcx.is_thread_local_static(did)); @@ -494,6 +501,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // If this allocation has size zero, there is no actual mutability here. let (size, _align, _alloc_kind) = self.ecx.get_alloc_info(alloc_id); if size != Size::ZERO { + let alloc_actual_mutbl = mutability(self.ecx, alloc_id); // Mutable pointer to immutable memory is no good. if ptr_expected_mutbl == Mutability::Mut && alloc_actual_mutbl == Mutability::Not @@ -830,6 +838,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> trace!("visit_value: {:?}, {:?}", *op, op.layout); // Check primitive types -- the leaves of our recursive descent. + // We assume that the Scalar validity range does not restrict these values + // any further than `try_visit_primitive` does! if self.try_visit_primitive(op)? { return Ok(()); } diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index 84557b8e2d6..692eb6c17e6 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -6,10 +6,11 @@ use rustc_middle::mir::interpret::InterpResult; use rustc_middle::ty::{self, Ty}; use rustc_target::abi::FieldIdx; use rustc_target::abi::{FieldsShape, VariantIdx, Variants}; +use tracing::trace; use std::num::NonZero; -use super::{InterpCx, MPlaceTy, Machine, Projectable}; +use super::{throw_inval, InterpCx, MPlaceTy, Machine, Projectable}; /// How to traverse a value and what to do when we are at the leaves. pub trait ValueVisitor<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized { diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs index d27d42737cd..3a7c87c1cad 100644 --- a/compiler/rustc_const_eval/src/lib.rs +++ b/compiler/rustc_const_eval/src/lib.rs @@ -1,9 +1,3 @@ -/*! - -Rust MIR: a lowered representation of Rust. - -*/ - #![allow(internal_features)] #![allow(rustc::diagnostic_outside_of_impl)] #![feature(rustdoc_internals)] @@ -20,15 +14,10 @@ Rust MIR: a lowered representation of Rust. #![feature(yeet_expr)] #![feature(if_let_guard)] -#[macro_use] -extern crate tracing; -#[macro_use] -extern crate rustc_middle; - +pub mod check_consts; pub mod const_eval; mod errors; pub mod interpret; -pub mod transform; pub mod util; use std::sync::atomic::AtomicBool; diff --git a/compiler/rustc_const_eval/src/transform/mod.rs b/compiler/rustc_const_eval/src/transform/mod.rs deleted file mode 100644 index e3582c7d317..00000000000 --- a/compiler/rustc_const_eval/src/transform/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod check_consts; -pub mod validate; diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs deleted file mode 100644 index 837c02a5b76..00000000000 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ /dev/null @@ -1,1421 +0,0 @@ -//! Validates the MIR to ensure that invariants are upheld. - -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_index::bit_set::BitSet; -use rustc_index::IndexVec; -use rustc_infer::traits::Reveal; -use rustc_middle::mir::coverage::CoverageKind; -use rustc_middle::mir::interpret::Scalar; -use rustc_middle::mir::visit::{NonUseContext, PlaceContext, Visitor}; -use rustc_middle::mir::*; -use rustc_middle::ty::{self, InstanceDef, ParamEnv, Ty, TyCtxt, TypeVisitableExt, Variance}; -use rustc_target::abi::{Size, FIRST_VARIANT}; -use rustc_target::spec::abi::Abi; - -use crate::util::is_within_packed; - -use crate::util::relate_types; - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum EdgeKind { - Unwind, - Normal, -} - -pub struct Validator { - /// Describes at which point in the pipeline this validation is happening. - pub when: String, - /// The phase for which we are upholding the dialect. If the given phase forbids a specific - /// element, this validator will now emit errors if that specific element is encountered. - /// Note that phases that change the dialect cause all *following* phases to check the - /// invariants of the new dialect. A phase that changes dialects never checks the new invariants - /// itself. - pub mir_phase: MirPhase, -} - -impl<'tcx> MirPass<'tcx> for Validator { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // FIXME(JakobDegen): These bodies never instantiated in codegend anyway, so it's not - // terribly important that they pass the validator. However, I think other passes might - // still see them, in which case they might be surprised. It would probably be better if we - // didn't put this through the MIR pipeline at all. - if matches!(body.source.instance, InstanceDef::Intrinsic(..) | InstanceDef::Virtual(..)) { - return; - } - let def_id = body.source.def_id(); - let mir_phase = self.mir_phase; - let param_env = match mir_phase.reveal() { - Reveal::UserFacing => tcx.param_env(def_id), - Reveal::All => tcx.param_env_reveal_all_normalized(def_id), - }; - - let can_unwind = if mir_phase <= MirPhase::Runtime(RuntimePhase::Initial) { - // In this case `AbortUnwindingCalls` haven't yet been executed. - true - } else if !tcx.def_kind(def_id).is_fn_like() { - true - } else { - let body_ty = tcx.type_of(def_id).skip_binder(); - let body_abi = match body_ty.kind() { - ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), - ty::Closure(..) => Abi::RustCall, - ty::CoroutineClosure(..) => Abi::RustCall, - ty::Coroutine(..) => Abi::Rust, - // No need to do MIR validation on error bodies - ty::Error(_) => return, - _ => { - span_bug!(body.span, "unexpected body ty: {:?} phase {:?}", body_ty, mir_phase) - } - }; - - ty::layout::fn_can_unwind(tcx, Some(def_id), body_abi) - }; - - let mut cfg_checker = CfgChecker { - when: &self.when, - body, - tcx, - mir_phase, - unwind_edge_count: 0, - reachable_blocks: traversal::reachable_as_bitset(body), - value_cache: FxHashSet::default(), - can_unwind, - }; - cfg_checker.visit_body(body); - cfg_checker.check_cleanup_control_flow(); - - // Also run the TypeChecker. - for (location, msg) in validate_types(tcx, self.mir_phase, param_env, body, body) { - cfg_checker.fail(location, msg); - } - - if let MirPhase::Runtime(_) = body.phase { - if let ty::InstanceDef::Item(_) = body.source.instance { - if body.has_free_regions() { - cfg_checker.fail( - Location::START, - format!("Free regions in optimized {} MIR", body.phase.name()), - ); - } - } - } - - // Enforce that coroutine-closure layouts are identical. - if let Some(layout) = body.coroutine_layout_raw() - && let Some(by_move_body) = body.coroutine_by_move_body() - && let Some(by_move_layout) = by_move_body.coroutine_layout_raw() - { - // FIXME(async_closures): We could do other validation here? - if layout.variant_fields.len() != by_move_layout.variant_fields.len() { - cfg_checker.fail( - Location::START, - format!( - "Coroutine layout has different number of variant fields from \ - by-move coroutine layout:\n\ - layout: {layout:#?}\n\ - by_move_layout: {by_move_layout:#?}", - ), - ); - } - } - } -} - -struct CfgChecker<'a, 'tcx> { - when: &'a str, - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - mir_phase: MirPhase, - unwind_edge_count: usize, - reachable_blocks: BitSet<BasicBlock>, - value_cache: FxHashSet<u128>, - // If `false`, then the MIR must not contain `UnwindAction::Continue` or - // `TerminatorKind::Resume`. - can_unwind: bool, -} - -impl<'a, 'tcx> CfgChecker<'a, 'tcx> { - #[track_caller] - fn fail(&self, location: Location, msg: impl AsRef<str>) { - // We might see broken MIR when other errors have already occurred. - assert!( - self.tcx.dcx().has_errors().is_some(), - "broken MIR in {:?} ({}) at {:?}:\n{}", - self.body.source.instance, - self.when, - location, - msg.as_ref(), - ); - } - - fn check_edge(&mut self, location: Location, bb: BasicBlock, edge_kind: EdgeKind) { - if bb == START_BLOCK { - self.fail(location, "start block must not have predecessors") - } - if let Some(bb) = self.body.basic_blocks.get(bb) { - let src = self.body.basic_blocks.get(location.block).unwrap(); - match (src.is_cleanup, bb.is_cleanup, edge_kind) { - // Non-cleanup blocks can jump to non-cleanup blocks along non-unwind edges - (false, false, EdgeKind::Normal) - // Cleanup blocks can jump to cleanup blocks along non-unwind edges - | (true, true, EdgeKind::Normal) => {} - // Non-cleanup blocks can jump to cleanup blocks along unwind edges - (false, true, EdgeKind::Unwind) => { - self.unwind_edge_count += 1; - } - // All other jumps are invalid - _ => { - self.fail( - location, - format!( - "{:?} edge to {:?} violates unwind invariants (cleanup {:?} -> {:?})", - edge_kind, - bb, - src.is_cleanup, - bb.is_cleanup, - ) - ) - } - } - } else { - self.fail(location, format!("encountered jump to invalid basic block {bb:?}")) - } - } - - fn check_cleanup_control_flow(&self) { - if self.unwind_edge_count <= 1 { - return; - } - let doms = self.body.basic_blocks.dominators(); - let mut post_contract_node = FxHashMap::default(); - // Reusing the allocation across invocations of the closure - let mut dom_path = vec![]; - let mut get_post_contract_node = |mut bb| { - let root = loop { - if let Some(root) = post_contract_node.get(&bb) { - break *root; - } - let parent = doms.immediate_dominator(bb).unwrap(); - dom_path.push(bb); - if !self.body.basic_blocks[parent].is_cleanup { - break bb; - } - bb = parent; - }; - for bb in dom_path.drain(..) { - post_contract_node.insert(bb, root); - } - root - }; - - let mut parent = IndexVec::from_elem(None, &self.body.basic_blocks); - for (bb, bb_data) in self.body.basic_blocks.iter_enumerated() { - if !bb_data.is_cleanup || !self.reachable_blocks.contains(bb) { - continue; - } - let bb = get_post_contract_node(bb); - for s in bb_data.terminator().successors() { - let s = get_post_contract_node(s); - if s == bb { - continue; - } - let parent = &mut parent[bb]; - match parent { - None => { - *parent = Some(s); - } - Some(e) if *e == s => (), - Some(e) => self.fail( - Location { block: bb, statement_index: 0 }, - format!( - "Cleanup control flow violation: The blocks dominated by {:?} have edges to both {:?} and {:?}", - bb, - s, - *e - ) - ), - } - } - } - - // Check for cycles - let mut stack = FxHashSet::default(); - for i in 0..parent.len() { - let mut bb = BasicBlock::from_usize(i); - stack.clear(); - stack.insert(bb); - loop { - let Some(parent) = parent[bb].take() else { break }; - let no_cycle = stack.insert(parent); - if !no_cycle { - self.fail( - Location { block: bb, statement_index: 0 }, - format!( - "Cleanup control flow violation: Cycle involving edge {bb:?} -> {parent:?}", - ), - ); - break; - } - bb = parent; - } - } - } - - fn check_unwind_edge(&mut self, location: Location, unwind: UnwindAction) { - let is_cleanup = self.body.basic_blocks[location.block].is_cleanup; - match unwind { - UnwindAction::Cleanup(unwind) => { - if is_cleanup { - self.fail(location, "`UnwindAction::Cleanup` in cleanup block"); - } - self.check_edge(location, unwind, EdgeKind::Unwind); - } - UnwindAction::Continue => { - if is_cleanup { - self.fail(location, "`UnwindAction::Continue` in cleanup block"); - } - - if !self.can_unwind { - self.fail(location, "`UnwindAction::Continue` in no-unwind function"); - } - } - UnwindAction::Terminate(UnwindTerminateReason::InCleanup) => { - if !is_cleanup { - self.fail( - location, - "`UnwindAction::Terminate(InCleanup)` in a non-cleanup block", - ); - } - } - // These are allowed everywhere. - UnwindAction::Unreachable | UnwindAction::Terminate(UnwindTerminateReason::Abi) => (), - } - } - - fn is_critical_call_edge(&self, target: Option<BasicBlock>, unwind: UnwindAction) -> bool { - let Some(target) = target else { return false }; - matches!(unwind, UnwindAction::Cleanup(_) | UnwindAction::Terminate(_)) - && self.body.basic_blocks.predecessors()[target].len() > 1 - } -} - -impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { - fn visit_local(&mut self, local: Local, _context: PlaceContext, location: Location) { - if self.body.local_decls.get(local).is_none() { - self.fail( - location, - format!("local {local:?} has no corresponding declaration in `body.local_decls`"), - ); - } - } - - fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { - match &statement.kind { - StatementKind::AscribeUserType(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`AscribeUserType` should have been removed after drop lowering phase", - ); - } - } - StatementKind::FakeRead(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`FakeRead` should have been removed after drop lowering phase", - ); - } - } - StatementKind::SetDiscriminant { .. } => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { - self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); - } - } - StatementKind::Deinit(..) => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { - self.fail(location, "`Deinit`is not allowed until deaggregation"); - } - } - StatementKind::Retag(kind, _) => { - // FIXME(JakobDegen) The validator should check that `self.mir_phase < - // DropsLowered`. However, this causes ICEs with generation of drop shims, which - // seem to fail to set their `MirPhase` correctly. - if matches!(kind, RetagKind::TwoPhase) { - self.fail(location, format!("explicit `{kind:?}` is forbidden")); - } - } - StatementKind::Coverage(kind) => { - if self.mir_phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) - && let CoverageKind::BlockMarker { .. } | CoverageKind::SpanMarker { .. } = kind - { - self.fail( - location, - format!("{kind:?} should have been removed after analysis"), - ); - } - } - StatementKind::Assign(..) - | StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::Intrinsic(_) - | StatementKind::ConstEvalCounter - | StatementKind::PlaceMention(..) - | StatementKind::Nop => {} - } - - self.super_statement(statement, location); - } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { - match &terminator.kind { - TerminatorKind::Goto { target } => { - self.check_edge(location, *target, EdgeKind::Normal); - } - TerminatorKind::SwitchInt { targets, discr: _ } => { - for (_, target) in targets.iter() { - self.check_edge(location, target, EdgeKind::Normal); - } - self.check_edge(location, targets.otherwise(), EdgeKind::Normal); - - self.value_cache.clear(); - self.value_cache.extend(targets.iter().map(|(value, _)| value)); - let has_duplicates = targets.iter().len() != self.value_cache.len(); - if has_duplicates { - self.fail( - location, - format!( - "duplicated values in `SwitchInt` terminator: {:?}", - terminator.kind, - ), - ); - } - } - TerminatorKind::Drop { target, unwind, .. } => { - self.check_edge(location, *target, EdgeKind::Normal); - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::Call { args, destination, target, unwind, .. } => { - if let Some(target) = target { - self.check_edge(location, *target, EdgeKind::Normal); - } - self.check_unwind_edge(location, *unwind); - - // The code generation assumes that there are no critical call edges. The assumption - // is used to simplify inserting code that should be executed along the return edge - // from the call. FIXME(tmiasko): Since this is a strictly code generation concern, - // the code generation should be responsible for handling it. - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) - && self.is_critical_call_edge(*target, *unwind) - { - self.fail( - location, - format!( - "encountered critical edge in `Call` terminator {:?}", - terminator.kind, - ), - ); - } - - // The call destination place and Operand::Move place used as an argument might be - // passed by a reference to the callee. Consequently they cannot be packed. - if is_within_packed(self.tcx, &self.body.local_decls, *destination).is_some() { - // This is bad! The callee will expect the memory to be aligned. - self.fail( - location, - format!( - "encountered packed place in `Call` terminator destination: {:?}", - terminator.kind, - ), - ); - } - for arg in args { - if let Operand::Move(place) = &arg.node { - if is_within_packed(self.tcx, &self.body.local_decls, *place).is_some() { - // This is bad! The callee will expect the memory to be aligned. - self.fail( - location, - format!( - "encountered `Move` of a packed place in `Call` terminator: {:?}", - terminator.kind, - ), - ); - } - } - } - } - TerminatorKind::Assert { target, unwind, .. } => { - self.check_edge(location, *target, EdgeKind::Normal); - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::Yield { resume, drop, .. } => { - if self.body.coroutine.is_none() { - self.fail(location, "`Yield` cannot appear outside coroutine bodies"); - } - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail(location, "`Yield` should have been replaced by coroutine lowering"); - } - self.check_edge(location, *resume, EdgeKind::Normal); - if let Some(drop) = drop { - self.check_edge(location, *drop, EdgeKind::Normal); - } - } - TerminatorKind::FalseEdge { real_target, imaginary_target } => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`FalseEdge` should have been removed after drop elaboration", - ); - } - self.check_edge(location, *real_target, EdgeKind::Normal); - self.check_edge(location, *imaginary_target, EdgeKind::Normal); - } - TerminatorKind::FalseUnwind { real_target, unwind } => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`FalseUnwind` should have been removed after drop elaboration", - ); - } - self.check_edge(location, *real_target, EdgeKind::Normal); - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::InlineAsm { targets, unwind, .. } => { - for &target in targets { - self.check_edge(location, target, EdgeKind::Normal); - } - self.check_unwind_edge(location, *unwind); - } - TerminatorKind::CoroutineDrop => { - if self.body.coroutine.is_none() { - self.fail(location, "`CoroutineDrop` cannot appear outside coroutine bodies"); - } - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`CoroutineDrop` should have been replaced by coroutine lowering", - ); - } - } - TerminatorKind::UnwindResume => { - let bb = location.block; - if !self.body.basic_blocks[bb].is_cleanup { - self.fail(location, "Cannot `UnwindResume` from non-cleanup basic block") - } - if !self.can_unwind { - self.fail(location, "Cannot `UnwindResume` in a function that cannot unwind") - } - } - TerminatorKind::UnwindTerminate(_) => { - let bb = location.block; - if !self.body.basic_blocks[bb].is_cleanup { - self.fail(location, "Cannot `UnwindTerminate` from non-cleanup basic block") - } - } - TerminatorKind::Return => { - let bb = location.block; - if self.body.basic_blocks[bb].is_cleanup { - self.fail(location, "Cannot `Return` from cleanup basic block") - } - } - TerminatorKind::Unreachable => {} - } - - self.super_terminator(terminator, location); - } - - fn visit_source_scope(&mut self, scope: SourceScope) { - if self.body.source_scopes.get(scope).is_none() { - self.tcx.dcx().span_bug( - self.body.span, - format!( - "broken MIR in {:?} ({}):\ninvalid source scope {:?}", - self.body.source.instance, self.when, scope, - ), - ); - } - } -} - -/// A faster version of the validation pass that only checks those things which may break when -/// instantiating any generic parameters. -/// -/// `caller_body` is used to detect cycles in MIR inlining and MIR validation before -/// `optimized_mir` is available. -pub fn validate_types<'tcx>( - tcx: TyCtxt<'tcx>, - mir_phase: MirPhase, - param_env: ty::ParamEnv<'tcx>, - body: &Body<'tcx>, - caller_body: &Body<'tcx>, -) -> Vec<(Location, String)> { - let mut type_checker = - TypeChecker { body, caller_body, tcx, param_env, mir_phase, failures: Vec::new() }; - type_checker.visit_body(body); - type_checker.failures -} - -struct TypeChecker<'a, 'tcx> { - body: &'a Body<'tcx>, - caller_body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, - mir_phase: MirPhase, - failures: Vec<(Location, String)>, -} - -impl<'a, 'tcx> TypeChecker<'a, 'tcx> { - fn fail(&mut self, location: Location, msg: impl Into<String>) { - self.failures.push((location, msg.into())); - } - - /// Check if src can be assigned into dest. - /// This is not precise, it will accept some incorrect assignments. - fn mir_assign_valid_types(&self, src: Ty<'tcx>, dest: Ty<'tcx>) -> bool { - // Fast path before we normalize. - if src == dest { - // Equal types, all is good. - return true; - } - - // We sometimes have to use `defining_opaque_types` for subtyping - // to succeed here and figuring out how exactly that should work - // is annoying. It is harmless enough to just not validate anything - // in that case. We still check this after analysis as all opaque - // types have been revealed at this point. - if (src, dest).has_opaque_types() { - return true; - } - - // After borrowck subtyping should be fully explicit via - // `Subtype` projections. - let variance = if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - Variance::Invariant - } else { - Variance::Covariant - }; - - crate::util::relate_types(self.tcx, self.param_env, variance, src, dest) - } -} - -impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { - fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { - // This check is somewhat expensive, so only run it when -Zvalidate-mir is passed. - if self.tcx.sess.opts.unstable_opts.validate_mir - && self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) - { - // `Operand::Copy` is only supposed to be used with `Copy` types. - if let Operand::Copy(place) = operand { - let ty = place.ty(&self.body.local_decls, self.tcx).ty; - - if !ty.is_copy_modulo_regions(self.tcx, self.param_env) { - self.fail(location, format!("`Operand::Copy` with non-`Copy` type {ty}")); - } - } - } - - self.super_operand(operand, location); - } - - fn visit_projection_elem( - &mut self, - place_ref: PlaceRef<'tcx>, - elem: PlaceElem<'tcx>, - context: PlaceContext, - location: Location, - ) { - match elem { - ProjectionElem::OpaqueCast(ty) - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) => - { - self.fail( - location, - format!("explicit opaque type cast to `{ty}` after `RevealAll`"), - ) - } - ProjectionElem::Index(index) => { - let index_ty = self.body.local_decls[index].ty; - if index_ty != self.tcx.types.usize { - self.fail(location, format!("bad index ({index_ty:?} != usize)")) - } - } - ProjectionElem::Deref - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup) => - { - let base_ty = place_ref.ty(&self.body.local_decls, self.tcx).ty; - - if base_ty.is_box() { - self.fail( - location, - format!("{base_ty:?} dereferenced after ElaborateBoxDerefs"), - ) - } - } - ProjectionElem::Field(f, ty) => { - let parent_ty = place_ref.ty(&self.body.local_decls, self.tcx); - let fail_out_of_bounds = |this: &mut Self, location| { - this.fail(location, format!("Out of bounds field {f:?} for {parent_ty:?}")); - }; - let check_equal = |this: &mut Self, location, f_ty| { - if !this.mir_assign_valid_types(ty, f_ty) { - this.fail( - location, - format!( - "Field projection `{place_ref:?}.{f:?}` specified type `{ty:?}`, but actual type is `{f_ty:?}`" - ) - ) - } - }; - - let kind = match parent_ty.ty.kind() { - &ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) => { - self.tcx.type_of(def_id).instantiate(self.tcx, args).kind() - } - kind => kind, - }; - - match kind { - ty::Tuple(fields) => { - let Some(f_ty) = fields.get(f.as_usize()) else { - fail_out_of_bounds(self, location); - return; - }; - check_equal(self, location, *f_ty); - } - ty::Adt(adt_def, args) => { - let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT); - let Some(field) = adt_def.variant(var).fields.get(f) else { - fail_out_of_bounds(self, location); - return; - }; - check_equal(self, location, field.ty(self.tcx, args)); - } - ty::Closure(_, args) => { - let args = args.as_closure(); - let Some(&f_ty) = args.upvar_tys().get(f.as_usize()) else { - fail_out_of_bounds(self, location); - return; - }; - check_equal(self, location, f_ty); - } - ty::CoroutineClosure(_, args) => { - let args = args.as_coroutine_closure(); - let Some(&f_ty) = args.upvar_tys().get(f.as_usize()) else { - fail_out_of_bounds(self, location); - return; - }; - check_equal(self, location, f_ty); - } - &ty::Coroutine(def_id, args) => { - let f_ty = if let Some(var) = parent_ty.variant_index { - // If we're currently validating an inlined copy of this body, - // then it will no longer be parameterized over the original - // args of the coroutine. Otherwise, we prefer to use this body - // since we may be in the process of computing this MIR in the - // first place. - let layout = if def_id == self.caller_body.source.def_id() { - // FIXME: This is not right for async closures. - self.caller_body.coroutine_layout_raw() - } else { - self.tcx.coroutine_layout(def_id, args.as_coroutine().kind_ty()) - }; - - let Some(layout) = layout else { - self.fail( - location, - format!("No coroutine layout for {parent_ty:?}"), - ); - return; - }; - - let Some(&local) = layout.variant_fields[var].get(f) else { - fail_out_of_bounds(self, location); - return; - }; - - let Some(f_ty) = layout.field_tys.get(local) else { - self.fail( - location, - format!("Out of bounds local {local:?} for {parent_ty:?}"), - ); - return; - }; - - ty::EarlyBinder::bind(f_ty.ty).instantiate(self.tcx, args) - } else { - let Some(&f_ty) = args.as_coroutine().prefix_tys().get(f.index()) - else { - fail_out_of_bounds(self, location); - return; - }; - - f_ty - }; - - check_equal(self, location, f_ty); - } - _ => { - self.fail(location, format!("{:?} does not have fields", parent_ty.ty)); - } - } - } - ProjectionElem::Subtype(ty) => { - if !relate_types( - self.tcx, - self.param_env, - Variance::Covariant, - ty, - place_ref.ty(&self.body.local_decls, self.tcx).ty, - ) { - self.fail( - location, - format!( - "Failed subtyping {ty:#?} and {:#?}", - place_ref.ty(&self.body.local_decls, self.tcx).ty - ), - ) - } - } - _ => {} - } - self.super_projection_elem(place_ref, elem, context, location); - } - - fn visit_var_debug_info(&mut self, debuginfo: &VarDebugInfo<'tcx>) { - if let Some(box VarDebugInfoFragment { ty, ref projection }) = debuginfo.composite { - if ty.is_union() || ty.is_enum() { - self.fail( - START_BLOCK.start_location(), - format!("invalid type {ty:?} in debuginfo for {:?}", debuginfo.name), - ); - } - if projection.is_empty() { - self.fail( - START_BLOCK.start_location(), - format!("invalid empty projection in debuginfo for {:?}", debuginfo.name), - ); - } - if projection.iter().any(|p| !matches!(p, PlaceElem::Field(..))) { - self.fail( - START_BLOCK.start_location(), - format!( - "illegal projection {:?} in debuginfo for {:?}", - projection, debuginfo.name - ), - ); - } - } - match debuginfo.value { - VarDebugInfoContents::Const(_) => {} - VarDebugInfoContents::Place(place) => { - if place.projection.iter().any(|p| !p.can_use_in_debuginfo()) { - self.fail( - START_BLOCK.start_location(), - format!("illegal place {:?} in debuginfo for {:?}", place, debuginfo.name), - ); - } - } - } - self.super_var_debug_info(debuginfo); - } - - fn visit_place(&mut self, place: &Place<'tcx>, cntxt: PlaceContext, location: Location) { - // Set off any `bug!`s in the type computation code - let _ = place.ty(&self.body.local_decls, self.tcx); - - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) - && place.projection.len() > 1 - && cntxt != PlaceContext::NonUse(NonUseContext::VarDebugInfo) - && place.projection[1..].contains(&ProjectionElem::Deref) - { - self.fail(location, format!("{place:?}, has deref at the wrong place")); - } - - self.super_place(place, cntxt, location); - } - - fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { - macro_rules! check_kinds { - ($t:expr, $text:literal, $typat:pat) => { - if !matches!(($t).kind(), $typat) { - self.fail(location, format!($text, $t)); - } - }; - } - match rvalue { - Rvalue::Use(_) | Rvalue::CopyForDeref(_) => {} - Rvalue::Aggregate(kind, fields) => match **kind { - AggregateKind::Tuple => {} - AggregateKind::Array(dest) => { - for src in fields { - if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest) { - self.fail(location, "array field has the wrong type"); - } - } - } - AggregateKind::Adt(def_id, idx, args, _, Some(field)) => { - let adt_def = self.tcx.adt_def(def_id); - assert!(adt_def.is_union()); - assert_eq!(idx, FIRST_VARIANT); - let dest_ty = self.tcx.normalize_erasing_regions( - self.param_env, - adt_def.non_enum_variant().fields[field].ty(self.tcx, args), - ); - if fields.len() == 1 { - let src_ty = fields.raw[0].ty(self.body, self.tcx); - if !self.mir_assign_valid_types(src_ty, dest_ty) { - self.fail(location, "union field has the wrong type"); - } - } else { - self.fail(location, "unions should have one initialized field"); - } - } - AggregateKind::Adt(def_id, idx, args, _, None) => { - let adt_def = self.tcx.adt_def(def_id); - assert!(!adt_def.is_union()); - let variant = &adt_def.variants()[idx]; - if variant.fields.len() != fields.len() { - self.fail(location, "adt has the wrong number of initialized fields"); - } - for (src, dest) in std::iter::zip(fields, &variant.fields) { - let dest_ty = self - .tcx - .normalize_erasing_regions(self.param_env, dest.ty(self.tcx, args)); - if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest_ty) { - self.fail(location, "adt field has the wrong type"); - } - } - } - AggregateKind::Closure(_, args) => { - let upvars = args.as_closure().upvar_tys(); - if upvars.len() != fields.len() { - self.fail(location, "closure has the wrong number of initialized fields"); - } - for (src, dest) in std::iter::zip(fields, upvars) { - if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest) { - self.fail(location, "closure field has the wrong type"); - } - } - } - AggregateKind::Coroutine(_, args) => { - let upvars = args.as_coroutine().upvar_tys(); - if upvars.len() != fields.len() { - self.fail(location, "coroutine has the wrong number of initialized fields"); - } - for (src, dest) in std::iter::zip(fields, upvars) { - if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest) { - self.fail(location, "coroutine field has the wrong type"); - } - } - } - AggregateKind::CoroutineClosure(_, args) => { - let upvars = args.as_coroutine_closure().upvar_tys(); - if upvars.len() != fields.len() { - self.fail( - location, - "coroutine-closure has the wrong number of initialized fields", - ); - } - for (src, dest) in std::iter::zip(fields, upvars) { - if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest) { - self.fail(location, "coroutine-closure field has the wrong type"); - } - } - } - AggregateKind::RawPtr(pointee_ty, mutability) => { - if !matches!(self.mir_phase, MirPhase::Runtime(_)) { - // It would probably be fine to support this in earlier phases, - // but at the time of writing it's only ever introduced from intrinsic lowering, - // so earlier things just `bug!` on it. - self.fail(location, "RawPtr should be in runtime MIR only"); - } - - if fields.len() != 2 { - self.fail(location, "raw pointer aggregate must have 2 fields"); - } else { - let data_ptr_ty = fields.raw[0].ty(self.body, self.tcx); - let metadata_ty = fields.raw[1].ty(self.body, self.tcx); - if let ty::RawPtr(in_pointee, in_mut) = data_ptr_ty.kind() { - if *in_mut != mutability { - self.fail(location, "input and output mutability must match"); - } - - // FIXME: check `Thin` instead of `Sized` - if !in_pointee.is_sized(self.tcx, self.param_env) { - self.fail(location, "input pointer must be thin"); - } - } else { - self.fail( - location, - "first operand to raw pointer aggregate must be a raw pointer", - ); - } - - // FIXME: Check metadata more generally - if pointee_ty.is_slice() { - if !self.mir_assign_valid_types(metadata_ty, self.tcx.types.usize) { - self.fail(location, "slice metadata must be usize"); - } - } else if pointee_ty.is_sized(self.tcx, self.param_env) { - if metadata_ty != self.tcx.types.unit { - self.fail(location, "metadata for pointer-to-thin must be unit"); - } - } - } - } - }, - Rvalue::Ref(_, BorrowKind::Fake(_), _) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`Assign` statement with a `Fake` borrow should have been removed in runtime MIR", - ); - } - } - Rvalue::Ref(..) => {} - Rvalue::Len(p) => { - let pty = p.ty(&self.body.local_decls, self.tcx).ty; - check_kinds!( - pty, - "Cannot compute length of non-array type {:?}", - ty::Array(..) | ty::Slice(..) - ); - } - Rvalue::BinaryOp(op, vals) => { - use BinOp::*; - let a = vals.0.ty(&self.body.local_decls, self.tcx); - let b = vals.1.ty(&self.body.local_decls, self.tcx); - if crate::util::binop_right_homogeneous(*op) { - if let Eq | Lt | Le | Ne | Ge | Gt = op { - // The function pointer types can have lifetimes - if !self.mir_assign_valid_types(a, b) { - self.fail( - location, - format!("Cannot {op:?} compare incompatible types {a:?} and {b:?}"), - ); - } - } else if a != b { - self.fail( - location, - format!( - "Cannot perform binary op {op:?} on unequal types {a:?} and {b:?}" - ), - ); - } - } - - match op { - Offset => { - check_kinds!(a, "Cannot offset non-pointer type {:?}", ty::RawPtr(..)); - if b != self.tcx.types.isize && b != self.tcx.types.usize { - self.fail(location, format!("Cannot offset by non-isize type {b:?}")); - } - } - Eq | Lt | Le | Ne | Ge | Gt => { - for x in [a, b] { - check_kinds!( - x, - "Cannot {op:?} compare type {:?}", - ty::Bool - | ty::Char - | ty::Int(..) - | ty::Uint(..) - | ty::Float(..) - | ty::RawPtr(..) - | ty::FnPtr(..) - ) - } - } - Cmp => { - for x in [a, b] { - check_kinds!( - x, - "Cannot three-way compare non-integer type {:?}", - ty::Char | ty::Uint(..) | ty::Int(..) - ) - } - } - AddUnchecked | SubUnchecked | MulUnchecked | Shl | ShlUnchecked | Shr - | ShrUnchecked => { - for x in [a, b] { - check_kinds!( - x, - "Cannot {op:?} non-integer type {:?}", - ty::Uint(..) | ty::Int(..) - ) - } - } - BitAnd | BitOr | BitXor => { - for x in [a, b] { - check_kinds!( - x, - "Cannot perform bitwise op {op:?} on type {:?}", - ty::Uint(..) | ty::Int(..) | ty::Bool - ) - } - } - Add | Sub | Mul | Div | Rem => { - for x in [a, b] { - check_kinds!( - x, - "Cannot perform arithmetic {op:?} on type {:?}", - ty::Uint(..) | ty::Int(..) | ty::Float(..) - ) - } - } - } - } - Rvalue::CheckedBinaryOp(op, vals) => { - use BinOp::*; - let a = vals.0.ty(&self.body.local_decls, self.tcx); - let b = vals.1.ty(&self.body.local_decls, self.tcx); - match op { - Add | Sub | Mul => { - for x in [a, b] { - check_kinds!( - x, - "Cannot perform checked arithmetic on type {:?}", - ty::Uint(..) | ty::Int(..) - ) - } - if a != b { - self.fail( - location, - format!( - "Cannot perform checked arithmetic on unequal types {a:?} and {b:?}" - ), - ); - } - } - _ => self.fail(location, format!("There is no checked version of {op:?}")), - } - } - Rvalue::UnaryOp(op, operand) => { - let a = operand.ty(&self.body.local_decls, self.tcx); - match op { - UnOp::Neg => { - check_kinds!(a, "Cannot negate type {:?}", ty::Int(..) | ty::Float(..)) - } - UnOp::Not => { - check_kinds!( - a, - "Cannot binary not type {:?}", - ty::Int(..) | ty::Uint(..) | ty::Bool - ); - } - } - } - Rvalue::ShallowInitBox(operand, _) => { - let a = operand.ty(&self.body.local_decls, self.tcx); - check_kinds!(a, "Cannot shallow init type {:?}", ty::RawPtr(..)); - } - Rvalue::Cast(kind, operand, target_type) => { - let op_ty = operand.ty(self.body, self.tcx); - match kind { - CastKind::DynStar => { - // FIXME(dyn-star): make sure nothing needs to be done here. - } - // FIXME: Add Checks for these - CastKind::PointerWithExposedProvenance - | CastKind::PointerExposeProvenance - | CastKind::PointerCoercion(_) => {} - CastKind::IntToInt | CastKind::IntToFloat => { - let input_valid = op_ty.is_integral() || op_ty.is_char() || op_ty.is_bool(); - let target_valid = target_type.is_numeric() || target_type.is_char(); - if !input_valid || !target_valid { - self.fail( - location, - format!("Wrong cast kind {kind:?} for the type {op_ty}",), - ); - } - } - CastKind::FnPtrToPtr | CastKind::PtrToPtr => { - if !(op_ty.is_any_ptr() && target_type.is_unsafe_ptr()) { - self.fail(location, "Can't cast {op_ty} into 'Ptr'"); - } - } - CastKind::FloatToFloat | CastKind::FloatToInt => { - if !op_ty.is_floating_point() || !target_type.is_numeric() { - self.fail( - location, - format!( - "Trying to cast non 'Float' as {kind:?} into {target_type:?}" - ), - ); - } - } - CastKind::Transmute => { - if let MirPhase::Runtime(..) = self.mir_phase { - // Unlike `mem::transmute`, a MIR `Transmute` is well-formed - // for any two `Sized` types, just potentially UB to run. - - if !self - .tcx - .normalize_erasing_regions(self.param_env, op_ty) - .is_sized(self.tcx, self.param_env) - { - self.fail( - location, - format!("Cannot transmute from non-`Sized` type {op_ty:?}"), - ); - } - if !self - .tcx - .normalize_erasing_regions(self.param_env, *target_type) - .is_sized(self.tcx, self.param_env) - { - self.fail( - location, - format!("Cannot transmute to non-`Sized` type {target_type:?}"), - ); - } - } else { - self.fail( - location, - format!( - "Transmute is not supported in non-runtime phase {:?}.", - self.mir_phase - ), - ); - } - } - } - } - Rvalue::NullaryOp(NullOp::OffsetOf(indices), container) => { - let fail_out_of_bounds = |this: &mut Self, location, field, ty| { - this.fail(location, format!("Out of bounds field {field:?} for {ty:?}")); - }; - - let mut current_ty = *container; - - for (variant, field) in indices.iter() { - match current_ty.kind() { - ty::Tuple(fields) => { - if variant != FIRST_VARIANT { - self.fail( - location, - format!("tried to get variant {variant:?} of tuple"), - ); - return; - } - let Some(&f_ty) = fields.get(field.as_usize()) else { - fail_out_of_bounds(self, location, field, current_ty); - return; - }; - - current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty); - } - ty::Adt(adt_def, args) => { - let Some(field) = adt_def.variant(variant).fields.get(field) else { - fail_out_of_bounds(self, location, field, current_ty); - return; - }; - - let f_ty = field.ty(self.tcx, args); - current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty); - } - _ => { - self.fail( - location, - format!("Cannot get offset ({variant:?}, {field:?}) from type {current_ty:?}"), - ); - return; - } - } - } - } - Rvalue::Repeat(_, _) - | Rvalue::ThreadLocalRef(_) - | Rvalue::AddressOf(_, _) - | Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::UbChecks, _) - | Rvalue::Discriminant(_) => {} - } - self.super_rvalue(rvalue, location); - } - - fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { - match &statement.kind { - StatementKind::Assign(box (dest, rvalue)) => { - // LHS and RHS of the assignment must have the same type. - let left_ty = dest.ty(&self.body.local_decls, self.tcx).ty; - let right_ty = rvalue.ty(&self.body.local_decls, self.tcx); - - if !self.mir_assign_valid_types(right_ty, left_ty) { - self.fail( - location, - format!( - "encountered `{:?}` with incompatible types:\n\ - left-hand side has type: {}\n\ - right-hand side has type: {}", - statement.kind, left_ty, right_ty, - ), - ); - } - if let Rvalue::CopyForDeref(place) = rvalue { - if place.ty(&self.body.local_decls, self.tcx).ty.builtin_deref(true).is_none() { - self.fail( - location, - "`CopyForDeref` should only be used for dereferenceable types", - ) - } - } - } - StatementKind::AscribeUserType(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`AscribeUserType` should have been removed after drop lowering phase", - ); - } - } - StatementKind::FakeRead(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { - self.fail( - location, - "`FakeRead` should have been removed after drop lowering phase", - ); - } - } - StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => { - let ty = op.ty(&self.body.local_decls, self.tcx); - if !ty.is_bool() { - self.fail( - location, - format!("`assume` argument must be `bool`, but got: `{ty}`"), - ); - } - } - StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping( - CopyNonOverlapping { src, dst, count }, - )) => { - let src_ty = src.ty(&self.body.local_decls, self.tcx); - let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) { - src_deref.ty - } else { - self.fail( - location, - format!("Expected src to be ptr in copy_nonoverlapping, got: {src_ty}"), - ); - return; - }; - let dst_ty = dst.ty(&self.body.local_decls, self.tcx); - let op_dst_ty = if let Some(dst_deref) = dst_ty.builtin_deref(true) { - dst_deref.ty - } else { - self.fail( - location, - format!("Expected dst to be ptr in copy_nonoverlapping, got: {dst_ty}"), - ); - return; - }; - // since CopyNonOverlapping is parametrized by 1 type, - // we only need to check that they are equal and not keep an extra parameter. - if !self.mir_assign_valid_types(op_src_ty, op_dst_ty) { - self.fail(location, format!("bad arg ({op_src_ty:?} != {op_dst_ty:?})")); - } - - let op_cnt_ty = count.ty(&self.body.local_decls, self.tcx); - if op_cnt_ty != self.tcx.types.usize { - self.fail(location, format!("bad arg ({op_cnt_ty:?} != usize)")) - } - } - StatementKind::SetDiscriminant { place, .. } => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { - self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); - } - let pty = place.ty(&self.body.local_decls, self.tcx).ty.kind(); - if !matches!(pty, ty::Adt(..) | ty::Coroutine(..) | ty::Alias(ty::Opaque, ..)) { - self.fail( - location, - format!( - "`SetDiscriminant` is only allowed on ADTs and coroutines, not {pty:?}" - ), - ); - } - } - StatementKind::Deinit(..) => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { - self.fail(location, "`Deinit`is not allowed until deaggregation"); - } - } - StatementKind::Retag(kind, _) => { - // FIXME(JakobDegen) The validator should check that `self.mir_phase < - // DropsLowered`. However, this causes ICEs with generation of drop shims, which - // seem to fail to set their `MirPhase` correctly. - if matches!(kind, RetagKind::TwoPhase) { - self.fail(location, format!("explicit `{kind:?}` is forbidden")); - } - } - StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::Coverage(_) - | StatementKind::ConstEvalCounter - | StatementKind::PlaceMention(..) - | StatementKind::Nop => {} - } - - self.super_statement(statement, location); - } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { - match &terminator.kind { - TerminatorKind::SwitchInt { targets, discr } => { - let switch_ty = discr.ty(&self.body.local_decls, self.tcx); - - let target_width = self.tcx.sess.target.pointer_width; - - let size = Size::from_bits(match switch_ty.kind() { - ty::Uint(uint) => uint.normalize(target_width).bit_width().unwrap(), - ty::Int(int) => int.normalize(target_width).bit_width().unwrap(), - ty::Char => 32, - ty::Bool => 1, - other => bug!("unhandled type: {:?}", other), - }); - - for (value, _) in targets.iter() { - if Scalar::<()>::try_from_uint(value, size).is_none() { - self.fail( - location, - format!("the value {value:#x} is not a proper {switch_ty:?}"), - ) - } - } - } - TerminatorKind::Call { func, .. } => { - let func_ty = func.ty(&self.body.local_decls, self.tcx); - match func_ty.kind() { - ty::FnPtr(..) | ty::FnDef(..) => {} - _ => self.fail( - location, - format!("encountered non-callable type {func_ty} in `Call` terminator"), - ), - } - } - TerminatorKind::Assert { cond, .. } => { - let cond_ty = cond.ty(&self.body.local_decls, self.tcx); - if cond_ty != self.tcx.types.bool { - self.fail( - location, - format!( - "encountered non-boolean condition of type {cond_ty} in `Assert` terminator" - ), - ); - } - } - TerminatorKind::Goto { .. } - | TerminatorKind::Drop { .. } - | TerminatorKind::Yield { .. } - | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } - | TerminatorKind::InlineAsm { .. } - | TerminatorKind::CoroutineDrop - | TerminatorKind::UnwindResume - | TerminatorKind::UnwindTerminate(_) - | TerminatorKind::Return - | TerminatorKind::Unreachable => {} - } - - self.super_terminator(terminator, location); - } -} diff --git a/compiler/rustc_const_eval/src/util/alignment.rs b/compiler/rustc_const_eval/src/util/alignment.rs index 8642dfccd78..528274e6aba 100644 --- a/compiler/rustc_const_eval/src/util/alignment.rs +++ b/compiler/rustc_const_eval/src/util/alignment.rs @@ -1,6 +1,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; use rustc_target::abi::Align; +use tracing::debug; /// Returns `true` if this place is allowed to be less aligned /// than its containing struct (because it is within a packed diff --git a/compiler/rustc_const_eval/src/util/caller_location.rs b/compiler/rustc_const_eval/src/util/caller_location.rs index af9a4a4271d..bc5fe39f867 100644 --- a/compiler/rustc_const_eval/src/util/caller_location.rs +++ b/compiler/rustc_const_eval/src/util/caller_location.rs @@ -1,9 +1,11 @@ use rustc_hir::LangItem; +use rustc_middle::bug; use rustc_middle::mir; use rustc_middle::query::TyCtxtAt; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Mutability}; use rustc_span::symbol::Symbol; +use tracing::trace; use crate::const_eval::{mk_eval_cx_to_read_const_val, CanAccessMutGlobal, CompileTimeEvalContext}; use crate::interpret::*; diff --git a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs index 8c4af5e5132..68fb122a765 100644 --- a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs +++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs @@ -1,3 +1,4 @@ +use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutCx, LayoutError, LayoutOf, TyAndLayout, ValidityRequirement}; use rustc_middle::ty::{ParamEnv, ParamEnvAnd, Ty, TyCtxt}; use rustc_target::abi::{Abi, FieldsShape, Scalar, Variants}; @@ -115,7 +116,7 @@ fn might_permit_raw_init_lax<'tcx>( // Special magic check for references and boxes (i.e., special pointer types). if let Some(pointee) = this.ty.builtin_deref(false) { - let pointee = cx.layout_of(pointee.ty)?; + let pointee = cx.layout_of(pointee)?; // We need to ensure that the LLVM attributes `aligned` and `dereferenceable(size)` are satisfied. if pointee.align.abi.bytes() > 1 { // 0x01-filling is not aligned. diff --git a/compiler/rustc_const_eval/src/util/mod.rs b/compiler/rustc_const_eval/src/util/mod.rs index 0c3b59a0e78..66a1addfb52 100644 --- a/compiler/rustc_const_eval/src/util/mod.rs +++ b/compiler/rustc_const_eval/src/util/mod.rs @@ -19,7 +19,9 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool { match op { Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor | BitAnd | BitOr | Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => true, - Eq | Ne | Lt | Le | Gt | Ge | Cmp => false, + AddWithOverflow | SubWithOverflow | MulWithOverflow | Eq | Ne | Lt | Le | Gt | Ge | Cmp => { + false + } } } @@ -29,8 +31,9 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool { pub fn binop_right_homogeneous(op: mir::BinOp) -> bool { use rustc_middle::mir::BinOp::*; match op { - Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor - | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge | Cmp => true, + Add | AddUnchecked | AddWithOverflow | Sub | SubUnchecked | SubWithOverflow | Mul + | MulUnchecked | MulWithOverflow | Div | Rem | BitXor | BitAnd | BitOr | Eq | Ne | Lt + | Le | Gt | Ge | Cmp => true, Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => false, } } diff --git a/compiler/rustc_const_eval/src/util/type_name.rs b/compiler/rustc_const_eval/src/util/type_name.rs index e474b952938..01e517250f7 100644 --- a/compiler/rustc_const_eval/src/util/type_name.rs +++ b/compiler/rustc_const_eval/src/util/type_name.rs @@ -1,6 +1,7 @@ use rustc_data_structures::intern::Interned; use rustc_hir::def_id::CrateNum; use rustc_hir::definitions::DisambiguatedDefPathData; +use rustc_middle::bug; use rustc_middle::ty::{ self, print::{PrettyPrinter, Print, PrintError, Printer}, |