diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
52 files changed, 1345 insertions, 1259 deletions
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs index 11980382ffd..d8f85d2e379 100644 --- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs +++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs @@ -1,6 +1,5 @@ use crate::MirPass; use rustc_ast::InlineAsmOptions; -use rustc_hir::def::DefKind; use rustc_middle::mir::*; use rustc_middle::ty::layout; use rustc_middle::ty::{self, TyCtxt}; @@ -31,11 +30,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls { // We don't simplify the MIR of constants at this time because that // namely results in a cyclic query when we call `tcx.type_of` below. - let is_function = match kind { - DefKind::Fn | DefKind::AssocFn | DefKind::Ctor(..) => true, - _ => tcx.is_closure(def_id), - }; - if !is_function { + if !kind.is_fn_like() { return; } @@ -61,7 +56,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls { // example. let mut calls_to_terminate = Vec::new(); let mut cleanups_to_remove = Vec::new(); - for (id, block) in body.basic_blocks().iter_enumerated() { + for (id, block) in body.basic_blocks.iter_enumerated() { if block.is_cleanup { continue; } @@ -80,7 +75,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls { layout::fn_can_unwind(tcx, fn_def_id, sig.abi()) } TerminatorKind::Drop { .. } | TerminatorKind::DropAndReplace { .. } => { - tcx.sess.opts.debugging_opts.panic_in_drop == PanicStrategy::Unwind + tcx.sess.opts.unstable_opts.panic_in_drop == PanicStrategy::Unwind && layout::fn_can_unwind(tcx, None, Abi::Rust) } TerminatorKind::Assert { .. } | TerminatorKind::FalseUnwind { .. } => { diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs index 10d52271734..30966d22e2f 100644 --- a/compiler/rustc_mir_transform/src/add_call_guards.rs +++ b/compiler/rustc_mir_transform/src/add_call_guards.rs @@ -39,13 +39,13 @@ impl<'tcx> MirPass<'tcx> for AddCallGuards { impl AddCallGuards { pub fn add_call_guards(&self, body: &mut Body<'_>) { let mut pred_count: IndexVec<_, _> = - body.predecessors().iter().map(|ps| ps.len()).collect(); + body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect(); pred_count[START_BLOCK] += 1; // We need a place to store the new blocks generated let mut new_blocks = Vec::new(); - let cur_len = body.basic_blocks().len(); + let cur_len = body.basic_blocks.len(); for block in body.basic_blocks_mut() { match block.terminator { diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs index 8de0aad041c..ffb5d8c6d95 100644 --- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs +++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs @@ -55,7 +55,7 @@ fn add_moves_for_packed_drops_patch<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) let mut patch = MirPatch::new(body); let param_env = tcx.param_env(def_id); - for (bb, data) in body.basic_blocks().iter_enumerated() { + for (bb, data) in body.basic_blocks.iter_enumerated() { let loc = Location { block: bb, statement_index: data.statements.len() }; let terminator = data.terminator(); diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index 0495439385b..036b5589849 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -15,26 +15,14 @@ pub struct AddRetag; /// (Concurrent accesses by other threads are no problem as these are anyway non-atomic /// copies. Data races are UB.) fn is_stable(place: PlaceRef<'_>) -> bool { - place.projection.iter().all(|elem| { - match elem { - // Which place this evaluates to can change with any memory write, - // so cannot assume this to be stable. - ProjectionElem::Deref => false, - // Array indices are interesting, but MIR building generates a *fresh* - // temporary for every array access, so the index cannot be changed as - // a side-effect. - ProjectionElem::Index { .. } | - // The rest is completely boring, they just offset by a constant. - ProjectionElem::Field { .. } | - ProjectionElem::ConstantIndex { .. } | - ProjectionElem::Subslice { .. } | - ProjectionElem::Downcast { .. } => true, - } - }) + // Which place this evaluates to can change with any memory write, + // so cannot assume deref to be stable. + !place.has_deref() } -/// Determine whether this type may be a reference (or box), and thus needs retagging. -fn may_be_reference(ty: Ty<'_>) -> bool { +/// Determine whether this type may contain a reference (or box), and thus needs retagging. +/// We will only recurse `depth` times into Tuples/ADTs to bound the cost of this. +fn may_contain_reference<'tcx>(ty: Ty<'tcx>, depth: u32, tcx: TyCtxt<'tcx>) -> bool { match ty.kind() { // Primitive types that are not references ty::Bool @@ -50,49 +38,47 @@ fn may_be_reference(ty: Ty<'_>) -> bool { // References ty::Ref(..) => true, ty::Adt(..) if ty.is_box() => true, - // Compound types are not references - ty::Array(..) | ty::Slice(..) | ty::Tuple(..) | ty::Adt(..) => false, + // Compound types: recurse + ty::Array(ty, _) | ty::Slice(ty) => { + // This does not branch so we keep the depth the same. + may_contain_reference(*ty, depth, tcx) + } + ty::Tuple(tys) => { + depth == 0 || tys.iter().any(|ty| may_contain_reference(ty, depth - 1, tcx)) + } + ty::Adt(adt, subst) => { + depth == 0 + || adt.variants().iter().any(|v| { + v.fields.iter().any(|f| may_contain_reference(f.ty(tcx, subst), depth - 1, tcx)) + }) + } // Conservative fallback _ => true, } } -/// Determines whether or not this LocalDecl is temp, if not it needs retagging. -fn is_not_temp<'tcx>(local_decl: &LocalDecl<'tcx>) -> bool { - if let Some(local_info) = &local_decl.local_info { - match local_info.as_ref() { - LocalInfo::DerefTemp => return false, - _ => (), - }; - } - return true; -} - impl<'tcx> MirPass<'tcx> for AddRetag { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.opts.debugging_opts.mir_emit_retag + sess.opts.unstable_opts.mir_emit_retag } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // We need an `AllCallEdges` pass before we can do any work. super::add_call_guards::AllCallEdges.run_pass(tcx, body); - let (span, arg_count) = (body.span, body.arg_count); - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); + let basic_blocks = body.basic_blocks.as_mut(); + let local_decls = &body.local_decls; let needs_retag = |place: &Place<'tcx>| { // FIXME: Instead of giving up for unstable places, we should introduce // a temporary and retag on that. is_stable(place.as_ref()) - && may_be_reference(place.ty(&*local_decls, tcx).ty) - && is_not_temp(&local_decls[place.local]) + && may_contain_reference(place.ty(&*local_decls, tcx).ty, /*depth*/ 3, tcx) + && !local_decls[place.local].is_deref_temp() }; let place_base_raw = |place: &Place<'tcx>| { // If this is a `Deref`, get the type of what we are deref'ing. - let deref_base = - place.projection.iter().rposition(|p| matches!(p, ProjectionElem::Deref)); - if let Some(deref_base) = deref_base { - let base_proj = &place.projection[..deref_base]; - let ty = Place::ty_from(place.local, base_proj, &*local_decls, tcx).ty; + if place.has_deref() { + let ty = &local_decls[place.local].ty; ty.is_unsafe_ptr() } else { // Not a deref, and thus not raw. @@ -103,20 +89,18 @@ impl<'tcx> MirPass<'tcx> for AddRetag { // PART 1 // Retag arguments at the beginning of the start block. { - // FIXME: Consider using just the span covering the function - // argument declaration. - let source_info = SourceInfo::outermost(span); // Gather all arguments, skip return value. - let places = local_decls - .iter_enumerated() - .skip(1) - .take(arg_count) - .map(|(local, _)| Place::from(local)) - .filter(needs_retag); + let places = local_decls.iter_enumerated().skip(1).take(body.arg_count).filter_map( + |(local, decl)| { + let place = Place::from(local); + needs_retag(&place).then_some((place, decl.source_info)) + }, + ); + // Emit their retags. basic_blocks[START_BLOCK].statements.splice( 0..0, - places.map(|place| Statement { + places.map(|(place, source_info)| Statement { source_info, kind: StatementKind::Retag(RetagKind::FnEntry, Box::new(place)), }), diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index 097a6186cd5..8838b14c53a 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -1,5 +1,4 @@ -use rustc_errors::DiagnosticBuilder; -use rustc_middle::lint::LintDiagnosticBuilder; +use rustc_errors::{DiagnosticBuilder, LintDiagnosticBuilder}; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index 4bf66cd4c9f..3b7ba3f9a67 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -36,16 +36,17 @@ fn unsafe_derive_on_repr_packed(tcx: TyCtxt<'_>, def_id: LocalDefId) { tcx.struct_span_lint_hir(UNALIGNED_REFERENCES, lint_hir_id, tcx.def_span(def_id), |lint| { // FIXME: when we make this a hard error, this should have its // own error code. - let message = if tcx.generics_of(def_id).own_requires_monomorphization() { - "`#[derive]` can't be used on a `#[repr(packed)]` struct with \ - type or const parameters (error E0133)" - .to_string() + let extra = if tcx.generics_of(def_id).own_requires_monomorphization() { + "with type or const parameters" } else { - "`#[derive]` can't be used on a `#[repr(packed)]` struct that \ - does not derive Copy (error E0133)" - .to_string() + "that does not derive `Copy`" }; - lint.build(&message).emit(); + let message = format!( + "`{}` can't be derived on this `#[repr(packed)]` struct {}", + tcx.item_name(tcx.trait_id_of_impl(def_id.to_def_id()).expect("derived trait name")), + extra + ); + lint.build(message).emit(); }); } diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 1f73b7da815..beff19a3ab2 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -1,17 +1,16 @@ -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxHashSet; use rustc_errors::struct_span_err; use rustc_hir as hir; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::hir_id::HirId; use rustc_hir::intravisit; use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::*; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt}; -use rustc_middle::{lint, mir::*}; use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE}; use rustc_session::lint::Level; -use std::collections::hash_map; use std::ops::Bound; pub struct UnsafetyChecker<'a, 'tcx> { @@ -23,10 +22,7 @@ pub struct UnsafetyChecker<'a, 'tcx> { param_env: ty::ParamEnv<'tcx>, /// Used `unsafe` blocks in this function. This is used for the "unused_unsafe" lint. - /// - /// The keys are the used `unsafe` blocks, the UnusedUnsafeKind indicates whether - /// or not any of the usages happen at a place that doesn't allow `unsafe_op_in_unsafe_fn`. - used_unsafe_blocks: FxHashMap<HirId, UsedUnsafeBlockData>, + used_unsafe_blocks: FxHashSet<HirId>, } impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { @@ -109,7 +105,8 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { // safe (at least as emitted during MIR construction) } - StatementKind::CopyNonOverlapping(..) => unreachable!(), + // Move to above list once mir construction uses it. + StatementKind::Intrinsic(..) => unreachable!(), } self.super_statement(statement, location); } @@ -129,11 +126,8 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { } &AggregateKind::Closure(def_id, _) | &AggregateKind::Generator(def_id, _, _) => { let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } = - self.tcx.unsafety_check_result(def_id.expect_local()); - self.register_violations( - violations, - used_unsafe_blocks.iter().map(|(&h, &d)| (h, d)), - ); + self.tcx.unsafety_check_result(def_id); + self.register_violations(violations, used_unsafe_blocks.iter().copied()); } }, _ => {} @@ -219,22 +213,12 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { // We have to check the actual type of the assignment, as that determines if the // old value is being dropped. let assigned_ty = place.ty(&self.body.local_decls, self.tcx).ty; - // To avoid semver hazard, we only consider `Copy` and `ManuallyDrop` non-dropping. - let manually_drop = assigned_ty - .ty_adt_def() - .map_or(false, |adt_def| adt_def.is_manually_drop()); - let nodrop = manually_drop - || assigned_ty.is_copy_modulo_regions( - self.tcx.at(self.source_info.span), - self.param_env, - ); - if !nodrop { - self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::AssignToDroppingUnionField, + if assigned_ty.needs_drop(self.tcx, self.param_env) { + // This would be unsafe, but should be outright impossible since we reject such unions. + self.tcx.sess.delay_span_bug( + self.source_info.span, + format!("union fields that need dropping should be impossible: {assigned_ty}") ); - } else { - // write to non-drop union field, safe } } else { self.require_unsafe( @@ -267,22 +251,8 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> { fn register_violations<'a>( &mut self, violations: impl IntoIterator<Item = &'a UnsafetyViolation>, - new_used_unsafe_blocks: impl IntoIterator<Item = (HirId, UsedUnsafeBlockData)>, + new_used_unsafe_blocks: impl IntoIterator<Item = HirId>, ) { - use UsedUnsafeBlockData::{AllAllowedInUnsafeFn, SomeDisallowedInUnsafeFn}; - - let update_entry = |this: &mut Self, hir_id, new_usage| { - match this.used_unsafe_blocks.entry(hir_id) { - hash_map::Entry::Occupied(mut entry) => { - if new_usage == SomeDisallowedInUnsafeFn { - *entry.get_mut() = SomeDisallowedInUnsafeFn; - } - } - hash_map::Entry::Vacant(entry) => { - entry.insert(new_usage); - } - }; - }; let safety = self.body.source_scopes[self.source_info.scope] .local_data .as_ref() @@ -309,22 +279,14 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> { } }), Safety::BuiltinUnsafe => {} - Safety::ExplicitUnsafe(hir_id) => violations.into_iter().for_each(|violation| { - update_entry( - self, - hir_id, - match self.tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, violation.lint_root).0 - { - Level::Allow => AllAllowedInUnsafeFn(violation.lint_root), - _ => SomeDisallowedInUnsafeFn, - }, - ) + Safety::ExplicitUnsafe(hir_id) => violations.into_iter().for_each(|_violation| { + self.used_unsafe_blocks.insert(hir_id); }), }; - new_used_unsafe_blocks - .into_iter() - .for_each(|(hir_id, usage_data)| update_entry(self, hir_id, usage_data)); + new_used_unsafe_blocks.into_iter().for_each(|hir_id| { + self.used_unsafe_blocks.insert(hir_id); + }); } fn check_mut_borrowing_layout_constrained_field( &mut self, @@ -421,34 +383,28 @@ enum Context { struct UnusedUnsafeVisitor<'a, 'tcx> { tcx: TyCtxt<'tcx>, - used_unsafe_blocks: &'a FxHashMap<HirId, UsedUnsafeBlockData>, + used_unsafe_blocks: &'a FxHashSet<HirId>, context: Context, unused_unsafes: &'a mut Vec<(HirId, UnusedUnsafe)>, } impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> { fn visit_block(&mut self, block: &'tcx hir::Block<'tcx>) { - use UsedUnsafeBlockData::{AllAllowedInUnsafeFn, SomeDisallowedInUnsafeFn}; - if let hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::UserProvided) = block.rules { let used = match self.tcx.lint_level_at_node(UNUSED_UNSAFE, block.hir_id) { - (Level::Allow, _) => Some(SomeDisallowedInUnsafeFn), - _ => self.used_unsafe_blocks.get(&block.hir_id).copied(), + (Level::Allow, _) => true, + _ => self.used_unsafe_blocks.contains(&block.hir_id), }; let unused_unsafe = match (self.context, used) { - (_, None) => UnusedUnsafe::Unused, - (Context::Safe, Some(_)) - | (Context::UnsafeFn(_), Some(SomeDisallowedInUnsafeFn)) => { + (_, false) => UnusedUnsafe::Unused, + (Context::Safe, true) | (Context::UnsafeFn(_), true) => { let previous_context = self.context; self.context = Context::UnsafeBlock(block.hir_id); intravisit::walk_block(self, block); self.context = previous_context; return; } - (Context::UnsafeFn(hir_id), Some(AllAllowedInUnsafeFn(lint_root))) => { - UnusedUnsafe::InUnsafeFn(hir_id, lint_root) - } - (Context::UnsafeBlock(hir_id), Some(_)) => UnusedUnsafe::InUnsafeBlock(hir_id), + (Context::UnsafeBlock(hir_id), true) => UnusedUnsafe::InUnsafeBlock(hir_id), }; self.unused_unsafes.push((block.hir_id, unused_unsafe)); } @@ -472,17 +428,17 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> { fn check_unused_unsafe( tcx: TyCtxt<'_>, def_id: LocalDefId, - used_unsafe_blocks: &FxHashMap<HirId, UsedUnsafeBlockData>, + used_unsafe_blocks: &FxHashSet<HirId>, ) -> Vec<(HirId, UnusedUnsafe)> { - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - let body_id = tcx.hir().maybe_body_owned_by(hir_id); + let body_id = tcx.hir().maybe_body_owned_by(def_id); let Some(body_id) = body_id else { debug!("check_unused_unsafe({:?}) - no body found", def_id); return vec![]; }; - let body = tcx.hir().body(body_id); + let body = tcx.hir().body(body_id); + let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); let context = match tcx.hir().fn_sig_by_hir_id(hir_id) { Some(sig) if sig.header.unsafety == hir::Unsafety::Unsafe => Context::UnsafeFn(hir_id), _ => Context::Safe, @@ -545,25 +501,6 @@ fn report_unused_unsafe(tcx: TyCtxt<'_>, kind: UnusedUnsafe, id: HirId) { "because it's nested under this `unsafe` block", ); } - UnusedUnsafe::InUnsafeFn(id, usage_lint_root) => { - db.span_label( - tcx.sess.source_map().guess_head_span(tcx.hir().span(id)), - "because it's nested under this `unsafe` fn", - ) - .note( - "this `unsafe` block does contain unsafe operations, \ - but those are already allowed in an `unsafe fn`", - ); - let (level, source) = - tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, usage_lint_root); - assert_eq!(level, Level::Allow); - lint::explain_lint_level_source( - UNSAFE_OP_IN_UNSAFE_FN, - Level::Allow, - source, - &mut db, - ); - } } db.emit(); diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs index 611d29a4ee2..3378923c22c 100644 --- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs +++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs @@ -33,7 +33,7 @@ pub struct DeleteNonCodegenStatements<'tcx> { impl<'tcx> MirPass<'tcx> for CleanupNonCodegenStatements { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let mut delete = DeleteNonCodegenStatements { tcx }; - delete.visit_body(body); + delete.visit_body_preserves_cfg(body); body.user_type_annotations.raw.clear(); for decl in &mut body.local_decls { diff --git a/compiler/rustc_mir_transform/src/const_debuginfo.rs b/compiler/rustc_mir_transform/src/const_debuginfo.rs index 8944ebed9a7..6f0ae4f07ab 100644 --- a/compiler/rustc_mir_transform/src/const_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/const_debuginfo.rs @@ -16,7 +16,7 @@ pub struct ConstDebugInfo; impl<'tcx> MirPass<'tcx> for ConstDebugInfo { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() > 0 + sess.opts.unstable_opts.unsound_mir_opts && sess.mir_opt_level() > 0 } fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -88,12 +88,12 @@ fn find_optimization_oportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, Consta } impl Visitor<'_> for LocalUseVisitor { - fn visit_local(&mut self, local: &Local, context: PlaceContext, location: Location) { + fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { if context.is_mutating_use() { - self.local_mutating_uses[*local] = self.local_mutating_uses[*local].saturating_add(1); + self.local_mutating_uses[local] = self.local_mutating_uses[local].saturating_add(1); if context.is_place_assignment() { - self.local_assignment_locations[*local] = Some(location); + self.local_assignment_locations[local] = Some(location); } } } diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs index 5acf939f06b..0a305a40209 100644 --- a/compiler/rustc_mir_transform/src/const_goto.rs +++ b/compiler/rustc_mir_transform/src/const_goto.rs @@ -61,14 +61,14 @@ impl<'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'_, 'tcx> { let _: Option<_> = try { let target = terminator.kind.as_goto()?; // We only apply this optimization if the last statement is a const assignment - let last_statement = self.body.basic_blocks()[location.block].statements.last()?; + let last_statement = self.body.basic_blocks[location.block].statements.last()?; if let (place, Rvalue::Use(Operand::Constant(_const))) = last_statement.kind.as_assign()? { // We found a constant being assigned to `place`. // Now check that the target of this Goto switches on this place. - let target_bb = &self.body.basic_blocks()[target]; + let target_bb = &self.body.basic_blocks[target]; // The `StorageDead(..)` statement does not affect the functionality of mir. // We can move this part of the statement up to the predecessor. diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 412a5b4fc91..53f33a7a0ba 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -18,20 +18,17 @@ use rustc_middle::mir::{ }; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; use rustc_middle::ty::subst::{InternalSubsts, Subst}; -use rustc_middle::ty::{ - self, ConstKind, EarlyBinder, Instance, ParamEnv, Ty, TyCtxt, TypeFoldable, -}; +use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisitable}; use rustc_span::{def_id::DefId, Span}; -use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout}; -use rustc_target::spec::abi::Abi; +use rustc_target::abi::{self, HasDataLayout, Size, TargetDataLayout}; +use rustc_target::spec::abi::Abi as CallAbi; use rustc_trait_selection::traits; use crate::MirPass; use rustc_const_eval::interpret::{ self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame, - ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, MemoryKind, OpTy, - Operand as InterpOperand, PlaceTy, Pointer, Scalar, ScalarMaybeUninit, StackPopCleanup, - StackPopUnwind, + ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, PlaceTy, + Pointer, Scalar, StackPopCleanup, StackPopUnwind, }; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -60,11 +57,8 @@ macro_rules! throw_machine_stop_str { pub struct ConstProp; impl<'tcx> MirPass<'tcx> for ConstProp { - fn is_enabled(&self, _sess: &rustc_session::Session) -> bool { - // FIXME(#70073): Unlike the other passes in "optimizations", this one emits errors, so it - // runs even when MIR optimizations are disabled. We should separate the lint out from the - // transform and move the lint as early in the pipeline as possible. - true + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 1 } #[instrument(skip(self, tcx), level = "debug")] @@ -137,7 +131,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp { let dummy_body = &Body::new( body.source, - body.basic_blocks().clone(), + (*body.basic_blocks).clone(), body.source_scopes.clone(), body.local_decls.clone(), Default::default(), @@ -159,18 +153,18 @@ impl<'tcx> MirPass<'tcx> for ConstProp { } } -struct ConstPropMachine<'mir, 'tcx> { +pub struct ConstPropMachine<'mir, 'tcx> { /// The virtual call stack. stack: Vec<Frame<'mir, 'tcx>>, /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end. - written_only_inside_own_block_locals: FxHashSet<Local>, + pub written_only_inside_own_block_locals: FxHashSet<Local>, /// Locals that need to be cleared after every block terminates. - only_propagate_inside_block_locals: BitSet<Local>, - can_const_prop: IndexVec<Local, ConstPropMode>, + pub only_propagate_inside_block_locals: BitSet<Local>, + pub can_const_prop: IndexVec<Local, ConstPropMode>, } impl ConstPropMachine<'_, '_> { - fn new( + pub fn new( only_propagate_inside_block_locals: BitSet<Local>, can_const_prop: IndexVec<Local, ConstPropMode>, ) -> Self { @@ -189,6 +183,18 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> type MemoryKind = !; + #[inline(always)] + fn enforce_alignment(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + // We do not check for alignment to avoid having to carry an `Align` + // in `ConstValue::ByRef`. + false + } + + #[inline(always)] + fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + false // for now, we don't enforce validity + } + fn load_mir( _ecx: &InterpCx<'mir, 'tcx, Self>, _instance: ty::InstanceDef<'tcx>, @@ -199,7 +205,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> fn find_mir_or_eval_fn( _ecx: &mut InterpCx<'mir, 'tcx, Self>, _instance: ty::Instance<'tcx>, - _abi: Abi, + _abi: CallAbi, _args: &[OpTy<'tcx>], _destination: &PlaceTy<'tcx>, _target: Option<BasicBlock>, @@ -237,26 +243,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") } - fn access_local( - _ecx: &InterpCx<'mir, 'tcx, Self>, - frame: &Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>, - local: Local, - ) -> InterpResult<'tcx, InterpOperand<Self::PointerTag>> { - let l = &frame.locals[local]; - - if l.value == LocalValue::Unallocated { - throw_machine_stop_str!("tried to access an unallocated local") - } - - l.access() - } - fn access_local_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, frame: usize, local: Local, - ) -> InterpResult<'tcx, Result<&'a mut LocalValue<Self::PointerTag>, MemPlace<Self::PointerTag>>> - { + ) -> InterpResult<'tcx, &'a mut interpret::Operand<Self::Provenance>> { if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation { throw_machine_stop_str!("tried to write to a local that is marked as not propagatable") } @@ -275,7 +266,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> _tcx: TyCtxt<'tcx>, _machine: &Self, _alloc_id: AllocId, - alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>, + alloc: ConstAllocation<'tcx, Self::Provenance, Self::AllocExtra>, _static_def_id: Option<DefId>, is_write: bool, ) -> InterpResult<'tcx> { @@ -310,14 +301,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> #[inline(always)] fn stack<'a>( ecx: &'a InterpCx<'mir, 'tcx, Self>, - ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { + ) -> &'a [Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] { &ecx.machine.stack } #[inline(always)] fn stack_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, - ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { + ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>> { &mut ecx.machine.stack } } @@ -388,10 +379,14 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ); let ret_layout = ecx - .layout_of(EarlyBinder(body.return_ty()).subst(tcx, substs)) + .layout_of(body.bound_return_ty().subst(tcx, substs)) .ok() // Don't bother allocating memory for large values. - .filter(|ret_layout| ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT)) + // I don't know how return types can seem to be unsized but this happens in the + // `type/type-unsatisfiable.rs` test. + .filter(|ret_layout| { + !ret_layout.is_unsized() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) + }) .unwrap_or_else(|| ecx.layout_of(tcx.types.unit).unwrap()); let ret = ecx @@ -418,7 +413,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { let op = match self.ecx.eval_place_to_op(place, None) { - Ok(op) => op, + Ok(op) => { + if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) { + // Make sure nobody accidentally uses this value. + return None; + } + op + } Err(e) => { trace!("get_const failed: {}", e); return None; @@ -427,7 +428,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Try to read the local as an immediate so that if it is representable as a scalar, we can // handle it as such, but otherwise, just return the value as is. - Some(match self.ecx.read_immediate_raw(&op, /*force*/ false) { + Some(match self.ecx.read_immediate_raw(&op) { Ok(Ok(imm)) => imm.into(), _ => op, }) @@ -436,8 +437,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local] = - LocalState { value: LocalValue::Unallocated, layout: Cell::new(None) }; + ecx.frame_mut().locals[local] = LocalState { + value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)), + layout: Cell::new(None), + }; } fn use_ecx<F, T>(&mut self, f: F) -> Option<T> @@ -511,14 +514,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let l = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?)); // Check for exceeding shifts *even if* we cannot evaluate the LHS. if op == BinOp::Shr || op == BinOp::Shl { - let r = r?; + let r = r.clone()?; // We need the type of the LHS. We cannot use `place_layout` as that is the type // of the result, which for checked binops is not the same! let left_ty = left.ty(self.local_decls, self.tcx); let left_size = self.ecx.layout_of(left_ty).ok()?.size; let right_size = r.layout.size; - let r_bits = r.to_scalar().ok(); - let r_bits = r_bits.and_then(|r| r.to_bits(right_size).ok()); + let r_bits = r.to_scalar().to_bits(right_size).ok(); if r_bits.map_or(false, |b| b >= left_size.bits() as u128) { return None; } @@ -547,7 +549,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // and use it to do const-prop here and everywhere else // where it makes sense. if let interpret::Operand::Immediate(interpret::Immediate::Scalar( - ScalarMaybeUninit::Scalar(scalar), + scalar, )) = *value { *operand = self.operand_from_scalar( @@ -616,6 +618,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // There's no other checking to do at this time. Rvalue::Aggregate(..) | Rvalue::Use(..) + | Rvalue::CopyForDeref(..) | Rvalue::Repeat(..) | Rvalue::Len(..) | Rvalue::Cast(..) @@ -628,6 +631,14 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if rvalue.needs_subst() { return None; } + if !rvalue + .ty(&self.ecx.frame().body.local_decls, *self.ecx.tcx) + .is_sized(self.ecx.tcx, self.param_env) + { + // the interpreter doesn't support unsized locals (only unsized arguments), + // but rustc does (in a kinda broken way), so we have to skip them here + return None; + } if self.tcx.sess.mir_opt_level() >= 4 { self.eval_rvalue_with_identities(rvalue, place) @@ -645,16 +656,23 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { self.use_ecx(|this| match rvalue { Rvalue::BinaryOp(op, box (left, right)) | Rvalue::CheckedBinaryOp(op, box (left, right)) => { - let l = this.ecx.eval_operand(left, None); - let r = this.ecx.eval_operand(right, None); + let l = this.ecx.eval_operand(left, None).and_then(|x| this.ecx.read_immediate(&x)); + let r = + this.ecx.eval_operand(right, None).and_then(|x| this.ecx.read_immediate(&x)); let const_arg = match (l, r) { - (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?, - (Err(e), Err(_)) => return Err(e), - (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place), + (Ok(x), Err(_)) | (Err(_), Ok(x)) => x, // exactly one side is known + (Err(e), Err(_)) => return Err(e), // neither side is known + (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place), // both sides are known }; - let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?; + if !matches!(const_arg.layout.abi, abi::Abi::Scalar(..)) { + // We cannot handle Scalar Pair stuff. + // No point in calling `eval_rvalue_into_place`, since only one side is known + throw_machine_stop_str!("cannot optimize this") + } + + let arg_value = const_arg.to_scalar().to_bits(const_arg.layout.size)?; let dest = this.ecx.eval_place(place)?; match op { @@ -668,7 +686,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { BinOp::Mul if const_arg.layout.ty.is_integral() && arg_value == 0 => { if let Rvalue::CheckedBinaryOp(_, _) = rvalue { let val = Immediate::ScalarPair( - const_arg.to_scalar()?.into(), + const_arg.to_scalar().into(), Scalar::from_bool(false).into(), ); this.ecx.write_immediate(val, &dest) @@ -676,7 +694,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { this.ecx.write_immediate(*const_arg, &dest) } } - _ => this.ecx.eval_rvalue_into_place(rvalue, place), + _ => throw_machine_stop_str!("cannot optimize this"), } } _ => this.ecx.eval_rvalue_into_place(rvalue, place), @@ -722,21 +740,18 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } // FIXME> figure out what to do when read_immediate_raw fails - let imm = self.use_ecx(|this| this.ecx.read_immediate_raw(value, /*force*/ false)); + let imm = self.use_ecx(|this| this.ecx.read_immediate_raw(value)); if let Some(Ok(imm)) = imm { match *imm { - interpret::Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar)) => { + interpret::Immediate::Scalar(scalar) => { *rval = Rvalue::Use(self.operand_from_scalar( scalar, value.layout.ty, source_info.span, )); } - Immediate::ScalarPair( - ScalarMaybeUninit::Scalar(_), - ScalarMaybeUninit::Scalar(_), - ) => { + Immediate::ScalarPair(..) => { // Found a value represented as a pair. For now only do const-prop if the type // of `rvalue` is also a tuple with two scalars. // FIXME: enable the general case stated above ^. @@ -786,24 +801,15 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Returns `true` if and only if this `op` should be const-propagated into. fn should_const_prop(&mut self, op: &OpTy<'tcx>) -> bool { - let mir_opt_level = self.tcx.sess.mir_opt_level(); - - if mir_opt_level == 0 { - return false; - } - if !self.tcx.consider_optimizing(|| format!("ConstantPropagation - OpTy: {:?}", op)) { return false; } match **op { - interpret::Operand::Immediate(Immediate::Scalar(ScalarMaybeUninit::Scalar(s))) => { - s.try_to_int().is_ok() + interpret::Operand::Immediate(Immediate::Scalar(s)) => s.try_to_int().is_ok(), + interpret::Operand::Immediate(Immediate::ScalarPair(l, r)) => { + l.try_to_int().is_ok() && r.try_to_int().is_ok() } - interpret::Operand::Immediate(Immediate::ScalarPair( - ScalarMaybeUninit::Scalar(l), - ScalarMaybeUninit::Scalar(r), - )) => l.try_to_int().is_ok() && r.try_to_int().is_ok(), _ => false, } } @@ -811,7 +817,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// The mode that `ConstProp` is allowed to run in for a given `Local`. #[derive(Clone, Copy, Debug, PartialEq)] -enum ConstPropMode { +pub enum ConstPropMode { /// The `Local` can be propagated into and reads of this `Local` can also be propagated. FullConstProp, /// The `Local` can only be propagated into and from its own block. @@ -823,7 +829,7 @@ enum ConstPropMode { NoPropagation, } -struct CanConstProp { +pub struct CanConstProp { can_const_prop: IndexVec<Local, ConstPropMode>, // False at the beginning. Once set, no more assignments are allowed to that local. found_assignment: BitSet<Local>, @@ -833,7 +839,7 @@ struct CanConstProp { impl CanConstProp { /// Returns true if `local` can be propagated - fn check<'tcx>( + pub fn check<'tcx>( tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &Body<'tcx>, @@ -882,7 +888,7 @@ impl CanConstProp { } impl Visitor<'_> for CanConstProp { - fn visit_local(&mut self, &local: &Local, context: PlaceContext, _: Location) { + fn visit_local(&mut self, local: Local, context: PlaceContext, _: Location) { use rustc_middle::mir::visit::PlaceContext::*; match context { // Projections are fine, because `&mut foo.x` will be caught by @@ -948,7 +954,7 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } fn visit_body(&mut self, body: &mut Body<'tcx>) { - for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { + for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { self.visit_basic_block_data(bb, data); } } @@ -1042,7 +1048,9 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { let frame = self.ecx.frame_mut(); frame.locals[local].value = if let StatementKind::StorageLive(_) = statement.kind { - LocalValue::Unallocated + LocalValue::Live(interpret::Operand::Immediate( + interpret::Immediate::Uninit, + )) } else { LocalValue::Dead }; @@ -1062,8 +1070,12 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { TerminatorKind::Assert { expected, ref mut cond, .. } => { if let Some(ref value) = self.eval_operand(&cond) { trace!("assertion on {:?} should be {:?}", value, expected); - let expected = ScalarMaybeUninit::from(Scalar::from_bool(*expected)); - let value_const = self.ecx.read_scalar(&value).unwrap(); + let expected = Scalar::from_bool(*expected); + let Ok(value_const) = self.ecx.read_scalar(&value) else { + // FIXME should be used use_ecx rather than a local match... but we have + // quite a few of these read_scalar/read_immediate that need fixing. + return + }; if expected != value_const { // Poison all places this operand references so that further code // doesn't use the invalid value @@ -1075,13 +1087,11 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } } else { if self.should_const_prop(value) { - if let ScalarMaybeUninit::Scalar(scalar) = value_const { - *cond = self.operand_from_scalar( - scalar, - self.tcx.types.bool, - source_info.span, - ); - } + *cond = self.operand_from_scalar( + value_const, + self.tcx.types.bool, + source_info.span, + ); } } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 15ad13009e5..37e78f4ac07 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -1,62 +1,38 @@ //! Propagates constants for early reporting of statically known //! assertion failures -use std::cell::Cell; - -use rustc_ast::Mutability; -use rustc_data_structures::fx::FxHashSet; +use crate::const_prop::CanConstProp; +use crate::const_prop::ConstPropMachine; +use crate::const_prop::ConstPropMode; +use crate::MirLint; +use rustc_const_eval::const_eval::ConstEvalErr; +use rustc_const_eval::interpret::Immediate; +use rustc_const_eval::interpret::{ + self, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup, +}; use rustc_hir::def::DefKind; use rustc_hir::HirId; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; -use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::{ - AssertKind, BasicBlock, BinOp, Body, Constant, ConstantKind, Local, LocalDecl, LocalKind, - Location, Operand, Place, Rvalue, SourceInfo, SourceScope, SourceScopeData, Statement, - StatementKind, Terminator, TerminatorKind, UnOp, RETURN_PLACE, + AssertKind, BinOp, Body, Constant, ConstantKind, Local, LocalDecl, Location, Operand, Place, + Rvalue, SourceInfo, SourceScope, SourceScopeData, Statement, StatementKind, Terminator, + TerminatorKind, UnOp, RETURN_PLACE, }; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; use rustc_middle::ty::subst::{InternalSubsts, Subst}; -use rustc_middle::ty::{ - self, ConstInt, ConstKind, EarlyBinder, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeFoldable, -}; +use rustc_middle::ty::{self, ConstInt, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitable}; use rustc_session::lint; -use rustc_span::{def_id::DefId, Span}; +use rustc_span::Span; use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout}; -use rustc_target::spec::abi::Abi; use rustc_trait_selection::traits; - -use crate::MirLint; -use rustc_const_eval::const_eval::ConstEvalErr; -use rustc_const_eval::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult, - LocalState, LocalValue, MemPlace, MemoryKind, OpTy, Operand as InterpOperand, PlaceTy, Pointer, - Scalar, ScalarMaybeUninit, StackPopCleanup, StackPopUnwind, -}; +use std::cell::Cell; /// The maximum number of bytes that we'll allocate space for a local or the return value. /// Needed for #66397, because otherwise we eval into large places and that can cause OOM or just /// Severely regress performance. const MAX_ALLOC_LIMIT: u64 = 1024; - -/// Macro for machine-specific `InterpError` without allocation. -/// (These will never be shown to the user, but they help diagnose ICEs.) -macro_rules! throw_machine_stop_str { - ($($tt:tt)*) => {{ - // We make a new local type for it. The type itself does not carry any information, - // but its vtable (for the `MachineStopType` trait) does. - struct Zst; - // Printing this type shows the desired string. - impl std::fmt::Display for Zst { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, $($tt)*) - } - } - impl rustc_middle::mir::interpret::MachineStopType for Zst {} - throw_machine_stop!(Zst) - }}; -} - pub struct ConstProp; impl<'tcx> MirLint<'tcx> for ConstProp { @@ -128,7 +104,7 @@ impl<'tcx> MirLint<'tcx> for ConstProp { let dummy_body = &Body::new( body.source, - body.basic_blocks().clone(), + (*body.basic_blocks).clone(), body.source_scopes.clone(), body.local_decls.clone(), Default::default(), @@ -150,169 +126,6 @@ impl<'tcx> MirLint<'tcx> for ConstProp { } } -struct ConstPropMachine<'mir, 'tcx> { - /// The virtual call stack. - stack: Vec<Frame<'mir, 'tcx>>, - /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end. - written_only_inside_own_block_locals: FxHashSet<Local>, - /// Locals that need to be cleared after every block terminates. - only_propagate_inside_block_locals: BitSet<Local>, - can_const_prop: IndexVec<Local, ConstPropMode>, -} - -impl ConstPropMachine<'_, '_> { - fn new( - only_propagate_inside_block_locals: BitSet<Local>, - can_const_prop: IndexVec<Local, ConstPropMode>, - ) -> Self { - Self { - stack: Vec::new(), - written_only_inside_own_block_locals: Default::default(), - only_propagate_inside_block_locals, - can_const_prop, - } - } -} - -impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { - compile_time_machine!(<'mir, 'tcx>); - const PANIC_ON_ALLOC_FAIL: bool = true; // all allocations are small (see `MAX_ALLOC_LIMIT`) - - type MemoryKind = !; - - fn load_mir( - _ecx: &InterpCx<'mir, 'tcx, Self>, - _instance: ty::InstanceDef<'tcx>, - ) -> InterpResult<'tcx, &'tcx Body<'tcx>> { - throw_machine_stop_str!("calling functions isn't supported in ConstProp") - } - - fn find_mir_or_eval_fn( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _instance: ty::Instance<'tcx>, - _abi: Abi, - _args: &[OpTy<'tcx>], - _destination: &PlaceTy<'tcx>, - _target: Option<BasicBlock>, - _unwind: StackPopUnwind, - ) -> InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> { - Ok(None) - } - - fn call_intrinsic( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _instance: ty::Instance<'tcx>, - _args: &[OpTy<'tcx>], - _destination: &PlaceTy<'tcx>, - _target: Option<BasicBlock>, - _unwind: StackPopUnwind, - ) -> InterpResult<'tcx> { - throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp") - } - - fn assert_panic( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _msg: &rustc_middle::mir::AssertMessage<'tcx>, - _unwind: Option<rustc_middle::mir::BasicBlock>, - ) -> InterpResult<'tcx> { - bug!("panics terminators are not evaluated in ConstProp") - } - - fn binary_ptr_op( - _ecx: &InterpCx<'mir, 'tcx, Self>, - _bin_op: BinOp, - _left: &ImmTy<'tcx>, - _right: &ImmTy<'tcx>, - ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { - // We can't do this because aliasing of memory can differ between const eval and llvm - throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") - } - - fn access_local( - _ecx: &InterpCx<'mir, 'tcx, Self>, - frame: &Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>, - local: Local, - ) -> InterpResult<'tcx, InterpOperand<Self::PointerTag>> { - let l = &frame.locals[local]; - - if l.value == LocalValue::Unallocated { - throw_machine_stop_str!("tried to access an uninitialized local") - } - - l.access() - } - - fn access_local_mut<'a>( - ecx: &'a mut InterpCx<'mir, 'tcx, Self>, - frame: usize, - local: Local, - ) -> InterpResult<'tcx, Result<&'a mut LocalValue<Self::PointerTag>, MemPlace<Self::PointerTag>>> - { - if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation { - throw_machine_stop_str!("tried to write to a local that is marked as not propagatable") - } - if frame == 0 && ecx.machine.only_propagate_inside_block_locals.contains(local) { - trace!( - "mutating local {:?} which is restricted to its block. \ - Will remove it from const-prop after block is finished.", - local - ); - ecx.machine.written_only_inside_own_block_locals.insert(local); - } - ecx.machine.stack[frame].locals[local].access_mut() - } - - fn before_access_global( - _tcx: TyCtxt<'tcx>, - _machine: &Self, - _alloc_id: AllocId, - alloc: ConstAllocation<'tcx, Self::PointerTag, Self::AllocExtra>, - _static_def_id: Option<DefId>, - is_write: bool, - ) -> InterpResult<'tcx> { - if is_write { - throw_machine_stop_str!("can't write to global"); - } - // If the static allocation is mutable, then we can't const prop it as its content - // might be different at runtime. - if alloc.inner().mutability == Mutability::Mut { - throw_machine_stop_str!("can't access mutable globals in ConstProp"); - } - - Ok(()) - } - - #[inline(always)] - fn expose_ptr( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _ptr: Pointer<AllocId>, - ) -> InterpResult<'tcx> { - throw_machine_stop_str!("exposing pointers isn't supported in ConstProp") - } - - #[inline(always)] - fn init_frame_extra( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - frame: Frame<'mir, 'tcx>, - ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { - Ok(frame) - } - - #[inline(always)] - fn stack<'a>( - ecx: &'a InterpCx<'mir, 'tcx, Self>, - ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { - &ecx.machine.stack - } - - #[inline(always)] - fn stack_mut<'a>( - ecx: &'a mut InterpCx<'mir, 'tcx, Self>, - ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { - &mut ecx.machine.stack - } -} - /// Finds optimization opportunities on the MIR. struct ConstPropagator<'mir, 'tcx> { ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, @@ -380,10 +193,14 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ); let ret_layout = ecx - .layout_of(EarlyBinder(body.return_ty()).subst(tcx, substs)) + .layout_of(body.bound_return_ty().subst(tcx, substs)) .ok() // Don't bother allocating memory for large values. - .filter(|ret_layout| ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT)) + // I don't know how return types can seem to be unsized but this happens in the + // `type/type-unsatisfiable.rs` test. + .filter(|ret_layout| { + !ret_layout.is_unsized() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) + }) .unwrap_or_else(|| ecx.layout_of(tcx.types.unit).unwrap()); let ret = ecx @@ -411,7 +228,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { let op = match self.ecx.eval_place_to_op(place, None) { - Ok(op) => op, + Ok(op) => { + if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) { + // Make sure nobody accidentally uses this value. + return None; + } + op + } Err(e) => { trace!("get_const failed: {}", e); return None; @@ -420,7 +243,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Try to read the local as an immediate so that if it is representable as a scalar, we can // handle it as such, but otherwise, just return the value as is. - Some(match self.ecx.read_immediate_raw(&op, /*force*/ false) { + Some(match self.ecx.read_immediate_raw(&op) { Ok(Ok(imm)) => imm.into(), _ => op, }) @@ -429,8 +252,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local] = - LocalState { value: LocalValue::Unallocated, layout: Cell::new(None) }; + ecx.frame_mut().locals[local] = LocalState { + value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)), + layout: Cell::new(None), + }; } fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> { @@ -474,18 +299,15 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let err = ConstEvalErr::new(&self.ecx, error, Some(c.span)); if let Some(lint_root) = self.lint_root(source_info) { let lint_only = match c.literal { - ConstantKind::Ty(ct) => match ct.kind() { + ConstantKind::Ty(ct) => ct.needs_subst(), + ConstantKind::Unevaluated( + ty::Unevaluated { def: _, substs: _, promoted: Some(_) }, + _, + ) => { // Promoteds must lint and not error as the user didn't ask for them - ConstKind::Unevaluated(ty::Unevaluated { - def: _, - substs: _, - promoted: Some(_), - }) => true, - // Out of backwards compatibility we cannot report hard errors in unused - // generic functions using associated constants of the generic parameters. - _ => c.literal.needs_subst(), - }, - ConstantKind::Val(_, ty) => ty.needs_subst(), + true + } + ConstantKind::Unevaluated(..) | ConstantKind::Val(..) => c.needs_subst(), }; if lint_only { // Out of backwards compatibility we cannot report hard errors in unused @@ -574,14 +396,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { }); // Check for exceeding shifts *even if* we cannot evaluate the LHS. if op == BinOp::Shr || op == BinOp::Shl { - let r = r?; + let r = r.clone()?; // We need the type of the LHS. We cannot use `place_layout` as that is the type // of the result, which for checked binops is not the same! let left_ty = left.ty(self.local_decls, self.tcx); let left_size = self.ecx.layout_of(left_ty).ok()?.size; let right_size = r.layout.size; - let r_bits = r.to_scalar().ok(); - let r_bits = r_bits.and_then(|r| r.to_bits(right_size).ok()); + let r_bits = r.to_scalar().to_bits(right_size).ok(); if r_bits.map_or(false, |b| b >= left_size.bits() as u128) { debug!("check_binary_op: reporting assert for {:?}", source_info); self.report_assert_as_lint( @@ -606,10 +427,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } } - if let (Some(l), Some(r)) = (&l, &r) { + if let (Some(l), Some(r)) = (l, r) { // The remaining operators are handled through `overflowing_binary_op`. if self.use_ecx(source_info, |this| { - let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, l, r)?; + let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, &l, &r)?; Ok(overflow) })? { self.report_assert_as_lint( @@ -683,6 +504,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // There's no other checking to do at this time. Rvalue::Aggregate(..) | Rvalue::Use(..) + | Rvalue::CopyForDeref(..) | Rvalue::Repeat(..) | Rvalue::Len(..) | Rvalue::Cast(..) @@ -695,147 +517,22 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if rvalue.needs_subst() { return None; } - - self.use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, place)) - } -} - -/// The mode that `ConstProp` is allowed to run in for a given `Local`. -#[derive(Clone, Copy, Debug, PartialEq)] -enum ConstPropMode { - /// The `Local` can be propagated into and reads of this `Local` can also be propagated. - FullConstProp, - /// The `Local` can only be propagated into and from its own block. - OnlyInsideOwnBlock, - /// The `Local` can be propagated into but reads cannot be propagated. - OnlyPropagateInto, - /// The `Local` cannot be part of propagation at all. Any statement - /// referencing it either for reading or writing will not get propagated. - NoPropagation, -} - -struct CanConstProp { - can_const_prop: IndexVec<Local, ConstPropMode>, - // False at the beginning. Once set, no more assignments are allowed to that local. - found_assignment: BitSet<Local>, - // Cache of locals' information - local_kinds: IndexVec<Local, LocalKind>, -} - -impl CanConstProp { - /// Returns true if `local` can be propagated - fn check<'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, - body: &Body<'tcx>, - ) -> IndexVec<Local, ConstPropMode> { - let mut cpv = CanConstProp { - can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls), - found_assignment: BitSet::new_empty(body.local_decls.len()), - local_kinds: IndexVec::from_fn_n( - |local| body.local_kind(local), - body.local_decls.len(), - ), - }; - for (local, val) in cpv.can_const_prop.iter_enumerated_mut() { - let ty = body.local_decls[local].ty; - match tcx.layout_of(param_env.and(ty)) { - Ok(layout) if layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) => {} - // Either the layout fails to compute, then we can't use this local anyway - // or the local is too large, then we don't want to. - _ => { - *val = ConstPropMode::NoPropagation; - continue; - } - } - // Cannot use args at all - // Cannot use locals because if x < y { y - x } else { x - y } would - // lint for x != y - // FIXME(oli-obk): lint variables until they are used in a condition - // FIXME(oli-obk): lint if return value is constant - if cpv.local_kinds[local] == LocalKind::Arg { - *val = ConstPropMode::OnlyPropagateInto; - trace!( - "local {:?} can't be const propagated because it's a function argument", - local - ); - } else if cpv.local_kinds[local] == LocalKind::Var { - *val = ConstPropMode::OnlyInsideOwnBlock; - trace!( - "local {:?} will only be propagated inside its block, because it's a user variable", - local - ); - } + if !rvalue + .ty(&self.ecx.frame().body.local_decls, *self.ecx.tcx) + .is_sized(self.ecx.tcx, self.param_env) + { + // the interpreter doesn't support unsized locals (only unsized arguments), + // but rustc does (in a kinda broken way), so we have to skip them here + return None; } - cpv.visit_body(&body); - cpv.can_const_prop - } -} - -impl Visitor<'_> for CanConstProp { - fn visit_local(&mut self, &local: &Local, context: PlaceContext, _: Location) { - use rustc_middle::mir::visit::PlaceContext::*; - match context { - // Projections are fine, because `&mut foo.x` will be caught by - // `MutatingUseContext::Borrow` elsewhere. - MutatingUse(MutatingUseContext::Projection) - // These are just stores, where the storing is not propagatable, but there may be later - // mutations of the same local via `Store` - | MutatingUse(MutatingUseContext::Call) - | MutatingUse(MutatingUseContext::AsmOutput) - | MutatingUse(MutatingUseContext::Deinit) - // Actual store that can possibly even propagate a value - | MutatingUse(MutatingUseContext::SetDiscriminant) - | MutatingUse(MutatingUseContext::Store) => { - if !self.found_assignment.insert(local) { - match &mut self.can_const_prop[local] { - // If the local can only get propagated in its own block, then we don't have - // to worry about multiple assignments, as we'll nuke the const state at the - // end of the block anyway, and inside the block we overwrite previous - // states as applicable. - ConstPropMode::OnlyInsideOwnBlock => {} - ConstPropMode::NoPropagation => {} - ConstPropMode::OnlyPropagateInto => {} - other @ ConstPropMode::FullConstProp => { - trace!( - "local {:?} can't be propagated because of multiple assignments. Previous state: {:?}", - local, other, - ); - *other = ConstPropMode::OnlyInsideOwnBlock; - } - } - } - } - // Reading constants is allowed an arbitrary number of times - NonMutatingUse(NonMutatingUseContext::Copy) - | NonMutatingUse(NonMutatingUseContext::Move) - | NonMutatingUse(NonMutatingUseContext::Inspect) - | NonMutatingUse(NonMutatingUseContext::Projection) - | NonUse(_) => {} - // These could be propagated with a smarter analysis or just some careful thinking about - // whether they'd be fine right now. - MutatingUse(MutatingUseContext::Yield) - | MutatingUse(MutatingUseContext::Drop) - | MutatingUse(MutatingUseContext::Retag) - // These can't ever be propagated under any scheme, as we can't reason about indirect - // mutation. - | NonMutatingUse(NonMutatingUseContext::SharedBorrow) - | NonMutatingUse(NonMutatingUseContext::ShallowBorrow) - | NonMutatingUse(NonMutatingUseContext::UniqueBorrow) - | NonMutatingUse(NonMutatingUseContext::AddressOf) - | MutatingUse(MutatingUseContext::Borrow) - | MutatingUse(MutatingUseContext::AddressOf) => { - trace!("local {:?} can't be propagaged because it's used: {:?}", local, context); - self.can_const_prop[local] = ConstPropMode::NoPropagation; - } - } + self.use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, place)) } } impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { fn visit_body(&mut self, body: &Body<'tcx>) { - for (bb, data) in body.basic_blocks().iter_enumerated() { + for (bb, data) in body.basic_blocks.iter_enumerated() { self.visit_basic_block_data(bb, data); } } @@ -914,7 +611,9 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { let frame = self.ecx.frame_mut(); frame.locals[local].value = if let StatementKind::StorageLive(_) = statement.kind { - LocalValue::Unallocated + LocalValue::Live(interpret::Operand::Immediate( + interpret::Immediate::Uninit, + )) } else { LocalValue::Dead }; @@ -934,8 +633,12 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { TerminatorKind::Assert { expected, ref msg, ref cond, .. } => { if let Some(ref value) = self.eval_operand(&cond, source_info) { trace!("assertion on {:?} should be {:?}", value, expected); - let expected = ScalarMaybeUninit::from(Scalar::from_bool(*expected)); - let value_const = self.ecx.read_scalar(&value).unwrap(); + let expected = Scalar::from_bool(*expected); + let Ok(value_const) = self.ecx.read_scalar(&value) else { + // FIXME should be used use_ecx rather than a local match... but we have + // quite a few of these read_scalar/read_immediate that need fixing. + return + }; if expected != value_const { enum DbgVal<T> { Val(T), @@ -952,9 +655,9 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { let mut eval_to_int = |op| { // This can be `None` if the lhs wasn't const propagated and we just // triggered the assert on the value of the rhs. - self.eval_operand(op, source_info).map_or(DbgVal::Underscore, |op| { - DbgVal::Val(self.ecx.read_immediate(&op).unwrap().to_const_int()) - }) + self.eval_operand(op, source_info) + .and_then(|op| self.ecx.read_immediate(&op).ok()) + .map_or(DbgVal::Underscore, |op| DbgVal::Val(op.to_const_int())) }; let msg = match msg { AssertKind::DivisionByZero(op) => { diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 510f1e64ed1..782129be088 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -80,7 +80,7 @@ impl CoverageGraph { IndexVec<BasicCoverageBlock, BasicCoverageBlockData>, IndexVec<BasicBlock, Option<BasicCoverageBlock>>, ) { - let num_basic_blocks = mir_body.num_nodes(); + let num_basic_blocks = mir_body.basic_blocks.len(); let mut bcbs = IndexVec::with_capacity(num_basic_blocks); let mut bb_to_bcb = IndexVec::from_elem_n(None, num_basic_blocks); @@ -95,7 +95,7 @@ impl CoverageGraph { let mut basic_blocks = Vec::new(); for (bb, data) in mir_cfg_without_unwind { if let Some(last) = basic_blocks.last() { - let predecessors = &mir_body.predecessors()[bb]; + let predecessors = &mir_body.basic_blocks.predecessors()[bb]; if predecessors.len() > 1 || !predecessors.contains(last) { // The `bb` has more than one _incoming_ edge, and should start its own // `BasicCoverageBlockData`. (Note, the `basic_blocks` vector does not yet @@ -713,7 +713,7 @@ impl< ShortCircuitPreorder { body, - visited: BitSet::new_empty(body.basic_blocks().len()), + visited: BitSet::new_empty(body.basic_blocks.len()), worklist, filtered_successors, } @@ -747,7 +747,7 @@ impl< } fn size_hint(&self) -> (usize, Option<usize>) { - let size = self.body.basic_blocks().len() - self.visited.count(); + let size = self.body.basic_blocks.len() - self.visited.count(); (size, Some(size)) } } diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 782b620e28f..60481014488 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -15,7 +15,6 @@ use spans::{CoverageSpan, CoverageSpans}; use crate::MirPass; use rustc_data_structures::graph::WithNumNodes; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::Lrc; use rustc_index::vec::IndexVec; use rustc_middle::hir; @@ -81,7 +80,7 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage { return; } - match mir_body.basic_blocks()[mir::START_BLOCK].terminator().kind { + match mir_body.basic_blocks[mir::START_BLOCK].terminator().kind { TerminatorKind::Unreachable => { trace!("InstrumentCoverage skipped for unreachable `START_BLOCK`"); return; @@ -161,8 +160,8 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { let mut debug_used_expressions = debug::UsedExpressions::new(); let dump_mir = dump_enabled(tcx, self.pass_name, def_id); - let dump_graphviz = dump_mir && tcx.sess.opts.debugging_opts.dump_mir_graphviz; - let dump_spanview = dump_mir && tcx.sess.opts.debugging_opts.dump_mir_spanview.is_some(); + let dump_graphviz = dump_mir && tcx.sess.opts.unstable_opts.dump_mir_graphviz; + let dump_spanview = dump_mir && tcx.sess.opts.unstable_opts.dump_mir_spanview.is_some(); if dump_graphviz { graphviz_data.enable(); @@ -542,7 +541,7 @@ fn fn_sig_and_body<'tcx>( // to HIR for it. let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local"); let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body"); - (hir::map::fn_sig(hir_node), tcx.hir().body(fn_body_id)) + (hir_node.fn_sig(), tcx.hir().body(fn_body_id)) } fn get_body_span<'tcx>( @@ -576,12 +575,6 @@ fn get_body_span<'tcx>( fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 { // FIXME(cjgillot) Stop hashing HIR manually here. - let mut hcx = tcx.create_no_span_stable_hashing_context(); - let mut stable_hasher = StableHasher::new(); let owner = hir_body.id().hir_id.owner; - let bodies = &tcx.hir_owner_nodes(owner).unwrap().bodies; - hcx.with_hir_bodies(false, owner, bodies, |hcx| { - hir_body.value.hash_stable(hcx, &mut stable_hasher) - }); - stable_hasher.finish() + tcx.hir_owner_nodes(owner).unwrap().hash_including_bodies.to_smaller_hash() } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 9d02f58ae65..dc1e68b253e 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -84,7 +84,7 @@ impl CoverageVisitor { } fn visit_body(&mut self, body: &Body<'_>) { - for bb_data in body.basic_blocks().iter() { + for bb_data in body.basic_blocks.iter() { for statement in bb_data.statements.iter() { if let StatementKind::Coverage(box ref coverage) = statement.kind { if is_inlined(body, statement) { @@ -138,7 +138,7 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) -> fn covered_code_regions<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Vec<&'tcx CodeRegion> { let body = mir_body(tcx, def_id); - body.basic_blocks() + body.basic_blocks .iter() .flat_map(|data| { data.statements.iter().filter_map(|statement| match statement.kind { diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 82070b90325..9f842c929dc 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -321,7 +321,8 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { } fn mir_to_initial_sorted_coverage_spans(&self) -> Vec<CoverageSpan> { - let mut initial_spans = Vec::<CoverageSpan>::with_capacity(self.mir_body.num_nodes() * 2); + let mut initial_spans = + Vec::<CoverageSpan>::with_capacity(self.mir_body.basic_blocks.len() * 2); for (bcb, bcb_data) in self.basic_coverage_blocks.iter_enumerated() { initial_spans.extend(self.bcb_to_initial_coverage_spans(bcb, bcb_data)); } @@ -824,7 +825,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> // Retain spans from all other statements StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding` - | StatementKind::CopyNonOverlapping(..) + | StatementKind::Intrinsic(..) | StatementKind::Assign(_) | StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(..) diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 213bb6608e1..9c9ed5fa510 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -176,7 +176,7 @@ fn debug_basic_blocks<'tcx>(mir_body: &Body<'tcx>) -> String { format!( "{:?}", mir_body - .basic_blocks() + .basic_blocks .iter_enumerated() .map(|(bb, data)| { let term = &data.terminator(); @@ -213,7 +213,7 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) { "digraph {} {{\n{}\n}}", name, mir_body - .basic_blocks() + .basic_blocks .iter_enumerated() .map(|(bb, data)| { format!( @@ -222,6 +222,7 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) { bb, debug::term_type(&data.terminator().kind), mir_body + .basic_blocks .successors(bb) .map(|successor| { format!(" {:?} -> {:?};", bb, successor) }) .join("\n") @@ -652,7 +653,7 @@ fn test_traverse_coverage_with_loops() { fn synthesize_body_span_from_terminators(mir_body: &Body<'_>) -> Span { let mut some_span: Option<Span> = None; - for (_, data) in mir_body.basic_blocks().iter_enumerated() { + for (_, data) in mir_body.basic_blocks.iter_enumerated() { let term_span = data.terminator().source_info.span; if let Some(span) = some_span.as_mut() { *span = span.to(term_span); diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index 28f3790914b..3f3870cc7ba 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -52,7 +52,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Coverage(_) - | StatementKind::CopyNonOverlapping(_) + | StatementKind::Intrinsic(_) | StatementKind::Nop => (), StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { @@ -66,7 +66,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS return; } - let bbs = body.basic_blocks_mut(); + let bbs = body.basic_blocks.as_mut_preserves_cfg(); for Location { block, statement_index } in patch { bbs[block].statements[statement_index].make_nop(); } diff --git a/compiler/rustc_mir_transform/src/deaggregator.rs b/compiler/rustc_mir_transform/src/deaggregator.rs index 616ba819982..fe272de20f8 100644 --- a/compiler/rustc_mir_transform/src/deaggregator.rs +++ b/compiler/rustc_mir_transform/src/deaggregator.rs @@ -6,13 +6,8 @@ use rustc_middle::ty::TyCtxt; pub struct Deaggregator; impl<'tcx> MirPass<'tcx> for Deaggregator { - fn phase_change(&self) -> Option<MirPhase> { - Some(MirPhase::Deaggregated) - } - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); - let local_decls = &*local_decls; + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); for bb in basic_blocks { bb.expand_statements(|stmt| { // FIXME(eddyb) don't match twice on `stmt.kind` (post-NLL). @@ -37,7 +32,7 @@ impl<'tcx> MirPass<'tcx> for Deaggregator { Some(expand_aggregate( lhs, operands.into_iter().map(|op| { - let ty = op.ty(local_decls, tcx); + let ty = op.ty(&body.local_decls, tcx); (op, ty) }), *kind, diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs index d1977ed49fe..909116a77f5 100644 --- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs +++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs @@ -58,7 +58,7 @@ fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> { let mut duplicates = FxHashMap::default(); let bbs_to_go_through = - body.basic_blocks().iter_enumerated().filter(|(_, bbd)| !bbd.is_cleanup).count(); + body.basic_blocks.iter_enumerated().filter(|(_, bbd)| !bbd.is_cleanup).count(); let mut same_hashes = FxHashMap::with_capacity_and_hasher(bbs_to_go_through, Default::default()); @@ -71,8 +71,7 @@ fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> { // When we see bb1, we see that it is a duplicate of bb3, and therefore insert it in the duplicates list // with replacement bb3. // When the duplicates are removed, we will end up with only bb3. - for (bb, bbd) in body.basic_blocks().iter_enumerated().rev().filter(|(_, bbd)| !bbd.is_cleanup) - { + for (bb, bbd) in body.basic_blocks.iter_enumerated().rev().filter(|(_, bbd)| !bbd.is_cleanup) { // Basic blocks can get really big, so to avoid checking for duplicates in basic blocks // that are unlikely to have duplicates, we stop early. The early bail number has been // found experimentally by eprintln while compiling the crates in the rustc-perf suite. diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index bfb3ad1be27..7508df92df1 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -28,24 +28,21 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { let mut last_len = 0; let mut last_deref_idx = 0; - let mut prev_temp: Option<Local> = None; - for (idx, elem) in place.projection[0..].iter().enumerate() { if *elem == ProjectionElem::Deref { last_deref_idx = idx; } } + for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() { if !p_ref.projection.is_empty() && p_elem == ProjectionElem::Deref { let ty = p_ref.ty(&self.local_decls, self.tcx).ty; - let temp = self.patcher.new_local_with_info( + let temp = self.patcher.new_internal_with_info( ty, self.local_decls[p_ref.local].source_info.span, Some(Box::new(LocalInfo::DerefTemp)), ); - self.patcher.add_statement(loc, StatementKind::StorageLive(temp)); - // We are adding current p_ref's projections to our // temp value, excluding projections we already covered. let deref_place = Place::from(place_local) @@ -54,7 +51,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { self.patcher.add_assign( loc, Place::from(temp), - Rvalue::Use(Operand::Move(deref_place)), + Rvalue::CopyForDeref(deref_place), ); place_local = temp; last_len = p_ref.projection.len(); @@ -65,22 +62,8 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { Place::from(temp).project_deeper(&place.projection[idx..], self.tcx); *place = temp_place; } - - // We are destroying the previous temp since it's no longer used. - if let Some(prev_temp) = prev_temp { - self.patcher.add_statement(loc, StatementKind::StorageDead(prev_temp)); - } - - prev_temp = Some(temp); } } - - // Since we won't be able to reach final temp, we destroy it outside the loop. - if let Some(prev_temp) = prev_temp { - let last_loc = - Location { block: loc.block, statement_index: loc.statement_index + 1 }; - self.patcher.add_statement(last_loc, StatementKind::StorageDead(prev_temp)); - } } } } @@ -89,7 +72,7 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let patch = MirPatch::new(body); let mut checker = DerefChecker { tcx, patcher: patch, local_decls: body.local_decls.clone() }; - for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() { + for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { checker.visit_basic_block_data(bb, data); } @@ -99,6 +82,5 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { impl<'tcx> MirPass<'tcx> for Derefer { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { deref_finder(tcx, body); - body.phase = MirPhase::Derefered; } } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 84c7aada5e5..9bc47613e4c 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -122,7 +122,7 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { // // Only run at mir-opt-level=3 or higher for now (we don't fix up debuginfo and remove // storage statements at the moment). - sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() >= 3 + sess.opts.unstable_opts.unsound_mir_opts && sess.mir_opt_level() >= 3 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -150,7 +150,7 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { def_id, body.local_decls.len(), relevant, - body.basic_blocks().len() + body.basic_blocks.len() ); if relevant > MAX_LOCALS { warn!( @@ -159,11 +159,11 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { ); return; } - if body.basic_blocks().len() > MAX_BLOCKS { + if body.basic_blocks.len() > MAX_BLOCKS { warn!( "too many blocks in {:?} ({}, max is {}), not optimizing", def_id, - body.basic_blocks().len(), + body.basic_blocks.len(), MAX_BLOCKS ); return; @@ -537,7 +537,7 @@ impl<'a> Conflicts<'a> { | StatementKind::FakeRead(..) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index 33f201cbd28..32e738bbcea 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -95,7 +95,7 @@ pub struct EarlyOtherwiseBranch; impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 3 && sess.opts.debugging_opts.unsound_mir_opts + sess.mir_opt_level() >= 3 && sess.opts.unstable_opts.unsound_mir_opts } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -104,8 +104,8 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { let mut should_cleanup = false; // Also consider newly generated bbs in the same pass - for i in 0..body.basic_blocks().len() { - let bbs = body.basic_blocks(); + for i in 0..body.basic_blocks.len() { + let bbs = &*body.basic_blocks; let parent = BasicBlock::from_usize(i); let Some(opt_data) = evaluate_candidate(tcx, body, parent) else { continue @@ -316,7 +316,7 @@ fn evaluate_candidate<'tcx>( body: &Body<'tcx>, parent: BasicBlock, ) -> Option<OptimizationData<'tcx>> { - let bbs = body.basic_blocks(); + let bbs = &body.basic_blocks; let TerminatorKind::SwitchInt { targets, switch_ty: parent_ty, diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs index a80d2fbd644..294af2455d0 100644 --- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs +++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs @@ -69,10 +69,7 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> { let (unique_ty, nonnull_ty, ptr_ty) = build_ptr_tys(tcx, base_ty.boxed_ty(), self.unique_did, self.nonnull_did); - let ptr_local = self.patch.new_temp(ptr_ty, source_info.span); - self.local_decls.push(LocalDecl::new(ptr_ty, source_info.span)); - - self.patch.add_statement(location, StatementKind::StorageLive(ptr_local)); + let ptr_local = self.patch.new_internal(ptr_ty, source_info.span); self.patch.add_assign( location, @@ -84,11 +81,6 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> { ); place.local = ptr_local; - - self.patch.add_statement( - Location { block: location.block, statement_index: location.statement_index + 1 }, - StatementKind::StorageDead(ptr_local), - ); } self.super_place(place, context, location); @@ -110,39 +102,13 @@ impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs { let patch = MirPatch::new(body); - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); + let local_decls = &mut body.local_decls; let mut visitor = ElaborateBoxDerefVisitor { tcx, unique_did, nonnull_did, local_decls, patch }; - for (block, BasicBlockData { statements, terminator, .. }) in - basic_blocks.iter_enumerated_mut() - { - let mut index = 0; - for statement in statements { - let location = Location { block, statement_index: index }; - visitor.visit_statement(statement, location); - index += 1; - } - - if let Some(terminator) = terminator - && !matches!(terminator.kind, TerminatorKind::Yield{..}) - { - let location = Location { block, statement_index: index }; - visitor.visit_terminator(terminator, location); - } - - let location = Location { block, statement_index: index }; - match terminator { - // yielding into a box is handled when lowering generators - Some(Terminator { kind: TerminatorKind::Yield { value, .. }, .. }) => { - visitor.visit_operand(value, location); - } - Some(terminator) => { - visitor.visit_terminator(terminator, location); - } - None => {} - } + for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + visitor.visit_basic_block_data(block, data); } visitor.patch.apply(body); diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index e0e27c53f18..65f4956d23a 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -1,3 +1,4 @@ +use crate::deref_separator::deref_finder; use crate::MirPass; use rustc_data_structures::fx::FxHashMap; use rustc_index::bit_set::BitSet; @@ -9,6 +10,7 @@ use rustc_mir_dataflow::elaborate_drops::{DropElaborator, DropFlagMode, DropStyl use rustc_mir_dataflow::impls::{MaybeInitializedPlaces, MaybeUninitializedPlaces}; use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; use rustc_mir_dataflow::on_lookup_result_bits; +use rustc_mir_dataflow::un_derefer::UnDerefer; use rustc_mir_dataflow::MoveDataParamEnv; use rustc_mir_dataflow::{on_all_children_bits, on_all_drop_children_bits}; use rustc_mir_dataflow::{Analysis, ResultsCursor}; @@ -19,29 +21,26 @@ use std::fmt; pub struct ElaborateDrops; impl<'tcx> MirPass<'tcx> for ElaborateDrops { - fn phase_change(&self) -> Option<MirPhase> { - Some(MirPhase::DropsLowered) - } - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("elaborate_drops({:?} @ {:?})", body.source, body.span); let def_id = body.source.def_id(); let param_env = tcx.param_env_reveal_all_normalized(def_id); - let move_data = match MoveData::gather_moves(body, tcx, param_env) { + let (side_table, move_data) = match MoveData::gather_moves(body, tcx, param_env) { Ok(move_data) => move_data, Err((move_data, _)) => { tcx.sess.delay_span_bug( body.span, "No `move_errors` should be allowed in MIR borrowck", ); - move_data + (Default::default(), move_data) } }; + let un_derefer = UnDerefer { tcx: tcx, derefer_sidetable: side_table }; let elaborate_patch = { let body = &*body; let env = MoveDataParamEnv { move_data, param_env }; - let dead_unwinds = find_dead_unwinds(tcx, body, &env); + let dead_unwinds = find_dead_unwinds(tcx, body, &env, &un_derefer); let inits = MaybeInitializedPlaces::new(tcx, body, &env) .into_engine(tcx, body) @@ -65,10 +64,12 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops { init_data: InitializationData { inits, uninits }, drop_flags: Default::default(), patch: MirPatch::new(body), + un_derefer: un_derefer, } .elaborate() }; elaborate_patch.apply(body); + deref_finder(tcx, body); } } @@ -79,20 +80,23 @@ fn find_dead_unwinds<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, env: &MoveDataParamEnv<'tcx>, + und: &UnDerefer<'tcx>, ) -> BitSet<BasicBlock> { debug!("find_dead_unwinds({:?})", body.span); // We only need to do this pass once, because unwind edges can only // reach cleanup blocks, which can't have unwind edges themselves. - let mut dead_unwinds = BitSet::new_empty(body.basic_blocks().len()); + let mut dead_unwinds = BitSet::new_empty(body.basic_blocks.len()); let mut flow_inits = MaybeInitializedPlaces::new(tcx, body, &env) .into_engine(tcx, body) .pass_name("find_dead_unwinds") .iterate_to_fixpoint() .into_results_cursor(body); - for (bb, bb_data) in body.basic_blocks().iter_enumerated() { + for (bb, bb_data) in body.basic_blocks.iter_enumerated() { let place = match bb_data.terminator().kind { TerminatorKind::Drop { ref place, unwind: Some(_), .. } - | TerminatorKind::DropAndReplace { ref place, unwind: Some(_), .. } => place, + | TerminatorKind::DropAndReplace { ref place, unwind: Some(_), .. } => { + und.derefer(place.as_ref(), body).unwrap_or(*place) + } _ => continue, }; @@ -256,6 +260,7 @@ struct ElaborateDropsCtxt<'a, 'tcx> { init_data: InitializationData<'a, 'tcx>, drop_flags: FxHashMap<MovePathIndex, Local>, patch: MirPatch<'tcx>, + un_derefer: UnDerefer<'tcx>, } impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { @@ -294,11 +299,13 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } fn collect_drop_flags(&mut self) { - for (bb, data) in self.body.basic_blocks().iter_enumerated() { + for (bb, data) in self.body.basic_blocks.iter_enumerated() { let terminator = data.terminator(); let place = match terminator.kind { TerminatorKind::Drop { ref place, .. } - | TerminatorKind::DropAndReplace { ref place, .. } => place, + | TerminatorKind::DropAndReplace { ref place, .. } => { + self.un_derefer.derefer(place.as_ref(), self.body).unwrap_or(*place) + } _ => continue, }; @@ -312,12 +319,17 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { LookupResult::Parent(None) => continue, LookupResult::Parent(Some(parent)) => { let (_maybe_live, maybe_dead) = self.init_data.maybe_live_dead(parent); + + if self.body.local_decls[place.local].is_deref_temp() { + continue; + } + if maybe_dead { self.tcx.sess.delay_span_bug( terminator.source_info.span, &format!( "drop of untracked, uninitialized value {:?}, place {:?} ({:?})", - bb, place, path, + bb, place, path ), ); } @@ -342,13 +354,17 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } fn elaborate_drops(&mut self) { - for (bb, data) in self.body.basic_blocks().iter_enumerated() { + for (bb, data) in self.body.basic_blocks.iter_enumerated() { let loc = Location { block: bb, statement_index: data.statements.len() }; let terminator = data.terminator(); let resume_block = self.patch.resume_block(); match terminator.kind { - TerminatorKind::Drop { place, target, unwind } => { + TerminatorKind::Drop { mut place, target, unwind } => { + if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) { + place = new_place; + } + self.init_data.seek_before(loc); match self.move_data().rev_lookup.find(place.as_ref()) { LookupResult::Exact(path) => elaborate_drop( @@ -372,9 +388,12 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } } } - TerminatorKind::DropAndReplace { place, ref value, target, unwind } => { + TerminatorKind::DropAndReplace { mut place, ref value, target, unwind } => { assert!(!data.is_cleanup); + if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) { + place = new_place; + } self.elaborate_replace(loc, place, value, target, unwind); } _ => continue, @@ -492,7 +511,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } fn drop_flags_for_fn_rets(&mut self) { - for (bb, data) in self.body.basic_blocks().iter_enumerated() { + for (bb, data) in self.body.basic_blocks.iter_enumerated() { if let TerminatorKind::Call { destination, target: Some(tgt), cleanup: Some(_), .. } = data.terminator().kind @@ -527,7 +546,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { // drop flags by themselves, to avoid the drop flags being // clobbered before they are read. - for (bb, data) in self.body.basic_blocks().iter_enumerated() { + for (bb, data) in self.body.basic_blocks.iter_enumerated() { debug!("drop_flags_for_locs({:?})", data); for i in 0..(data.statements.len() + 1) { debug!("drop_flag_for_locs: stmt {}", i); diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs new file mode 100644 index 00000000000..7522a50a8c6 --- /dev/null +++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs @@ -0,0 +1,170 @@ +use rustc_hir::def_id::{CrateNum, LocalDefId, LOCAL_CRATE}; +use rustc_middle::mir::*; +use rustc_middle::ty::layout; +use rustc_middle::ty::query::Providers; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_session::lint::builtin::FFI_UNWIND_CALLS; +use rustc_target::spec::abi::Abi; +use rustc_target::spec::PanicStrategy; + +fn abi_can_unwind(abi: Abi) -> bool { + use Abi::*; + match abi { + C { unwind } + | System { unwind } + | Cdecl { unwind } + | Stdcall { unwind } + | Fastcall { unwind } + | Vectorcall { unwind } + | Thiscall { unwind } + | Aapcs { unwind } + | Win64 { unwind } + | SysV64 { unwind } => unwind, + PtxKernel + | Msp430Interrupt + | X86Interrupt + | AmdGpuKernel + | EfiApi + | AvrInterrupt + | AvrNonBlockingInterrupt + | CCmseNonSecureCall + | Wasm + | RustIntrinsic + | PlatformIntrinsic + | Unadjusted => false, + Rust | RustCall | RustCold => true, + } +} + +// Check if the body of this def_id can possibly leak a foreign unwind into Rust code. +fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool { + debug!("has_ffi_unwind_calls({local_def_id:?})"); + + // Only perform check on functions because constants cannot call FFI functions. + let def_id = local_def_id.to_def_id(); + let kind = tcx.def_kind(def_id); + if !kind.is_fn_like() { + return false; + } + + let body = &*tcx.mir_built(ty::WithOptConstParam::unknown(local_def_id)).borrow(); + + let body_ty = tcx.type_of(def_id); + let body_abi = match body_ty.kind() { + ty::FnDef(..) => body_ty.fn_sig(tcx).abi(), + ty::Closure(..) => Abi::RustCall, + ty::Generator(..) => Abi::Rust, + _ => span_bug!(body.span, "unexpected body ty: {:?}", body_ty), + }; + let body_can_unwind = layout::fn_can_unwind(tcx, Some(def_id), body_abi); + + // Foreign unwinds cannot leak past functions that themselves cannot unwind. + if !body_can_unwind { + return false; + } + + let mut tainted = false; + + for block in body.basic_blocks.iter() { + if block.is_cleanup { + continue; + } + let Some(terminator) = &block.terminator else { continue }; + let TerminatorKind::Call { func, .. } = &terminator.kind else { continue }; + + let ty = func.ty(body, tcx); + let sig = ty.fn_sig(tcx); + + // Rust calls cannot themselves create foreign unwinds. + if let Abi::Rust | Abi::RustCall | Abi::RustCold = sig.abi() { + continue; + }; + + let fn_def_id = match ty.kind() { + ty::FnPtr(_) => None, + &ty::FnDef(def_id, _) => { + // Rust calls cannot themselves create foreign unwinds. + if !tcx.is_foreign_item(def_id) { + continue; + } + Some(def_id) + } + _ => bug!("invalid callee of type {:?}", ty), + }; + + if layout::fn_can_unwind(tcx, fn_def_id, sig.abi()) && abi_can_unwind(sig.abi()) { + // We have detected a call that can possibly leak foreign unwind. + // + // Because the function body itself can unwind, we are not aborting this function call + // upon unwind, so this call can possibly leak foreign unwind into Rust code if the + // panic runtime linked is panic-abort. + + let lint_root = body.source_scopes[terminator.source_info.scope] + .local_data + .as_ref() + .assert_crate_local() + .lint_root; + let span = terminator.source_info.span; + + tcx.struct_span_lint_hir(FFI_UNWIND_CALLS, lint_root, span, |lint| { + let msg = match fn_def_id { + Some(_) => "call to foreign function with FFI-unwind ABI", + None => "call to function pointer with FFI-unwind ABI", + }; + let mut db = lint.build(msg); + db.span_label(span, msg); + db.emit(); + }); + + tainted = true; + } + } + + tainted +} + +fn required_panic_strategy(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option<PanicStrategy> { + assert_eq!(cnum, LOCAL_CRATE); + + if tcx.is_panic_runtime(LOCAL_CRATE) { + return Some(tcx.sess.panic_strategy()); + } + + if tcx.sess.panic_strategy() == PanicStrategy::Abort { + return Some(PanicStrategy::Abort); + } + + for def_id in tcx.hir().body_owners() { + if tcx.has_ffi_unwind_calls(def_id) { + // Given that this crate is compiled in `-C panic=unwind`, the `AbortUnwindingCalls` + // MIR pass will not be run on FFI-unwind call sites, therefore a foreign exception + // can enter Rust through these sites. + // + // On the other hand, crates compiled with `-C panic=abort` expects that all Rust + // functions cannot unwind (whether it's caused by Rust panic or foreign exception), + // and this expectation mismatch can cause unsoundness (#96926). + // + // To address this issue, we enforce that if FFI-unwind calls are used in a crate + // compiled with `panic=unwind`, then the final panic strategy must be `panic=unwind`. + // This will ensure that no crates will have wrong unwindability assumption. + // + // It should be noted that it is okay to link `panic=unwind` into a `panic=abort` + // program if it contains no FFI-unwind calls. In such case foreign exception can only + // enter Rust in a `panic=abort` crate, which will lead to an abort. There will also + // be no exceptions generated from Rust, so the assumption which `panic=abort` crates + // make, that no Rust function can unwind, indeed holds for crates compiled with + // `panic=unwind` as well. In such case this function returns `None`, indicating that + // the crate does not require a particular final panic strategy, and can be freely + // linked to crates with either strategy (we need such ability for libstd and its + // dependencies). + return Some(PanicStrategy::Unwind); + } + } + + // This crate can be linked with either runtime. + None +} + +pub(crate) fn provide(providers: &mut Providers) { + *providers = Providers { has_ffi_unwind_calls, required_panic_strategy, ..*providers }; +} diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 9078b8a1cb7..705cf776fb2 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -67,7 +67,7 @@ use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt}; use rustc_mir_dataflow::impls::{ MaybeBorrowedLocals, MaybeLiveLocals, MaybeRequiresStorage, MaybeStorageLive, }; -use rustc_mir_dataflow::storage; +use rustc_mir_dataflow::storage::always_storage_live_locals; use rustc_mir_dataflow::{self, Analysis}; use rustc_target::abi::VariantIdx; use rustc_target::spec::PanicStrategy; @@ -490,12 +490,12 @@ fn locals_live_across_suspend_points<'tcx>( .iterate_to_fixpoint() .into_results_cursor(body_ref); - let mut storage_liveness_map = IndexVec::from_elem(None, body.basic_blocks()); + let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); let mut live_locals_at_suspension_points = Vec::new(); let mut source_info_at_suspension_points = Vec::new(); let mut live_locals_at_any_suspension_point = BitSet::new_empty(body.local_decls.len()); - for (block, data) in body.basic_blocks().iter_enumerated() { + for (block, data) in body.basic_blocks.iter_enumerated() { if let TerminatorKind::Yield { .. } = data.terminator().kind { let loc = Location { block, statement_index: data.statements.len() }; @@ -704,7 +704,7 @@ impl<'mir, 'tcx> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> impl StorageConflictVisitor<'_, '_, '_> { fn apply_state(&mut self, flow_state: &BitSet<Local>, loc: Location) { // Ignore unreachable blocks. - if self.body.basic_blocks()[loc.block].terminator().kind == TerminatorKind::Unreachable { + if self.body.basic_blocks[loc.block].terminator().kind == TerminatorKind::Unreachable { return; } @@ -886,7 +886,7 @@ fn elaborate_generator_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let mut elaborator = DropShimElaborator { body, patch: MirPatch::new(body), tcx, param_env }; - for (block, block_data) in body.basic_blocks().iter_enumerated() { + for (block, block_data) in body.basic_blocks.iter_enumerated() { let (target, unwind, source_info) = match block_data.terminator() { Terminator { source_info, kind: TerminatorKind::Drop { place, target, unwind } } => { if let Some(local) = place.as_local() { @@ -957,7 +957,7 @@ fn create_generator_drop_shim<'tcx>( tcx.mk_ptr(ty::TypeAndMut { ty: gen_ty, mutbl: hir::Mutability::Mut }), source_info, ); - if tcx.sess.opts.debugging_opts.mir_emit_retag { + if tcx.sess.opts.unstable_opts.mir_emit_retag { // Alias tracking must know we changed the type body.basic_blocks_mut()[START_BLOCK].statements.insert( 0, @@ -991,7 +991,7 @@ fn insert_panic_block<'tcx>( body: &mut Body<'tcx>, message: AssertMessage<'tcx>, ) -> BasicBlock { - let assert_block = BasicBlock::new(body.basic_blocks().len()); + let assert_block = BasicBlock::new(body.basic_blocks.len()); let term = TerminatorKind::Assert { cond: Operand::Constant(Box::new(Constant { span: body.span, @@ -1021,7 +1021,7 @@ fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, param_env: ty::ParamEn } // If there's a return terminator the function may return. - for block in body.basic_blocks() { + for block in body.basic_blocks.iter() { if let TerminatorKind::Return = block.terminator().kind { return true; } @@ -1038,7 +1038,7 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool { } // Unwinds can only start at certain terminators. - for block in body.basic_blocks() { + for block in body.basic_blocks.iter() { match block.terminator().kind { // These never unwind. TerminatorKind::Goto { .. } @@ -1182,8 +1182,6 @@ fn create_cases<'tcx>( transform: &TransformVisitor<'tcx>, operation: Operation, ) -> Vec<(usize, BasicBlock)> { - let tcx = transform.tcx; - let source_info = SourceInfo::outermost(body.span); transform @@ -1216,85 +1214,13 @@ fn create_cases<'tcx>( if operation == Operation::Resume { // Move the resume argument to the destination place of the `Yield` terminator let resume_arg = Local::new(2); // 0 = return, 1 = self - - // handle `box yield` properly - let box_place = if let [projection @ .., ProjectionElem::Deref] = - &**point.resume_arg.projection - { - let box_place = - Place::from(point.resume_arg.local).project_deeper(projection, tcx); - - let box_ty = box_place.ty(&body.local_decls, tcx).ty; - - if box_ty.is_box() { Some((box_place, box_ty)) } else { None } - } else { - None - }; - - if let Some((box_place, box_ty)) = box_place { - let unique_did = box_ty - .ty_adt_def() - .expect("expected Box to be an Adt") - .non_enum_variant() - .fields[0] - .did; - - let Some(nonnull_def) = tcx.type_of(unique_did).ty_adt_def() else { - span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique") - }; - - let nonnull_did = nonnull_def.non_enum_variant().fields[0].did; - - let (unique_ty, nonnull_ty, ptr_ty) = - crate::elaborate_box_derefs::build_ptr_tys( - tcx, - box_ty.boxed_ty(), - unique_did, - nonnull_did, - ); - - let ptr_local = body.local_decls.push(LocalDecl::new(ptr_ty, body.span)); - - statements.push(Statement { - source_info, - kind: StatementKind::StorageLive(ptr_local), - }); - - statements.push(Statement { - source_info, - kind: StatementKind::Assign(Box::new(( - Place::from(ptr_local), - Rvalue::Use(Operand::Copy(box_place.project_deeper( - &crate::elaborate_box_derefs::build_projection( - unique_ty, nonnull_ty, ptr_ty, - ), - tcx, - ))), - ))), - }); - - statements.push(Statement { - source_info, - kind: StatementKind::Assign(Box::new(( - Place::from(ptr_local) - .project_deeper(&[ProjectionElem::Deref], tcx), - Rvalue::Use(Operand::Move(resume_arg.into())), - ))), - }); - - statements.push(Statement { - source_info, - kind: StatementKind::StorageDead(ptr_local), - }); - } else { - statements.push(Statement { - source_info, - kind: StatementKind::Assign(Box::new(( - point.resume_arg, - Rvalue::Use(Operand::Move(resume_arg.into())), - ))), - }); - } + statements.push(Statement { + source_info, + kind: StatementKind::Assign(Box::new(( + point.resume_arg, + Rvalue::Use(Operand::Move(resume_arg.into())), + ))), + }); } // Then jump to the real target @@ -1314,10 +1240,6 @@ fn create_cases<'tcx>( } impl<'tcx> MirPass<'tcx> for StateTransform { - fn phase_change(&self) -> Option<MirPhase> { - Some(MirPhase::GeneratorsLowered) - } - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let Some(yield_ty) = body.yield_ty() else { // This only applies to generators @@ -1379,14 +1301,14 @@ impl<'tcx> MirPass<'tcx> for StateTransform { }, ); - let always_live_locals = storage::always_live_locals(&body); + let always_live_locals = always_storage_live_locals(&body); let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); sanitize_witness(tcx, body, interior, upvars, &liveness_info.saved_locals); - if tcx.sess.opts.debugging_opts.validate_mir { + if tcx.sess.opts.unstable_opts.validate_mir { let mut vis = EnsureGeneratorFieldAssignmentsNeverAlias { assigned_local: None, saved_locals: &liveness_info.saved_locals, @@ -1530,7 +1452,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { | StatementKind::Retag(..) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) - | StatementKind::CopyNonOverlapping(..) + | StatementKind::Intrinsic(..) | StatementKind::Nop => {} } } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 49403ba03a4..d00a384cb44 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -1,15 +1,18 @@ //! Inlining pass for MIR functions use crate::deref_separator::deref_finder; use rustc_attr::InlineAttr; +use rustc_const_eval::transform::validate::equal_up_to_regions; use rustc_index::bit_set::BitSet; use rustc_index::vec::Idx; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::traits::ObligationCause; use rustc_middle::ty::subst::Subst; -use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; +use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; +use rustc_session::config::OptLevel; +use rustc_span::def_id::DefId; use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span}; +use rustc_target::abi::VariantIdx; use rustc_target::spec::abi::Abi; use super::simplify::{remove_dead_blocks, CfgSimplifier}; @@ -39,11 +42,19 @@ struct CallSite<'tcx> { impl<'tcx> MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - if let Some(enabled) = sess.opts.debugging_opts.inline_mir { + if let Some(enabled) = sess.opts.unstable_opts.inline_mir { return enabled; } - sess.opts.mir_opt_level() >= 3 + match sess.mir_opt_level() { + 0 | 1 => false, + 2 => { + (sess.opts.optimize == OptLevel::Default + || sess.opts.optimize == OptLevel::Aggressive) + && sess.opts.incremental == None + } + _ => true, + } } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -76,13 +87,6 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { } let param_env = tcx.param_env_reveal_all_normalized(def_id); - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - let param_env = rustc_trait_selection::traits::normalize_param_env_or_error( - tcx, - def_id.to_def_id(), - param_env, - ObligationCause::misc(body.span, hir_id), - ); let mut this = Inliner { tcx, @@ -91,7 +95,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { history: Vec::new(), changed: false, }; - let blocks = BasicBlock::new(0)..body.basic_blocks().next_index(); + let blocks = BasicBlock::new(0)..body.basic_blocks.next_index(); this.process_blocks(body, blocks); this.changed } @@ -101,8 +105,12 @@ struct Inliner<'tcx> { param_env: ParamEnv<'tcx>, /// Caller codegen attributes. codegen_fn_attrs: &'tcx CodegenFnAttrs, - /// Stack of inlined Instances. - history: Vec<ty::Instance<'tcx>>, + /// Stack of inlined instances. + /// We only check the `DefId` and not the substs because we want to + /// avoid inlining cases of polymorphic recursion. + /// The number of `DefId`s is finite, so checking history is enough + /// to ensure that we do not loop endlessly while inlining. + history: Vec<DefId>, /// Indicates that the caller body has been modified. changed: bool, } @@ -130,7 +138,7 @@ impl<'tcx> Inliner<'tcx> { Ok(new_blocks) => { debug!("inlined {}", callsite.callee); self.changed = true; - self.history.push(callsite.callee); + self.history.push(callsite.callee.def_id()); self.process_blocks(caller_body, new_blocks); self.history.pop(); } @@ -166,9 +174,52 @@ impl<'tcx> Inliner<'tcx> { return Err("failed to normalize callee body"); }; - let old_blocks = caller_body.basic_blocks().next_index(); + // Check call signature compatibility. + // Normally, this shouldn't be required, but trait normalization failure can create a + // validation ICE. + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; + let output_type = callee_body.return_ty(); + if !equal_up_to_regions(self.tcx, self.param_env, output_type, destination_ty) { + trace!(?output_type, ?destination_ty); + return Err("failed to normalize return type"); + } + if callsite.fn_sig.abi() == Abi::RustCall { + let (arg_tuple, skipped_args) = match &args[..] { + [arg_tuple] => (arg_tuple, 0), + [_, arg_tuple] => (arg_tuple, 1), + _ => bug!("Expected `rust-call` to have 1 or 2 args"), + }; + + let arg_tuple_ty = arg_tuple.ty(&caller_body.local_decls, self.tcx); + let ty::Tuple(arg_tuple_tys) = arg_tuple_ty.kind() else { + bug!("Closure arguments are not passed as a tuple"); + }; + + for (arg_ty, input) in + arg_tuple_tys.iter().zip(callee_body.args_iter().skip(skipped_args)) + { + let input_type = callee_body.local_decls[input].ty; + if !equal_up_to_regions(self.tcx, self.param_env, arg_ty, input_type) { + trace!(?arg_ty, ?input_type); + return Err("failed to normalize tuple argument type"); + } + } + } else { + for (arg, input) in args.iter().zip(callee_body.args_iter()) { + let input_type = callee_body.local_decls[input].ty; + let arg_ty = arg.ty(&caller_body.local_decls, self.tcx); + if !equal_up_to_regions(self.tcx, self.param_env, arg_ty, input_type) { + trace!(?arg_ty, ?input_type); + return Err("failed to normalize argument type"); + } + } + } + + let old_blocks = caller_body.basic_blocks.next_index(); self.inline_call(caller_body, &callsite, callee_body); - let new_blocks = old_blocks..caller_body.basic_blocks().next_index(); + let new_blocks = old_blocks..caller_body.basic_blocks.next_index(); Ok(new_blocks) } @@ -201,7 +252,7 @@ impl<'tcx> Inliner<'tcx> { // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we // do not need to catch this here, we can wait until the inliner decides to continue // inlining a second time. - InstanceDef::VtableShim(_) + InstanceDef::VTableShim(_) | InstanceDef::ReifyShim(_) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } @@ -263,6 +314,10 @@ impl<'tcx> Inliner<'tcx> { return None; } + if self.history.contains(&callee.def_id()) { + return None; + } + let fn_sig = self.tcx.bound_fn_sig(def_id).subst(self.tcx, substs); return Some(CallSite { @@ -285,8 +340,14 @@ impl<'tcx> Inliner<'tcx> { callsite: &CallSite<'tcx>, callee_attrs: &CodegenFnAttrs, ) -> Result<(), &'static str> { - if let InlineAttr::Never = callee_attrs.inline { - return Err("never inline hint"); + match callee_attrs.inline { + InlineAttr::Never => return Err("never inline hint"), + InlineAttr::Always | InlineAttr::Hint => {} + InlineAttr::None => { + if self.tcx.sess.mir_opt_level() <= 2 { + return Err("at mir-opt-level=2, only #[inline] is inlined"); + } + } } // Only inline local functions if they would be eligible for cross-crate @@ -340,156 +401,83 @@ impl<'tcx> Inliner<'tcx> { let tcx = self.tcx; let mut threshold = if callee_attrs.requests_inline() { - self.tcx.sess.opts.debugging_opts.inline_mir_hint_threshold.unwrap_or(100) + self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) } else { - self.tcx.sess.opts.debugging_opts.inline_mir_threshold.unwrap_or(50) + self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) }; // Give a bonus functions with a small number of blocks, // We normally have two or three blocks for even // very small functions. - if callee_body.basic_blocks().len() <= 3 { + if callee_body.basic_blocks.len() <= 3 { threshold += threshold / 4; } debug!(" final inline threshold = {}", threshold); // FIXME: Give a bonus to functions with only a single caller - let mut first_block = true; - let mut cost = 0; + let diverges = matches!( + callee_body.basic_blocks[START_BLOCK].terminator().kind, + TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. } + ); + if diverges && !matches!(callee_attrs.inline, InlineAttr::Always) { + return Err("callee diverges unconditionally"); + } - // Traverse the MIR manually so we can account for the effects of - // inlining on the CFG. + let mut checker = CostChecker { + tcx: self.tcx, + param_env: self.param_env, + instance: callsite.callee, + callee_body, + cost: 0, + validation: Ok(()), + }; + + // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; - let mut visited = BitSet::new_empty(callee_body.basic_blocks().len()); + let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); while let Some(bb) = work_list.pop() { if !visited.insert(bb.index()) { continue; } - let blk = &callee_body.basic_blocks()[bb]; - for stmt in &blk.statements { - // Don't count StorageLive/StorageDead in the inlining cost. - match stmt.kind { - StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::Deinit(_) - | StatementKind::Nop => {} - _ => cost += INSTR_COST, - } - } - let term = blk.terminator(); - let mut is_drop = false; - match term.kind { - TerminatorKind::Drop { ref place, target, unwind } - | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } => { - is_drop = true; - work_list.push(target); - // If the place doesn't actually need dropping, treat it like - // a regular goto. - let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty); - if ty.needs_drop(tcx, self.param_env) { - cost += CALL_PENALTY; - if let Some(unwind) = unwind { - cost += LANDINGPAD_PENALTY; - work_list.push(unwind); - } - } else { - cost += INSTR_COST; - } - } - - TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. } - if first_block => - { - // If the function always diverges, don't inline - // unless the cost is zero - threshold = 0; - } - - TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => { - if let ty::FnDef(def_id, substs) = - *callsite.callee.subst_mir(self.tcx, &f.literal.ty()).kind() - { - if let Ok(substs) = - self.tcx.try_normalize_erasing_regions(self.param_env, substs) - { - if let Ok(Some(instance)) = - Instance::resolve(self.tcx, self.param_env, def_id, substs) - { - if callsite.callee.def_id() == instance.def_id() { - return Err("self-recursion"); - } else if self.history.contains(&instance) { - return Err("already inlined"); - } - } - } - // Don't give intrinsics the extra penalty for calls - if tcx.is_intrinsic(def_id) { - cost += INSTR_COST; - } else { - cost += CALL_PENALTY; - } - } else { - cost += CALL_PENALTY; - } - if cleanup.is_some() { - cost += LANDINGPAD_PENALTY; - } - } - TerminatorKind::Assert { cleanup, .. } => { - cost += CALL_PENALTY; + let blk = &callee_body.basic_blocks[bb]; + checker.visit_basic_block_data(bb, blk); - if cleanup.is_some() { - cost += LANDINGPAD_PENALTY; - } - } - TerminatorKind::Resume => cost += RESUME_PENALTY, - TerminatorKind::InlineAsm { cleanup, .. } => { - cost += INSTR_COST; + let term = blk.terminator(); + if let TerminatorKind::Drop { ref place, target, unwind } + | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } = term.kind + { + work_list.push(target); - if cleanup.is_some() { - cost += LANDINGPAD_PENALTY; + // If the place doesn't actually need dropping, treat it like a regular goto. + let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty); + if ty.needs_drop(tcx, self.param_env) && let Some(unwind) = unwind { + work_list.push(unwind); } - } - _ => cost += INSTR_COST, - } - - if !is_drop { - for succ in term.successors() { - work_list.push(succ); - } + } else { + work_list.extend(term.successors()) } - - first_block = false; } // Count up the cost of local variables and temps, if we know the size // use that, otherwise we use a moderately-large dummy cost. - - let ptr_size = tcx.data_layout.pointer_size.bytes(); - for v in callee_body.vars_and_temps_iter() { - let ty = callsite.callee.subst_mir(self.tcx, &callee_body.local_decls[v].ty); - // Cost of the var is the size in machine-words, if we know - // it. - if let Some(size) = type_size_of(tcx, self.param_env, ty) { - cost += ((size + ptr_size - 1) / ptr_size) as usize; - } else { - cost += UNKNOWN_SIZE_COST; - } + checker.visit_local_decl(v, &callee_body.local_decls[v]); } + // Abort if type validation found anything fishy. + checker.validation?; + + let cost = checker.cost; if let InlineAttr::Always = callee_attrs.inline { debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost); Ok(()) + } else if cost <= threshold { + debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold); + Ok(()) } else { - if cost <= threshold { - debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold); - Ok(()) - } else { - debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold); - Err("cost above threshold") - } + debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold); + Err("cost above threshold") } } @@ -548,12 +536,12 @@ impl<'tcx> Inliner<'tcx> { ); expn_data.def_site = callee_body.span; let expn_data = - LocalExpnId::fresh(expn_data, self.tcx.create_stable_hashing_context()); + self.tcx.with_stable_hashing_context(|hcx| LocalExpnId::fresh(expn_data, hcx)); let mut integrator = Integrator { args: &args, new_locals: Local::new(caller_body.local_decls.len()).., new_scopes: SourceScope::new(caller_body.source_scopes.len()).., - new_blocks: BasicBlock::new(caller_body.basic_blocks().len()).., + new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., destination: dest, callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), callsite, @@ -571,7 +559,9 @@ impl<'tcx> Inliner<'tcx> { // If there are any locals without storage markers, give them storage only for the // duration of the call. for local in callee_body.vars_and_temps_iter() { - if integrator.always_live_locals.contains(local) { + if !callee_body.local_decls[local].internal + && integrator.always_live_locals.contains(local) + { let new_local = integrator.map_local(local); caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, @@ -584,7 +574,9 @@ impl<'tcx> Inliner<'tcx> { // the slice once. let mut n = 0; for local in callee_body.vars_and_temps_iter().rev() { - if integrator.always_live_locals.contains(local) { + if !callee_body.local_decls[local].internal + && integrator.always_live_locals.contains(local) + { let new_local = integrator.map_local(local); caller_body[block].statements.push(Statement { source_info: callsite.source_info, @@ -612,11 +604,11 @@ impl<'tcx> Inliner<'tcx> { // `required_consts`, here we may not only have `ConstKind::Unevaluated` // because we are calling `subst_and_normalize_erasing_regions`. caller_body.required_consts.extend( - callee_body.required_consts.iter().copied().filter(|&ct| { - match ct.literal.const_for_ty() { - Some(ct) => matches!(ct.kind(), ConstKind::Unevaluated(_)), - None => true, + callee_body.required_consts.iter().copied().filter(|&ct| match ct.literal { + ConstantKind::Ty(_) => { + bug!("should never encounter ty::Unevaluated in `required_consts`") } + ConstantKind::Val(..) | ConstantKind::Unevaluated(..) => true, }), ); } @@ -750,6 +742,193 @@ fn type_size_of<'tcx>( tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes()) } +/// Verify that the callee body is compatible with the caller. +/// +/// This visitor mostly computes the inlining cost, +/// but also needs to verify that types match because of normalization failure. +struct CostChecker<'b, 'tcx> { + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + cost: usize, + callee_body: &'b Body<'tcx>, + instance: ty::Instance<'tcx>, + validation: Result<(), &'static str>, +} + +impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + // Don't count StorageLive/StorageDead in the inlining cost. + match statement.kind { + StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Deinit(_) + | StatementKind::Nop => {} + _ => self.cost += INSTR_COST, + } + + self.super_statement(statement, location); + } + + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { + let tcx = self.tcx; + match terminator.kind { + TerminatorKind::Drop { ref place, unwind, .. } + | TerminatorKind::DropAndReplace { ref place, unwind, .. } => { + // If the place doesn't actually need dropping, treat it like a regular goto. + let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty); + if ty.needs_drop(tcx, self.param_env) { + self.cost += CALL_PENALTY; + if unwind.is_some() { + self.cost += LANDINGPAD_PENALTY; + } + } else { + self.cost += INSTR_COST; + } + } + TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => { + let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty()); + self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { + // Don't give intrinsics the extra penalty for calls + INSTR_COST + } else { + CALL_PENALTY + }; + if cleanup.is_some() { + self.cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::Assert { cleanup, .. } => { + self.cost += CALL_PENALTY; + if cleanup.is_some() { + self.cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::Resume => self.cost += RESUME_PENALTY, + TerminatorKind::InlineAsm { cleanup, .. } => { + self.cost += INSTR_COST; + if cleanup.is_some() { + self.cost += LANDINGPAD_PENALTY; + } + } + _ => self.cost += INSTR_COST, + } + + self.super_terminator(terminator, location); + } + + /// Count up the cost of local variables and temps, if we know the size + /// use that, otherwise we use a moderately-large dummy cost. + fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) { + let tcx = self.tcx; + let ptr_size = tcx.data_layout.pointer_size.bytes(); + + let ty = self.instance.subst_mir(tcx, &local_decl.ty); + // Cost of the var is the size in machine-words, if we know + // it. + if let Some(size) = type_size_of(tcx, self.param_env, ty) { + self.cost += ((size + ptr_size - 1) / ptr_size) as usize; + } else { + self.cost += UNKNOWN_SIZE_COST; + } + + self.super_local_decl(local, local_decl) + } + + /// This method duplicates code from MIR validation in an attempt to detect type mismatches due + /// to normalization failure. + fn visit_projection_elem( + &mut self, + local: Local, + proj_base: &[PlaceElem<'tcx>], + elem: PlaceElem<'tcx>, + context: PlaceContext, + location: Location, + ) { + if let ProjectionElem::Field(f, ty) = elem { + let parent = Place { local, projection: self.tcx.intern_place_elems(proj_base) }; + let parent_ty = parent.ty(&self.callee_body.local_decls, self.tcx); + let check_equal = |this: &mut Self, f_ty| { + if !equal_up_to_regions(this.tcx, this.param_env, ty, f_ty) { + trace!(?ty, ?f_ty); + this.validation = Err("failed to normalize projection type"); + return; + } + }; + + let kind = match parent_ty.ty.kind() { + &ty::Opaque(def_id, substs) => { + self.tcx.bound_type_of(def_id).subst(self.tcx, substs).kind() + } + kind => kind, + }; + + match kind { + ty::Tuple(fields) => { + let Some(f_ty) = fields.get(f.as_usize()) else { + self.validation = Err("malformed MIR"); + return; + }; + check_equal(self, *f_ty); + } + ty::Adt(adt_def, substs) => { + let var = parent_ty.variant_index.unwrap_or(VariantIdx::from_u32(0)); + let Some(field) = adt_def.variant(var).fields.get(f.as_usize()) else { + self.validation = Err("malformed MIR"); + return; + }; + check_equal(self, field.ty(self.tcx, substs)); + } + ty::Closure(_, substs) => { + let substs = substs.as_closure(); + let Some(f_ty) = substs.upvar_tys().nth(f.as_usize()) else { + self.validation = Err("malformed MIR"); + return; + }; + check_equal(self, f_ty); + } + &ty::Generator(def_id, substs, _) => { + let f_ty = if let Some(var) = parent_ty.variant_index { + let gen_body = if def_id == self.callee_body.source.def_id() { + self.callee_body + } else { + self.tcx.optimized_mir(def_id) + }; + + let Some(layout) = gen_body.generator_layout() else { + self.validation = Err("malformed MIR"); + return; + }; + + let Some(&local) = layout.variant_fields[var].get(f) else { + self.validation = Err("malformed MIR"); + return; + }; + + let Some(&f_ty) = layout.field_tys.get(local) else { + self.validation = Err("malformed MIR"); + return; + }; + + f_ty + } else { + let Some(f_ty) = substs.as_generator().prefix_tys().nth(f.index()) else { + self.validation = Err("malformed MIR"); + return; + }; + + f_ty + }; + + check_equal(self, f_ty); + } + _ => self.validation = Err("malformed MIR"), + } + } + + self.super_projection_elem(local, proj_base, elem, context, location); + } +} + /** * Integrator. * diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index fd7de2bd1dc..b027f94925d 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -2,7 +2,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::mir::TerminatorKind; -use rustc_middle::ty::TypeFoldable; +use rustc_middle::ty::TypeVisitable; use rustc_middle::ty::{self, subst::SubstsRef, InstanceDef, TyCtxt}; use rustc_session::Limit; @@ -48,7 +48,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( trace!(?caller, ?param_env, ?substs, "cannot normalize, skipping"); continue; }; - let Some(callee) = ty::Instance::resolve(tcx, param_env, callee, substs).unwrap() else { + let Ok(Some(callee)) = ty::Instance::resolve(tcx, param_env, callee, substs) else { trace!(?callee, "cannot resolve, skipping"); continue; }; @@ -79,7 +79,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( // These have MIR and if that MIR is inlined, substituted and then inlining is run // again, a function item can end up getting inlined. Thus we'll be able to cause // a cycle that way - InstanceDef::VtableShim(_) + InstanceDef::VTableShim(_) | InstanceDef::ReifyShim(_) | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } @@ -153,7 +153,7 @@ pub(crate) fn mir_inliner_callees<'tcx>( _ => tcx.instance_mir(instance), }; let mut calls = FxIndexSet::default(); - for bb_data in body.basic_blocks() { + for bb_data in body.basic_blocks.iter() { let terminator = bb_data.terminator(); if let TerminatorKind::Call { func, .. } = &terminator.kind { let ty = func.ty(&body.local_decls, tcx); diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs index ea10ec5f25c..2f3c65869ef 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instcombine.rs @@ -16,9 +16,8 @@ impl<'tcx> MirPass<'tcx> for InstCombine { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); - let ctx = InstCombineContext { tcx, local_decls }; - for block in basic_blocks.iter_mut() { + let ctx = InstCombineContext { tcx, local_decls: &body.local_decls }; + for block in body.basic_blocks.as_mut() { for statement in block.statements.iter_mut() { match statement.kind { StatementKind::Assign(box (_place, ref mut rvalue)) => { diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index b7caa61ef07..e6fc8559571 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,7 +1,7 @@ #![allow(rustc::potential_query_instability)] #![feature(box_patterns)] #![feature(let_chains)] -#![feature(let_else)] +#![cfg_attr(bootstrap, feature(let_else))] #![feature(map_try_insert)] #![feature(min_specialization)] #![feature(never_type)] @@ -10,6 +10,7 @@ #![feature(trusted_step)] #![feature(try_blocks)] #![feature(yeet_expr)] +#![feature(if_let_guard)] #![recursion_limit = "256"] #[macro_use] @@ -26,10 +27,14 @@ use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::{self, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; -use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted}; +use rustc_middle::mir::{ + traversal, AnalysisPhase, Body, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand, + Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind, + TerminatorKind, +}; use rustc_middle::ty::query::Providers; -use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; -use rustc_span::{Span, Symbol}; +use rustc_middle::ty::{self, TyCtxt, TypeVisitable}; +use rustc_span::sym; #[macro_use] mod pass_manager; @@ -59,6 +64,7 @@ pub mod dump_mir; mod early_otherwise_branch; mod elaborate_box_derefs; mod elaborate_drops; +mod ffi_unwind_calls; mod function_item_references; mod generator; mod inline; @@ -98,6 +104,7 @@ pub fn provide(providers: &mut Providers) { check_unsafety::provide(providers); check_packed_ref::provide(providers); coverage::query::provide(providers); + ffi_unwind_calls::provide(providers); shim::provide(providers); *providers = Providers { mir_keys, @@ -137,6 +144,64 @@ pub fn provide(providers: &mut Providers) { }; } +fn remap_mir_for_const_eval_select<'tcx>( + tcx: TyCtxt<'tcx>, + mut body: Body<'tcx>, + context: hir::Constness, +) -> Body<'tcx> { + for bb in body.basic_blocks.as_mut().iter_mut() { + let terminator = bb.terminator.as_mut().expect("invalid terminator"); + match terminator.kind { + TerminatorKind::Call { + func: Operand::Constant(box Constant { ref literal, .. }), + ref mut args, + destination, + target, + cleanup, + fn_span, + .. + } if let ty::FnDef(def_id, _) = *literal.ty().kind() + && tcx.item_name(def_id) == sym::const_eval_select + && tcx.is_intrinsic(def_id) => + { + let [tupled_args, called_in_const, called_at_rt]: [_; 3] = std::mem::take(args).try_into().unwrap(); + let ty = tupled_args.ty(&body.local_decls, tcx); + let fields = ty.tuple_fields(); + let num_args = fields.len(); + let func = if context == hir::Constness::Const { called_in_const } else { called_at_rt }; + let (method, place): (fn(Place<'tcx>) -> Operand<'tcx>, Place<'tcx>) = match tupled_args { + Operand::Constant(_) => { + // there is no good way of extracting a tuple arg from a constant (const generic stuff) + // so we just create a temporary and deconstruct that. + let local = body.local_decls.push(LocalDecl::new(ty, fn_span)); + bb.statements.push(Statement { + source_info: SourceInfo::outermost(fn_span), + kind: StatementKind::Assign(Box::new((local.into(), Rvalue::Use(tupled_args.clone())))), + }); + (Operand::Move, local.into()) + } + Operand::Move(place) => (Operand::Move, place), + Operand::Copy(place) => (Operand::Copy, place), + }; + let place_elems = place.projection; + let arguments = (0..num_args).map(|x| { + let mut place_elems = place_elems.to_vec(); + place_elems.push(ProjectionElem::Field(x.into(), fields[x])); + let projection = tcx.intern_place_elems(&place_elems); + let place = Place { + local: place.local, + projection, + }; + method(place) + }).collect(); + terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, cleanup, from_hir_call: false, fn_span }; + } + _ => {} + } + } + body +} + fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { let def_id = def_id.expect_local(); tcx.mir_keys(()).contains(&def_id) @@ -157,21 +222,14 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { set: &'a mut FxIndexSet<LocalDefId>, } impl<'tcx> Visitor<'tcx> for GatherCtors<'_, 'tcx> { - fn visit_variant_data( - &mut self, - v: &'tcx hir::VariantData<'tcx>, - _: Symbol, - _: &'tcx hir::Generics<'tcx>, - _: hir::HirId, - _: Span, - ) { + fn visit_variant_data(&mut self, v: &'tcx hir::VariantData<'tcx>) { if let hir::VariantData::Tuple(_, hir_id) = *v { self.set.insert(self.tcx.hir().local_def_id(hir_id)); } intravisit::walk_struct_def(self, v) } } - tcx.hir().deep_visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }); + tcx.hir().visit_all_item_likes_in_crate(&mut GatherCtors { tcx, set: &mut set }); set } @@ -206,6 +264,8 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> } /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts! +/// FIXME(oli-obk): it's unclear whether we still need this phase (and its corresponding query). +/// We used to have this for pre-miri MIR based const eval. fn mir_const<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, @@ -215,7 +275,7 @@ fn mir_const<'tcx>( } // Unsafety check uses the raw mir, so make sure it is run. - if !tcx.sess.opts.debugging_opts.thir_unsafeck { + if !tcx.sess.opts.unstable_opts.thir_unsafeck { if let Some(param_did) = def.const_param_did { tcx.ensure().unsafety_check_result_for_const_arg((def.did, param_did)); } else { @@ -223,6 +283,9 @@ fn mir_const<'tcx>( } } + // has_ffi_unwind_calls query uses the raw mir, so make sure it is run. + tcx.ensure().has_ffi_unwind_calls(def.did); + let mut body = tcx.mir_built(def).steal(); rustc_middle::mir::dump_mir(tcx, None, "mir_map", &0, &body, |_, _| Ok(())); @@ -238,7 +301,6 @@ fn mir_const<'tcx>( // What we need to do constant evaluation. &simplify::SimplifyCfg::new("initial"), &rustc_peek::SanityCheck, // Just a lint - &marker::PhaseChange(MirPhase::Const), ], ); tcx.alloc_steal_mir(body) @@ -325,7 +387,9 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - .body_const_context(def.did) .expect("mir_for_ctfe should not be used for runtime functions"); - let mut body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); + let body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); + + let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::Const); match context { // Do not const prop functions, either they get executed at runtime or exported to metadata, @@ -344,7 +408,10 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - pm::run_passes( tcx, &mut body, - &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimized)], + &[ + &const_prop::ConstProp, + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)), + ], ); } } @@ -384,37 +451,61 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( body.tainted_by_errors = Some(error_reported); } - // IMPORTANT - pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]); + run_analysis_to_runtime_passes(tcx, &mut body); + + tcx.alloc_steal_mir(body) +} + +fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial)); + let did = body.source.def_id(); + + debug!("analysis_mir_cleanup({:?})", did); + run_analysis_cleanup_passes(tcx, body); + assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup)); // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled. if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) { pm::run_passes( tcx, - &mut body, + body, &[ - &simplify::SimplifyCfg::new("remove-false-edges"), &remove_uninit_drops::RemoveUninitDrops, + &simplify::SimplifyCfg::new("remove-false-edges"), ], ); check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint } - run_post_borrowck_cleanup_passes(tcx, &mut body); - assert!(body.phase == MirPhase::Deaggregated); - tcx.alloc_steal_mir(body) + debug!("runtime_mir_lowering({:?})", did); + run_runtime_lowering_passes(tcx, body); + assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial)); + + debug!("runtime_mir_cleanup({:?})", did); + run_runtime_cleanup_passes(tcx, body); + assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup)); } -/// After this series of passes, no lifetime analysis based on borrowing can be done. -fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - debug!("post_borrowck_cleanup({:?})", body.source.def_id()); +// FIXME(JakobDegen): Can we make these lists of passes consts? - let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ - // Remove all things only needed by analysis +/// After this series of passes, no lifetime analysis based on borrowing can be done. +fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let passes: &[&dyn MirPass<'tcx>] = &[ + &remove_false_edges::RemoveFalseEdges, &simplify_branches::SimplifyConstCondition::new("initial"), &remove_noop_landing_pads::RemoveNoopLandingPads, &cleanup_post_borrowck::CleanupNonCodegenStatements, &simplify::SimplifyCfg::new("early-opt"), + &deref_separator::Derefer, + &marker::PhaseChange(MirPhase::Analysis(AnalysisPhase::PostCleanup)), + ]; + + pm::run_passes(tcx, body, passes); +} + +/// Returns the sequence of passes that lowers analysis to runtime MIR. +fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let passes: &[&dyn MirPass<'tcx>] = &[ // These next passes must be executed together &add_call_guards::CriticalCallEdges, &elaborate_drops::ElaborateDrops, @@ -427,18 +518,28 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc &add_moves_for_packed_drops::AddMovesForPackedDrops, // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late, // but before optimizations begin. - &deref_separator::Derefer, &elaborate_box_derefs::ElaborateBoxDerefs, + &generator::StateTransform, &add_retag::AddRetag, - &lower_intrinsics::LowerIntrinsics, - &simplify::SimplifyCfg::new("elaborate-drops"), - // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening - // and it can help optimizations. + // Deaggregator is necessary for const prop. We may want to consider implementing + // CTFE support for aggregates. &deaggregator::Deaggregator, &Lint(const_prop_lint::ConstProp), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Initial)), + ]; + pm::run_passes_no_validate(tcx, body, passes); +} + +/// Returns the sequence of passes that do the initial cleanup of runtime MIR. +fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let passes: &[&dyn MirPass<'tcx>] = &[ + &elaborate_box_derefs::ElaborateBoxDerefs, + &lower_intrinsics::LowerIntrinsics, + &simplify::SimplifyCfg::new("elaborate-drops"), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::PostCleanup)), ]; - pm::run_passes(tcx, body, post_borrowck_cleanup); + pm::run_passes(tcx, body, passes); } fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -446,9 +547,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { WithMinOptLevel(1, x) } - // Lowering generator control-flow and variables has to happen before we do anything else - // to them. We run some optimizations before that, because they may be harder to do on the state - // machine than on MIR with async primitives. + // The main optimizations that we do on MIR. pm::run_passes( tcx, body, @@ -460,17 +559,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &uninhabited_enum_branching::UninhabitedEnumBranching, &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")), &inline::Inline, - &generator::StateTransform, - ], - ); - - assert!(body.phase == MirPhase::GeneratorsLowered); - - // The main optimizations that we do on MIR. - pm::run_passes( - tcx, - body, - &[ &remove_storage_markers::RemoveStorageMarkers, &remove_zsts::RemoveZsts, &const_goto::ConstGoto, @@ -502,7 +590,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &deduplicate_blocks::DeduplicateBlocks, // Some cleanup necessary at least for LLVM and potentially other codegen backends. &add_call_guards::CriticalCallEdges, - &marker::PhaseChange(MirPhase::Optimized), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)), // Dump the end result for testing and debugging purposes. &dump_mir::Marker("PreCodegen"), ], @@ -534,8 +622,9 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), } debug!("about to call mir_drops_elaborated..."); - let mut body = + let body = tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); + let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst); debug!("body: {:#?}", body); run_optimization_passes(tcx, &mut body); @@ -561,7 +650,7 @@ fn promoted_mir<'tcx>( if let Some(error_reported) = tainted_by_errors { body.tainted_by_errors = Some(error_reported); } - run_post_borrowck_cleanup_passes(tcx, body); + run_analysis_to_runtime_passes(tcx, body); } debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 989b94b68c1..9892580e63d 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -11,8 +11,8 @@ pub struct LowerIntrinsics; impl<'tcx> MirPass<'tcx> for LowerIntrinsics { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); - for block in basic_blocks { + let local_decls = &body.local_decls; + for block in body.basic_blocks.as_mut() { let terminator = block.terminator.as_mut().unwrap(); if let TerminatorKind::Call { func, args, destination, target, .. } = &mut terminator.kind @@ -46,12 +46,31 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { let mut args = args.drain(..); block.statements.push(Statement { source_info: terminator.source_info, - kind: StatementKind::CopyNonOverlapping(Box::new( - rustc_middle::mir::CopyNonOverlapping { - src: args.next().unwrap(), - dst: args.next().unwrap(), - count: args.next().unwrap(), - }, + kind: StatementKind::Intrinsic(Box::new( + NonDivergingIntrinsic::CopyNonOverlapping( + rustc_middle::mir::CopyNonOverlapping { + src: args.next().unwrap(), + dst: args.next().unwrap(), + count: args.next().unwrap(), + }, + ), + )), + }); + assert_eq!( + args.next(), + None, + "Extra argument for copy_non_overlapping intrinsic" + ); + drop(args); + terminator.kind = TerminatorKind::Goto { target }; + } + sym::assume => { + let target = target.unwrap(); + let mut args = args.drain(..); + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Intrinsic(Box::new( + NonDivergingIntrinsic::Assume(args.next().unwrap()), )), }); assert_eq!( diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index 07163cfe575..2f02d00ec9f 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -11,7 +11,7 @@ pub struct LowerSliceLenCalls; impl<'tcx> MirPass<'tcx> for LowerSliceLenCalls { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.opts.mir_opt_level() > 0 + sess.mir_opt_level() > 0 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -26,11 +26,11 @@ pub fn lower_slice_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { return; }; - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); - + // The one successor remains unchanged, so no need to invalidate + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); for block in basic_blocks { // lower `<[_]>::len` calls - lower_slice_len_call(tcx, block, &*local_decls, slice_len_fn_item_def_id); + lower_slice_len_call(tcx, block, &body.local_decls, slice_len_fn_item_def_id); } } diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 7cd7d26328a..a0ba69c89b0 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -48,7 +48,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { let def_id = body.source.def_id(); let param_env = tcx.param_env(def_id); - let (bbs, local_decls) = body.basic_blocks_and_local_decls_mut(); + let bbs = body.basic_blocks.as_mut(); let mut should_cleanup = false; 'outer: for bb_idx in bbs.indices() { if !tcx.consider_optimizing(|| format!("MatchBranchSimplification {:?} ", def_id)) { @@ -108,7 +108,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { // Introduce a temporary for the discriminant value. let source_info = bbs[bb_idx].terminator().source_info; - let discr_local = local_decls.push(LocalDecl::new(switch_ty, source_info.span)); + let discr_local = body.local_decls.push(LocalDecl::new(switch_ty, source_info.span)); // We already checked that first and second are different blocks, // and bb_idx has a different terminator from both of them. diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index 22b6dead99c..3957cd92c4e 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -15,7 +15,7 @@ impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // find basic blocks with no statement and a return terminator - let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks().len()); + let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks.len()); let def_id = body.source.def_id(); let bbs = body.basic_blocks_mut(); for idx in bbs.indices() { diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs index 0f45711baa3..a159e617178 100644 --- a/compiler/rustc_mir_transform/src/normalize_array_len.rs +++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs @@ -21,10 +21,10 @@ impl<'tcx> MirPass<'tcx> for NormalizeArrayLen { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // early returns for edge cases of highly unrolled functions - if body.basic_blocks().len() > MAX_NUM_BLOCKS { + if body.basic_blocks.len() > MAX_NUM_BLOCKS { return; } - if body.local_decls().len() > MAX_NUM_LOCALS { + if body.local_decls.len() > MAX_NUM_LOCALS { return; } normalize_array_len_calls(tcx, body) @@ -32,7 +32,9 @@ impl<'tcx> MirPass<'tcx> for NormalizeArrayLen { } pub fn normalize_array_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); + // We don't ever touch terminators, so no need to invalidate the CFG cache + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); + let local_decls = &mut body.local_decls; // do a preliminary analysis to see if we ever have locals of type `[T;N]` or `&[T;N]` let mut interesting_locals = BitSet::new_empty(local_decls.len()); diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 444b4126e88..4291e81c78c 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -53,10 +53,10 @@ impl<'tcx> MirPass<'tcx> for RenameReturnPlace { def_id, returned_local ); - RenameToReturnPlace { tcx, to_rename: returned_local }.visit_body(body); + RenameToReturnPlace { tcx, to_rename: returned_local }.visit_body_preserves_cfg(body); // Clean up the `NOP`s we inserted for statements made useless by our renaming. - for block_data in body.basic_blocks_mut() { + for block_data in body.basic_blocks.as_mut_preserves_cfg() { block_data.statements.retain(|stmt| stmt.kind != mir::StatementKind::Nop); } @@ -89,7 +89,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { } let mut copied_to_return_place = None; - for block in body.basic_blocks().indices() { + for block in body.basic_blocks.indices() { // Look for blocks with a `Return` terminator. if !matches!(body[block].terminator().kind, mir::TerminatorKind::Return) { continue; @@ -122,7 +122,7 @@ fn find_local_assigned_to_return_place( body: &mut mir::Body<'_>, ) -> Option<Local> { let mut block = start; - let mut seen = HybridBitSet::new_empty(body.basic_blocks().len()); + let mut seen = HybridBitSet::new_empty(body.basic_blocks.len()); // Iterate as long as `block` has exactly one predecessor that we have not yet visited. while seen.insert(block) { @@ -133,7 +133,7 @@ fn find_local_assigned_to_return_place( return local; } - match body.predecessors()[block].as_slice() { + match body.basic_blocks.predecessors()[block].as_slice() { &[pred] => block = pred, _ => return None, } @@ -219,7 +219,7 @@ impl IsReturnPlaceRead { } impl<'tcx> Visitor<'tcx> for IsReturnPlaceRead { - fn visit_local(&mut self, &l: &Local, ctxt: PlaceContext, _: Location) { + fn visit_local(&mut self, l: Local, ctxt: PlaceContext, _: Location) { if l == mir::RETURN_PLACE && ctxt.is_use() && !ctxt.is_place_assignment() { self.0 = true; } diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 2380391d09a..67dae71468f 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -1,6 +1,6 @@ use std::borrow::Cow; -use rustc_middle::mir::{self, Body, MirPhase}; +use rustc_middle::mir::{self, Body, MirPhase, RuntimePhase}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -72,48 +72,62 @@ where } } +/// Run the sequence of passes without validating the MIR after each pass. The MIR is still +/// validated at the end. +pub fn run_passes_no_validate<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + passes: &[&dyn MirPass<'tcx>], +) { + run_passes_inner(tcx, body, passes, false); +} + pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>]) { + run_passes_inner(tcx, body, passes, true); +} + +fn run_passes_inner<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + passes: &[&dyn MirPass<'tcx>], + validate_each: bool, +) { let start_phase = body.phase; let mut cnt = 0; - let validate = tcx.sess.opts.debugging_opts.validate_mir; - let overridden_passes = &tcx.sess.opts.debugging_opts.mir_enable_passes; + let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir; + let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes; trace!(?overridden_passes); - if validate { - validate_body(tcx, body, format!("start of phase transition from {:?}", start_phase)); - } - for pass in passes { let name = pass.name(); - if let Some((_, polarity)) = overridden_passes.iter().rev().find(|(s, _)| s == &*name) { - trace!( - pass = %name, - "{} as requested by flag", - if *polarity { "Running" } else { "Not running" }, - ); - if !polarity { - continue; - } - } else { - if !pass.is_enabled(&tcx.sess) { - continue; - } - } - let dump_enabled = pass.is_mir_dump_enabled(); + // Gather information about what we should be doing for this pass + let overridden = + overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| { + trace!( + pass = %name, + "{} as requested by flag", + if *polarity { "Running" } else { "Not running" }, + ); + *polarity + }); + let is_enabled = overridden.unwrap_or_else(|| pass.is_enabled(&tcx.sess)); + let new_phase = pass.phase_change(); + let dump_enabled = (is_enabled && pass.is_mir_dump_enabled()) || new_phase.is_some(); + let validate = (validate && is_enabled) + || new_phase == Some(MirPhase::Runtime(RuntimePhase::Optimized)); if dump_enabled { dump_mir(tcx, body, start_phase, &name, cnt, false); } - - pass.run_pass(tcx, body); - + if is_enabled { + pass.run_pass(tcx, body); + } if dump_enabled { dump_mir(tcx, body, start_phase, &name, cnt, true); cnt += 1; } - if let Some(new_phase) = pass.phase_change() { if body.phase >= new_phase { panic!("Invalid MIR phase transition from {:?} to {:?}", body.phase, new_phase); @@ -121,15 +135,10 @@ pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn body.phase = new_phase; } - if validate { - validate_body(tcx, body, format!("after pass {}", pass.name())); + validate_body(tcx, body, format!("after pass {}", name)); } } - - if validate || body.phase == MirPhase::Optimized { - validate_body(tcx, body, format!("end of phase transition to {:?}", body.phase)); - } } pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { @@ -144,7 +153,7 @@ pub fn dump_mir<'tcx>( cnt: usize, is_after: bool, ) { - let phase_index = phase as u32; + let phase_index = phase.phase_index(); mir::dump_mir( tcx, diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index 89808d3d4cd..f1bbf2ea7e8 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -51,7 +51,7 @@ impl RemoveNoopLandingPads { StatementKind::Assign { .. } | StatementKind::SetDiscriminant { .. } | StatementKind::Deinit(..) - | StatementKind::CopyNonOverlapping(..) + | StatementKind::Intrinsic(..) | StatementKind::Retag { .. } => { return false; } @@ -83,9 +83,9 @@ impl RemoveNoopLandingPads { fn remove_nop_landing_pads(&self, body: &mut Body<'_>) { debug!("body: {:#?}", body); - // make sure there's a single resume block + // make sure there's a resume block let resume_block = { - let patch = MirPatch::new(body); + let mut patch = MirPatch::new(body); let resume_block = patch.resume_block(); patch.apply(body); resume_block @@ -94,7 +94,7 @@ impl RemoveNoopLandingPads { let mut jumps_folded = 0; let mut landing_pads_removed = 0; - let mut nop_landing_pads = BitSet::new_empty(body.basic_blocks().len()); + let mut nop_landing_pads = BitSet::new_empty(body.basic_blocks.len()); // This is a post-order traversal, so that if A post-dominates B // then A will be visited before B. diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs index c9b6e1459d3..dbe082e9093 100644 --- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs +++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs @@ -17,7 +17,7 @@ impl<'tcx> MirPass<'tcx> for RemoveStorageMarkers { } trace!("Running RemoveStorageMarkers on {:?}", body.source); - for data in body.basic_blocks_mut() { + for data in body.basic_blocks.as_mut_preserves_cfg() { data.statements.retain(|statement| match statement.kind { StatementKind::StorageLive(..) | StatementKind::StorageDead(..) diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs index efa45883eab..78b6f714a9b 100644 --- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs @@ -21,7 +21,7 @@ pub struct RemoveUninitDrops; impl<'tcx> MirPass<'tcx> for RemoveUninitDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let param_env = tcx.param_env(body.source.def_id()); - let Ok(move_data) = MoveData::gather_moves(body, tcx, param_env) else { + let Ok((_,move_data)) = MoveData::gather_moves(body, tcx, param_env) else { // We could continue if there are move errors, but there's not much point since our // init data isn't complete. return; @@ -35,7 +35,7 @@ impl<'tcx> MirPass<'tcx> for RemoveUninitDrops { .into_results_cursor(body); let mut to_remove = vec![]; - for (bb, block) in body.basic_blocks().iter_enumerated() { + for (bb, block) in body.basic_blocks.iter_enumerated() { let terminator = block.terminator(); let (TerminatorKind::Drop { place, .. } | TerminatorKind::DropAndReplace { place, .. }) = &terminator.kind @@ -102,7 +102,7 @@ fn is_needs_drop_and_init<'tcx>( let field_needs_drop_and_init = |(f, f_ty, mpi)| { let child = move_path_children_matching(move_data, mpi, |x| x.is_field_to(f)); let Some(mpi) = child else { - return f_ty.needs_drop(tcx, param_env); + return Ty::needs_drop(f_ty, tcx, param_env); }; is_needs_drop_and_init(tcx, param_env, maybe_inits, move_data, f_ty, mpi) diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs index 921a11a3a06..84ccf6e1f61 100644 --- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs @@ -20,11 +20,10 @@ impl<'tcx> MirPass<'tcx> for RemoveUnneededDrops { let param_env = tcx.param_env_reveal_all_normalized(did); let mut should_simplify = false; - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); - for block in basic_blocks { + for block in body.basic_blocks.as_mut() { let terminator = block.terminator_mut(); if let TerminatorKind::Drop { place, target, .. } = terminator.kind { - let ty = place.ty(local_decls, tcx); + let ty = place.ty(&body.local_decls, tcx); if ty.ty.needs_drop(tcx, param_env) { continue; } diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index aaee6f491cd..40be4f146db 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -18,8 +18,9 @@ impl<'tcx> MirPass<'tcx> for RemoveZsts { return; } let param_env = tcx.param_env(body.source.def_id()); - let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); - for block in basic_blocks.iter_mut() { + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); + let local_decls = &body.local_decls; + for block in basic_blocks { for statement in block.statements.iter_mut() { if let StatementKind::Assign(box (place, _)) | StatementKind::Deinit(box place) = statement.kind diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index 827ce0c02ac..cc75947d9dd 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -1,5 +1,5 @@ use rustc_middle::mir::visit::Visitor; -use rustc_middle::mir::{Constant, Location}; +use rustc_middle::mir::{Constant, ConstantKind, Location}; use rustc_middle::ty::ConstKind; pub struct RequiredConstsVisitor<'a, 'tcx> { @@ -15,8 +15,13 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> { impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) { let literal = constant.literal; - if let Some(ct) = literal.const_for_ty() && let ConstKind::Unevaluated(_) = ct.kind() { - self.required_consts.push(*constant); + match literal { + ConstantKind::Ty(c) => match c.kind() { + ConstKind::Param(_) => {} + _ => bug!("only ConstKind::Param should be encountered here, got {:#?}", c), + }, + ConstantKind::Unevaluated(..) => self.required_consts.push(*constant), + ConstantKind::Val(..) => {} } } } diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index 8ea550fa123..abe6cb285f5 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -9,7 +9,7 @@ pub struct RevealAll; impl<'tcx> MirPass<'tcx> for RevealAll { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.opts.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess) + sess.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess) } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -19,7 +19,7 @@ impl<'tcx> MirPass<'tcx> for RevealAll { } let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - RevealAllVisitor { tcx, param_env }.visit_body(body); + RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body); } } diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 33ea1c4ba2f..2f116aaa958 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -61,8 +61,8 @@ impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { /// Returns the amount of blocks that were duplicated pub fn separate_const_switch(body: &mut Body<'_>) -> usize { let mut new_blocks: SmallVec<[(BasicBlock, BasicBlock); 6]> = SmallVec::new(); - let predecessors = body.predecessors(); - 'block_iter: for (block_id, block) in body.basic_blocks().iter_enumerated() { + let predecessors = body.basic_blocks.predecessors(); + 'block_iter: for (block_id, block) in body.basic_blocks.iter_enumerated() { if let TerminatorKind::SwitchInt { discr: Operand::Copy(switch_place) | Operand::Move(switch_place), .. @@ -90,7 +90,7 @@ pub fn separate_const_switch(body: &mut Body<'_>) -> usize { let mut predecessors_left = predecessors[block_id].len(); 'predec_iter: for predecessor_id in predecessors[block_id].iter().copied() { - let predecessor = &body.basic_blocks()[predecessor_id]; + let predecessor = &body.basic_blocks[predecessor_id]; // First we make sure the predecessor jumps // in a reasonable way @@ -218,6 +218,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData< // These rvalues move the place to track Rvalue::Cast(_, Operand::Copy(place) | Operand::Move(place), _) | Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) + | Rvalue::CopyForDeref(place) | Rvalue::UnaryOp(_, Operand::Copy(place) | Operand::Move(place)) | Rvalue::Discriminant(place) => tracked_place = place, } @@ -248,7 +249,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData< | StatementKind::AscribeUserType(_, _) | StatementKind::Coverage(_) | StatementKind::StorageDead(_) - | StatementKind::CopyNonOverlapping(_) + | StatementKind::Intrinsic(_) | StatementKind::Nop => {} } } @@ -279,6 +280,7 @@ fn find_determining_place<'tcx>( // that may be const in the predecessor Rvalue::Use(Operand::Move(new) | Operand::Copy(new)) | Rvalue::UnaryOp(_, Operand::Copy(new) | Operand::Move(new)) + | Rvalue::CopyForDeref(new) | Rvalue::Cast(_, Operand::Move(new) | Operand::Copy(new), _) | Rvalue::Repeat(Operand::Move(new) | Operand::Copy(new), _) | Rvalue::Discriminant(new) @@ -315,7 +317,7 @@ fn find_determining_place<'tcx>( | StatementKind::Retag(_, _) | StatementKind::AscribeUserType(_, _) | StatementKind::Coverage(_) - | StatementKind::CopyNonOverlapping(_) + | StatementKind::Intrinsic(_) | StatementKind::Nop => {} // If the discriminant is set, it is always set diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 3be1783ae33..88213620029 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -4,7 +4,7 @@ use rustc_hir::lang_items::LangItem; use rustc_middle::mir::*; use rustc_middle::ty::query::Providers; use rustc_middle::ty::subst::{InternalSubsts, Subst}; -use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt}; +use rustc_middle::ty::{self, EarlyBinder, GeneratorSubsts, Ty, TyCtxt}; use rustc_target::abi::VariantIdx; use rustc_index::vec::{Idx, IndexVec}; @@ -17,8 +17,8 @@ use std::iter; use crate::util::expand_aggregate; use crate::{ - abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, marker, pass_manager as pm, - remove_noop_landing_pads, simplify, + abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, marker, + pass_manager as pm, remove_noop_landing_pads, simplify, }; use rustc_middle::mir::patch::MirPatch; use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle}; @@ -32,7 +32,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' let mut result = match instance { ty::InstanceDef::Item(..) => bug!("item {:?} passed to make_shim", instance), - ty::InstanceDef::VtableShim(def_id) => { + ty::InstanceDef::VTableShim(def_id) => { build_call_shim(tcx, instance, Some(Adjustment::Deref), CallKind::Direct(def_id)) } ty::InstanceDef::FnPtrShim(def_id, ty) => { @@ -92,11 +92,12 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' &mut result, &[ &add_moves_for_packed_drops::AddMovesForPackedDrops, + &deref_separator::Derefer, &remove_noop_landing_pads::RemoveNoopLandingPads, &simplify::SimplifyCfg::new("make_shim"), &add_call_guards::CriticalCallEdges, &abort_unwinding_calls::AbortUnwindingCalls, - &marker::PhaseChange(MirPhase::Const), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)), ], ); @@ -113,7 +114,7 @@ enum Adjustment { /// We get passed `&[mut] self` and call the target with `*self`. /// /// This either copies `self` (if `Self: Copy`, eg. for function items), or moves out of it - /// (for `VtableShim`, which effectively is passed `&own Self`). + /// (for `VTableShim`, which effectively is passed `&own Self`). Deref, /// We get passed `self: Self` and call the target with `&mut self`. @@ -176,7 +177,7 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) if ty.is_some() { // The first argument (index 0), but add 1 for the return value. let dropee_ptr = Place::from(Local::new(1 + 0)); - if tcx.sess.opts.debugging_opts.mir_emit_retag { + if tcx.sess.opts.unstable_opts.mir_emit_retag { // Function arguments should be retagged, and we make this one raw. body.basic_blocks_mut()[START_BLOCK].statements.insert( 0, @@ -322,6 +323,9 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) - builder.tuple_like_shim(dest, src, substs.as_closure().upvar_tys()) } ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()), + ty::Generator(gen_def_id, substs, hir::Movability::Movable) => { + builder.generator_shim(dest, src, *gen_def_id, substs.as_generator()) + } _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty), }; @@ -387,7 +391,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { /// offset=0 will give you the index of the next BasicBlock, /// offset=1 will give the index of the next-to-next block, /// offset=-1 will give you the index of the last-created block - fn block_index_offset(&mut self, offset: usize) -> BasicBlock { + fn block_index_offset(&self, offset: usize) -> BasicBlock { BasicBlock::new(self.blocks.len() + offset) } @@ -460,49 +464,106 @@ impl<'tcx> CloneShimBuilder<'tcx> { ); } - fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I) + fn clone_fields<I>( + &mut self, + dest: Place<'tcx>, + src: Place<'tcx>, + target: BasicBlock, + mut unwind: BasicBlock, + tys: I, + ) -> BasicBlock where I: IntoIterator<Item = Ty<'tcx>>, { - let mut previous_field = None; + // For an iterator of length n, create 2*n + 1 blocks. for (i, ity) in tys.into_iter().enumerate() { + // Each iteration creates two blocks, referred to here as block 2*i and block 2*i + 1. + // + // Block 2*i attempts to clone the field. If successful it branches to 2*i + 2 (the + // next clone block). If unsuccessful it branches to the previous unwind block, which + // is initially the `unwind` argument passed to this function. + // + // Block 2*i + 1 is the unwind block for this iteration. It drops the cloned value + // created by block 2*i. We store this block in `unwind` so that the next clone block + // will unwind to it if cloning fails. + let field = Field::new(i); let src_field = self.tcx.mk_place_field(src, field, ity); let dest_field = self.tcx.mk_place_field(dest, field, ity); - // #(2i + 1) is the cleanup block for the previous clone operation - let cleanup_block = self.block_index_offset(1); - // #(2i + 2) is the next cloning block - // (or the Return terminator if this is the last block) + let next_unwind = self.block_index_offset(1); let next_block = self.block_index_offset(2); + self.make_clone_call(dest_field, src_field, ity, next_block, unwind); + self.block( + vec![], + TerminatorKind::Drop { place: dest_field, target: unwind, unwind: None }, + true, + ); + unwind = next_unwind; + } + // If all clones succeed then we end up here. + self.block(vec![], TerminatorKind::Goto { target }, false); + unwind + } - // BB #(2i) - // `dest.i = Clone::clone(&src.i);` - // Goto #(2i + 2) if ok, #(2i + 1) if unwinding happens. - self.make_clone_call(dest_field, src_field, ity, next_block, cleanup_block); - - // BB #(2i + 1) (cleanup) - if let Some((previous_field, previous_cleanup)) = previous_field.take() { - // Drop previous field and goto previous cleanup block. - self.block( - vec![], - TerminatorKind::Drop { - place: previous_field, - target: previous_cleanup, - unwind: None, - }, - true, - ); - } else { - // Nothing to drop, just resume. - self.block(vec![], TerminatorKind::Resume, true); - } + fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I) + where + I: IntoIterator<Item = Ty<'tcx>>, + { + self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false); + let unwind = self.block(vec![], TerminatorKind::Resume, true); + let target = self.block(vec![], TerminatorKind::Return, false); - previous_field = Some((dest_field, cleanup_block)); - } + let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys); + } - self.block(vec![], TerminatorKind::Return, false); + fn generator_shim( + &mut self, + dest: Place<'tcx>, + src: Place<'tcx>, + gen_def_id: DefId, + substs: GeneratorSubsts<'tcx>, + ) { + self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false); + let unwind = self.block(vec![], TerminatorKind::Resume, true); + // This will get overwritten with a switch once we know the target blocks + let switch = self.block(vec![], TerminatorKind::Unreachable, false); + let unwind = self.clone_fields(dest, src, switch, unwind, substs.upvar_tys()); + let target = self.block(vec![], TerminatorKind::Return, false); + let unreachable = self.block(vec![], TerminatorKind::Unreachable, false); + let mut cases = Vec::with_capacity(substs.state_tys(gen_def_id, self.tcx).count()); + for (index, state_tys) in substs.state_tys(gen_def_id, self.tcx).enumerate() { + let variant_index = VariantIdx::new(index); + let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index); + let src = self.tcx.mk_place_downcast_unnamed(src, variant_index); + let clone_block = self.block_index_offset(1); + let start_block = self.block( + vec![self.make_statement(StatementKind::SetDiscriminant { + place: Box::new(Place::return_place()), + variant_index, + })], + TerminatorKind::Goto { target: clone_block }, + false, + ); + cases.push((index as u128, start_block)); + let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys); + } + let discr_ty = substs.discr_ty(self.tcx); + let temp = self.make_place(Mutability::Mut, discr_ty); + let rvalue = Rvalue::Discriminant(src); + let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue)))); + match &mut self.blocks[switch] { + BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => { + statements.push(statement); + *kind = TerminatorKind::SwitchInt { + discr: Operand::Move(temp), + switch_ty: discr_ty, + targets: SwitchTargets::new(cases.into_iter(), unreachable), + }; + } + BasicBlockData { terminator: None, .. } => unreachable!(), + } } } @@ -537,13 +598,12 @@ fn build_call_shim<'tcx>( }; let def_id = instance.def_id(); - let sig = tcx.fn_sig(def_id); - let mut sig = tcx.erase_late_bound_regions(sig); + let sig = tcx.bound_fn_sig(def_id); + let sig = sig.map_bound(|sig| tcx.erase_late_bound_regions(sig)); assert_eq!(sig_substs.is_some(), !instance.has_polymorphic_mir_body()); - if let Some(sig_substs) = sig_substs { - sig = EarlyBinder(sig).subst(tcx, sig_substs); - } + let mut sig = + if let Some(sig_substs) = sig_substs { sig.subst(tcx, sig_substs) } else { sig.0 }; if let CallKind::Indirect(fnty) = call_kind { // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This @@ -570,7 +630,7 @@ fn build_call_shim<'tcx>( // FIXME(eddyb) avoid having this snippet both here and in // `Instance::fn_sig` (introduce `InstanceDef::fn_sig`?). - if let ty::InstanceDef::VtableShim(..) = instance { + if let ty::InstanceDef::VTableShim(..) = instance { // Modify fn(self, ...) to fn(self: *mut Self, ...) let mut inputs_and_output = sig.inputs_and_output.to_vec(); let self_arg = &mut inputs_and_output[0]; diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 8a78ea5c82b..57d372fda56 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -28,6 +28,7 @@ //! return. use crate::MirPass; +use rustc_data_structures::fx::FxHashSet; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::mir::coverage::*; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; @@ -73,7 +74,7 @@ pub struct CfgSimplifier<'a, 'tcx> { impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { pub fn new(body: &'a mut Body<'tcx>) -> Self { - let mut pred_count = IndexVec::from_elem(0u32, body.basic_blocks()); + let mut pred_count = IndexVec::from_elem(0u32, &body.basic_blocks); // we can't use mir.predecessors() here because that counts // dead blocks, which we don't want to. @@ -262,12 +263,13 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let reachable = traversal::reachable_as_bitset(body); - let num_blocks = body.basic_blocks().len(); + let num_blocks = body.basic_blocks.len(); if num_blocks == reachable.count() { return; } - let basic_blocks = body.basic_blocks_mut(); + let basic_blocks = body.basic_blocks.as_mut(); + let source_scopes = &body.source_scopes; let mut replacements: Vec<_> = (0..num_blocks).map(BasicBlock::new).collect(); let mut used_blocks = 0; for alive_index in reachable.iter() { @@ -282,7 +284,7 @@ pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } if tcx.sess.instrument_coverage() { - save_unreachable_coverage(basic_blocks, used_blocks); + save_unreachable_coverage(basic_blocks, source_scopes, used_blocks); } basic_blocks.raw.truncate(used_blocks); @@ -311,56 +313,72 @@ pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { /// `Unreachable` coverage statements. These are non-executable statements whose /// code regions are still recorded in the coverage map, representing regions /// with `0` executions. +/// +/// If there are no live `Counter` `Coverage` statements remaining, we remove +/// `Coverage` statements along with the dead blocks. Since at least one +/// counter per function is required by LLVM (and necessary, to add the +/// `function_hash` to the counter's call to the LLVM intrinsic +/// `instrprof.increment()`). +/// +/// The `generator::StateTransform` MIR pass and MIR inlining can create +/// atypical conditions, where all live `Counter`s are dropped from the MIR. +/// +/// With MIR inlining we can have coverage counters belonging to different +/// instances in a single body, so the strategy described above is applied to +/// coverage counters from each instance individually. fn save_unreachable_coverage( basic_blocks: &mut IndexVec<BasicBlock, BasicBlockData<'_>>, + source_scopes: &IndexVec<SourceScope, SourceScopeData<'_>>, first_dead_block: usize, ) { - let has_live_counters = basic_blocks.raw[0..first_dead_block].iter().any(|live_block| { - live_block.statements.iter().any(|statement| { - if let StatementKind::Coverage(coverage) = &statement.kind { - matches!(coverage.kind, CoverageKind::Counter { .. }) - } else { - false + // Identify instances that still have some live coverage counters left. + let mut live = FxHashSet::default(); + for basic_block in &basic_blocks.raw[0..first_dead_block] { + for statement in &basic_block.statements { + let StatementKind::Coverage(coverage) = &statement.kind else { continue }; + let CoverageKind::Counter { .. } = coverage.kind else { continue }; + let instance = statement.source_info.scope.inlined_instance(source_scopes); + live.insert(instance); + } + } + + for block in &mut basic_blocks.raw[..first_dead_block] { + for statement in &mut block.statements { + let StatementKind::Coverage(_) = &statement.kind else { continue }; + let instance = statement.source_info.scope.inlined_instance(source_scopes); + if !live.contains(&instance) { + statement.make_nop(); } - }) - }); - if !has_live_counters { - // If there are no live `Counter` `Coverage` statements anymore, don't - // move dead coverage to the `START_BLOCK`. Just allow the dead - // `Coverage` statements to be dropped with the dead blocks. - // - // The `generator::StateTransform` MIR pass can create atypical - // conditions, where all live `Counter`s are dropped from the MIR. - // - // At least one Counter per function is required by LLVM (and necessary, - // to add the `function_hash` to the counter's call to the LLVM - // intrinsic `instrprof.increment()`). + } + } + + if live.is_empty() { return; } - // Retain coverage info for dead blocks, so coverage reports will still - // report `0` executions for the uncovered code regions. - let mut dropped_coverage = Vec::new(); - for dead_block in basic_blocks.raw[first_dead_block..].iter() { - for statement in dead_block.statements.iter() { - if let StatementKind::Coverage(coverage) = &statement.kind { - if let Some(code_region) = &coverage.code_region { - dropped_coverage.push((statement.source_info, code_region.clone())); - } + // Retain coverage for instances that still have some live counters left. + let mut retained_coverage = Vec::new(); + for dead_block in &basic_blocks.raw[first_dead_block..] { + for statement in &dead_block.statements { + let StatementKind::Coverage(coverage) = &statement.kind else { continue }; + let Some(code_region) = &coverage.code_region else { continue }; + let instance = statement.source_info.scope.inlined_instance(source_scopes); + if live.contains(&instance) { + retained_coverage.push((statement.source_info, code_region.clone())); } } } let start_block = &mut basic_blocks[START_BLOCK]; - for (source_info, code_region) in dropped_coverage { - start_block.statements.push(Statement { + start_block.statements.extend(retained_coverage.into_iter().map( + |(source_info, code_region)| Statement { source_info, kind: StatementKind::Coverage(Box::new(Coverage { kind: CoverageKind::Unreachable, code_region: Some(code_region), })), - }) - } + }, + )); } pub struct SimplifyLocals; @@ -394,7 +412,7 @@ pub fn simplify_locals<'tcx>(body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>) { if map.iter().any(Option::is_none) { // Update references to all vars and tmps now let mut updater = LocalUpdater { map, tcx }; - updater.visit_body(body); + updater.visit_body_preserves_cfg(body); body.local_decls.shrink_to_fit(); } @@ -481,7 +499,7 @@ impl UsedLocals { impl<'tcx> Visitor<'tcx> for UsedLocals { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match statement.kind { - StatementKind::CopyNonOverlapping(..) + StatementKind::Intrinsic(..) | StatementKind::Retag(..) | StatementKind::Coverage(..) | StatementKind::FakeRead(..) @@ -509,12 +527,12 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { } } - fn visit_local(&mut self, local: &Local, _ctx: PlaceContext, _location: Location) { + fn visit_local(&mut self, local: Local, _ctx: PlaceContext, _location: Location) { if self.increment { - self.use_count[*local] += 1; + self.use_count[local] += 1; } else { - assert_ne!(self.use_count[*local], 0); - self.use_count[*local] -= 1; + assert_ne!(self.use_count[local], 0); + self.use_count[local] -= 1; } } } @@ -530,7 +548,7 @@ fn remove_unused_definitions(used_locals: &mut UsedLocals, body: &mut Body<'_>) while modified { modified = false; - for data in body.basic_blocks_mut() { + for data in body.basic_blocks.as_mut_preserves_cfg() { // Remove unnecessary StorageLive and StorageDead annotations. data.statements.retain(|statement| { let keep = match &statement.kind { diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index bbfaace7041..321d8c63b6e 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -151,7 +151,7 @@ struct OptimizationFinder<'a, 'tcx> { impl<'tcx> OptimizationFinder<'_, 'tcx> { fn find_optimizations(&self) -> Vec<OptimizationInfo<'tcx>> { self.body - .basic_blocks() + .basic_blocks .iter_enumerated() .filter_map(|(bb_idx, bb)| { // find switch diff --git a/compiler/rustc_mir_transform/src/simplify_try.rs b/compiler/rustc_mir_transform/src/simplify_try.rs index b3d45c7a221..baeb620ef24 100644 --- a/compiler/rustc_mir_transform/src/simplify_try.rs +++ b/compiler/rustc_mir_transform/src/simplify_try.rs @@ -378,7 +378,7 @@ fn optimization_applies<'tcx>( impl<'tcx> MirPass<'tcx> for SimplifyArmIdentity { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // FIXME(77359): This optimization can result in unsoundness. - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { + if !tcx.sess.opts.unstable_opts.unsound_mir_opts { return; } @@ -386,14 +386,17 @@ impl<'tcx> MirPass<'tcx> for SimplifyArmIdentity { trace!("running SimplifyArmIdentity on {:?}", source); let local_uses = LocalUseCounter::get_local_uses(body); - let (basic_blocks, local_decls, debug_info) = - body.basic_blocks_local_decls_mut_and_var_debug_info(); - for bb in basic_blocks { + for bb in body.basic_blocks.as_mut() { if let Some(opt_info) = - get_arm_identity_info(&bb.statements, local_decls.len(), debug_info) + get_arm_identity_info(&bb.statements, body.local_decls.len(), &body.var_debug_info) { trace!("got opt_info = {:#?}", opt_info); - if !optimization_applies(&opt_info, local_decls, &local_uses, &debug_info) { + if !optimization_applies( + &opt_info, + &body.local_decls, + &local_uses, + &body.var_debug_info, + ) { debug!("optimization skipped for {:?}", source); continue; } @@ -431,7 +434,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyArmIdentity { // Fix the debug info to point to the right local for dbg_index in opt_info.dbg_info_to_adjust { - let dbg_info = &mut debug_info[dbg_index]; + let dbg_info = &mut body.var_debug_info[dbg_index]; assert!( matches!(dbg_info.value, VarDebugInfoContents::Place(_)), "value was not a Place" @@ -462,14 +465,14 @@ impl LocalUseCounter { } impl Visitor<'_> for LocalUseCounter { - fn visit_local(&mut self, local: &Local, context: PlaceContext, _location: Location) { + fn visit_local(&mut self, local: Local, context: PlaceContext, _location: Location) { if context.is_storage_marker() || context == PlaceContext::NonUse(NonUseContext::VarDebugInfo) { return; } - self.local_uses[*local] += 1; + self.local_uses[local] += 1; } } @@ -548,7 +551,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranchSame { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // This optimization is disabled by default for now due to // soundness concerns; see issue #89485 and PR #89489. - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { + if !tcx.sess.opts.unstable_opts.unsound_mir_opts { return; } @@ -593,7 +596,7 @@ struct SimplifyBranchSameOptimizationFinder<'a, 'tcx> { impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> { fn find(&self) -> Vec<SimplifyBranchSameOptimization> { self.body - .basic_blocks() + .basic_blocks .iter_enumerated() .filter_map(|(bb_idx, bb)| { let (discr_switched_on, targets_and_values) = match &bb.terminator().kind { @@ -629,7 +632,7 @@ impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> { let mut iter_bbs_reachable = targets_and_values .iter() - .map(|target_and_value| (target_and_value, &self.body.basic_blocks()[target_and_value.target])) + .map(|target_and_value| (target_and_value, &self.body.basic_blocks[target_and_value.target])) .filter(|(_, bb)| { // Reaching `unreachable` is UB so assume it doesn't happen. bb.terminator().kind != TerminatorKind::Unreachable diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs index bd196f11879..96ea15f1b80 100644 --- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs @@ -1,7 +1,7 @@ //! A pass that eliminates branches on uninhabited enum variants. use crate::MirPass; -use rustc_data_structures::stable_set::FxHashSet; +use rustc_data_structures::fx::FxHashSet; use rustc_middle::mir::{ BasicBlockData, Body, Local, Operand, Rvalue, StatementKind, SwitchTargets, Terminator, TerminatorKind, @@ -79,7 +79,7 @@ fn ensure_otherwise_unreachable<'tcx>( targets: &SwitchTargets, ) -> Option<BasicBlockData<'tcx>> { let otherwise = targets.otherwise(); - let bb = &body.basic_blocks()[otherwise]; + let bb = &body.basic_blocks[otherwise]; if bb.terminator().kind == TerminatorKind::Unreachable && bb.statements.iter().all(|s| matches!(&s.kind, StatementKind::StorageDead(_))) { @@ -102,10 +102,10 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("UninhabitedEnumBranching starting for {:?}", body.source); - for bb in body.basic_blocks().indices() { + for bb in body.basic_blocks.indices() { trace!("processing block {:?}", bb); - let Some(discriminant_ty) = get_switched_on_type(&body.basic_blocks()[bb], tcx, body) else { + let Some(discriminant_ty) = get_switched_on_type(&body.basic_blocks[bb], tcx, body) else { continue; }; diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs index f916ca36217..95fda2eafe8 100644 --- a/compiler/rustc_mir_transform/src/unreachable_prop.rs +++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs @@ -12,9 +12,8 @@ pub struct UnreachablePropagation; impl MirPass<'_> for UnreachablePropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt - // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code. - sess.mir_opt_level() >= 4 + // Enable only under -Zmir-opt-level=2 as this can make programs less debuggable. + sess.mir_opt_level() >= 2 } fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -38,7 +37,19 @@ impl MirPass<'_> for UnreachablePropagation { } } + // We do want do keep some unreachable blocks, but make them empty. + for bb in unreachable_blocks { + if !tcx.consider_optimizing(|| { + format!("UnreachablePropagation {:?} ", body.source.def_id()) + }) { + break; + } + + body.basic_blocks_mut()[bb].statements.clear(); + } + let replaced = !replacements.is_empty(); + for (bb, terminator_kind) in replacements { if !tcx.consider_optimizing(|| { format!("UnreachablePropagation {:?} ", body.source.def_id()) @@ -57,42 +68,55 @@ impl MirPass<'_> for UnreachablePropagation { fn remove_successors<'tcx, F>( terminator_kind: &TerminatorKind<'tcx>, - predicate: F, + is_unreachable: F, ) -> Option<TerminatorKind<'tcx>> where F: Fn(BasicBlock) -> bool, { - let terminator = match *terminator_kind { - TerminatorKind::Goto { target } if predicate(target) => TerminatorKind::Unreachable, - TerminatorKind::SwitchInt { ref discr, switch_ty, ref targets } => { + let terminator = match terminator_kind { + // This will unconditionally run into an unreachable and is therefore unreachable as well. + TerminatorKind::Goto { target } if is_unreachable(*target) => TerminatorKind::Unreachable, + TerminatorKind::SwitchInt { targets, discr, switch_ty } => { let otherwise = targets.otherwise(); - let original_targets_len = targets.iter().len() + 1; - let (mut values, mut targets): (Vec<_>, Vec<_>) = - targets.iter().filter(|(_, bb)| !predicate(*bb)).unzip(); + // If all targets are unreachable, we can be unreachable as well. + if targets.all_targets().iter().all(|bb| is_unreachable(*bb)) { + TerminatorKind::Unreachable + } else if is_unreachable(otherwise) { + // If there are multiple targets, don't delete unreachable branches (like an unreachable otherwise) + // unless otherwise is unreachable, in which case deleting a normal branch causes it to be merged with + // the otherwise, keeping its unreachable. + // This looses information about reachability causing worse codegen. + // For example (see src/test/codegen/match-optimizes-away.rs) + // + // pub enum Two { A, B } + // pub fn identity(x: Two) -> Two { + // match x { + // Two::A => Two::A, + // Two::B => Two::B, + // } + // } + // + // This generates a `switchInt() -> [0: 0, 1: 1, otherwise: unreachable]`, which allows us or LLVM to + // turn it into just `x` later. Without the unreachable, such a transformation would be illegal. + // If the otherwise branch is unreachable, we can delete all other unreacahble targets, as they will + // still point to the unreachable and therefore not lose reachability information. + let reachable_iter = targets.iter().filter(|(_, bb)| !is_unreachable(*bb)); - if !predicate(otherwise) { - targets.push(otherwise); - } else { - values.pop(); - } + let new_targets = SwitchTargets::new(reachable_iter, otherwise); - let retained_targets_len = targets.len(); + // No unreachable branches were removed. + if new_targets.all_targets().len() == targets.all_targets().len() { + return None; + } - if targets.is_empty() { - TerminatorKind::Unreachable - } else if targets.len() == 1 { - TerminatorKind::Goto { target: targets[0] } - } else if original_targets_len != retained_targets_len { TerminatorKind::SwitchInt { discr: discr.clone(), - switch_ty, - targets: SwitchTargets::new( - values.iter().copied().zip(targets.iter().copied()), - *targets.last().unwrap(), - ), + switch_ty: *switch_ty, + targets: new_targets, } } else { + // If the otherwise branch is reachable, we don't want to delete any unreachable branches. return None; } } |