diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
19 files changed, 345 insertions, 359 deletions
diff --git a/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs b/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs index ed3b1ae4f42..daddb5dedbc 100644 --- a/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs +++ b/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs @@ -42,7 +42,7 @@ impl<'a, 'tcx> UndefinedTransmutesChecker<'a, 'tcx> { if self.tcx.is_const_fn(def_id) || matches!( self.tcx.opt_associated_item(def_id), - Some(AssocItem { kind: AssocKind::Const, .. }) + Some(AssocItem { kind: AssocKind::Const { .. }, .. }) ) { let fn_sig = function.ty(self.body, self.tcx).fn_sig(self.tcx).skip_binder(); diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs index cb844019857..4be67b873f7 100644 --- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs +++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs @@ -35,7 +35,8 @@ impl<'tcx> crate::MirPass<'tcx> for CleanupPostBorrowck { // MIR building, and are not needed after InstrumentCoverage. CoverageKind::BlockMarker { .. } | CoverageKind::SpanMarker { .. }, ) - | StatementKind::FakeRead(..) => statement.make_nop(), + | StatementKind::FakeRead(..) + | StatementKind::BackwardIncompatibleDropHint { .. } => statement.make_nop(), StatementKind::Assign(box ( _, Rvalue::Cast( diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 04d96f11707..92d5e152f58 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -203,7 +203,7 @@ struct TransformVisitor<'tcx> { impl<'tcx> TransformVisitor<'tcx> { fn insert_none_ret_block(&self, body: &mut Body<'tcx>) -> BasicBlock { - let block = BasicBlock::new(body.basic_blocks.len()); + let block = body.basic_blocks.next_index(); let source_info = SourceInfo::outermost(body.span); let none_value = match self.coroutine_kind { @@ -547,7 +547,7 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let get_context_def_id = tcx.require_lang_item(LangItem::GetContext, None); - for bb in START_BLOCK..body.basic_blocks.next_index() { + for bb in body.basic_blocks.indices() { let bb_data = &body[bb]; if bb_data.is_cleanup { continue; @@ -556,11 +556,11 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { match &bb_data.terminator().kind { TerminatorKind::Call { func, .. } => { let func_ty = func.ty(body, tcx); - if let ty::FnDef(def_id, _) = *func_ty.kind() { - if def_id == get_context_def_id { - let local = eliminate_get_context_call(&mut body[bb]); - replace_resume_ty_local(tcx, body, local, context_mut_ref); - } + if let ty::FnDef(def_id, _) = *func_ty.kind() + && def_id == get_context_def_id + { + let local = eliminate_get_context_call(&mut body[bb]); + replace_resume_ty_local(tcx, body, local, context_mut_ref); } } TerminatorKind::Yield { resume_arg, .. } => { @@ -1057,7 +1057,7 @@ fn insert_switch<'tcx>( let blocks = body.basic_blocks_mut().iter_mut(); for target in blocks.flat_map(|b| b.terminator_mut().successors_mut()) { - *target = BasicBlock::new(target.index() + 1); + *target += 1; } } @@ -1169,6 +1169,13 @@ fn create_coroutine_drop_shim<'tcx>( dump_mir(tcx, false, "coroutine_drop", &0, &body, |_, _| Ok(())); body.source.instance = drop_instance; + // Creating a coroutine drop shim happens on `Analysis(PostCleanup) -> Runtime(Initial)` + // but the pass manager doesn't update the phase of the coroutine drop shim. Update the + // phase of the drop shim so that later on when we run the pass manager on the shim, in + // the `mir_shims` query, we don't ICE on the intra-pass validation before we've updated + // the phase of the body from analysis. + body.phase = MirPhase::Runtime(RuntimePhase::Initial); + body } @@ -1186,7 +1193,7 @@ fn insert_panic_block<'tcx>( body: &mut Body<'tcx>, message: AssertMessage<'tcx>, ) -> BasicBlock { - let assert_block = BasicBlock::new(body.basic_blocks.len()); + let assert_block = body.basic_blocks.next_index(); let kind = TerminatorKind::Assert { cond: Operand::Constant(Box::new(ConstOperand { span: body.span, @@ -1209,14 +1216,8 @@ fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, typing_env: ty::Typing } // If there's a return terminator the function may return. - for block in body.basic_blocks.iter() { - if let TerminatorKind::Return = block.terminator().kind { - return true; - } - } - + body.basic_blocks.iter().any(|block| matches!(block.terminator().kind, TerminatorKind::Return)) // Otherwise the function can't return. - false } fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool { @@ -1293,12 +1294,12 @@ fn create_coroutine_resume_function<'tcx>( kind: TerminatorKind::Goto { target: poison_block }, }; } - } else if !block.is_cleanup { + } else if !block.is_cleanup // Any terminators that *can* unwind but don't have an unwind target set are also // pointed at our poisoning block (unless they're part of the cleanup path). - if let Some(unwind @ UnwindAction::Continue) = block.terminator_mut().unwind_mut() { - *unwind = UnwindAction::Cleanup(poison_block); - } + && let Some(unwind @ UnwindAction::Continue) = block.terminator_mut().unwind_mut() + { + *unwind = UnwindAction::Cleanup(poison_block); } } } @@ -1340,12 +1341,14 @@ fn create_coroutine_resume_function<'tcx>( make_coroutine_state_argument_indirect(tcx, body); match transform.coroutine_kind { + CoroutineKind::Coroutine(_) + | CoroutineKind::Desugared(CoroutineDesugaring::Async | CoroutineDesugaring::AsyncGen, _) => + { + make_coroutine_state_argument_pinned(tcx, body); + } // Iterator::next doesn't accept a pinned argument, // unlike for all other coroutine kinds. CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) => {} - _ => { - make_coroutine_state_argument_pinned(tcx, body); - } } // Make sure we remove dead blocks to remove @@ -1408,8 +1411,7 @@ fn create_cases<'tcx>( let mut statements = Vec::new(); // Create StorageLive instructions for locals with live storage - for i in 0..(body.local_decls.len()) { - let l = Local::new(i); + for l in body.local_decls.indices() { let needs_storage_live = point.storage_liveness.contains(l) && !transform.remap.contains(l) && !transform.always_live_locals.contains(l); @@ -1535,15 +1537,10 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { let coroutine_kind = body.coroutine_kind().unwrap(); // Get the discriminant type and args which typeck computed - let (discr_ty, movable) = match *coroutine_ty.kind() { - ty::Coroutine(_, args) => { - let args = args.as_coroutine(); - (args.discr_ty(tcx), coroutine_kind.movability() == hir::Movability::Movable) - } - _ => { - tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); - } + let ty::Coroutine(_, args) = coroutine_ty.kind() else { + tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); }; + let discr_ty = args.as_coroutine().discr_ty(tcx); let new_ret_ty = match coroutine_kind { CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => { @@ -1610,6 +1607,7 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { let always_live_locals = always_storage_live_locals(body); + let movable = coroutine_kind.movability() == hir::Movability::Movable; let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index 57f7893be1b..d49f5d9f9c3 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -103,9 +103,8 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { let mut should_cleanup = false; // Also consider newly generated bbs in the same pass - for i in 0..body.basic_blocks.len() { + for parent in body.basic_blocks.indices() { let bbs = &*body.basic_blocks; - let parent = BasicBlock::from_usize(i); let Some(opt_data) = evaluate_candidate(tcx, body, parent) else { continue }; trace!("SUCCESS: found optimization possibility to apply: {opt_data:?}"); diff --git a/compiler/rustc_mir_transform/src/elaborate_drop.rs b/compiler/rustc_mir_transform/src/elaborate_drop.rs index 0d8cf524661..e3057a2f648 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drop.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drop.rs @@ -258,31 +258,27 @@ where ) -> Vec<(Place<'tcx>, Option<D::Path>)> { variant .fields - .iter() - .enumerate() - .map(|(i, f)| { - let field = FieldIdx::new(i); - let subpath = self.elaborator.field_subpath(variant_path, field); + .iter_enumerated() + .map(|(field_idx, field)| { + let subpath = self.elaborator.field_subpath(variant_path, field_idx); let tcx = self.tcx(); assert_eq!(self.elaborator.typing_env().typing_mode, ty::TypingMode::PostAnalysis); - // The type error for normalization may have been in dropck: see - // `compute_drop_data` in rustc_borrowck, in which case we wouldn't have - // deleted the MIR body and could have an error here as well. - let field_ty = match tcx - .try_normalize_erasing_regions(self.elaborator.typing_env(), f.ty(tcx, args)) - { + let field_ty = match tcx.try_normalize_erasing_regions( + self.elaborator.typing_env(), + field.ty(tcx, args), + ) { Ok(t) => t, Err(_) => Ty::new_error( self.tcx(), - self.elaborator - .body() - .tainted_by_errors - .expect("Error in drop elaboration not found by dropck."), + self.tcx().dcx().span_delayed_bug( + self.elaborator.body().span, + "Error normalizing in drop elaboration.", + ), ), }; - (tcx.mk_place_field(base_place, field, field_ty), subpath) + (tcx.mk_place_field(base_place, field_idx, field_ty), subpath) }) .collect() } diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 68bc0ffce6b..8b8d1efbbd2 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -980,27 +980,17 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } } - let tcx = self.tcx; - let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new(); - loop { - if let Some(local) = self.try_as_local(copy_from_local_value, location) { - projection.reverse(); - let place = Place { local, projection: tcx.mk_place_elems(projection.as_slice()) }; - if rvalue.ty(self.local_decls, tcx) == place.ty(self.local_decls, tcx).ty { - self.reused_locals.insert(local); - *rvalue = Rvalue::Use(Operand::Copy(place)); - return Some(copy_from_value); - } - return None; - } else if let Value::Projection(pointer, proj) = *self.get(copy_from_local_value) - && let Some(proj) = self.try_as_place_elem(proj, location) - { - projection.push(proj); - copy_from_local_value = pointer; - } else { - return None; + // Allow introducing places with non-constant offsets, as those are still better than + // reconstructing an aggregate. + if let Some(place) = self.try_as_place(copy_from_local_value, location, true) { + if rvalue.ty(self.local_decls, self.tcx) == place.ty(self.local_decls, self.tcx).ty { + self.reused_locals.insert(place.local); + *rvalue = Rvalue::Use(Operand::Copy(place)); + return Some(copy_from_local_value); } } + + None } fn simplify_aggregate( @@ -1672,14 +1662,14 @@ fn op_to_prop_const<'tcx>( } impl<'tcx> VnState<'_, 'tcx> { - /// If either [`Self::try_as_constant`] as [`Self::try_as_local`] succeeds, + /// If either [`Self::try_as_constant`] as [`Self::try_as_place`] succeeds, /// returns that result as an [`Operand`]. fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option<Operand<'tcx>> { if let Some(const_) = self.try_as_constant(index) { Some(Operand::Constant(Box::new(const_))) - } else if let Some(local) = self.try_as_local(index, location) { - self.reused_locals.insert(local); - Some(Operand::Copy(local.into())) + } else if let Some(place) = self.try_as_place(index, location, false) { + self.reused_locals.insert(place.local); + Some(Operand::Copy(place)) } else { None } @@ -1712,6 +1702,35 @@ impl<'tcx> VnState<'_, 'tcx> { Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_ }) } + /// Construct a place which holds the same value as `index` and for which all locals strictly + /// dominate `loc`. If you used this place, add its base local to `reused_locals` to remove + /// storage statements. + #[instrument(level = "trace", skip(self), ret)] + fn try_as_place( + &mut self, + mut index: VnIndex, + loc: Location, + allow_complex_projection: bool, + ) -> Option<Place<'tcx>> { + let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new(); + loop { + if let Some(local) = self.try_as_local(index, loc) { + projection.reverse(); + let place = + Place { local, projection: self.tcx.mk_place_elems(projection.as_slice()) }; + return Some(place); + } else if let Value::Projection(pointer, proj) = *self.get(index) + && (allow_complex_projection || proj.is_stable_offset()) + && let Some(proj) = self.try_as_place_elem(proj, loc) + { + projection.push(proj); + index = pointer; + } else { + return None; + } + } + } + /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`, /// return it. If you used this local, add it to `reused_locals` to remove storage statements. fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option<Local> { @@ -1762,11 +1781,12 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> { if let Some(value) = value { if let Some(const_) = self.try_as_constant(value) { *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_))); - } else if let Some(local) = self.try_as_local(value, location) - && *rvalue != Rvalue::Use(Operand::Move(local.into())) + } else if let Some(place) = self.try_as_place(value, location, false) + && *rvalue != Rvalue::Use(Operand::Move(place)) + && *rvalue != Rvalue::Use(Operand::Copy(place)) { - *rvalue = Rvalue::Use(Operand::Copy(local.into())); - self.reused_locals.insert(local); + *rvalue = Rvalue::Use(Operand::Copy(place)); + self.reused_locals.insert(place.local); } } } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 0ab24e48d44..69e80ed54ea 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -903,9 +903,9 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( let mut integrator = Integrator { args: &args, - new_locals: Local::new(caller_body.local_decls.len()).., - new_scopes: SourceScope::new(caller_body.source_scopes.len()).., - new_blocks: BasicBlock::new(caller_body.basic_blocks.len()).., + new_locals: caller_body.local_decls.next_index().., + new_scopes: caller_body.source_scopes.next_index().., + new_blocks: caller_body.basic_blocks.next_index().., destination: destination_local, callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), callsite, @@ -1169,7 +1169,7 @@ impl Integrator<'_, '_> { if idx < self.args.len() { self.args[idx] } else { - Local::new(self.new_locals.start.index() + (idx - self.args.len())) + self.new_locals.start + (idx - self.args.len()) } }; trace!("mapping local `{:?}` to `{:?}`", local, new); @@ -1177,13 +1177,13 @@ impl Integrator<'_, '_> { } fn map_scope(&self, scope: SourceScope) -> SourceScope { - let new = SourceScope::new(self.new_scopes.start.index() + scope.index()); + let new = self.new_scopes.start + scope.index(); trace!("mapping scope `{:?}` to `{:?}`", scope, new); new } fn map_block(&self, block: BasicBlock) -> BasicBlock { - let new = BasicBlock::new(self.new_blocks.start.index() + block.index()); + let new = self.new_blocks.start + block.index(); trace!("mapping block `{:?}` to `{:?}`", block, new); new } diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 2eff6b31372..5f0c55ddc09 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -10,7 +10,6 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, layout}; use rustc_span::{DUMMY_SP, Symbol, sym}; use crate::simplify::simplify_duplicate_switch_targets; -use crate::take_array; pub(super) enum InstSimplify { BeforeInline, @@ -39,26 +38,26 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { attr::contains_name(tcx.hir_krate_attrs(), sym::rustc_preserve_ub_checks); for block in body.basic_blocks.as_mut() { for statement in block.statements.iter_mut() { - match statement.kind { - StatementKind::Assign(box (_place, ref mut rvalue)) => { - if !preserve_ub_checks { - ctx.simplify_ub_check(rvalue); - } - ctx.simplify_bool_cmp(rvalue); - ctx.simplify_ref_deref(rvalue); - ctx.simplify_ptr_aggregate(rvalue); - ctx.simplify_cast(rvalue); - ctx.simplify_repeated_aggregate(rvalue); - ctx.simplify_repeat_once(rvalue); - } - _ => {} + let StatementKind::Assign(box (.., rvalue)) = &mut statement.kind else { + continue; + }; + + if !preserve_ub_checks { + ctx.simplify_ub_check(rvalue); } + ctx.simplify_bool_cmp(rvalue); + ctx.simplify_ref_deref(rvalue); + ctx.simplify_ptr_aggregate(rvalue); + ctx.simplify_cast(rvalue); + ctx.simplify_repeated_aggregate(rvalue); + ctx.simplify_repeat_once(rvalue); } - ctx.simplify_primitive_clone(block.terminator.as_mut().unwrap(), &mut block.statements); - ctx.simplify_intrinsic_assert(block.terminator.as_mut().unwrap()); - ctx.simplify_nounwind_call(block.terminator.as_mut().unwrap()); - simplify_duplicate_switch_targets(block.terminator.as_mut().unwrap()); + let terminator = block.terminator.as_mut().unwrap(); + ctx.simplify_primitive_clone(terminator, &mut block.statements); + ctx.simplify_intrinsic_assert(terminator); + ctx.simplify_nounwind_call(terminator); + simplify_duplicate_switch_targets(terminator); } } @@ -105,43 +104,34 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { /// Transform boolean comparisons into logical operations. fn simplify_bool_cmp(&self, rvalue: &mut Rvalue<'tcx>) { - match rvalue { - Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), box (a, b)) => { - let new = match (op, self.try_eval_bool(a), self.try_eval_bool(b)) { - // Transform "Eq(a, true)" ==> "a" - (BinOp::Eq, _, Some(true)) => Some(Rvalue::Use(a.clone())), + let Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), box (a, b)) = &*rvalue else { return }; + *rvalue = match (op, self.try_eval_bool(a), self.try_eval_bool(b)) { + // Transform "Eq(a, true)" ==> "a" + (BinOp::Eq, _, Some(true)) => Rvalue::Use(a.clone()), - // Transform "Ne(a, false)" ==> "a" - (BinOp::Ne, _, Some(false)) => Some(Rvalue::Use(a.clone())), + // Transform "Ne(a, false)" ==> "a" + (BinOp::Ne, _, Some(false)) => Rvalue::Use(a.clone()), - // Transform "Eq(true, b)" ==> "b" - (BinOp::Eq, Some(true), _) => Some(Rvalue::Use(b.clone())), + // Transform "Eq(true, b)" ==> "b" + (BinOp::Eq, Some(true), _) => Rvalue::Use(b.clone()), - // Transform "Ne(false, b)" ==> "b" - (BinOp::Ne, Some(false), _) => Some(Rvalue::Use(b.clone())), + // Transform "Ne(false, b)" ==> "b" + (BinOp::Ne, Some(false), _) => Rvalue::Use(b.clone()), - // Transform "Eq(false, b)" ==> "Not(b)" - (BinOp::Eq, Some(false), _) => Some(Rvalue::UnaryOp(UnOp::Not, b.clone())), + // Transform "Eq(false, b)" ==> "Not(b)" + (BinOp::Eq, Some(false), _) => Rvalue::UnaryOp(UnOp::Not, b.clone()), - // Transform "Ne(true, b)" ==> "Not(b)" - (BinOp::Ne, Some(true), _) => Some(Rvalue::UnaryOp(UnOp::Not, b.clone())), + // Transform "Ne(true, b)" ==> "Not(b)" + (BinOp::Ne, Some(true), _) => Rvalue::UnaryOp(UnOp::Not, b.clone()), - // Transform "Eq(a, false)" ==> "Not(a)" - (BinOp::Eq, _, Some(false)) => Some(Rvalue::UnaryOp(UnOp::Not, a.clone())), + // Transform "Eq(a, false)" ==> "Not(a)" + (BinOp::Eq, _, Some(false)) => Rvalue::UnaryOp(UnOp::Not, a.clone()), - // Transform "Ne(a, true)" ==> "Not(a)" - (BinOp::Ne, _, Some(true)) => Some(Rvalue::UnaryOp(UnOp::Not, a.clone())), + // Transform "Ne(a, true)" ==> "Not(a)" + (BinOp::Ne, _, Some(true)) => Rvalue::UnaryOp(UnOp::Not, a.clone()), - _ => None, - }; - - if let Some(new) = new { - *rvalue = new; - } - } - - _ => {} - } + _ => return, + }; } fn try_eval_bool(&self, a: &Operand<'_>) -> Option<bool> { @@ -151,64 +141,58 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { /// Transform `&(*a)` ==> `a`. fn simplify_ref_deref(&self, rvalue: &mut Rvalue<'tcx>) { - if let Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) = rvalue { - if let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() { - if rvalue.ty(self.local_decls, self.tcx) != base.ty(self.local_decls, self.tcx).ty { - return; - } - - *rvalue = Rvalue::Use(Operand::Copy(Place { - local: base.local, - projection: self.tcx.mk_place_elems(base.projection), - })); - } + if let Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) = rvalue + && let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() + && rvalue.ty(self.local_decls, self.tcx) == base.ty(self.local_decls, self.tcx).ty + { + *rvalue = Rvalue::Use(Operand::Copy(Place { + local: base.local, + projection: self.tcx.mk_place_elems(base.projection), + })); } } /// Transform `Aggregate(RawPtr, [p, ()])` ==> `Cast(PtrToPtr, p)`. fn simplify_ptr_aggregate(&self, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::Aggregate(box AggregateKind::RawPtr(pointee_ty, mutability), fields) = rvalue + && let meta_ty = fields.raw[1].ty(self.local_decls, self.tcx) + && meta_ty.is_unit() { - let meta_ty = fields.raw[1].ty(self.local_decls, self.tcx); - if meta_ty.is_unit() { - // The mutable borrows we're holding prevent printing `rvalue` here - let mut fields = std::mem::take(fields); - let _meta = fields.pop().unwrap(); - let data = fields.pop().unwrap(); - let ptr_ty = Ty::new_ptr(self.tcx, *pointee_ty, *mutability); - *rvalue = Rvalue::Cast(CastKind::PtrToPtr, data, ptr_ty); - } + // The mutable borrows we're holding prevent printing `rvalue` here + let mut fields = std::mem::take(fields); + let _meta = fields.pop().unwrap(); + let data = fields.pop().unwrap(); + let ptr_ty = Ty::new_ptr(self.tcx, *pointee_ty, *mutability); + *rvalue = Rvalue::Cast(CastKind::PtrToPtr, data, ptr_ty); } } fn simplify_ub_check(&self, rvalue: &mut Rvalue<'tcx>) { - if let Rvalue::NullaryOp(NullOp::UbChecks, _) = *rvalue { - let const_ = Const::from_bool(self.tcx, self.tcx.sess.ub_checks()); - let constant = ConstOperand { span: DUMMY_SP, const_, user_ty: None }; - *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); - } + let Rvalue::NullaryOp(NullOp::UbChecks, _) = *rvalue else { return }; + + let const_ = Const::from_bool(self.tcx, self.tcx.sess.ub_checks()); + let constant = ConstOperand { span: DUMMY_SP, const_, user_ty: None }; + *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); } fn simplify_cast(&self, rvalue: &mut Rvalue<'tcx>) { - if let Rvalue::Cast(kind, operand, cast_ty) = rvalue { - let operand_ty = operand.ty(self.local_decls, self.tcx); - if operand_ty == *cast_ty { - *rvalue = Rvalue::Use(operand.clone()); - } else if *kind == CastKind::Transmute { - // Transmuting an integer to another integer is just a signedness cast - if let (ty::Int(int), ty::Uint(uint)) | (ty::Uint(uint), ty::Int(int)) = - (operand_ty.kind(), cast_ty.kind()) - && int.bit_width() == uint.bit_width() - { - // The width check isn't strictly necessary, as different widths - // are UB and thus we'd be allowed to turn it into a cast anyway. - // But let's keep the UB around for codegen to exploit later. - // (If `CastKind::Transmute` ever becomes *not* UB for mismatched sizes, - // then the width check is necessary for big-endian correctness.) - *kind = CastKind::IntToInt; - return; - } - } + let Rvalue::Cast(kind, operand, cast_ty) = rvalue else { return }; + + let operand_ty = operand.ty(self.local_decls, self.tcx); + if operand_ty == *cast_ty { + *rvalue = Rvalue::Use(operand.clone()); + } else if *kind == CastKind::Transmute + // Transmuting an integer to another integer is just a signedness cast + && let (ty::Int(int), ty::Uint(uint)) | (ty::Uint(uint), ty::Int(int)) = + (operand_ty.kind(), cast_ty.kind()) + && int.bit_width() == uint.bit_width() + { + // The width check isn't strictly necessary, as different widths + // are UB and thus we'd be allowed to turn it into a cast anyway. + // But let's keep the UB around for codegen to exploit later. + // (If `CastKind::Transmute` ever becomes *not* UB for mismatched sizes, + // then the width check is necessary for big-endian correctness.) + *kind = CastKind::IntToInt; } } @@ -229,7 +213,9 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { terminator: &mut Terminator<'tcx>, statements: &mut Vec<Statement<'tcx>>, ) { - let TerminatorKind::Call { func, args, destination, target, .. } = &mut terminator.kind + let TerminatorKind::Call { + func, args, destination, target: Some(destination_block), .. + } = &terminator.kind else { return; }; @@ -237,15 +223,8 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { // It's definitely not a clone if there are multiple arguments let [arg] = &args[..] else { return }; - let Some(destination_block) = *target else { return }; - // Only bother looking more if it's easy to know what we're calling - let Some((fn_def_id, fn_args)) = func.const_fn_def() else { return }; - - // Clone needs one arg, so we can cheaply rule out other stuff - if fn_args.len() != 1 { - return; - } + let Some((fn_def_id, ..)) = func.const_fn_def() else { return }; // These types are easily available from locals, so check that before // doing DefId lookups to figure out what we're actually calling. @@ -253,15 +232,12 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { let ty::Ref(_region, inner_ty, Mutability::Not) = *arg_ty.kind() else { return }; - if !inner_ty.is_trivially_pure_clone_copy() { - return; - } - - if !self.tcx.is_lang_item(fn_def_id, LangItem::CloneFn) { + if !self.tcx.is_lang_item(fn_def_id, LangItem::CloneFn) + || !inner_ty.is_trivially_pure_clone_copy() + { return; } - let Ok([arg]) = take_array(args) else { return }; let Some(arg_place) = arg.node.place() else { return }; statements.push(Statement { @@ -273,11 +249,11 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { )), ))), }); - terminator.kind = TerminatorKind::Goto { target: destination_block }; + terminator.kind = TerminatorKind::Goto { target: *destination_block }; } fn simplify_nounwind_call(&self, terminator: &mut Terminator<'tcx>) { - let TerminatorKind::Call { func, unwind, .. } = &mut terminator.kind else { + let TerminatorKind::Call { ref func, ref mut unwind, .. } = terminator.kind else { return; }; @@ -290,7 +266,7 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { ty::FnDef(..) => body_ty.fn_sig(self.tcx).abi(), ty::Closure(..) => ExternAbi::RustCall, ty::Coroutine(..) => ExternAbi::Rust, - _ => bug!("unexpected body ty: {:?}", body_ty), + _ => bug!("unexpected body ty: {body_ty:?}"), }; if !layout::fn_can_unwind(self.tcx, Some(def_id), body_abi) { @@ -299,10 +275,9 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } fn simplify_intrinsic_assert(&self, terminator: &mut Terminator<'tcx>) { - let TerminatorKind::Call { func, target, .. } = &mut terminator.kind else { - return; - }; - let Some(target_block) = target else { + let TerminatorKind::Call { ref func, target: ref mut target @ Some(target_block), .. } = + terminator.kind + else { return; }; let func_ty = func.ty(self.local_decls, self.tcx); @@ -310,12 +285,10 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { return; }; // The intrinsics we are interested in have one generic parameter - if args.is_empty() { - return; - } + let [arg, ..] = args[..] else { return }; let known_is_valid = - intrinsic_assert_panics(self.tcx, self.typing_env, args[0], intrinsic_name); + intrinsic_assert_panics(self.tcx, self.typing_env, arg, intrinsic_name); match known_is_valid { // We don't know the layout or it's not validity assertion at all, don't touch it None => {} @@ -325,7 +298,7 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } Some(false) => { // If we know the assert does not panic, turn the call into a Goto - terminator.kind = TerminatorKind::Goto { target: *target_block }; + terminator.kind = TerminatorKind::Goto { target: target_block }; } } } @@ -346,9 +319,7 @@ fn resolve_rust_intrinsic<'tcx>( tcx: TyCtxt<'tcx>, func_ty: Ty<'tcx>, ) -> Option<(Symbol, GenericArgsRef<'tcx>)> { - if let ty::FnDef(def_id, args) = *func_ty.kind() { - let intrinsic = tcx.intrinsic(def_id)?; - return Some((intrinsic.name, args)); - } - None + let ty::FnDef(def_id, args) = *func_ty.kind() else { return None }; + let intrinsic = tcx.intrinsic(def_id)?; + Some((intrinsic.name, args)) } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 0a72a9d669f..8b4b214a3d4 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -90,7 +90,11 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { }; for bb in body.basic_blocks.indices() { - finder.start_from_switch(bb); + let old_len = finder.opportunities.len(); + // If we have any const-eval errors discard any opportunities found + if finder.start_from_switch(bb).is_none() { + finder.opportunities.truncate(old_len); + } } let opportunities = finder.opportunities; @@ -150,14 +154,6 @@ impl Condition { fn matches(&self, value: ScalarInt) -> bool { (self.value == value) == (self.polarity == Polarity::Eq) } - - fn inv(mut self) -> Self { - self.polarity = match self.polarity { - Polarity::Eq => Polarity::Ne, - Polarity::Ne => Polarity::Eq, - }; - self - } } #[derive(Copy, Clone, Debug)] @@ -180,8 +176,21 @@ impl<'a> ConditionSet<'a> { self.iter().filter(move |c| c.matches(value)) } - fn map(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> Condition) -> ConditionSet<'a> { - ConditionSet(arena.alloc_from_iter(self.iter().map(f))) + fn map( + self, + arena: &'a DroplessArena, + f: impl Fn(Condition) -> Option<Condition>, + ) -> Option<ConditionSet<'a>> { + let mut all_ok = true; + let set = arena.alloc_from_iter(self.iter().map_while(|c| { + if let Some(c) = f(c) { + Some(c) + } else { + all_ok = false; + None + } + })); + all_ok.then_some(ConditionSet(set)) } } @@ -192,28 +201,28 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { /// Recursion entry point to find threading opportunities. #[instrument(level = "trace", skip(self))] - fn start_from_switch(&mut self, bb: BasicBlock) { + fn start_from_switch(&mut self, bb: BasicBlock) -> Option<()> { let bbdata = &self.body[bb]; if bbdata.is_cleanup || self.loop_headers.contains(bb) { - return; + return Some(()); } - let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return }; - let Some(discr) = discr.place() else { return }; + let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return Some(()) }; + let Some(discr) = discr.place() else { return Some(()) }; debug!(?discr, ?bb); let discr_ty = discr.ty(self.body, self.tcx).ty; let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { - return; + return Some(()); }; - let Some(discr) = self.map.find(discr.as_ref()) else { return }; + let Some(discr) = self.map.find(discr.as_ref()) else { return Some(()) }; debug!(?discr); let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); let mut state = State::new_reachable(); let conds = if let Some((value, then, else_)) = targets.as_static_if() { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; + let value = ScalarInt::try_from_uint(value, discr_layout.size)?; self.arena.alloc_from_iter([ Condition { value, polarity: Polarity::Eq, target: then }, Condition { value, polarity: Polarity::Ne, target: else_ }, @@ -227,7 +236,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let conds = ConditionSet(conds); state.insert_value_idx(discr, conds, &self.map); - self.find_opportunity(bb, state, cost, 0); + self.find_opportunity(bb, state, cost, 0) } /// Recursively walk statements backwards from this bb's terminator to find threading @@ -239,10 +248,10 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { mut state: State<ConditionSet<'a>>, mut cost: CostChecker<'_, 'tcx>, depth: usize, - ) { + ) -> Option<()> { // Do not thread through loop headers. if self.loop_headers.contains(bb) { - return; + return Some(()); } debug!(cost = ?cost.cost()); @@ -250,16 +259,16 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { self.body.basic_blocks[bb].statements.iter().enumerate().rev() { if self.is_empty(&state) { - return; + return Some(()); } cost.visit_statement(stmt, Location { block: bb, statement_index }); if cost.cost() > MAX_COST { - return; + return Some(()); } // Attempt to turn the `current_condition` on `lhs` into a condition on another place. - self.process_statement(bb, stmt, &mut state); + self.process_statement(bb, stmt, &mut state)?; // When a statement mutates a place, assignments to that place that happen // above the mutation cannot fulfill a condition. @@ -271,7 +280,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } if self.is_empty(&state) || depth >= MAX_BACKTRACK { - return; + return Some(()); } let last_non_rec = self.opportunities.len(); @@ -284,9 +293,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { match term.kind { TerminatorKind::SwitchInt { ref discr, ref targets } => { self.process_switch_int(discr, targets, bb, &mut state); - self.find_opportunity(pred, state, cost, depth + 1); + self.find_opportunity(pred, state, cost, depth + 1)?; } - _ => self.recurse_through_terminator(pred, || state, &cost, depth), + _ => self.recurse_through_terminator(pred, || state, &cost, depth)?, } } else if let &[ref predecessors @ .., last_pred] = &predecessors[..] { for &pred in predecessors { @@ -311,12 +320,13 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let first = &mut new_tos[0]; *first = ThreadingOpportunity { chain: vec![bb], target: first.target }; self.opportunities.truncate(last_non_rec + 1); - return; + return Some(()); } for op in self.opportunities[last_non_rec..].iter_mut() { op.chain.push(bb); } + Some(()) } /// Extract the mutated place from a statement. @@ -430,23 +440,23 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { lhs: PlaceIndex, rhs: &Operand<'tcx>, state: &mut State<ConditionSet<'a>>, - ) { + ) -> Option<()> { match rhs { // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. Operand::Constant(constant) => { - let Some(constant) = - self.ecx.eval_mir_constant(&constant.const_, constant.span, None).discard_err() - else { - return; - }; + let constant = self + .ecx + .eval_mir_constant(&constant.const_, constant.span, None) + .discard_err()?; self.process_constant(bb, lhs, constant, state); } // Transfer the conditions on the copied rhs. Operand::Move(rhs) | Operand::Copy(rhs) => { - let Some(rhs) = self.map.find(rhs.as_ref()) else { return }; + let Some(rhs) = self.map.find(rhs.as_ref()) else { return Some(()) }; state.insert_place_idx(rhs, lhs, &self.map); } } + Some(()) } #[instrument(level = "trace", skip(self))] @@ -456,14 +466,18 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { lhs_place: &Place<'tcx>, rhs: &Rvalue<'tcx>, state: &mut State<ConditionSet<'a>>, - ) { - let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return }; + ) -> Option<()> { + let Some(lhs) = self.map.find(lhs_place.as_ref()) else { + return Some(()); + }; match rhs { - Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state), + Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state)?, // Transfer the conditions on the copy rhs. - Rvalue::CopyForDeref(rhs) => self.process_operand(bb, lhs, &Operand::Copy(*rhs), state), + Rvalue::CopyForDeref(rhs) => { + self.process_operand(bb, lhs, &Operand::Copy(*rhs), state)? + } Rvalue::Discriminant(rhs) => { - let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return }; + let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return Some(()) }; state.insert_place_idx(rhs, lhs, &self.map); } // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. @@ -471,7 +485,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let agg_ty = lhs_place.ty(self.body, self.tcx).ty; let lhs = match kind { // Do not support unions. - AggregateKind::Adt(.., Some(_)) => return, + AggregateKind::Adt(.., Some(_)) => return Some(()), AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => { if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant) && let Some(discr_value) = self @@ -484,30 +498,31 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) { idx } else { - return; + return Some(()); } } _ => lhs, }; for (field_index, operand) in operands.iter_enumerated() { if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) { - self.process_operand(bb, field, operand, state); + self.process_operand(bb, field, operand, state)?; } } } - // Transfer the conditions on the copy rhs, after inversing polarity. + // Transfer the conditions on the copy rhs, after inverting the value of the condition. Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => { - if !place.ty(self.body, self.tcx).ty.is_bool() { - // Constructing the conditions by inverting the polarity - // of equality is only correct for bools. That is to say, - // `!a == b` is not `a != b` for integers greater than 1 bit. - return; - } - let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return }; - let Some(place) = self.map.find(place.as_ref()) else { return }; - // FIXME: I think This could be generalized to not bool if we - // actually perform a logical not on the condition's value. - let conds = conditions.map(self.arena, Condition::inv); + let layout = self.ecx.layout_of(place.ty(self.body, self.tcx).ty).unwrap(); + let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return Some(()) }; + let Some(place) = self.map.find(place.as_ref()) else { return Some(()) }; + let conds = conditions.map(self.arena, |mut cond| { + cond.value = self + .ecx + .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout)) + .discard_err()? + .to_scalar_int() + .discard_err()?; + Some(cond) + })?; state.insert_value_idx(place, conds, &self.map); } // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`. @@ -517,34 +532,34 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value)) | box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)), ) => { - let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return }; - let Some(place) = self.map.find(place.as_ref()) else { return }; + let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return Some(()) }; + let Some(place) = self.map.find(place.as_ref()) else { return Some(()) }; let equals = match op { BinOp::Eq => ScalarInt::TRUE, BinOp::Ne => ScalarInt::FALSE, - _ => return, + _ => return Some(()), }; if value.const_.ty().is_floating_point() { // Floating point equality does not follow bit-patterns. // -0.0 and NaN both have special rules for equality, // and therefore we cannot use integer comparisons for them. // Avoid handling them, though this could be extended in the future. - return; + return Some(()); } - let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env) - else { - return; - }; - let conds = conditions.map(self.arena, |c| Condition { - value, - polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne }, - ..c - }); + let value = value.const_.try_eval_scalar_int(self.tcx, self.typing_env)?; + let conds = conditions.map(self.arena, |c| { + Some(Condition { + value, + polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne }, + ..c + }) + })?; state.insert_value_idx(place, conds, &self.map); } _ => {} } + Some(()) } #[instrument(level = "trace", skip(self))] @@ -553,7 +568,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { bb: BasicBlock, stmt: &Statement<'tcx>, state: &mut State<ConditionSet<'a>>, - ) { + ) -> Option<()> { let register_opportunity = |c: Condition| { debug!(?bb, ?c.target, "register"); self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) @@ -566,30 +581,32 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // If we expect `discriminant(place) ?= A`, // we have an opportunity if `variant_index ?= A`. StatementKind::SetDiscriminant { box place, variant_index } => { - let Some(discr_target) = self.map.find_discr(place.as_ref()) else { return }; + let Some(discr_target) = self.map.find_discr(place.as_ref()) else { + return Some(()); + }; let enum_ty = place.ty(self.body, self.tcx).ty; // `SetDiscriminant` guarantees that the discriminant is now `variant_index`. // Even if the discriminant write does nothing due to niches, it is UB to set the // discriminant when the data does not encode the desired discriminant. - let Some(discr) = - self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err() - else { - return; - }; + let discr = + self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err()?; self.process_immediate(bb, discr_target, discr, state); } // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( Operand::Copy(place) | Operand::Move(place), )) => { - let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { return }; + let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { + return Some(()); + }; conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity); } StatementKind::Assign(box (lhs_place, rhs)) => { - self.process_assign(bb, lhs_place, rhs, state); + self.process_assign(bb, lhs_place, rhs, state)?; } _ => {} } + Some(()) } #[instrument(level = "trace", skip(self, state, cost))] @@ -600,7 +617,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { state: impl FnOnce() -> State<ConditionSet<'a>>, cost: &CostChecker<'_, 'tcx>, depth: usize, - ) { + ) -> Option<()> { let term = self.body.basic_blocks[bb].terminator(); let place_to_flood = match term.kind { // We come from a target, so those are not possible. @@ -615,9 +632,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { | TerminatorKind::FalseUnwind { .. } | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"), // Cannot reason about inline asm. - TerminatorKind::InlineAsm { .. } => return, + TerminatorKind::InlineAsm { .. } => return Some(()), // `SwitchInt` is handled specially. - TerminatorKind::SwitchInt { .. } => return, + TerminatorKind::SwitchInt { .. } => return Some(()), // We can recurse, no thing particular to do. TerminatorKind::Goto { .. } => None, // Flood the overwritten place, and progress through. @@ -632,7 +649,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { if let Some(place_to_flood) = place_to_flood { state.flood_with(place_to_flood.as_ref(), &self.map, ConditionSet::BOTTOM); } - self.find_opportunity(bb, state, cost.clone(), depth + 1); + self.find_opportunity(bb, state, cost.clone(), depth + 1) } #[instrument(level = "trace", skip(self))] diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 6429d3f67ec..4d74ecddfb0 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,5 +1,4 @@ // tidy-alphabetical-start -#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141 #![feature(array_windows)] #![feature(assert_matches)] #![feature(box_patterns)] @@ -498,8 +497,11 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & } // We only need to borrowck non-synthetic MIR. - let tainted_by_errors = - if !tcx.is_synthetic_mir(def) { tcx.mir_borrowck(def).tainted_by_errors } else { None }; + let tainted_by_errors = if !tcx.is_synthetic_mir(def) { + tcx.mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local()).err() + } else { + None + }; let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { @@ -795,7 +797,7 @@ fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_ } if !tcx.is_synthetic_mir(def) { - tcx.ensure_done().mir_borrowck(def); + tcx.ensure_done().mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local()); } let mut promoted = tcx.mir_promoted(def).1.steal(); diff --git a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs index 29a9133abe9..537f152938e 100644 --- a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs +++ b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs @@ -512,23 +512,17 @@ struct LocalLabel<'a> { /// A custom `Subdiagnostic` implementation so that the notes are delivered in a specific order impl Subdiagnostic for LocalLabel<'_> { - fn add_to_diag_with< - G: rustc_errors::EmissionGuarantee, - F: rustc_errors::SubdiagMessageOp<G>, - >( - self, - diag: &mut rustc_errors::Diag<'_, G>, - f: &F, - ) { + fn add_to_diag<G: rustc_errors::EmissionGuarantee>(self, diag: &mut rustc_errors::Diag<'_, G>) { diag.arg("name", self.name); diag.arg("is_generated_name", self.is_generated_name); diag.arg("is_dropped_first_edition_2024", self.is_dropped_first_edition_2024); - let msg = f(diag, crate::fluent_generated::mir_transform_tail_expr_local.into()); + let msg = diag.eagerly_translate(crate::fluent_generated::mir_transform_tail_expr_local); diag.span_label(self.span, msg); for dtor in self.destructors { - dtor.add_to_diag_with(diag, f); + dtor.add_to_diag(diag); } - let msg = f(diag, crate::fluent_generated::mir_transform_label_local_epilogue); + let msg = + diag.eagerly_translate(crate::fluent_generated::mir_transform_label_local_epilogue); diag.span_label(self.span, msg); } } diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 0d9d0368d37..5059837328e 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -20,13 +20,11 @@ impl<'tcx> crate::MirPass<'tcx> for MatchBranchSimplification { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let typing_env = body.typing_env(tcx); let mut should_cleanup = false; - for i in 0..body.basic_blocks.len() { - let bbs = &*body.basic_blocks; - let bb_idx = BasicBlock::from_usize(i); - match bbs[bb_idx].terminator().kind { + for bb_idx in body.basic_blocks.indices() { + match &body.basic_blocks[bb_idx].terminator().kind { TerminatorKind::SwitchInt { - discr: ref _discr @ (Operand::Copy(_) | Operand::Move(_)), - ref targets, + discr: Operand::Copy(_) | Operand::Move(_), + targets, .. // We require that the possible target blocks don't contain this block. } if !targets.all_targets().contains(&bb_idx) => {} @@ -66,9 +64,10 @@ trait SimplifyMatch<'tcx> { typing_env: ty::TypingEnv<'tcx>, ) -> Option<()> { let bbs = &body.basic_blocks; - let (discr, targets) = match bbs[switch_bb_idx].terminator().kind { - TerminatorKind::SwitchInt { ref discr, ref targets, .. } => (discr, targets), - _ => unreachable!(), + let TerminatorKind::SwitchInt { discr, targets, .. } = + &bbs[switch_bb_idx].terminator().kind + else { + unreachable!(); }; let discr_ty = discr.ty(body.local_decls(), tcx); diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index c63bfdcee85..f59b849e85c 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -18,19 +18,17 @@ impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators { // find basic blocks with no statement and a return terminator let mut bbs_simple_returns = DenseBitSet::new_empty(body.basic_blocks.len()); let bbs = body.basic_blocks_mut(); - for idx in bbs.indices() { - if bbs[idx].statements.is_empty() - && bbs[idx].terminator().kind == TerminatorKind::Return - { + for (idx, bb) in bbs.iter_enumerated() { + if bb.statements.is_empty() && bb.terminator().kind == TerminatorKind::Return { bbs_simple_returns.insert(idx); } } for bb in bbs { - if let TerminatorKind::Goto { target } = bb.terminator().kind { - if bbs_simple_returns.contains(target) { - bb.terminator_mut().kind = TerminatorKind::Return; - } + if let TerminatorKind::Goto { target } = bb.terminator().kind + && bbs_simple_returns.contains(target) + { + bb.terminator_mut().kind = TerminatorKind::Return; } } diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index 6a177faeac8..12ace04c5e2 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -181,7 +181,7 @@ impl<'tcx> MirPatch<'tcx> { /// Queues the addition of a new basic block. pub(crate) fn new_block(&mut self, data: BasicBlockData<'tcx>) -> BasicBlock { - let block = BasicBlock::new(self.term_patch_map.len()); + let block = self.term_patch_map.next_index(); debug!("MirPatch: new_block: {:?}: {:?}", block, data); self.new_blocks.push(data); self.term_patch_map.push(None); diff --git a/compiler/rustc_mir_transform/src/post_analysis_normalize.rs b/compiler/rustc_mir_transform/src/post_analysis_normalize.rs index 76c2f082c0b..5599dee4cca 100644 --- a/compiler/rustc_mir_transform/src/post_analysis_normalize.rs +++ b/compiler/rustc_mir_transform/src/post_analysis_normalize.rs @@ -39,20 +39,22 @@ impl<'tcx> MutVisitor<'tcx> for PostAnalysisNormalizeVisitor<'tcx> { _context: PlaceContext, _location: Location, ) { - // Performance optimization: don't reintern if there is no `OpaqueCast` to remove. - if place.projection.iter().all(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) { - return; + if !self.tcx.next_trait_solver_globally() { + // `OpaqueCast` projections are only needed if there are opaque types on which projections + // are performed. After the `PostAnalysisNormalize` pass, all opaque types are replaced with their + // hidden types, so we don't need these projections anymore. + // + // Performance optimization: don't reintern if there is no `OpaqueCast` to remove. + if place.projection.iter().any(|elem| matches!(elem, ProjectionElem::OpaqueCast(_))) { + place.projection = self.tcx.mk_place_elems( + &place + .projection + .into_iter() + .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) + .collect::<Vec<_>>(), + ); + }; } - // `OpaqueCast` projections are only needed if there are opaque types on which projections - // are performed. After the `PostAnalysisNormalize` pass, all opaque types are replaced with their - // hidden types, so we don't need these projections anymore. - place.projection = self.tcx.mk_place_elems( - &place - .projection - .into_iter() - .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) - .collect::<Vec<_>>(), - ); self.super_place(place, _context, _location); } diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index c8d8dc147e9..47d43830970 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -18,7 +18,7 @@ use either::{Left, Right}; use rustc_const_eval::check_consts::{ConstCx, qualifs}; use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; -use rustc_index::{Idx, IndexSlice, IndexVec}; +use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::{self, GenericArgs, List, Ty, TyCtxt, TypeVisitableExt}; @@ -864,17 +864,21 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { new_temp } - fn promote_candidate(mut self, candidate: Candidate, next_promoted_id: usize) -> Body<'tcx> { + fn promote_candidate( + mut self, + candidate: Candidate, + next_promoted_index: Promoted, + ) -> Body<'tcx> { let def = self.source.source.def_id(); let (mut rvalue, promoted_op) = { let promoted = &mut self.promoted; - let promoted_id = Promoted::new(next_promoted_id); let tcx = self.tcx; let mut promoted_operand = |ty, span| { promoted.span = span; promoted.local_decls[RETURN_PLACE] = LocalDecl::new(ty, span); let args = tcx.erase_regions(GenericArgs::identity_for_item(tcx, def)); - let uneval = mir::UnevaluatedConst { def, args, promoted: Some(promoted_id) }; + let uneval = + mir::UnevaluatedConst { def, args, promoted: Some(next_promoted_index) }; ConstOperand { span, user_ty: None, const_: Const::Unevaluated(uneval, ty) } }; @@ -1034,7 +1038,7 @@ fn promote_candidates<'tcx>( required_consts: Vec::new(), }; - let mut promoted = promoter.promote_candidate(candidate, promotions.len()); + let mut promoted = promoter.promote_candidate(candidate, promotions.next_index()); promoted.source.promoted = Some(promotions.next_index()); promotions.push(promoted); } diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index c9771467e49..c13ffae3649 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -66,7 +66,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< let call_mut = tcx .associated_items(fn_mut) .in_definition_order() - .find(|it| it.kind == ty::AssocKind::Fn) + .find(|it| it.is_fn()) .unwrap() .def_id; diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 84905f4a400..5947637cded 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -597,20 +597,6 @@ impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { self.tcx } - fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { - if let StatementKind::BackwardIncompatibleDropHint { place, reason: _ } = - &mut statement.kind - { - self.visit_local( - &mut place.local, - PlaceContext::MutatingUse(MutatingUseContext::Store), - location, - ); - } else { - self.super_statement(statement, location); - } - } - fn visit_local(&mut self, l: &mut Local, _: PlaceContext, _: Location) { *l = self.map[*l].unwrap(); } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index e7930f0a1e3..66fe3ef4141 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -221,12 +221,11 @@ impl<'a, 'tcx> CfgChecker<'a, 'tcx> { // Check for cycles let mut stack = FxHashSet::default(); - for i in 0..parent.len() { - let mut bb = BasicBlock::from_usize(i); + for (mut bb, parent) in parent.iter_enumerated_mut() { stack.clear(); stack.insert(bb); loop { - let Some(parent) = parent[bb].take() else { break }; + let Some(parent) = parent.take() else { break }; let no_cycle = stack.insert(parent); if !no_cycle { self.fail( |