diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
19 files changed, 251 insertions, 196 deletions
diff --git a/compiler/rustc_mir_transform/src/const_debuginfo.rs b/compiler/rustc_mir_transform/src/const_debuginfo.rs index 3577b3d2d80..8944ebed9a7 100644 --- a/compiler/rustc_mir_transform/src/const_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/const_debuginfo.rs @@ -19,7 +19,7 @@ impl<'tcx> MirPass<'tcx> for ConstDebugInfo { sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() > 0 } - fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running ConstDebugInfo on {:?}", body.source); for (local, constant) in find_optimization_oportunities(body) { diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index b17485fd542..412a5b4fc91 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -67,6 +67,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp { true } + #[instrument(skip(self, tcx), level = "debug")] fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // will be evaluated by miri and produce its errors there if body.source.promoted.is_some() { @@ -687,7 +688,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { Operand::Constant(Box::new(Constant { span, user_ty: None, - literal: ty::Const::from_scalar(self.tcx, scalar, ty).into(), + literal: ConstantKind::from_scalar(self.tcx, scalar, ty), })) } @@ -699,7 +700,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ) { if let Rvalue::Use(Operand::Constant(c)) = rval { match c.literal { - ConstantKind::Ty(c) if matches!(c.val(), ConstKind::Unevaluated(..)) => {} + ConstantKind::Ty(c) if matches!(c.kind(), ConstKind::Unevaluated(..)) => {} _ => { trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c); return; @@ -765,20 +766,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if let Some(Some(alloc)) = alloc { // Assign entire constant in a single statement. // We can't use aggregates, as we run after the aggregate-lowering `MirPhase`. + let const_val = ConstValue::ByRef { alloc, offset: Size::ZERO }; + let literal = ConstantKind::Val(const_val, ty); *rval = Rvalue::Use(Operand::Constant(Box::new(Constant { span: source_info.span, user_ty: None, - literal: self - .ecx - .tcx - .mk_const(ty::ConstS { - ty, - val: ty::ConstKind::Value(ConstValue::ByRef { - alloc, - offset: Size::ZERO, - }), - }) - .into(), + literal, }))); } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 5fd9db3989d..15ad13009e5 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -437,10 +437,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { source_info.scope.lint_root(self.source_scopes) } - fn use_ecx<F, T>(&mut self, f: F) -> Option<T> + fn use_ecx<F, T>(&mut self, source_info: SourceInfo, f: F) -> Option<T> where F: FnOnce(&mut Self) -> InterpResult<'tcx, T>, { + // Overwrite the PC -- whatever the interpreter does to it does not make any sense anyway. + self.ecx.frame_mut().loc = Err(source_info.span); match f(self) { Ok(val) => Some(val), Err(error) => { @@ -472,7 +474,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let err = ConstEvalErr::new(&self.ecx, error, Some(c.span)); if let Some(lint_root) = self.lint_root(source_info) { let lint_only = match c.literal { - ConstantKind::Ty(ct) => match ct.val() { + ConstantKind::Ty(ct) => match ct.kind() { // Promoteds must lint and not error as the user didn't ask for them ConstKind::Unevaluated(ty::Unevaluated { def: _, @@ -501,9 +503,9 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } /// Returns the value, if any, of evaluating `place`. - fn eval_place(&mut self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { + fn eval_place(&mut self, place: Place<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { trace!("eval_place(place={:?})", place); - self.use_ecx(|this| this.ecx.eval_place_to_op(place, None)) + self.use_ecx(source_info, |this| this.ecx.eval_place_to_op(place, None)) } /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant` @@ -511,7 +513,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn eval_operand(&mut self, op: &Operand<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { match *op { Operand::Constant(ref c) => self.eval_constant(c, source_info), - Operand::Move(place) | Operand::Copy(place) => self.eval_place(place), + Operand::Move(place) | Operand::Copy(place) => self.eval_place(place, source_info), } } @@ -537,7 +539,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { arg: &Operand<'tcx>, source_info: SourceInfo, ) -> Option<()> { - if let (val, true) = self.use_ecx(|this| { + if let (val, true) = self.use_ecx(source_info, |this| { let val = this.ecx.read_immediate(&this.ecx.eval_operand(arg, None)?)?; let (_res, overflow, _ty) = this.ecx.overflowing_unary_op(op, &val)?; Ok((val, overflow)) @@ -564,8 +566,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { right: &Operand<'tcx>, source_info: SourceInfo, ) -> Option<()> { - let r = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?)); - let l = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?)); + let r = self.use_ecx(source_info, |this| { + this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?) + }); + let l = self.use_ecx(source_info, |this| { + this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?) + }); // Check for exceeding shifts *even if* we cannot evaluate the LHS. if op == BinOp::Shr || op == BinOp::Shl { let r = r?; @@ -602,7 +608,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if let (Some(l), Some(r)) = (&l, &r) { // The remaining operators are handled through `overflowing_binary_op`. - if self.use_ecx(|this| { + if self.use_ecx(source_info, |this| { let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, l, r)?; Ok(overflow) })? { @@ -690,7 +696,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { return None; } - self.use_ecx(|this| this.ecx.eval_rvalue_into_place(rvalue, place)) + self.use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, place)) } } @@ -890,7 +896,10 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { StatementKind::SetDiscriminant { ref place, .. } => { match self.ecx.machine.can_const_prop[place.local] { ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { - if self.use_ecx(|this| this.ecx.statement(statement)).is_some() { + if self + .use_ecx(source_info, |this| this.ecx.statement(statement)) + .is_some() + { trace!("propped discriminant into {:?}", place); } else { Self::remove_const(&mut self.ecx, place.local); diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index 6726b669ff2..45de0c28035 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -43,7 +43,7 @@ impl CoverageCounters { pub fn make_bcb_counters( &mut self, basic_coverage_blocks: &mut CoverageGraph, - coverage_spans: &Vec<CoverageSpan>, + coverage_spans: &[CoverageSpan], ) -> Result<Vec<CoverageKind>, Error> { let mut bcb_counters = BcbCounters::new(self, basic_coverage_blocks); bcb_counters.make_bcb_counters(coverage_spans) @@ -349,7 +349,7 @@ impl<'a> BcbCounters<'a> { // counters and/or expressions of its incoming edges. This will recursively get or create // counters for those incoming edges first, then call `make_expression()` to sum them up, // with additional intermediate expressions as needed. - let mut predecessors = self.bcb_predecessors(bcb).clone().into_iter(); + let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter(); debug!( "{}{:?} has multiple incoming edges and will get an expression that sums them up...", NESTED_INDENT.repeat(debug_indent_level), @@ -571,12 +571,12 @@ impl<'a> BcbCounters<'a> { } #[inline] - fn bcb_predecessors(&self, bcb: BasicCoverageBlock) -> &Vec<BasicCoverageBlock> { + fn bcb_predecessors(&self, bcb: BasicCoverageBlock) -> &[BasicCoverageBlock] { &self.basic_coverage_blocks.predecessors[bcb] } #[inline] - fn bcb_successors(&self, bcb: BasicCoverageBlock) -> &Vec<BasicCoverageBlock> { + fn bcb_successors(&self, bcb: BasicCoverageBlock) -> &[BasicCoverageBlock] { &self.basic_coverage_blocks.successors[bcb] } diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs index 434bf9d849e..0f8679b0bd6 100644 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ b/compiler/rustc_mir_transform/src/coverage/debug.rs @@ -123,14 +123,15 @@ use rustc_middle::ty::TyCtxt; use rustc_span::Span; use std::iter; -use std::lazy::SyncOnceCell; +use std::ops::Deref; +use std::sync::OnceLock; pub const NESTED_INDENT: &str = " "; const RUSTC_COVERAGE_DEBUG_OPTIONS: &str = "RUSTC_COVERAGE_DEBUG_OPTIONS"; pub(super) fn debug_options<'a>() -> &'a DebugOptions { - static DEBUG_OPTIONS: SyncOnceCell<DebugOptions> = SyncOnceCell::new(); + static DEBUG_OPTIONS: OnceLock<DebugOptions> = OnceLock::new(); &DEBUG_OPTIONS.get_or_init(DebugOptions::from_env) } @@ -434,11 +435,11 @@ impl GraphvizData { pub fn get_bcb_coverage_spans_with_counters( &self, bcb: BasicCoverageBlock, - ) -> Option<&Vec<(CoverageSpan, CoverageKind)>> { + ) -> Option<&[(CoverageSpan, CoverageKind)]> { if let Some(bcb_to_coverage_spans_with_counters) = self.some_bcb_to_coverage_spans_with_counters.as_ref() { - bcb_to_coverage_spans_with_counters.get(&bcb) + bcb_to_coverage_spans_with_counters.get(&bcb).map(Deref::deref) } else { None } @@ -457,12 +458,9 @@ impl GraphvizData { } } - pub fn get_bcb_dependency_counters( - &self, - bcb: BasicCoverageBlock, - ) -> Option<&Vec<CoverageKind>> { + pub fn get_bcb_dependency_counters(&self, bcb: BasicCoverageBlock) -> Option<&[CoverageKind]> { if let Some(bcb_to_dependency_counters) = self.some_bcb_to_dependency_counters.as_ref() { - bcb_to_dependency_counters.get(&bcb) + bcb_to_dependency_counters.get(&bcb).map(Deref::deref) } else { None } @@ -571,11 +569,11 @@ impl UsedExpressions { /// associated with a coverage span). pub fn validate( &mut self, - bcb_counters_without_direct_coverage_spans: &Vec<( + bcb_counters_without_direct_coverage_spans: &[( Option<BasicCoverageBlock>, BasicCoverageBlock, CoverageKind, - )>, + )], ) { if self.is_enabled() { let mut not_validated = bcb_counters_without_direct_coverage_spans @@ -634,7 +632,7 @@ pub(super) fn dump_coverage_spanview<'tcx>( basic_coverage_blocks: &CoverageGraph, pass_name: &str, body_span: Span, - coverage_spans: &Vec<CoverageSpan>, + coverage_spans: &[CoverageSpan], ) { let mir_source = mir_body.source; let def_id = mir_source.def_id(); @@ -654,7 +652,7 @@ fn span_viewables<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, basic_coverage_blocks: &CoverageGraph, - coverage_spans: &Vec<CoverageSpan>, + coverage_spans: &[CoverageSpan], ) -> Vec<SpanViewable> { let mut span_viewables = Vec::new(); for coverage_span in coverage_spans { @@ -676,7 +674,7 @@ pub(super) fn dump_coverage_graphviz<'tcx>( basic_coverage_blocks: &CoverageGraph, debug_counters: &DebugCounters, graphviz_data: &GraphvizData, - intermediate_expressions: &Vec<CoverageKind>, + intermediate_expressions: &[CoverageKind], debug_used_expressions: &UsedExpressions, ) { let mir_source = mir_body.source; @@ -753,9 +751,9 @@ fn bcb_to_string_sections<'tcx>( mir_body: &mir::Body<'tcx>, debug_counters: &DebugCounters, bcb_data: &BasicCoverageBlockData, - some_coverage_spans_with_counters: Option<&Vec<(CoverageSpan, CoverageKind)>>, - some_dependency_counters: Option<&Vec<CoverageKind>>, - some_intermediate_expressions: Option<&Vec<CoverageKind>>, + some_coverage_spans_with_counters: Option<&[(CoverageSpan, CoverageKind)]>, + some_dependency_counters: Option<&[CoverageKind]>, + some_intermediate_expressions: Option<&[CoverageKind]>, ) -> Vec<String> { let len = bcb_data.basic_blocks.len(); let mut sections = Vec::new(); diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 4615f9be33f..213bb6608e1 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -37,7 +37,7 @@ use rustc_data_structures::graph::WithSuccessors; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::mir::*; -use rustc_middle::ty::{self, BOOL_TY}; +use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::{self, BytePos, Pos, Span, DUMMY_SP}; // All `TEMP_BLOCK` targets should be replaced before calling `to_body() -> mir::Body`. @@ -47,6 +47,7 @@ struct MockBlocks<'tcx> { blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>, dummy_place: Place<'tcx>, next_local: usize, + bool_ty: Ty<'tcx>, } impl<'tcx> MockBlocks<'tcx> { @@ -55,6 +56,7 @@ impl<'tcx> MockBlocks<'tcx> { blocks: IndexVec::new(), dummy_place: Place { local: RETURN_PLACE, projection: ty::List::empty() }, next_local: 0, + bool_ty: TyCtxt::BOOL_TY_FOR_UNIT_TESTING, } } @@ -155,7 +157,7 @@ impl<'tcx> MockBlocks<'tcx> { fn switchint(&mut self, some_from_block: Option<BasicBlock>) -> BasicBlock { let switchint_kind = TerminatorKind::SwitchInt { discr: Operand::Move(Place::from(self.new_temp())), - switch_ty: BOOL_TY, // just a dummy value + switch_ty: self.bool_ty, // just a dummy value targets: SwitchTargets::static_if(0, TEMP_BLOCK, TEMP_BLOCK), }; self.add_block_from(some_from_block, switchint_kind) diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs new file mode 100644 index 00000000000..28f3790914b --- /dev/null +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -0,0 +1,86 @@ +//! This module implements a dead store elimination (DSE) routine. +//! +//! This transformation was written specifically for the needs of dest prop. Although it is +//! perfectly sound to use it in any context that might need it, its behavior should not be changed +//! without analyzing the interaction this will have with dest prop. Specifically, in addition to +//! the soundness of this pass in general, dest prop needs it to satisfy two additional conditions: +//! +//! 1. It's idempotent, meaning that running this pass a second time immediately after running it a +//! first time will not cause any further changes. +//! 2. This idempotence persists across dest prop's main transform, in other words inserting any +//! number of iterations of dest prop between the first and second application of this transform +//! will still not cause any further changes. +//! + +use rustc_index::bit_set::BitSet; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::impls::{borrowed_locals, MaybeTransitiveLiveLocals}; +use rustc_mir_dataflow::Analysis; + +/// Performs the optimization on the body +/// +/// The `borrowed` set must be a `BitSet` of all the locals that are ever borrowed in this body. It +/// can be generated via the [`borrowed_locals`] function. +pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitSet<Local>) { + let mut live = MaybeTransitiveLiveLocals::new(borrowed) + .into_engine(tcx, body) + .iterate_to_fixpoint() + .into_results_cursor(body); + + let mut patch = Vec::new(); + for (bb, bb_data) in traversal::preorder(body) { + for (statement_index, statement) in bb_data.statements.iter().enumerate().rev() { + let loc = Location { block: bb, statement_index }; + if let StatementKind::Assign(assign) = &statement.kind { + if !assign.1.is_safe_to_remove() { + continue; + } + } + match &statement.kind { + StatementKind::Assign(box (place, _)) + | StatementKind::SetDiscriminant { place: box place, .. } + | StatementKind::Deinit(box place) => { + if !place.is_indirect() && !borrowed.contains(place.local) { + live.seek_before_primary_effect(loc); + if !live.get().contains(place.local) { + patch.push(loc); + } + } + } + StatementKind::Retag(_, _) + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Coverage(_) + | StatementKind::CopyNonOverlapping(_) + | StatementKind::Nop => (), + + StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { + bug!("{:?} not found in this MIR phase!", &statement.kind) + } + } + } + } + + if patch.is_empty() { + return; + } + + let bbs = body.basic_blocks_mut(); + for Location { block, statement_index } in patch { + bbs[block].statements[statement_index].make_nop(); + } +} + +pub struct DeadStoreElimination; + +impl<'tcx> MirPass<'tcx> for DeadStoreElimination { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 2 + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let borrowed = borrowed_locals(body); + eliminate(tcx, body, &borrowed); + } +} diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index 57a95a67df7..bfb3ad1be27 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -1,9 +1,11 @@ use crate::MirPass; use rustc_index::vec::IndexVec; use rustc_middle::mir::patch::MirPatch; +use rustc_middle::mir::visit::NonUseContext::VarDebugInfo; use rustc_middle::mir::visit::{MutVisitor, PlaceContext}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; + pub struct Derefer; pub struct DerefChecker<'tcx> { @@ -17,63 +19,68 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { self.tcx } - fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, loc: Location) { - let mut place_local = place.local; - let mut last_len = 0; - let mut last_deref_idx = 0; + fn visit_place(&mut self, place: &mut Place<'tcx>, cntxt: PlaceContext, loc: Location) { + if !place.projection.is_empty() + && cntxt != PlaceContext::NonUse(VarDebugInfo) + && place.projection[1..].contains(&ProjectionElem::Deref) + { + let mut place_local = place.local; + let mut last_len = 0; + let mut last_deref_idx = 0; - let mut prev_temp: Option<Local> = None; + let mut prev_temp: Option<Local> = None; - for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() { - if p_elem == ProjectionElem::Deref && !p_ref.projection.is_empty() { - last_deref_idx = idx; - } - } - - for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() { - if p_elem == ProjectionElem::Deref && !p_ref.projection.is_empty() { - let ty = p_ref.ty(&self.local_decls, self.tcx).ty; - let temp = self.patcher.new_local_with_info( - ty, - self.local_decls[p_ref.local].source_info.span, - Some(Box::new(LocalInfo::DerefTemp)), - ); - - self.patcher.add_statement(loc, StatementKind::StorageLive(temp)); - - // We are adding current p_ref's projections to our - // temp value, excluding projections we already covered. - let deref_place = Place::from(place_local) - .project_deeper(&p_ref.projection[last_len..], self.tcx); - - self.patcher.add_assign( - loc, - Place::from(temp), - Rvalue::Use(Operand::Move(deref_place)), - ); - place_local = temp; - last_len = p_ref.projection.len(); - - // Change `Place` only if we are actually at the Place's last deref - if idx == last_deref_idx { - let temp_place = - Place::from(temp).project_deeper(&place.projection[idx..], self.tcx); - *place = temp_place; + for (idx, elem) in place.projection[0..].iter().enumerate() { + if *elem == ProjectionElem::Deref { + last_deref_idx = idx; } - - // We are destroying the previous temp since it's no longer used. - if let Some(prev_temp) = prev_temp { - self.patcher.add_statement(loc, StatementKind::StorageDead(prev_temp)); + } + for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() { + if !p_ref.projection.is_empty() && p_elem == ProjectionElem::Deref { + let ty = p_ref.ty(&self.local_decls, self.tcx).ty; + let temp = self.patcher.new_local_with_info( + ty, + self.local_decls[p_ref.local].source_info.span, + Some(Box::new(LocalInfo::DerefTemp)), + ); + + self.patcher.add_statement(loc, StatementKind::StorageLive(temp)); + + // We are adding current p_ref's projections to our + // temp value, excluding projections we already covered. + let deref_place = Place::from(place_local) + .project_deeper(&p_ref.projection[last_len..], self.tcx); + + self.patcher.add_assign( + loc, + Place::from(temp), + Rvalue::Use(Operand::Move(deref_place)), + ); + place_local = temp; + last_len = p_ref.projection.len(); + + // Change `Place` only if we are actually at the Place's last deref + if idx == last_deref_idx { + let temp_place = + Place::from(temp).project_deeper(&place.projection[idx..], self.tcx); + *place = temp_place; + } + + // We are destroying the previous temp since it's no longer used. + if let Some(prev_temp) = prev_temp { + self.patcher.add_statement(loc, StatementKind::StorageDead(prev_temp)); + } + + prev_temp = Some(temp); } - - prev_temp = Some(temp); } - } - // Since we won't be able to reach final temp, we destroy it outside the loop. - if let Some(prev_temp) = prev_temp { - let last_loc = Location { block: loc.block, statement_index: loc.statement_index + 1 }; - self.patcher.add_statement(last_loc, StatementKind::StorageDead(prev_temp)); + // Since we won't be able to reach final temp, we destroy it outside the loop. + if let Some(prev_temp) = prev_temp { + let last_loc = + Location { block: loc.block, statement_index: loc.statement_index + 1 }; + self.patcher.add_statement(last_loc, StatementKind::StorageDead(prev_temp)); + } } } } @@ -92,5 +99,6 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { impl<'tcx> MirPass<'tcx> for Derefer { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { deref_finder(tcx, body); + body.phase = MirPhase::Derefered; } } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 182dd6f379c..84c7aada5e5 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -104,7 +104,7 @@ use rustc_middle::mir::{ Rvalue, Statement, StatementKind, Terminator, TerminatorKind, }; use rustc_middle::ty::TyCtxt; -use rustc_mir_dataflow::impls::{MaybeInitializedLocals, MaybeLiveLocals}; +use rustc_mir_dataflow::impls::{borrowed_locals, MaybeInitializedLocals, MaybeLiveLocals}; use rustc_mir_dataflow::Analysis; // Empirical measurements have resulted in some observations: @@ -805,7 +805,7 @@ fn find_candidates<'tcx>(body: &Body<'tcx>) -> Vec<CandidateAssignment<'tcx>> { let mut visitor = FindAssignments { body, candidates: Vec::new(), - ever_borrowed_locals: ever_borrowed_locals(body), + ever_borrowed_locals: borrowed_locals(body), locals_used_as_array_index: locals_used_as_array_index(body), }; visitor.visit_body(body); @@ -886,69 +886,6 @@ fn is_local_required(local: Local, body: &Body<'_>) -> bool { } } -/// Walks MIR to find all locals that have their address taken anywhere. -fn ever_borrowed_locals(body: &Body<'_>) -> BitSet<Local> { - let mut visitor = BorrowCollector { locals: BitSet::new_empty(body.local_decls.len()) }; - visitor.visit_body(body); - visitor.locals -} - -struct BorrowCollector { - locals: BitSet<Local>, -} - -impl<'tcx> Visitor<'tcx> for BorrowCollector { - fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { - self.super_rvalue(rvalue, location); - - match rvalue { - Rvalue::AddressOf(_, borrowed_place) | Rvalue::Ref(_, _, borrowed_place) => { - if !borrowed_place.is_indirect() { - self.locals.insert(borrowed_place.local); - } - } - - Rvalue::Cast(..) - | Rvalue::ShallowInitBox(..) - | Rvalue::Use(..) - | Rvalue::Repeat(..) - | Rvalue::Len(..) - | Rvalue::BinaryOp(..) - | Rvalue::CheckedBinaryOp(..) - | Rvalue::NullaryOp(..) - | Rvalue::UnaryOp(..) - | Rvalue::Discriminant(..) - | Rvalue::Aggregate(..) - | Rvalue::ThreadLocalRef(..) => {} - } - } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { - self.super_terminator(terminator, location); - - match terminator.kind { - TerminatorKind::Drop { place: dropped_place, .. } - | TerminatorKind::DropAndReplace { place: dropped_place, .. } => { - self.locals.insert(dropped_place.local); - } - - TerminatorKind::Abort - | TerminatorKind::Assert { .. } - | TerminatorKind::Call { .. } - | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } - | TerminatorKind::GeneratorDrop - | TerminatorKind::Goto { .. } - | TerminatorKind::Resume - | TerminatorKind::Return - | TerminatorKind::SwitchInt { .. } - | TerminatorKind::Unreachable - | TerminatorKind::Yield { .. } - | TerminatorKind::InlineAsm { .. } => {} - } - } -} - /// `PlaceElem::Index` only stores a `Local`, so we can't replace that with a full `Place`. /// /// Collect locals used as indices so we don't generate candidates that are impossible to apply diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 8a9f2107316..e0e27c53f18 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -470,7 +470,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { Rvalue::Use(Operand::Constant(Box::new(Constant { span, user_ty: None, - literal: ty::Const::from_bool(self.tcx, val).into(), + literal: ConstantKind::from_bool(self.tcx, val), }))) } diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 4a3505ca3ff..7f0d3b0a612 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -49,6 +49,7 @@ //! For generators with state 1 (returned) and state 2 (poisoned) it does nothing. //! Otherwise it drops all the values in scope at the last suspension point. +use crate::deref_separator::deref_finder; use crate::simplify; use crate::util::expand_aggregate; use crate::MirPass; @@ -227,7 +228,7 @@ struct TransformVisitor<'tcx> { suspension_points: Vec<SuspensionPoint<'tcx>>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. - always_live_locals: storage::AlwaysLiveLocals, + always_live_locals: BitSet<Local>, // The original RETURN_PLACE local new_ret_local: Local, @@ -449,7 +450,7 @@ struct LivenessInfo { fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &storage::AlwaysLiveLocals, + always_live_locals: &BitSet<Local>, movable: bool, ) -> LivenessInfo { let body_ref: &Body<'_> = &body; @@ -494,7 +495,8 @@ fn locals_live_across_suspend_points<'tcx>( let loc = Location { block, statement_index: data.statements.len() }; liveness.seek_to_block_end(block); - let mut live_locals = liveness.get().clone(); + let mut live_locals: BitSet<_> = BitSet::new_empty(body.local_decls.len()); + live_locals.union(liveness.get()); if !movable { // The `liveness` variable contains the liveness of MIR locals ignoring borrows. @@ -614,7 +616,7 @@ impl ops::Deref for GeneratorSavedLocals { fn compute_storage_conflicts<'mir, 'tcx>( body: &'mir Body<'tcx>, saved_locals: &GeneratorSavedLocals, - always_live_locals: storage::AlwaysLiveLocals, + always_live_locals: BitSet<Local>, requires_storage: rustc_mir_dataflow::Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>, ) -> BitMatrix<GeneratorSavedLocal, GeneratorSavedLocal> { assert_eq!(body.local_decls.len(), saved_locals.domain_size()); @@ -624,7 +626,7 @@ fn compute_storage_conflicts<'mir, 'tcx>( // Locals that are always live or ones that need to be stored across // suspension points are not eligible for overlap. - let mut ineligible_locals = always_live_locals.into_inner(); + let mut ineligible_locals = always_live_locals; ineligible_locals.intersect(&**saved_locals); // Compute the storage conflicts for all eligible locals. @@ -990,7 +992,7 @@ fn insert_panic_block<'tcx>( cond: Operand::Constant(Box::new(Constant { span: body.span, user_ty: None, - literal: ty::Const::from_bool(tcx, false).into(), + literal: ConstantKind::from_bool(tcx, false), })), expected: true, msg: message, @@ -1299,7 +1301,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform { }, ); - let always_live_locals = storage::AlwaysLiveLocals::new(&body); + let always_live_locals = storage::always_live_locals(&body); let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); @@ -1368,6 +1370,9 @@ impl<'tcx> MirPass<'tcx> for StateTransform { // Create the Generator::resume function create_generator_resume_function(tcx, transform, body, can_return); + + // Run derefer to fix Derefs that are not in the first place + deref_finder(tcx, body); } } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 66fb01bd464..49403ba03a4 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -1,5 +1,5 @@ //! Inlining pass for MIR functions - +use crate::deref_separator::deref_finder; use rustc_attr::InlineAttr; use rustc_index::bit_set::BitSet; use rustc_index::vec::Idx; @@ -53,6 +53,7 @@ impl<'tcx> MirPass<'tcx> for Inline { debug!("running simplify cfg on {:?}", body.source); CfgSimplifier::new(body).simplify(); remove_dead_blocks(tcx, body); + deref_finder(tcx, body); } } } @@ -157,11 +158,13 @@ impl<'tcx> Inliner<'tcx> { return Err("optimization fuel exhausted"); } - let callee_body = callsite.callee.subst_mir_and_normalize_erasing_regions( + let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( self.tcx, self.param_env, callee_body.clone(), - ); + ) else { + return Err("failed to normalize callee body"); + }; let old_blocks = caller_body.basic_blocks().next_index(); self.inline_call(caller_body, &callsite, callee_body); @@ -252,7 +255,7 @@ impl<'tcx> Inliner<'tcx> { let func_ty = func.ty(caller_body, self.tcx); if let ty::FnDef(def_id, substs) = *func_ty.kind() { // To resolve an instance its substs have to be fully normalized. - let substs = self.tcx.normalize_erasing_regions(self.param_env, substs); + let substs = self.tcx.try_normalize_erasing_regions(self.param_env, substs).ok()?; let callee = Instance::resolve(self.tcx, self.param_env, def_id, substs).ok().flatten()?; @@ -407,14 +410,17 @@ impl<'tcx> Inliner<'tcx> { if let ty::FnDef(def_id, substs) = *callsite.callee.subst_mir(self.tcx, &f.literal.ty()).kind() { - let substs = self.tcx.normalize_erasing_regions(self.param_env, substs); - if let Ok(Some(instance)) = - Instance::resolve(self.tcx, self.param_env, def_id, substs) + if let Ok(substs) = + self.tcx.try_normalize_erasing_regions(self.param_env, substs) { - if callsite.callee.def_id() == instance.def_id() { - return Err("self-recursion"); - } else if self.history.contains(&instance) { - return Err("already inlined"); + if let Ok(Some(instance)) = + Instance::resolve(self.tcx, self.param_env, def_id, substs) + { + if callsite.callee.def_id() == instance.def_id() { + return Err("self-recursion"); + } else if self.history.contains(&instance) { + return Err("already inlined"); + } } } // Don't give intrinsics the extra penalty for calls @@ -608,7 +614,7 @@ impl<'tcx> Inliner<'tcx> { caller_body.required_consts.extend( callee_body.required_consts.iter().copied().filter(|&ct| { match ct.literal.const_for_ty() { - Some(ct) => matches!(ct.val(), ConstKind::Unevaluated(_)), + Some(ct) => matches!(ct.kind(), ConstKind::Unevaluated(_)), None => true, } }), diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs index 4fbb7643378..ea10ec5f25c 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instcombine.rs @@ -3,8 +3,8 @@ use crate::MirPass; use rustc_hir::Mutability; use rustc_middle::mir::{ - BinOp, Body, Constant, LocalDecls, Operand, Place, ProjectionElem, Rvalue, SourceInfo, - Statement, StatementKind, Terminator, TerminatorKind, UnOp, + BinOp, Body, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem, Rvalue, + SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnOp, }; use rustc_middle::ty::{self, TyCtxt}; @@ -129,8 +129,8 @@ impl<'tcx> InstCombineContext<'tcx, '_> { return; } - let constant = - Constant { span: source_info.span, literal: len.into(), user_ty: None }; + let literal = ConstantKind::from_const(len, self.tcx); + let constant = Constant { span: source_info.span, literal, user_ty: None }; *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); } } diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 571f541072a..b8932251465 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -9,6 +9,7 @@ #![feature(option_get_or_insert_default)] #![feature(trusted_step)] #![feature(try_blocks)] +#![feature(yeet_expr)] #![recursion_limit = "256"] #[macro_use] @@ -49,6 +50,7 @@ mod const_goto; mod const_prop; mod const_prop_lint; mod coverage; +mod dead_store_elimination; mod deaggregator; mod deduplicate_blocks; mod deref_separator; @@ -481,17 +483,18 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &const_prop::ConstProp, // // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0. + &const_debuginfo::ConstDebugInfo, &o1(simplify_branches::SimplifyConstCondition::new("after-const-prop")), &early_otherwise_branch::EarlyOtherwiseBranch, &simplify_comparison_integral::SimplifyComparisonIntegral, &simplify_try::SimplifyArmIdentity, &simplify_try::SimplifyBranchSame, + &dead_store_elimination::DeadStoreElimination, &dest_prop::DestinationPropagation, &o1(simplify_branches::SimplifyConstCondition::new("final")), &o1(remove_noop_landing_pads::RemoveNoopLandingPads), &o1(simplify::SimplifyCfg::new("final")), &nrvo::RenameReturnPlace, - &const_debuginfo::ConstDebugInfo, &simplify::SimplifyLocals, &multiple_return_terminators::MultipleReturnTerminators, &deduplicate_blocks::DeduplicateBlocks, @@ -528,8 +531,10 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { None => {} Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), } + debug!("about to call mir_drops_elaborated..."); let mut body = tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); + debug!("body: {:#?}", body); run_optimization_passes(tcx, &mut body); debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 65801069560..989b94b68c1 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -34,7 +34,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { Rvalue::Use(Operand::Constant(Box::new(Constant { span: terminator.source_info.span, user_ty: None, - literal: ty::Const::zero_sized(tcx, tcx.types.unit).into(), + literal: ConstantKind::zero_sized(tcx.types.unit), }))), ))), }); diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index f925d13b2fb..89808d3d4cd 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -15,7 +15,7 @@ impl<'tcx> MirPass<'tcx> for RemoveNoopLandingPads { sess.panic_strategy() != PanicStrategy::Abort } - fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("remove_noop_landing_pads({:?})", body); self.remove_nop_landing_pads(body) } @@ -81,6 +81,8 @@ impl RemoveNoopLandingPads { } fn remove_nop_landing_pads(&self, body: &mut Body<'_>) { + debug!("body: {:#?}", body); + // make sure there's a single resume block let resume_block = { let patch = MirPatch::new(body); diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index b87220a3aa4..827ce0c02ac 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -15,7 +15,7 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> { impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) { let literal = constant.literal; - if let Some(ct) = literal.const_for_ty() && let ConstKind::Unevaluated(_) = ct.val() { + if let Some(ct) = literal.const_for_ty() && let ConstKind::Unevaluated(_) = ct.kind() { self.required_consts.push(*constant); } } diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 78a4ece2ecb..3be1783ae33 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -430,7 +430,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { let func = Operand::Constant(Box::new(Constant { span: self.span, user_ty: None, - literal: ty::Const::zero_sized(tcx, func_ty).into(), + literal: ConstantKind::zero_sized(func_ty), })); let ref_loc = self.make_place( @@ -630,7 +630,7 @@ fn build_call_shim<'tcx>( Operand::Constant(Box::new(Constant { span, user_ty: None, - literal: ty::Const::zero_sized(tcx, ty).into(), + literal: ConstantKind::zero_sized(ty), })), rcvr.into_iter().collect::<Vec<_>>(), ) diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 72e08343925..8a78ea5c82b 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -494,8 +494,12 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { StatementKind::StorageLive(_local) | StatementKind::StorageDead(_local) => {} StatementKind::Assign(box (ref place, ref rvalue)) => { - self.visit_lhs(place, location); - self.visit_rvalue(rvalue, location); + if rvalue.is_safe_to_remove() { + self.visit_lhs(place, location); + self.visit_rvalue(rvalue, location); + } else { + self.super_statement(statement, location); + } } StatementKind::SetDiscriminant { ref place, variant_index: _ } |