diff options
Diffstat (limited to 'src/librustc_mir/transform/inline.rs')
| -rw-r--r-- | src/librustc_mir/transform/inline.rs | 220 |
1 files changed, 92 insertions, 128 deletions
diff --git a/src/librustc_mir/transform/inline.rs b/src/librustc_mir/transform/inline.rs index dca14227946..aa0c71ff0f1 100644 --- a/src/librustc_mir/transform/inline.rs +++ b/src/librustc_mir/transform/inline.rs @@ -1,23 +1,23 @@ //! Inlining pass for MIR functions -use rustc::hir::CodegenFnAttrFlags; use rustc::hir::def_id::DefId; +use rustc::hir::CodegenFnAttrFlags; use rustc_index::bit_set::BitSet; use rustc_index::vec::{Idx, IndexVec}; -use rustc::mir::*; use rustc::mir::visit::*; -use rustc::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; +use rustc::mir::*; use rustc::ty::subst::{Subst, SubstsRef}; +use rustc::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; +use super::simplify::{remove_dead_blocks, CfgSimplifier}; +use crate::transform::{MirPass, MirSource}; use std::collections::VecDeque; use std::iter; -use crate::transform::{MirPass, MirSource}; -use super::simplify::{remove_dead_blocks, CfgSimplifier}; -use syntax::attr; use rustc_target::spec::abi::Abi; +use syntax::attr; const DEFAULT_THRESHOLD: usize = 50; const HINT_THRESHOLD: usize = 100; @@ -38,9 +38,7 @@ struct CallSite<'tcx> { } impl<'tcx> MirPass<'tcx> for Inline { - fn run_pass( - &self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut BodyAndCache<'tcx> - ) { + fn run_pass(&self, tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, body: &mut BodyAndCache<'tcx>) { if tcx.sess.opts.debugging_opts.mir_opt_level >= 2 { Inliner { tcx, source }.run_pass(body); } @@ -72,14 +70,11 @@ impl Inliner<'tcx> { // Only do inlining into fn bodies. let id = self.tcx.hir().as_local_hir_id(self.source.def_id()).unwrap(); - if self.tcx.hir().body_owner_kind(id).is_fn_or_closure() - && self.source.promoted.is_none() - { + if self.tcx.hir().body_owner_kind(id).is_fn_or_closure() && self.source.promoted.is_none() { for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated() { - if let Some(callsite) = self.get_valid_function_call(bb, - bb_data, - caller_body, - param_env) { + if let Some(callsite) = + self.get_valid_function_call(bb, bb_data, caller_body, param_env) + { callsites.push_back(callsite); } } @@ -108,7 +103,8 @@ impl Inliner<'tcx> { // not inline us. This trick only works without incremental compilation. // So don't do it if that is enabled. if !self.tcx.dep_graph.is_fully_enabled() - && self_node_id.as_u32() < callee_node_id.as_u32() { + && self_node_id.as_u32() < callee_node_id.as_u32() + { self.tcx.optimized_mir(callsite.callee) } else { continue; @@ -138,12 +134,10 @@ impl Inliner<'tcx> { debug!("attempting to inline callsite {:?} - success", callsite); // Add callsites from inlined function - for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) - { - if let Some(new_callsite) = self.get_valid_function_call(bb, - bb_data, - caller_body, - param_env) { + for (bb, bb_data) in caller_body.basic_blocks().iter_enumerated().skip(start) { + if let Some(new_callsite) = + self.get_valid_function_call(bb, bb_data, caller_body, param_env) + { // Don't inline the same function multiple times. if callsite.callee != new_callsite.callee { callsites.push_back(new_callsite); @@ -168,23 +162,23 @@ impl Inliner<'tcx> { } } - fn get_valid_function_call(&self, - bb: BasicBlock, - bb_data: &BasicBlockData<'tcx>, - caller_body: &Body<'tcx>, - param_env: ParamEnv<'tcx>, + fn get_valid_function_call( + &self, + bb: BasicBlock, + bb_data: &BasicBlockData<'tcx>, + caller_body: &Body<'tcx>, + param_env: ParamEnv<'tcx>, ) -> Option<CallSite<'tcx>> { // Don't inline calls that are in cleanup blocks. - if bb_data.is_cleanup { return None; } + if bb_data.is_cleanup { + return None; + } // Only consider direct calls to functions let terminator = bb_data.terminator(); if let TerminatorKind::Call { func: ref op, .. } = terminator.kind { if let ty::FnDef(callee_def_id, substs) = op.ty(caller_body, self.tcx).kind { - let instance = Instance::resolve(self.tcx, - param_env, - callee_def_id, - substs)?; + let instance = Instance::resolve(self.tcx, param_env, callee_def_id, substs)?; if let InstanceDef::Virtual(..) = instance.def { return None; @@ -194,7 +188,7 @@ impl Inliner<'tcx> { callee: instance.def_id(), substs: instance.substs, bb, - location: terminator.source_info + location: terminator.source_info, }); } } @@ -202,23 +196,15 @@ impl Inliner<'tcx> { None } - fn consider_optimizing(&self, - callsite: CallSite<'tcx>, - callee_body: &Body<'tcx>) - -> bool - { + fn consider_optimizing(&self, callsite: CallSite<'tcx>, callee_body: &Body<'tcx>) -> bool { debug!("consider_optimizing({:?})", callsite); self.should_inline(callsite, callee_body) - && self.tcx.consider_optimizing(|| format!("Inline {:?} into {:?}", - callee_body.span, - callsite)) + && self.tcx.consider_optimizing(|| { + format!("Inline {:?} into {:?}", callee_body.span, callsite) + }) } - fn should_inline(&self, - callsite: CallSite<'tcx>, - callee_body: &Body<'tcx>) - -> bool - { + fn should_inline(&self, callsite: CallSite<'tcx>, callee_body: &Body<'tcx>) -> bool { debug!("should_inline({:?})", callsite); let tcx = self.tcx; @@ -242,7 +228,7 @@ impl Inliner<'tcx> { attr::InlineAttr::Always => true, attr::InlineAttr::Never => { debug!("`#[inline(never)]` present - not inlining"); - return false + return false; } attr::InlineAttr::Hint => true, attr::InlineAttr::None => false, @@ -258,11 +244,7 @@ impl Inliner<'tcx> { } } - let mut threshold = if hinted { - HINT_THRESHOLD - } else { - DEFAULT_THRESHOLD - }; + let mut threshold = if hinted { HINT_THRESHOLD } else { DEFAULT_THRESHOLD }; // Significantly lower the threshold for inlining cold functions if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { @@ -289,23 +271,25 @@ impl Inliner<'tcx> { let mut work_list = vec![START_BLOCK]; let mut visited = BitSet::new_empty(callee_body.basic_blocks().len()); while let Some(bb) = work_list.pop() { - if !visited.insert(bb.index()) { continue; } + if !visited.insert(bb.index()) { + continue; + } let blk = &callee_body.basic_blocks()[bb]; for stmt in &blk.statements { // Don't count StorageLive/StorageDead in the inlining cost. match stmt.kind { - StatementKind::StorageLive(_) | - StatementKind::StorageDead(_) | - StatementKind::Nop => {} - _ => cost += INSTR_COST + StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Nop => {} + _ => cost += INSTR_COST, } } let term = blk.terminator(); let mut is_drop = false; match term.kind { - TerminatorKind::Drop { ref location, target, unwind } | - TerminatorKind::DropAndReplace { ref location, target, unwind, .. } => { + TerminatorKind::Drop { ref location, target, unwind } + | TerminatorKind::DropAndReplace { ref location, target, unwind, .. } => { is_drop = true; work_list.push(target); // If the location doesn't actually need dropping, treat it like @@ -321,14 +305,15 @@ impl Inliner<'tcx> { } } - TerminatorKind::Unreachable | - TerminatorKind::Call { destination: None, .. } if first_block => { + TerminatorKind::Unreachable | TerminatorKind::Call { destination: None, .. } + if first_block => + { // If the function always diverges, don't inline // unless the cost is zero threshold = 0; } - TerminatorKind::Call {func: Operand::Constant(ref f), .. } => { + TerminatorKind::Call { func: Operand::Constant(ref f), .. } => { if let ty::FnDef(def_id, _) = f.literal.ty.kind { // Don't give intrinsics the extra penalty for calls let f = tcx.fn_sig(def_id); @@ -340,7 +325,7 @@ impl Inliner<'tcx> { } } TerminatorKind::Assert { .. } => cost += CALL_PENALTY, - _ => cost += INSTR_COST + _ => cost += INSTR_COST, } if !is_drop { @@ -383,10 +368,12 @@ impl Inliner<'tcx> { } } - fn inline_call(&self, - callsite: CallSite<'tcx>, - caller_body: &mut BodyAndCache<'tcx>, - mut callee_body: BodyAndCache<'tcx>) -> bool { + fn inline_call( + &self, + callsite: CallSite<'tcx>, + caller_body: &mut BodyAndCache<'tcx>, + mut callee_body: BodyAndCache<'tcx>, + ) -> bool { let terminator = caller_body[callsite.bb].terminator.take().unwrap(); match terminator.kind { // FIXME: Handle inlining of diverging calls @@ -416,8 +403,7 @@ impl Inliner<'tcx> { for loc in callee_body.vars_and_temps_iter() { let mut local = callee_body.local_decls[loc].clone(); - local.source_info.scope = - scope_map[local.source_info.scope]; + local.source_info.scope = scope_map[local.source_info.scope]; local.source_info.span = callsite.location.span; let idx = caller_body.local_decls.push(local); @@ -432,8 +418,7 @@ impl Inliner<'tcx> { fn dest_needs_borrow(place: &Place<'_>) -> bool { for elem in place.projection.iter() { match elem { - ProjectionElem::Deref | - ProjectionElem::Index(_) => return true, + ProjectionElem::Deref | ProjectionElem::Index(_) => return true, _ => {} } } @@ -442,7 +427,7 @@ impl Inliner<'tcx> { // Static variables need a borrow because the callee // might modify the same static. PlaceBase::Static(_) => true, - _ => false + _ => false, } } @@ -451,7 +436,8 @@ impl Inliner<'tcx> { let dest = Rvalue::Ref( self.tcx.lifetimes.re_erased, BorrowKind::Mut { allow_two_phase_borrow: false }, - destination.0); + destination.0, + ); let ty = dest.ty(&**caller_body, self.tcx); @@ -462,10 +448,9 @@ impl Inliner<'tcx> { let stmt = Statement { source_info: callsite.location, - kind: StatementKind::Assign(box(tmp.clone(), dest)) + kind: StatementKind::Assign(box (tmp.clone(), dest)), }; - caller_body[callsite.bb] - .statements.push(stmt); + caller_body[callsite.bb].statements.push(stmt); self.tcx.mk_place_deref(tmp) } else { destination.0 @@ -501,7 +486,7 @@ impl Inliner<'tcx> { let terminator = Terminator { source_info: callsite.location, - kind: TerminatorKind::Goto { target: BasicBlock::new(bb_len) } + kind: TerminatorKind::Goto { target: BasicBlock::new(bb_len) }, }; caller_body[callsite.bb].terminator = Some(terminator); @@ -509,10 +494,8 @@ impl Inliner<'tcx> { true } kind => { - caller_body[callsite.bb].terminator = Some(Terminator { - source_info: terminator.source_info, - kind, - }); + caller_body[callsite.bb].terminator = + Some(Terminator { source_info: terminator.source_info, kind }); false } } @@ -551,10 +534,8 @@ impl Inliner<'tcx> { // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. if tcx.is_closure(callsite.callee) { let mut args = args.into_iter(); - let self_ - = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body); - let tuple - = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body); + let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body); + let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body); assert!(args.next().is_none()); let tuple = Place::from(tuple); @@ -568,18 +549,14 @@ impl Inliner<'tcx> { let closure_ref_arg = iter::once(self_); // The `tmp0`, `tmp1`, and `tmp2` in our example abonve. - let tuple_tmp_args = - tuple_tys.iter().enumerate().map(|(i, ty)| { - // This is e.g., `tuple_tmp.0` in our example above. - let tuple_field = Operand::Move(tcx.mk_place_field( - tuple.clone(), - Field::new(i), - ty.expect_ty(), - )); - - // Spill to a local to make e.g., `tmp0`. - self.create_temp_if_necessary(tuple_field, callsite, caller_body) - }); + let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { + // This is e.g., `tuple_tmp.0` in our example above. + let tuple_field = + Operand::Move(tcx.mk_place_field(tuple.clone(), Field::new(i), ty.expect_ty())); + + // Spill to a local to make e.g., `tmp0`. + self.create_temp_if_necessary(tuple_field, callsite, caller_body) + }); closure_ref_arg.chain(tuple_tmp_args).collect() } else { @@ -620,7 +597,7 @@ impl Inliner<'tcx> { let stmt = Statement { source_info: callsite.location, - kind: StatementKind::Assign(box(Place::from(arg_tmp), arg)), + kind: StatementKind::Assign(box (Place::from(arg_tmp), arg)), }; caller_body[callsite.bb].statements.push(stmt); arg_tmp @@ -683,12 +660,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { self.tcx } - fn visit_local( - &mut self, - local: &mut Local, - _ctxt: PlaceContext, - _location: Location, - ) { + fn visit_local(&mut self, local: &mut Local, _ctxt: PlaceContext, _location: Location) { *local = self.make_integrate_local(local); } @@ -699,7 +671,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { _location: Location, ) { match &mut place.base { - PlaceBase::Static(_) => {}, + PlaceBase::Static(_) => {} PlaceBase::Local(l) => { // If this is the `RETURN_PLACE`, we need to rebase any projections onto it. let dest_proj_len = self.destination.projection.len(); @@ -716,15 +688,12 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { } } - fn process_projection_elem( - &mut self, - elem: &PlaceElem<'tcx>, - ) -> Option<PlaceElem<'tcx>> { + fn process_projection_elem(&mut self, elem: &PlaceElem<'tcx>) -> Option<PlaceElem<'tcx>> { if let PlaceElem::Index(local) = elem { let new_local = self.make_integrate_local(local); if new_local != *local { - return Some(PlaceElem::Index(new_local)) + return Some(PlaceElem::Index(new_local)); } } @@ -737,12 +706,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { self.in_cleanup_block = false; } - fn visit_retag( - &mut self, - kind: &mut RetagKind, - place: &mut Place<'tcx>, - loc: Location, - ) { + fn visit_retag(&mut self, kind: &mut RetagKind, place: &mut Place<'tcx>, loc: Location) { self.super_retag(kind, place, loc); // We have to patch all inlined retags to be aware that they are no longer @@ -752,14 +716,12 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { } } - fn visit_terminator_kind(&mut self, - kind: &mut TerminatorKind<'tcx>, loc: Location) { + fn visit_terminator_kind(&mut self, kind: &mut TerminatorKind<'tcx>, loc: Location) { self.super_terminator_kind(kind, loc); match *kind { - TerminatorKind::GeneratorDrop | - TerminatorKind::Yield { .. } => bug!(), - TerminatorKind::Goto { ref mut target} => { + TerminatorKind::GeneratorDrop | TerminatorKind::Yield { .. } => bug!(), + TerminatorKind::Goto { ref mut target } => { *target = self.update_target(*target); } TerminatorKind::SwitchInt { ref mut targets, .. } => { @@ -767,8 +729,8 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { *tgt = self.update_target(*tgt); } } - TerminatorKind::Drop { ref mut target, ref mut unwind, .. } | - TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => { + TerminatorKind::Drop { ref mut target, ref mut unwind, .. } + | TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => { *target = self.update_target(*target); if let Some(tgt) = *unwind { *unwind = Some(self.update_target(tgt)); @@ -808,15 +770,17 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { *kind = TerminatorKind::Goto { target: tgt } } } - TerminatorKind::Abort => { } - TerminatorKind::Unreachable => { } + TerminatorKind::Abort => {} + TerminatorKind::Unreachable => {} TerminatorKind::FalseEdges { ref mut real_target, ref mut imaginary_target } => { *real_target = self.update_target(*real_target); *imaginary_target = self.update_target(*imaginary_target); } - TerminatorKind::FalseUnwind { real_target: _ , unwind: _ } => - // see the ordering of passes in the optimized_mir query. + TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => + // see the ordering of passes in the optimized_mir query. + { bug!("False unwinds should have been removed before inlining") + } } } |