const_eval: Predetermine the layout of all locals when pushing a stack frame

Usually the layout of any locals is required at least three times, once
when it becomes live, once when it is written to, and once it is read
from. By adding a cache for them, we can reduce the number of layout
queries speeding up code that is heavy on const_eval.

4 files changed, 24 insertions, 18 deletions

diff --git a/src/librustc_mir/const_eval.rs b/src/librustc_mir/const_eval.rs
index f5f40481679..105856fecc7 100644
--- a/src/librustc_mir/const_eval.rs
+++ b/src/librustc_mir/const_eval.rs
@@ -72,6 +72,7 @@ fn mk_eval_cx_inner<'a, 'mir, 'tcx>(
     ecx.stack.push(interpret::Frame {
         block: mir::START_BLOCK,
         locals: IndexVec::new(),
+        local_layouts: IndexVec::new(),
         instance,
         span,
         mir,
diff --git a/src/librustc_mir/interpret/eval_context.rs b/src/librustc_mir/interpret/eval_context.rs
index 19362b6cfdb..b2d3328a73f 100644
--- a/src/librustc_mir/interpret/eval_context.rs
+++ b/src/librustc_mir/interpret/eval_context.rs
@@ -1,3 +1,4 @@
+use std::cell::Cell;
 use std::fmt::Write;
 use std::mem;
 
@@ -76,6 +77,7 @@ pub struct Frame<'mir, 'tcx: 'mir, Tag=(), Extra=()> {
     /// `None` represents a local that is currently dead, while a live local
     /// can either directly contain `Scalar` or refer to some part of an `Allocation`.
     pub locals: IndexVec<mir::Local, LocalValue<Tag>>,
+    pub local_layouts: IndexVec<mir::Local, Cell<Option<TyLayout<'tcx>>>>,
 
     ////////////////////////////////////////////////////////////////////////////////
     // Current position within the function
@@ -290,9 +292,15 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tc
         frame: &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>,
         local: mir::Local
     ) -> EvalResult<'tcx, TyLayout<'tcx>> {
-        let local_ty = frame.mir.local_decls[local].ty;
-        let local_ty = self.monomorphize(local_ty, frame.instance.substs);
-        self.layout_of(local_ty)
+        let cell = &frame.local_layouts[local];
+        if cell.get().is_none() {
+            let local_ty = frame.mir.local_decls[local].ty;
+            let local_ty = self.monomorphize(local_ty, frame.instance.substs);
+            let layout = self.layout_of(local_ty)?;
+            cell.set(Some(layout));
+        }
+
+        Ok(cell.get().unwrap())
     }
 
     pub fn str_to_immediate(&mut self, s: &str) -> EvalResult<'tcx, Immediate<M::PointerTag>> {
@@ -426,6 +434,7 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tc
             // empty local array, we fill it in below, after we are inside the stack frame and
             // all methods actually know about the frame
             locals: IndexVec::new(),
+            local_layouts: IndexVec::from_elem_n(Default::default(), mir.local_decls.len()),
             span,
             instance,
             stmt: 0,
@@ -464,11 +473,11 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tc
                 },
             }
             // Finally, properly initialize all those that still have the dummy value
-            for (local, decl) in locals.iter_mut().zip(mir.local_decls.iter()) {
+            for (idx, local) in locals.iter_enumerated_mut() {
                 match *local {
                     LocalValue::Live(_) => {
                         // This needs to be peoperly initialized.
-                        let layout = self.layout_of(self.monomorphize(decl.ty, instance.substs))?;
+                        let layout = self.layout_of_local(self.frame(), idx)?;
                         *local = LocalValue::Live(self.uninit_operand(layout)?);
                     }
                     LocalValue::Dead => {
diff --git a/src/librustc_mir/interpret/operand.rs b/src/librustc_mir/interpret/operand.rs
index 04e0955ad61..b2648480f20 100644
--- a/src/librustc_mir/interpret/operand.rs
+++ b/src/librustc_mir/interpret/operand.rs
@@ -457,36 +457,30 @@ impl<'a, 'mir, 'tcx, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M>
     }
 
     /// This is used by [priroda](https://github.com/oli-obk/priroda) to get an OpTy from a local
-    ///
-    /// When you know the layout of the local in advance, you can pass it as last argument
-    pub fn access_local(
+    fn access_local(
         &self,
         frame: &super::Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>,
         local: mir::Local,
-        layout: Option<TyLayout<'tcx>>,
     ) -> EvalResult<'tcx, OpTy<'tcx, M::PointerTag>> {
         assert_ne!(local, mir::RETURN_PLACE);
         let op = *frame.locals[local].access()?;
-        let layout = from_known_layout(layout,
-                    || self.layout_of_local(frame, local))?;
+        let layout = self.layout_of_local(frame, local)?;
         Ok(OpTy { op, layout })
     }
 
     // Evaluate a place with the goal of reading from it.  This lets us sometimes
-    // avoid allocations.  If you already know the layout, you can pass it in
-    // to avoid looking it up again.
+    // avoid allocations.
     fn eval_place_to_op(
         &self,
         mir_place: &mir::Place<'tcx>,
-        layout: Option<TyLayout<'tcx>>,
     ) -> EvalResult<'tcx, OpTy<'tcx, M::PointerTag>> {
         use rustc::mir::Place::*;
         let op = match *mir_place {
             Local(mir::RETURN_PLACE) => return err!(ReadFromReturnPointer),
-            Local(local) => self.access_local(self.frame(), local, layout)?,
+            Local(local) => self.access_local(self.frame(), local)?,
 
             Projection(ref proj) => {
-                let op = self.eval_place_to_op(&proj.base, None)?;
+                let op = self.eval_place_to_op(&proj.base)?;
                 self.operand_projection(op, &proj.elem)?
             }
 
@@ -510,7 +504,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M>
             // FIXME: do some more logic on `move` to invalidate the old location
             Copy(ref place) |
             Move(ref place) =>
-                self.eval_place_to_op(place, layout)?,
+                self.eval_place_to_op(place)?,
 
             Constant(ref constant) => {
                 let layout = from_known_layout(layout, || {
diff --git a/src/librustc_mir/interpret/snapshot.rs b/src/librustc_mir/interpret/snapshot.rs
index f9ce7b4319f..53105266b39 100644
--- a/src/librustc_mir/interpret/snapshot.rs
+++ b/src/librustc_mir/interpret/snapshot.rs
@@ -314,13 +314,14 @@ struct FrameSnapshot<'a, 'tcx: 'a> {
     stmt: usize,
 }
 
-impl_stable_hash_for!(impl<'tcx, 'mir: 'tcx> for struct Frame<'mir, 'tcx> {
+impl_stable_hash_for!(impl<'mir, 'tcx: 'mir> for struct Frame<'mir, 'tcx> {
     mir,
     instance,
     span,
     return_to_block,
     return_place -> (return_place.as_ref().map(|r| &**r)),
     locals,
+    local_layouts -> _,
     block,
     stmt,
     extra,
@@ -339,6 +340,7 @@ impl<'a, 'mir, 'tcx, Ctx> Snapshot<'a, Ctx> for &'a Frame<'mir, 'tcx>
             return_to_block,
             return_place,
             locals,
+            local_layouts: _,
             block,
             stmt,
             extra: _,