Move all intrinsics out of `interpret` and fail CTFE on intrinsic calls

13 files changed, 148 insertions, 59 deletions

diff --git a/miri/helpers.rs b/miri/helpers.rs
new file mode 100644
index 00000000000..add6558bcc4
--- /dev/null
+++ b/miri/helpers.rs
@@ -0,0 +1,69 @@
+use rustc_miri::interpret::{
+    Pointer,
+    EvalResult, EvalError,
+    PrimVal,
+    EvalContext,
+};
+
+use rustc::ty::Ty;
+
+pub trait EvalContextExt<'tcx> {
+    fn wrapping_pointer_offset(
+        &self,
+        ptr: Pointer,
+        pointee_ty: Ty<'tcx>,
+        offset: i64,
+    ) -> EvalResult<'tcx, Pointer>;
+
+    fn pointer_offset(
+        &self,
+        ptr: Pointer,
+        pointee_ty: Ty<'tcx>,
+        offset: i64,
+    ) -> EvalResult<'tcx, Pointer>;
+}
+
+impl<'a, 'tcx> EvalContextExt<'tcx> for EvalContext<'a, 'tcx, super::Evaluator> {
+    fn wrapping_pointer_offset(
+        &self,
+        ptr: Pointer,
+        pointee_ty: Ty<'tcx>,
+        offset: i64,
+    ) -> EvalResult<'tcx, Pointer> {
+        // FIXME: assuming here that type size is < i64::max_value()
+        let pointee_size = self.type_size(pointee_ty)?.expect("cannot offset a pointer to an unsized type") as i64;
+        let offset = offset.overflowing_mul(pointee_size).0;
+        ptr.wrapping_signed_offset(offset, self)
+    }
+
+    fn pointer_offset(
+        &self,
+        ptr: Pointer,
+        pointee_ty: Ty<'tcx>,
+        offset: i64,
+    ) -> EvalResult<'tcx, Pointer> {
+        // This function raises an error if the offset moves the pointer outside of its allocation.  We consider
+        // ZSTs their own huge allocation that doesn't overlap with anything (and nothing moves in there because the size is 0).
+        // We also consider the NULL pointer its own separate allocation, and all the remaining integers pointers their own
+        // allocation.
+
+        if ptr.is_null()? { // NULL pointers must only be offset by 0
+            return if offset == 0 { Ok(ptr) } else { Err(EvalError::InvalidNullPointerUsage) };
+        }
+        // FIXME: assuming here that type size is < i64::max_value()
+        let pointee_size = self.type_size(pointee_ty)?.expect("cannot offset a pointer to an unsized type") as i64;
+        return if let Some(offset) = offset.checked_mul(pointee_size) {
+            let ptr = ptr.signed_offset(offset, self)?;
+            // Do not do bounds-checking for integers; they can never alias a normal pointer anyway.
+            if let PrimVal::Ptr(ptr) = ptr.into_inner_primval() {
+                self.memory.check_bounds(ptr, false)?;
+            } else if ptr.is_null()? {
+                // We moved *to* a NULL pointer.  That seems wrong, LLVM considers the NULL pointer its own small allocation.  Reject this, for now.
+                return Err(EvalError::InvalidNullPointerUsage);
+            }
+            Ok(ptr)
+        } else {
+            Err(EvalError::OverflowingMath)
+        }
+    }
+}
diff --git a/src/librustc_mir/interpret/terminator/intrinsic.rs b/miri/intrinsic.rs
index 7c81b76ba41..73caf64dbde 100644
--- a/src/librustc_mir/interpret/terminator/intrinsic.rs
+++ b/miri/intrinsic.rs
@@ -3,17 +3,30 @@ use rustc::traits::Reveal;
 use rustc::ty::layout::Layout;
 use rustc::ty::{self, Ty};
 
-use interpret::{
+use rustc_miri::interpret::{
     EvalError, EvalResult,
-    EvalContext,
     Lvalue, LvalueExtra,
     PrimVal, PrimValKind, Value, Pointer,
     HasMemory,
-    Machine,
+    EvalContext,
 };
 
-impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
-    pub(super) fn call_intrinsic(
+use helpers::EvalContextExt as HelperEvalContextExt;
+
+pub trait EvalContextExt<'tcx> {
+    fn call_intrinsic(
+        &mut self,
+        instance: ty::Instance<'tcx>,
+        args: &[mir::Operand<'tcx>],
+        dest: Lvalue<'tcx>,
+        dest_ty: Ty<'tcx>,
+        dest_layout: &'tcx Layout,
+        target: mir::BasicBlock,
+    ) -> EvalResult<'tcx>;
+}
+
+impl<'a, 'tcx> EvalContextExt<'tcx> for EvalContext<'a, 'tcx, super::Evaluator> {
+    fn call_intrinsic(
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[mir::Operand<'tcx>],
@@ -495,7 +508,7 @@ fn numeric_intrinsic<'tcx>(
 ) -> EvalResult<'tcx, PrimVal> {
     macro_rules! integer_intrinsic {
         ($method:ident) => ({
-            use interpret::PrimValKind::*;
+            use rustc_miri::interpret::PrimValKind::*;
             let result_bytes = match kind {
                 I8 => (bytes as i8).$method() as u128,
                 U8 => (bytes as u8).$method() as u128,
diff --git a/miri/lib.rs b/miri/lib.rs
index bd17ab6f2cb..c32c8105e33 100644
--- a/miri/lib.rs
+++ b/miri/lib.rs
@@ -14,6 +14,7 @@ extern crate rustc_data_structures;
 extern crate syntax;
 
 use rustc::ty::{self, TyCtxt};
+use rustc::ty::layout::Layout;
 use rustc::hir::def_id::DefId;
 use rustc::mir;
 
@@ -29,9 +30,12 @@ pub use rustc_miri::interpret::*;
 
 mod fn_call;
 mod operator;
+mod intrinsic;
+mod helpers;
 
 use fn_call::EvalContextExt as MissingFnsEvalContextExt;
 use operator::EvalContextExt as OperatorEvalContextExt;
+use intrinsic::EvalContextExt as IntrinsicEvalContextExt;
 
 pub fn eval_main<'a, 'tcx: 'a>(
     tcx: TyCtxt<'a, 'tcx, 'tcx>,
@@ -287,6 +291,18 @@ impl<'tcx> Machine<'tcx> for Evaluator {
         ecx.eval_fn_call(instance, destination, arg_operands, span, sig)
     }
 
+    fn call_intrinsic<'a>(
+        ecx: &mut rustc_miri::interpret::EvalContext<'a, 'tcx, Self>,
+        instance: ty::Instance<'tcx>,
+        args: &[mir::Operand<'tcx>],
+        dest: Lvalue<'tcx>,
+        dest_ty: ty::Ty<'tcx>,
+        dest_layout: &'tcx Layout,
+        target: mir::BasicBlock,
+    ) -> EvalResult<'tcx> {
+        ecx.call_intrinsic(instance, args, dest, dest_ty, dest_layout, target)
+    }
+
     fn ptr_op<'a>(
         ecx: &rustc_miri::interpret::EvalContext<'a, 'tcx, Self>,
         bin_op: mir::BinOp,
diff --git a/miri/operator.rs b/miri/operator.rs
index fcc3986015d..a01ba25cd75 100644
--- a/miri/operator.rs
+++ b/miri/operator.rs
@@ -3,6 +3,8 @@ use rustc::mir;
 
 use rustc_miri::interpret::*;
 
+use helpers::EvalContextExt as HelperEvalContextExt;
+
 pub trait EvalContextExt<'tcx> {
     fn ptr_op(
         &self,
diff --git a/src/librustc_mir/interpret/const_eval.rs b/src/librustc_mir/interpret/const_eval.rs
index 32f5a0a183e..604ef15e904 100644
--- a/src/librustc_mir/interpret/const_eval.rs
+++ b/src/librustc_mir/interpret/const_eval.rs
@@ -1,5 +1,5 @@
 use rustc::traits::Reveal;
-use rustc::ty::{self, TyCtxt, Ty, Instance};
+use rustc::ty::{self, TyCtxt, Ty, Instance, layout};
 use rustc::mir;
 
 use syntax::ast::Mutability;
@@ -163,6 +163,18 @@ impl<'tcx> super::Machine<'tcx> for CompileTimeFunctionEvaluator {
         Ok(false)
     }
 
+    fn call_intrinsic<'a>(
+        _ecx: &mut EvalContext<'a, 'tcx, Self>,
+        _instance: ty::Instance<'tcx>,
+        _args: &[mir::Operand<'tcx>],
+        _dest: Lvalue<'tcx>,
+        _dest_ty: Ty<'tcx>,
+        _dest_layout: &'tcx layout::Layout,
+        _target: mir::BasicBlock,
+    ) -> EvalResult<'tcx> {
+        Err(ConstEvalError::NeedsRfc("calling intrinsics".to_string()).into())
+    }
+
     fn ptr_op<'a>(
         _ecx: &EvalContext<'a, 'tcx, Self>,
         _bin_op: mir::BinOp,
diff --git a/src/librustc_mir/interpret/eval_context.rs b/src/librustc_mir/interpret/eval_context.rs
index 2b2e1f30c88..f5082a4d2d8 100644
--- a/src/librustc_mir/interpret/eval_context.rs
+++ b/src/librustc_mir/interpret/eval_context.rs
@@ -211,7 +211,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         false
     }
 
-    pub(crate) fn str_to_value(&mut self, s: &str) -> EvalResult<'tcx, Value> {
+    pub fn str_to_value(&mut self, s: &str) -> EvalResult<'tcx, Value> {
         let ptr = self.memory.allocate_cached(s.as_bytes())?;
         Ok(Value::ByValPair(PrimVal::Ptr(ptr), PrimVal::from_u128(s.len() as u128)))
     }
@@ -369,11 +369,11 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         self.tcx.normalize_associated_type(&f.ty(self.tcx, param_substs))
     }
 
-    pub(super) fn type_size(&self, ty: Ty<'tcx>) -> EvalResult<'tcx, Option<u64>> {
+    pub fn type_size(&self, ty: Ty<'tcx>) -> EvalResult<'tcx, Option<u64>> {
         self.type_size_with_substs(ty, self.substs())
     }
 
-    pub(super) fn type_align(&self, ty: Ty<'tcx>) -> EvalResult<'tcx, u64> {
+    pub fn type_align(&self, ty: Ty<'tcx>) -> EvalResult<'tcx, u64> {
         self.type_align_with_substs(ty, self.substs())
     }
 
@@ -1022,39 +1022,6 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         }
     }
 
-    pub(super) fn wrapping_pointer_offset(&self, ptr: Pointer, pointee_ty: Ty<'tcx>, offset: i64) -> EvalResult<'tcx, Pointer> {
-        // FIXME: assuming here that type size is < i64::max_value()
-        let pointee_size = self.type_size(pointee_ty)?.expect("cannot offset a pointer to an unsized type") as i64;
-        let offset = offset.overflowing_mul(pointee_size).0;
-        ptr.wrapping_signed_offset(offset, self)
-    }
-
-    pub fn pointer_offset(&self, ptr: Pointer, pointee_ty: Ty<'tcx>, offset: i64) -> EvalResult<'tcx, Pointer> {
-        // This function raises an error if the offset moves the pointer outside of its allocation.  We consider
-        // ZSTs their own huge allocation that doesn't overlap with anything (and nothing moves in there because the size is 0).
-        // We also consider the NULL pointer its own separate allocation, and all the remaining integers pointers their own
-        // allocation.
-
-        if ptr.is_null()? { // NULL pointers must only be offset by 0
-            return if offset == 0 { Ok(ptr) } else { Err(EvalError::InvalidNullPointerUsage) };
-        }
-        // FIXME: assuming here that type size is < i64::max_value()
-        let pointee_size = self.type_size(pointee_ty)?.expect("cannot offset a pointer to an unsized type") as i64;
-        return if let Some(offset) = offset.checked_mul(pointee_size) {
-            let ptr = ptr.signed_offset(offset, self)?;
-            // Do not do bounds-checking for integers; they can never alias a normal pointer anyway.
-            if let PrimVal::Ptr(ptr) = ptr.into_inner_primval() {
-                self.memory.check_bounds(ptr, false)?;
-            } else if ptr.is_null()? {
-                // We moved *to* a NULL pointer.  That seems wrong, LLVM considers the NULL pointer its own small allocation.  Reject this, for now.
-                return Err(EvalError::InvalidNullPointerUsage);
-            }
-            Ok(ptr)
-        } else {
-            Err(EvalError::OverflowingMath)
-        }
-    }
-
     pub(super) fn eval_operand_to_primval(&mut self, op: &mir::Operand<'tcx>) -> EvalResult<'tcx, PrimVal> {
         let value = self.eval_operand(op)?;
         let ty = self.operand_ty(op);
@@ -1103,7 +1070,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         Ok(())
     }
 
-    pub(super) fn force_allocation(
+    pub fn force_allocation(
         &mut self,
         lvalue: Lvalue<'tcx>,
     ) -> EvalResult<'tcx, Lvalue<'tcx>> {
@@ -1297,7 +1264,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         Ok(())
     }
 
-    pub(super) fn write_value_to_ptr(
+    pub fn write_value_to_ptr(
         &mut self,
         value: Value,
         dest: Pointer,
@@ -1315,7 +1282,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         }
     }
 
-    pub(super) fn write_pair_to_ptr(
+    pub fn write_pair_to_ptr(
         &mut self,
         a: PrimVal,
         b: PrimVal,
@@ -1445,7 +1412,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
         }
     }
 
-    pub(super) fn read_value(&self, ptr: Pointer, ty: Ty<'tcx>) -> EvalResult<'tcx, Value> {
+    pub fn read_value(&self, ptr: Pointer, ty: Ty<'tcx>) -> EvalResult<'tcx, Value> {
         if let Some(val) = self.try_read_value(ptr, ty)? {
             Ok(val)
         } else {
diff --git a/src/librustc_mir/interpret/lvalue.rs b/src/librustc_mir/interpret/lvalue.rs
index 0d0da53985e..5c10d2c1952 100644
--- a/src/librustc_mir/interpret/lvalue.rs
+++ b/src/librustc_mir/interpret/lvalue.rs
@@ -73,7 +73,7 @@ impl<'tcx> Lvalue<'tcx> {
         Self::from_primval_ptr(PrimVal::Undef.into())
     }
 
-    pub(crate) fn from_primval_ptr(ptr: Pointer) -> Self {
+    pub fn from_primval_ptr(ptr: Pointer) -> Self {
         Lvalue::Ptr { ptr, extra: LvalueExtra::None, aligned: true }
     }
 
@@ -89,7 +89,7 @@ impl<'tcx> Lvalue<'tcx> {
         }
     }
 
-    pub(super) fn to_ptr(self) -> EvalResult<'tcx, MemoryPointer> {
+    pub fn to_ptr(self) -> EvalResult<'tcx, MemoryPointer> {
         let (ptr, extra, _aligned) = self.to_ptr_extra_aligned();
         // At this point, we forget about the alignment information -- the lvalue has been turned into a reference,
         // and no matter where it came from, it now must be aligned.
diff --git a/src/librustc_mir/interpret/machine.rs b/src/librustc_mir/interpret/machine.rs
index 68eb8064f64..adb1054af1d 100644
--- a/src/librustc_mir/interpret/machine.rs
+++ b/src/librustc_mir/interpret/machine.rs
@@ -36,6 +36,17 @@ pub trait Machine<'tcx>: Sized {
         sig: ty::FnSig<'tcx>,
     ) -> EvalResult<'tcx, bool>;
 
+    /// directly process an intrinsic without pushing a stack frame.
+    fn call_intrinsic<'a>(
+        ecx: &mut EvalContext<'a, 'tcx, Self>,
+        instance: ty::Instance<'tcx>,
+        args: &[mir::Operand<'tcx>],
+        dest: Lvalue<'tcx>,
+        dest_ty: ty::Ty<'tcx>,
+        dest_layout: &'tcx ty::layout::Layout,
+        target: mir::BasicBlock,
+    ) -> EvalResult<'tcx>;
+
     /// Called when operating on the value of pointers.
     ///
     /// Returns `None` if the operation should be handled by the integer
diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs
index 28cc9b9a25e..e8701d1e64c 100644
--- a/src/librustc_mir/interpret/memory.rs
+++ b/src/librustc_mir/interpret/memory.rs
@@ -434,7 +434,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> Memory<'a, 'tcx, M> {
         }
     }
 
-    pub(crate) fn check_bounds(&self, ptr: MemoryPointer, access: bool) -> EvalResult<'tcx> {
+    pub fn check_bounds(&self, ptr: MemoryPointer, access: bool) -> EvalResult<'tcx> {
         let alloc = self.get(ptr.alloc_id)?;
         let allocation_size = alloc.bytes.len() as u64;
         if ptr.offset > allocation_size {
@@ -1311,7 +1311,7 @@ fn bit_index(bits: u64) -> (usize, usize) {
 // Unaligned accesses
 ////////////////////////////////////////////////////////////////////////////////
 
-pub(crate) trait HasMemory<'a, 'tcx, M: Machine<'tcx>> {
+pub trait HasMemory<'a, 'tcx, M: Machine<'tcx>> {
     fn memory_mut(&mut self) -> &mut Memory<'a, 'tcx, M>;
     fn memory(&self) -> &Memory<'a, 'tcx, M>;
 
diff --git a/src/librustc_mir/interpret/mod.rs b/src/librustc_mir/interpret/mod.rs
index 10a58ce3e00..236e708d96f 100644
--- a/src/librustc_mir/interpret/mod.rs
+++ b/src/librustc_mir/interpret/mod.rs
@@ -40,10 +40,10 @@ pub use self::memory::{
     Memory,
     MemoryPointer,
     Kind,
+    HasMemory,
 };
 
 use self::memory::{
-    HasMemory,
     PointerArithmetic,
     LockInfo,
     AccessKind,
diff --git a/src/librustc_mir/interpret/operator.rs b/src/librustc_mir/interpret/operator.rs
index 606761f371c..010531d96cc 100644
--- a/src/librustc_mir/interpret/operator.rs
+++ b/src/librustc_mir/interpret/operator.rs
@@ -34,7 +34,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
 
     /// Applies the binary operation `op` to the two operands and writes a tuple of the result
     /// and a boolean signifying the potential overflow to the destination.
-    pub(super) fn intrinsic_with_overflow(
+    pub fn intrinsic_with_overflow(
         &mut self,
         op: mir::BinOp,
         left: &mir::Operand<'tcx>,
@@ -49,7 +49,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
 
     /// Applies the binary operation `op` to the arguments and writes the result to the
     /// destination. Returns `true` if the operation overflowed.
-    pub(super) fn intrinsic_overflowing(
+    pub fn intrinsic_overflowing(
         &mut self,
         op: mir::BinOp,
         left: &mir::Operand<'tcx>,
diff --git a/src/librustc_mir/interpret/terminator/mod.rs b/src/librustc_mir/interpret/terminator/mod.rs
index 21e59e9d456..3ccc2ee0fb4 100644
--- a/src/librustc_mir/interpret/terminator/mod.rs
+++ b/src/librustc_mir/interpret/terminator/mod.rs
@@ -18,7 +18,6 @@ use super::eval_context::IntegerExt;
 use rustc_data_structures::indexed_vec::Idx;
 
 mod drop;
-mod intrinsic;
 
 impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
     pub fn goto_block(&mut self, target: mir::BasicBlock) {
@@ -222,7 +221,7 @@ impl<'a, 'tcx, M: Machine<'tcx>> EvalContext<'a, 'tcx, M> {
                     return Err(EvalError::Unreachable);
                 }
                 let layout = self.type_layout(ty)?;
-                self.call_intrinsic(instance, arg_operands, ret, ty, layout, target)?;
+                M::call_intrinsic(self, instance, arg_operands, ret, ty, layout, target)?;
                 self.dump_local(ret);
                 Ok(())
             },
diff --git a/src/librustc_mir/interpret/value.rs b/src/librustc_mir/interpret/value.rs
index fe109dbbd63..c88d1c22dc9 100644
--- a/src/librustc_mir/interpret/value.rs
+++ b/src/librustc_mir/interpret/value.rs
@@ -64,7 +64,7 @@ impl<'tcx> Pointer {
         self.primval
     }
 
-    pub(crate) fn signed_offset<C: HasDataLayout>(self, i: i64, cx: C) -> EvalResult<'tcx, Self> {
+    pub fn signed_offset<C: HasDataLayout>(self, i: i64, cx: C) -> EvalResult<'tcx, Self> {
         let layout = cx.data_layout();
         match self.primval {
             PrimVal::Bytes(b) => {
@@ -88,7 +88,7 @@ impl<'tcx> Pointer {
         }
     }
 
-    pub(crate) fn wrapping_signed_offset<C: HasDataLayout>(self, i: i64, cx: C) -> EvalResult<'tcx, Self> {
+    pub fn wrapping_signed_offset<C: HasDataLayout>(self, i: i64, cx: C) -> EvalResult<'tcx, Self> {
         let layout = cx.data_layout();
         match self.primval {
             PrimVal::Bytes(b) => {
@@ -165,7 +165,7 @@ pub enum PrimValKind {
 
 impl<'a, 'tcx: 'a> Value {
     #[inline]
-    pub(super) fn by_ref(ptr: Pointer) -> Self {
+    pub fn by_ref(ptr: Pointer) -> Self {
         Value::ByRef { ptr, aligned: true }
     }