Auto merge of #89247 - fee1-dead:const-eval-select, r=oli-obk

Add `const_eval_select` intrinsic

Adds an intrinsic that calls a given function when evaluated at compiler time, but generates a call to another function when called at runtime.

See https://github.com/rust-lang/const-eval/issues/7 for previous discussion.

r? `@oli-obk.`

14 files changed, 88 insertions, 39 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 15bb9067805..78fdf9c02d0 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -309,13 +309,13 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     span: Span,
     func: &Operand<'tcx>,
     args: &[Operand<'tcx>],
-    destination: Option<(Place<'tcx>, BasicBlock)>,
+    mir_dest: Option<(Place<'tcx>, BasicBlock)>,
 ) {
     let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
     let fn_sig =
         fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
 
-    let destination = destination.map(|(place, bb)| (codegen_place(fx, place), bb));
+    let destination = mir_dest.map(|(place, bb)| (codegen_place(fx, place), bb));
 
     // Handle special calls like instrinsics and empty drop glue.
     let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 48183b2d4f6..313b62c5770 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -407,11 +407,9 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
     span: Span,
 ) {
-    let def_id = instance.def_id();
+    let intrinsic = fx.tcx.item_name(instance.def_id());
     let substs = instance.substs;
 
-    let intrinsic = fx.tcx.item_name(def_id);
-
     let ret = match destination {
         Some((place, _)) => place,
         None => {
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index ae20f6f97b2..202c9cad8eb 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -26,14 +26,35 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
     /// "Intercept" a function call to a panic-related function
     /// because we have something special to do for it.
     /// If this returns successfully (`Ok`), the function should just be evaluated normally.
-    fn hook_panic_fn(
+    fn hook_special_const_fn(
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx>],
+        is_const_fn: bool,
     ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         // The list of functions we handle here must be in sync with
-        // `is_lang_panic_fn` in `transform/check_consts/mod.rs`.
+        // `is_lang_special_const_fn` in `transform/check_consts/mod.rs`.
         let def_id = instance.def_id();
+
+        if is_const_fn {
+            if Some(def_id) == self.tcx.lang_items().const_eval_select() {
+                // redirect to const_eval_select_ct
+                if let Some(const_eval_select) = self.tcx.lang_items().const_eval_select_ct() {
+                    return Ok(Some(
+                        ty::Instance::resolve(
+                            *self.tcx,
+                            ty::ParamEnv::reveal_all(),
+                            const_eval_select,
+                            instance.substs,
+                        )
+                        .unwrap()
+                        .unwrap(),
+                    ));
+                }
+            }
+            return Ok(None);
+        }
+
         if Some(def_id) == self.tcx.lang_items().panic_fn()
             || Some(def_id) == self.tcx.lang_items().panic_str()
             || Some(def_id) == self.tcx.lang_items().panic_display()
@@ -255,31 +276,31 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
 
         // Only check non-glue functions
         if let ty::InstanceDef::Item(def) = instance.def {
+            let mut is_const_fn = true;
+
             // Execution might have wandered off into other crates, so we cannot do a stability-
             // sensitive check here.  But we can at least rule out functions that are not const
             // at all.
             if !ecx.tcx.is_const_fn_raw(def.did) {
                 // allow calling functions marked with #[default_method_body_is_const].
                 if !ecx.tcx.has_attr(def.did, sym::default_method_body_is_const) {
-                    // Some functions we support even if they are non-const -- but avoid testing
-                    // that for const fn!
-                    if let Some(new_instance) = ecx.hook_panic_fn(instance, args)? {
-                        // We call another const fn instead.
-                        return Self::find_mir_or_eval_fn(
-                            ecx,
-                            new_instance,
-                            _abi,
-                            args,
-                            _ret,
-                            _unwind,
-                        );
-                    } else {
-                        // We certainly do *not* want to actually call the fn
-                        // though, so be sure we return here.
-                        throw_unsup_format!("calling non-const function `{}`", instance)
-                    }
+                    is_const_fn = false;
                 }
             }
+
+            // Some functions we support even if they are non-const -- but avoid testing
+            // that for const fn!
+            // `const_eval_select` is a const fn because it must use const trait bounds.
+            if let Some(new_instance) = ecx.hook_special_const_fn(instance, args, is_const_fn)? {
+                // We call another const fn instead.
+                return Self::find_mir_or_eval_fn(ecx, new_instance, _abi, args, _ret, _unwind);
+            }
+
+            if !is_const_fn {
+                // We certainly do *not* want to actually call the fn
+                // though, so be sure we return here.
+                throw_unsup_format!("calling non-const function `{}`", instance)
+            }
         }
         // This is a const fn. Call it.
         Ok(Some(ecx.load_mir(instance.def, None)?))
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
index a06903aedf6..8d3544d434a 100644
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -231,7 +231,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     }
 
     /// Call this function -- pushing the stack frame and initializing the arguments.
-    fn eval_fn_call(
+    pub(crate) fn eval_fn_call(
         &mut self,
         fn_val: FnVal<'tcx, M::ExtraFnVal>,
         caller_abi: Abi,
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index fd5cd269a3a..d704c4335c7 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -24,7 +24,7 @@ use std::ops::Deref;
 use super::ops::{self, NonConstOp, Status};
 use super::qualifs::{self, CustomEq, HasMutInterior, NeedsNonConstDrop};
 use super::resolver::FlowSensitiveAnalysis;
-use super::{is_lang_panic_fn, ConstCx, Qualif};
+use super::{is_lang_panic_fn, is_lang_special_const_fn, ConstCx, Qualif};
 use crate::const_eval::is_unstable_const_fn;
 
 // We are using `MaybeMutBorrowedLocals` as a proxy for whether an item may have been mutated
@@ -259,7 +259,9 @@ impl Checker<'mir, 'tcx> {
             self.check_local_or_return_ty(return_ty.skip_binder(), RETURN_PLACE);
         }
 
-        self.visit_body(&body);
+        if !tcx.has_attr(def_id.to_def_id(), sym::rustc_do_not_const_check) {
+            self.visit_body(&body);
+        }
 
         // Ensure that the end result is `Sync` in a non-thread local `static`.
         let should_check_for_sync = self.const_kind()
@@ -886,7 +888,7 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> {
                 }
 
                 // At this point, we are calling a function, `callee`, whose `DefId` is known...
-                if is_lang_panic_fn(tcx, callee) {
+                if is_lang_special_const_fn(tcx, callee) {
                     // `begin_panic` and `panic_display` are generic functions that accept
                     // types other than str. Check to enforce that only str can be used in
                     // const-eval.
@@ -908,7 +910,10 @@ impl Visitor<'tcx> for Checker<'mir, 'tcx> {
                         }
                     }
 
-                    return;
+                    if is_lang_panic_fn(tcx, callee) {
+                        // run stability check on non-panic special const fns.
+                        return;
+                    }
                 }
 
                 if Some(callee) == tcx.lang_items().exchange_malloc_fn() {
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
index d1fd3ceaa58..0a852282f8f 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
@@ -74,9 +74,6 @@ impl ConstCx<'mir, 'tcx> {
 
 /// Returns `true` if this `DefId` points to one of the official `panic` lang items.
 pub fn is_lang_panic_fn(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
-    // We can allow calls to these functions because `hook_panic_fn` in
-    // `const_eval/machine.rs` ensures the calls are handled specially.
-    // Keep in sync with what that function handles!
     Some(def_id) == tcx.lang_items().panic_fn()
         || Some(def_id) == tcx.lang_items().panic_str()
         || Some(def_id) == tcx.lang_items().panic_display()
@@ -85,6 +82,15 @@ pub fn is_lang_panic_fn(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
         || Some(def_id) == tcx.lang_items().begin_panic_fmt()
 }
 
+/// Returns `true` if this `DefId` points to one of the lang items that will be handled differently
+/// in const_eval.
+pub fn is_lang_special_const_fn(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
+    // We can allow calls to these functions because `hook_special_const_fn` in
+    // `const_eval/machine.rs` ensures the calls are handled specially.
+    // Keep in sync with what that function handles!
+    is_lang_panic_fn(tcx, def_id) || Some(def_id) == tcx.lang_items().const_eval_select()
+}
+
 pub fn rustc_allow_const_fn_unstable(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
index f2ba5a1ebb1..1a8c8b1c78d 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
@@ -1,7 +1,7 @@
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::{self, BasicBlock, Location};
 use rustc_middle::ty::TyCtxt;
-use rustc_span::Span;
+use rustc_span::{symbol::sym, Span};
 
 use super::check::Qualifs;
 use super::ops::{self, NonConstOp};
@@ -30,6 +30,10 @@ pub fn check_live_drops(tcx: TyCtxt<'tcx>, body: &mir::Body<'tcx>) {
         return;
     }
 
+    if tcx.has_attr(def_id.to_def_id(), sym::rustc_do_not_const_check) {
+        return;
+    }
+
     let ccx = ConstCx { body, tcx, const_kind, param_env: tcx.param_env(def_id) };
     if !checking_enabled(&ccx) {
         return;
diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs
index be1b827f235..7cfe3d7f809 100644
--- a/compiler/rustc_const_eval/src/transform/promote_consts.rs
+++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs
@@ -26,7 +26,7 @@ use rustc_index::vec::{Idx, IndexVec};
 use std::cell::Cell;
 use std::{cmp, iter, mem};
 
-use crate::transform::check_consts::{is_lang_panic_fn, qualifs, ConstCx};
+use crate::transform::check_consts::{is_lang_special_const_fn, qualifs, ConstCx};
 use crate::transform::MirPass;
 
 /// A `MirPass` for promotion.
@@ -657,7 +657,7 @@ impl<'tcx> Validator<'_, 'tcx> {
 
         let is_const_fn = match *fn_ty.kind() {
             ty::FnDef(def_id, _) => {
-                self.tcx.is_const_fn_raw(def_id) || is_lang_panic_fn(self.tcx, def_id)
+                self.tcx.is_const_fn_raw(def_id) || is_lang_special_const_fn(self.tcx, def_id)
             }
             _ => false,
         };
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index f3eaf2645f5..85b0db468d1 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -467,6 +467,8 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
 
     rustc_attr!(rustc_promotable, Normal, template!(Word), IMPL_DETAIL),
     rustc_attr!(rustc_legacy_const_generics, Normal, template!(List: "N"), INTERNAL_UNSTABLE),
+    // Do not const-check this function's body. It will always get replaced during CTFE.
+    rustc_attr!(rustc_do_not_const_check, Normal, template!(Word), INTERNAL_UNSTABLE),
 
     // ==========================================================================
     // Internal attributes, Layout related:
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 814054c5518..f35353dbfb5 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -299,6 +299,8 @@ language_item_table! {
     DropInPlace,             sym::drop_in_place,       drop_in_place_fn,           Target::Fn,             GenericRequirement::Minimum(1);
     Oom,                     sym::oom,                 oom,                        Target::Fn,             GenericRequirement::None;
     AllocLayout,             sym::alloc_layout,        alloc_layout,               Target::Struct,         GenericRequirement::None;
+    ConstEvalSelect,         sym::const_eval_select,   const_eval_select,          Target::Fn,             GenericRequirement::Exact(4);
+    ConstConstEvalSelect,    sym::const_eval_select_ct,const_eval_select_ct,       Target::Fn,             GenericRequirement::Exact(4);
 
     Start,                   sym::start,               start_fn,                   Target::Fn,             GenericRequirement::Exact(1);
 
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 6c889e88a59..fddb225345f 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -441,6 +441,8 @@ symbols! {
         const_compare_raw_pointers,
         const_constructor,
         const_eval_limit,
+        const_eval_select,
+        const_eval_select_ct,
         const_evaluatable_checked,
         const_extern_fn,
         const_fn,
@@ -1097,6 +1099,7 @@ symbols! {
         rustc_diagnostic_item,
         rustc_diagnostic_macros,
         rustc_dirty,
+        rustc_do_not_const_check,
         rustc_dummy,
         rustc_dump_env_program_clauses,
         rustc_dump_program_clauses,
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 0f6e2e0be52..856ea43b1ff 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -973,12 +973,16 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ty::Tuple(_) => stack.extend(ty.tuple_fields().map(|t| (t, depth + 1))),
 
                 ty::Closure(_, substs) => {
-                    stack.extend(substs.as_closure().upvar_tys().map(|t| (t, depth + 1)))
+                    let substs = substs.as_closure();
+                    let ty = self.infcx.shallow_resolve(substs.tupled_upvars_ty());
+                    stack.push((ty, depth + 1));
                 }
 
                 ty::Generator(_, substs, _) => {
                     let substs = substs.as_generator();
-                    stack.extend(substs.upvar_tys().map(|t| (t, depth + 1)));
+                    let ty = self.infcx.shallow_resolve(substs.tupled_upvars_ty());
+
+                    stack.push((ty, depth + 1));
                     stack.push((substs.witness(), depth + 1));
                 }
 
diff --git a/compiler/rustc_typeck/src/check/callee.rs b/compiler/rustc_typeck/src/check/callee.rs
index 51bbcbebcdc..06c42098791 100644
--- a/compiler/rustc_typeck/src/check/callee.rs
+++ b/compiler/rustc_typeck/src/check/callee.rs
@@ -17,7 +17,7 @@ use rustc_infer::{
 use rustc_middle::ty::adjustment::{
     Adjust, Adjustment, AllowTwoPhase, AutoBorrow, AutoBorrowMutability,
 };
-use rustc_middle::ty::subst::SubstsRef;
+use rustc_middle::ty::subst::{Subst, SubstsRef};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable};
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
@@ -317,6 +317,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let (fn_sig, def_id) = match *callee_ty.kind() {
             ty::FnDef(def_id, subst) => {
+                let fn_sig = self.tcx.fn_sig(def_id).subst(self.tcx, subst);
+
                 // Unit testing: function items annotated with
                 // `#[rustc_evaluate_where_clauses]` trigger special output
                 // to let us test the trait evaluation system.
@@ -342,7 +344,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             .emit();
                     }
                 }
-                (callee_ty.fn_sig(self.tcx), Some(def_id))
+                (fn_sig, Some(def_id))
             }
             ty::FnPtr(sig) => (sig, None),
             ref t => {
diff --git a/compiler/rustc_typeck/src/check/intrinsic.rs b/compiler/rustc_typeck/src/check/intrinsic.rs
index ff7a26853b1..b0cb8443bfb 100644
--- a/compiler/rustc_typeck/src/check/intrinsic.rs
+++ b/compiler/rustc_typeck/src/check/intrinsic.rs
@@ -390,6 +390,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
 
             sym::black_box => (1, vec![param(0)], param(0)),
 
+            sym::const_eval_select => (4, vec![param(0), param(1), param(2)], param(3)),
+
             other => {
                 tcx.sess.emit_err(UnrecognizedIntrinsicFunction { span: it.span, name: other });
                 return;