Make `intern_lazy_const` actually intern its argument.

Currently it just unconditionally allocates it in the arena.

For a "Clean Check" build of the the `packed-simd` benchmark, this
change reduces both the `max-rss` and `faults` counts by 59%; it
slightly (~3%) increases the instruction counts but the `wall-time` is
unchanged.

For the same builds of a few other benchmarks, `max-rss` and `faults`
drop by 1--5%, but instruction counts and `wall-time` changes are in the
noise.

Fixes #57432, fixes #57829.

17 files changed, 40 insertions, 38 deletions

diff --git a/src/librustc/mir/mod.rs b/src/librustc/mir/mod.rs
index 82083b4f699..eca01d78c2a 100644
--- a/src/librustc/mir/mod.rs
+++ b/src/librustc/mir/mod.rs
@@ -2154,7 +2154,7 @@ impl<'tcx> Operand<'tcx> {
             span,
             ty,
             user_ty: None,
-            literal: tcx.intern_lazy_const(
+            literal: tcx.mk_lazy_const(
                 ty::LazyConst::Evaluated(ty::Const::zero_sized(ty)),
             ),
         })
diff --git a/src/librustc/traits/project.rs b/src/librustc/traits/project.rs
index bec45046cb9..21151276e72 100644
--- a/src/librustc/traits/project.rs
+++ b/src/librustc/traits/project.rs
@@ -408,7 +408,7 @@ impl<'a, 'b, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for AssociatedTypeNormalizer<'a,
                         if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) {
                             let substs = tcx.lift_to_global(&substs).unwrap();
                             let evaluated = evaluated.subst(tcx, substs);
-                            return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated));
+                            return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated));
                         }
                     }
                 } else {
@@ -420,7 +420,7 @@ impl<'a, 'b, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for AssociatedTypeNormalizer<'a,
                                 promoted: None
                             };
                             if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) {
-                                return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated));
+                                return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated));
                             }
                         }
                     }
diff --git a/src/librustc/traits/query/normalize.rs b/src/librustc/traits/query/normalize.rs
index be05445cfc6..9f55bb95d25 100644
--- a/src/librustc/traits/query/normalize.rs
+++ b/src/librustc/traits/query/normalize.rs
@@ -203,7 +203,7 @@ impl<'cx, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for QueryNormalizer<'cx, 'gcx, 'tcx
                         if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) {
                             let substs = tcx.lift_to_global(&substs).unwrap();
                             let evaluated = evaluated.subst(tcx, substs);
-                            return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated));
+                            return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated));
                         }
                     }
                 } else {
@@ -215,7 +215,7 @@ impl<'cx, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for QueryNormalizer<'cx, 'gcx, 'tcx
                                 promoted: None,
                             };
                             if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) {
-                                return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated));
+                                return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated));
                             }
                         }
                     }
diff --git a/src/librustc/ty/codec.rs b/src/librustc/ty/codec.rs
index e0e4d9c362a..a4a24718527 100644
--- a/src/librustc/ty/codec.rs
+++ b/src/librustc/ty/codec.rs
@@ -252,7 +252,7 @@ pub fn decode_lazy_const<'a, 'tcx, D>(decoder: &mut D)
     where D: TyDecoder<'a, 'tcx>,
           'tcx: 'a,
 {
-    Ok(decoder.tcx().intern_lazy_const(Decodable::decode(decoder)?))
+    Ok(decoder.tcx().mk_lazy_const(Decodable::decode(decoder)?))
 }
 
 #[inline]
diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs
index b379b5ba024..87cf52af1d1 100644
--- a/src/librustc/ty/context.rs
+++ b/src/librustc/ty/context.rs
@@ -127,6 +127,7 @@ pub struct CtxtInterners<'tcx> {
     goal: InternedSet<'tcx, GoalKind<'tcx>>,
     goal_list: InternedSet<'tcx, List<Goal<'tcx>>>,
     projs: InternedSet<'tcx, List<ProjectionKind<'tcx>>>,
+    lazy_const: InternedSet<'tcx, LazyConst<'tcx>>,
 }
 
 impl<'gcx: 'tcx, 'tcx> CtxtInterners<'tcx> {
@@ -144,6 +145,7 @@ impl<'gcx: 'tcx, 'tcx> CtxtInterners<'tcx> {
             goal: Default::default(),
             goal_list: Default::default(),
             projs: Default::default(),
+            lazy_const: Default::default(),
         }
     }
 
@@ -1096,10 +1098,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         self.global_arenas.adt_def.alloc(def)
     }
 
-    pub fn intern_const_alloc(
-        self,
-        alloc: Allocation,
-    ) -> &'gcx Allocation {
+    pub fn intern_const_alloc(self, alloc: Allocation) -> &'gcx Allocation {
         self.allocation_interner.borrow_mut().intern(alloc, |alloc| {
             self.global_arenas.const_allocs.alloc(alloc)
         })
@@ -1119,10 +1118,6 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         })
     }
 
-    pub fn intern_lazy_const(self, c: ty::LazyConst<'tcx>) -> &'tcx ty::LazyConst<'tcx> {
-        self.global_interners.arena.alloc(c)
-    }
-
     pub fn intern_layout(self, layout: LayoutDetails) -> &'gcx LayoutDetails {
         self.layout_interner.borrow_mut().intern(layout, |layout| {
             self.global_arenas.layout.alloc(layout)
@@ -2271,6 +2266,12 @@ impl<'tcx: 'lcx, 'lcx> Borrow<GoalKind<'lcx>> for Interned<'tcx, GoalKind<'tcx>>
     }
 }
 
+impl<'tcx: 'lcx, 'lcx> Borrow<LazyConst<'lcx>> for Interned<'tcx, LazyConst<'tcx>> {
+    fn borrow<'a>(&'a self) -> &'a LazyConst<'lcx> {
+        &self.0
+    }
+}
+
 impl<'tcx: 'lcx, 'lcx> Borrow<[ExistentialPredicate<'lcx>]>
     for Interned<'tcx, List<ExistentialPredicate<'tcx>>> {
     fn borrow<'a>(&'a self) -> &'a [ExistentialPredicate<'lcx>] {
@@ -2377,7 +2378,8 @@ pub fn keep_local<'tcx, T: ty::TypeFoldable<'tcx>>(x: &T) -> bool {
 
 direct_interners!('tcx,
     region: mk_region(|r: &RegionKind| r.keep_in_local_tcx()) -> RegionKind,
-    goal: mk_goal(|c: &GoalKind<'_>| keep_local(c)) -> GoalKind<'tcx>
+    goal: mk_goal(|c: &GoalKind<'_>| keep_local(c)) -> GoalKind<'tcx>,
+    lazy_const: mk_lazy_const(|c: &LazyConst<'_>| keep_local(&c)) -> LazyConst<'tcx>
 );
 
 macro_rules! slice_interners {
@@ -2562,7 +2564,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
 
     #[inline]
     pub fn mk_array(self, ty: Ty<'tcx>, n: u64) -> Ty<'tcx> {
-        self.mk_ty(Array(ty, self.intern_lazy_const(
+        self.mk_ty(Array(ty, self.mk_lazy_const(
             ty::LazyConst::Evaluated(ty::Const::from_usize(self.global_tcx(), n))
         )))
     }
diff --git a/src/librustc/ty/structural_impls.rs b/src/librustc/ty/structural_impls.rs
index 28f5a65374d..f89a6ed2e76 100644
--- a/src/librustc/ty/structural_impls.rs
+++ b/src/librustc/ty/structural_impls.rs
@@ -1042,7 +1042,7 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::LazyConst<'tcx> {
                 ty::LazyConst::Unevaluated(*def_id, substs.fold_with(folder))
             }
         };
-        folder.tcx().intern_lazy_const(new)
+        folder.tcx().mk_lazy_const(new)
     }
 
     fn fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self {
diff --git a/src/librustc_mir/build/expr/as_rvalue.rs b/src/librustc_mir/build/expr/as_rvalue.rs
index 3de2f475786..508f6555f4a 100644
--- a/src/librustc_mir/build/expr/as_rvalue.rs
+++ b/src/librustc_mir/build/expr/as_rvalue.rs
@@ -268,7 +268,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
                             span: expr_span,
                             ty: this.hir.tcx().types.u32,
                             user_ty: None,
-                            literal: this.hir.tcx().intern_lazy_const(ty::LazyConst::Evaluated(
+                            literal: this.hir.tcx().mk_lazy_const(ty::LazyConst::Evaluated(
                                 ty::Const::from_bits(
                                     this.hir.tcx(),
                                     0,
diff --git a/src/librustc_mir/build/matches/test.rs b/src/librustc_mir/build/matches/test.rs
index 696c173b048..6e836f7059b 100644
--- a/src/librustc_mir/build/matches/test.rs
+++ b/src/librustc_mir/build/matches/test.rs
@@ -302,7 +302,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
                     }
                     let eq_def_id = self.hir.tcx().lang_items().eq_trait().unwrap();
                     let (mty, method) = self.hir.trait_method(eq_def_id, "eq", ty, &[ty.into()]);
-                    let method = self.hir.tcx().intern_lazy_const(ty::LazyConst::Evaluated(method));
+                    let method = self.hir.tcx().mk_lazy_const(ty::LazyConst::Evaluated(method));
 
                     let re_erased = self.hir.tcx().types.re_erased;
                     // take the argument by reference
diff --git a/src/librustc_mir/build/misc.rs b/src/librustc_mir/build/misc.rs
index c849c022428..a7b201fc0db 100644
--- a/src/librustc_mir/build/misc.rs
+++ b/src/librustc_mir/build/misc.rs
@@ -33,7 +33,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
             span,
             ty,
             user_ty: None,
-            literal: self.hir.tcx().intern_lazy_const(ty::LazyConst::Evaluated(literal)),
+            literal: self.hir.tcx().mk_lazy_const(ty::LazyConst::Evaluated(literal)),
         };
         Operand::Constant(constant)
     }
diff --git a/src/librustc_mir/hair/cx/expr.rs b/src/librustc_mir/hair/cx/expr.rs
index 8d64c9e9ada..88512fede33 100644
--- a/src/librustc_mir/hair/cx/expr.rs
+++ b/src/librustc_mir/hair/cx/expr.rs
@@ -342,7 +342,7 @@ fn make_mirror_unadjusted<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>,
         }
 
         hir::ExprKind::Lit(ref lit) => ExprKind::Literal {
-            literal: cx.tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: cx.tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                 cx.const_eval_literal(&lit.node, expr_ty, lit.span, false)
             )),
             user_ty: None,
@@ -442,7 +442,7 @@ fn make_mirror_unadjusted<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>,
             } else {
                 if let hir::ExprKind::Lit(ref lit) = arg.node {
                     ExprKind::Literal {
-                        literal: cx.tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+                        literal: cx.tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                             cx.const_eval_literal(&lit.node, expr_ty, lit.span, true)
                         )),
                         user_ty: None,
@@ -702,7 +702,7 @@ fn make_mirror_unadjusted<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>,
                         ty: var_ty,
                         span: expr.span,
                         kind: ExprKind::Literal {
-                            literal: cx.tcx.intern_lazy_const(literal),
+                            literal: cx.tcx.mk_lazy_const(literal),
                             user_ty: None
                         },
                     }.to_ref();
@@ -856,7 +856,7 @@ fn method_callee<'a, 'gcx, 'tcx>(
         ty,
         span,
         kind: ExprKind::Literal {
-            literal: cx.tcx().intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: cx.tcx().mk_lazy_const(ty::LazyConst::Evaluated(
                 ty::Const::zero_sized(ty)
             )),
             user_ty,
@@ -918,7 +918,7 @@ fn convert_path_expr<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>,
             let user_ty = user_substs_applied_to_def(cx, expr.hir_id, &def);
             debug!("convert_path_expr: user_ty={:?}", user_ty);
             ExprKind::Literal {
-                literal: cx.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::zero_sized(
+                literal: cx.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::zero_sized(
                     cx.tables().node_id_to_type(expr.hir_id),
                 ))),
                 user_ty,
@@ -930,7 +930,7 @@ fn convert_path_expr<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>,
             let user_ty = user_substs_applied_to_def(cx, expr.hir_id, &def);
             debug!("convert_path_expr: (const) user_ty={:?}", user_ty);
             ExprKind::Literal {
-                literal: cx.tcx.intern_lazy_const(ty::LazyConst::Unevaluated(def_id, substs)),
+                literal: cx.tcx.mk_lazy_const(ty::LazyConst::Unevaluated(def_id, substs)),
                 user_ty,
             }
         },
diff --git a/src/librustc_mir/hair/cx/mod.rs b/src/librustc_mir/hair/cx/mod.rs
index f514cac6326..a726b87afa8 100644
--- a/src/librustc_mir/hair/cx/mod.rs
+++ b/src/librustc_mir/hair/cx/mod.rs
@@ -110,7 +110,7 @@ impl<'a, 'gcx, 'tcx> Cx<'a, 'gcx, 'tcx> {
     }
 
     pub fn usize_literal(&mut self, value: u64) -> &'tcx ty::LazyConst<'tcx> {
-        self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_usize(self.tcx, value)))
+        self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_usize(self.tcx, value)))
     }
 
     pub fn bool_ty(&mut self) -> Ty<'tcx> {
@@ -122,11 +122,11 @@ impl<'a, 'gcx, 'tcx> Cx<'a, 'gcx, 'tcx> {
     }
 
     pub fn true_literal(&mut self) -> &'tcx ty::LazyConst<'tcx> {
-        self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, true)))
+        self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, true)))
     }
 
     pub fn false_literal(&mut self) -> &'tcx ty::LazyConst<'tcx> {
-        self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, false)))
+        self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, false)))
     }
 
     pub fn const_eval_literal(
diff --git a/src/librustc_mir/shim.rs b/src/librustc_mir/shim.rs
index 751815eab28..15796186063 100644
--- a/src/librustc_mir/shim.rs
+++ b/src/librustc_mir/shim.rs
@@ -459,7 +459,7 @@ impl<'a, 'tcx> CloneShimBuilder<'a, 'tcx> {
             span: self.span,
             ty: func_ty,
             user_ty: None,
-            literal: tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                 ty::Const::zero_sized(func_ty),
             )),
         });
@@ -521,7 +521,7 @@ impl<'a, 'tcx> CloneShimBuilder<'a, 'tcx> {
             span: self.span,
             ty: self.tcx.types.usize,
             user_ty: None,
-            literal: self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                 ty::Const::from_usize(self.tcx, value),
             )),
         }
@@ -759,7 +759,7 @@ fn build_call_shim<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                 span,
                 ty,
                 user_ty: None,
-                literal: tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+                literal: tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                     ty::Const::zero_sized(ty)
                 )),
              }),
diff --git a/src/librustc_mir/transform/elaborate_drops.rs b/src/librustc_mir/transform/elaborate_drops.rs
index 06e16de8b43..9b75a70ff41 100644
--- a/src/librustc_mir/transform/elaborate_drops.rs
+++ b/src/librustc_mir/transform/elaborate_drops.rs
@@ -533,7 +533,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
             span,
             ty: self.tcx.types.bool,
             user_ty: None,
-            literal: self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                 ty::Const::from_bool(self.tcx, val),
             )),
         })))
diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index f5cc6a43e28..2a1188ed870 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -198,7 +198,7 @@ impl<'a, 'tcx> TransformVisitor<'a, 'tcx> {
             span: source_info.span,
             ty: self.tcx.types.u32,
             user_ty: None,
-            literal: self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bits(
+            literal: self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bits(
                 self.tcx,
                 state_disc.into(),
                 ty::ParamEnv::empty().and(self.tcx.types.u32)
@@ -731,7 +731,7 @@ fn insert_panic_block<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
             span: mir.span,
             ty: tcx.types.bool,
             user_ty: None,
-            literal: tcx.intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: tcx.mk_lazy_const(ty::LazyConst::Evaluated(
                 ty::Const::from_bool(tcx, false),
             )),
         }),
diff --git a/src/librustc_mir/util/elaborate_drops.rs b/src/librustc_mir/util/elaborate_drops.rs
index 8b55a4424ae..eedc42927c1 100644
--- a/src/librustc_mir/util/elaborate_drops.rs
+++ b/src/librustc_mir/util/elaborate_drops.rs
@@ -963,7 +963,7 @@ impl<'l, 'b, 'tcx, D> DropCtxt<'l, 'b, 'tcx, D>
             span: self.source_info.span,
             ty: self.tcx().types.usize,
             user_ty: None,
-            literal: self.tcx().intern_lazy_const(ty::LazyConst::Evaluated(
+            literal: self.tcx().mk_lazy_const(ty::LazyConst::Evaluated(
                 ty::Const::from_usize(self.tcx(), val.into())
             )),
         })
diff --git a/src/librustc_typeck/astconv.rs b/src/librustc_typeck/astconv.rs
index 8da0b6dcbea..757385aeb3e 100644
--- a/src/librustc_typeck/astconv.rs
+++ b/src/librustc_typeck/astconv.rs
@@ -1793,7 +1793,7 @@ impl<'o, 'gcx: 'tcx, 'tcx> dyn AstConv<'gcx, 'tcx> + 'o {
                 let length_def_id = tcx.hir().local_def_id(length.id);
                 let substs = Substs::identity_for_item(tcx, length_def_id);
                 let length = ty::LazyConst::Unevaluated(length_def_id, substs);
-                let length = tcx.intern_lazy_const(length);
+                let length = tcx.mk_lazy_const(length);
                 let array_ty = tcx.mk_ty(ty::Array(self.ast_ty_to_ty(&ty), length));
                 self.normalize_ty(ast_ty.span, array_ty)
             }
diff --git a/src/librustc_typeck/check/mod.rs b/src/librustc_typeck/check/mod.rs
index 3e2a9d720f1..fb8f6088121 100644
--- a/src/librustc_typeck/check/mod.rs
+++ b/src/librustc_typeck/check/mod.rs
@@ -4597,7 +4597,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {
                 if element_ty.references_error() {
                     tcx.types.err
                 } else if let Ok(count) = count {
-                    tcx.mk_ty(ty::Array(t, tcx.intern_lazy_const(ty::LazyConst::Evaluated(count))))
+                    tcx.mk_ty(ty::Array(t, tcx.mk_lazy_const(ty::LazyConst::Evaluated(count))))
                 } else {
                     tcx.types.err
                 }