Merge from rustc

82 files changed, 674 insertions, 486 deletions

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index a3783db5f80..f414ff746bb 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -13,7 +13,7 @@ use rustc_session::parse::feature_err;
 use rustc_session::{RustcVersion, Session};
 use rustc_span::hygiene::Transparency;
 use rustc_span::{symbol::sym, symbol::Symbol, Span};
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
 
@@ -113,7 +113,7 @@ pub enum StabilityLevel {
         /// Reason for the current stability level.
         reason: UnstableReason,
         /// Relevant `rust-lang/rust` issue.
-        issue: Option<NonZeroU32>,
+        issue: Option<NonZero<u32>>,
         is_soft: bool,
         /// If part of a feature is stabilized and a new feature is added for the remaining parts,
         /// then the `implied_by` attribute is used to indicate which now-stable feature previously
@@ -442,7 +442,7 @@ fn parse_unstability(sess: &Session, attr: &Attribute) -> Option<(Symbol, Stabil
                 // is a name/value pair string literal.
                 issue_num = match issue.unwrap().as_str() {
                     "none" => None,
-                    issue => match issue.parse::<NonZeroU32>() {
+                    issue => match issue.parse::<NonZero<u32>>() {
                         Ok(num) => Some(num),
                         Err(err) => {
                             sess.dcx().emit_err(
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index dd87a5c4dc3..fada69c4e6d 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -7,6 +7,7 @@
 #![allow(internal_features)]
 #![feature(rustdoc_internals)]
 #![doc(rust_logo)]
+#![feature(generic_nonzero)]
 #![feature(let_chains)]
 
 #[macro_use]
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index b3d684086c2..65643e93d27 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -698,7 +698,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                                     ),
                                 ..
                             }) => {
-                                let hir::Ty { span, .. } = inputs[local.index() - 1];
+                                let hir::Ty { span, .. } = *inputs.get(local.index() - 1)?;
                                 Some(span)
                             }
                             _ => None,
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index ae4000f02a7..64469727d0d 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -1666,16 +1666,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 
         let func_ty = func.ty(body, self.infcx.tcx);
         if let ty::FnDef(def_id, _) = *func_ty.kind() {
-            if self.tcx().is_intrinsic(def_id) {
-                match self.tcx().item_name(def_id) {
-                    sym::simd_shuffle => {
-                        if !matches!(args[2], Spanned { node: Operand::Constant(_), .. }) {
-                            self.tcx()
-                                .dcx()
-                                .emit_err(SimdShuffleLastConst { span: term.source_info.span });
-                        }
-                    }
-                    _ => {}
+            if let Some(sym::simd_shuffle) = self.tcx().intrinsic(def_id) {
+                if !matches!(args[2], Spanned { node: Operand::Constant(_), .. }) {
+                    self.tcx().dcx().emit_err(SimdShuffleLastConst { span: term.source_info.span });
                 }
             }
         }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 0f0d828c8fc..6e846d721f2 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -387,15 +387,17 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
         match instance.def {
             InstanceDef::Intrinsic(_) => {
-                crate::intrinsics::codegen_intrinsic_call(
+                match crate::intrinsics::codegen_intrinsic_call(
                     fx,
                     instance,
                     args,
                     ret_place,
                     target,
                     source_info,
-                );
-                return;
+                ) {
+                    Ok(()) => return,
+                    Err(instance) => Some(instance),
+                }
             }
             InstanceDef::DropGlue(_, None) => {
                 // empty drop glue - a nop.
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 819cb5ef137..476752c7230 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -268,7 +268,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     destination: CPlace<'tcx>,
     target: Option<BasicBlock>,
     source_info: mir::SourceInfo,
-) {
+) -> Result<(), Instance<'tcx>> {
     let intrinsic = fx.tcx.item_name(instance.def_id());
     let instance_args = instance.args;
 
@@ -295,8 +295,9 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             destination,
             target,
             source_info,
-        );
+        )?;
     }
+    Ok(())
 }
 
 fn codegen_float_intrinsic_call<'tcx>(
@@ -430,25 +431,20 @@ fn codegen_regular_intrinsic_call<'tcx>(
     ret: CPlace<'tcx>,
     destination: Option<BasicBlock>,
     source_info: mir::SourceInfo,
-) {
+) -> Result<(), Instance<'tcx>> {
+    assert_eq!(generic_args, instance.args);
     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 
     match intrinsic {
         sym::abort => {
             fx.bcx.ins().trap(TrapCode::User(0));
-            return;
+            return Ok(());
         }
         sym::likely | sym::unlikely => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
             ret.write_cvalue(fx, a);
         }
-        sym::is_val_statically_known => {
-            intrinsic_args!(fx, args => (_a); intrinsic);
-
-            let res = fx.bcx.ins().iconst(types::I8, 0);
-            ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
-        }
         sym::breakpoint => {
             intrinsic_args!(fx, args => (); intrinsic);
 
@@ -697,7 +693,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                         })
                     });
                     crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span));
-                    return;
+                    return Ok(());
                 }
             }
         }
@@ -792,7 +788,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
                         // special case for compiler-builtins to avoid having to patch it
                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
-                        return;
+                        return Ok(());
                     } else {
                         fx.tcx
                             .dcx()
@@ -802,7 +798,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
-                    return;
+                    return Ok(());
                 }
             }
             let clif_ty = fx.clif_type(ty).unwrap();
@@ -823,7 +819,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
                         // special case for compiler-builtins to avoid having to patch it
                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
-                        return;
+                        return Ok(());
                     } else {
                         fx.tcx
                             .dcx()
@@ -833,7 +829,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
-                    return;
+                    return Ok(());
                 }
             }
 
@@ -850,7 +846,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -872,7 +868,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
 
@@ -895,7 +891,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -917,7 +913,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -939,7 +935,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -960,7 +956,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -981,7 +977,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1002,7 +998,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1023,7 +1019,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1044,7 +1040,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1065,7 +1061,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1086,7 +1082,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
-                    return;
+                    return Ok(());
                 }
             }
             let ty = fx.clif_type(layout.ty).unwrap();
@@ -1233,19 +1229,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(cmp, ret.layout()));
         }
 
-        sym::const_allocate => {
-            intrinsic_args!(fx, args => (_size, _align); intrinsic);
-
-            // returns a null pointer at runtime.
-            let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
-            ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
-        }
-
-        sym::const_deallocate => {
-            intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
-            // nop at runtime.
-        }
-
         sym::black_box => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
@@ -1261,13 +1244,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
             );
         }
 
-        _ => {
-            fx.tcx
-                .dcx()
-                .span_fatal(source_info.span, format!("unsupported intrinsic {}", intrinsic));
-        }
+        // Unimplemented intrinsics must have a fallback body. The fallback body is obtained
+        // by converting the `InstanceDef::Intrinsic` to an `InstanceDef::Item`.
+        _ => return Err(Instance::new(instance.def_id(), instance.args)),
     }
 
     let ret_block = fx.get_block(destination.unwrap());
     fx.bcx.ins().jump(ret_block, &[]);
+    Ok(())
 }
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
index eac8cb43779..f162ef831b7 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
@@ -90,7 +90,7 @@ fn get_simple_intrinsic<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, name: Symbol) ->
 }
 
 impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
-    fn codegen_intrinsic_call(&mut self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], llresult: RValue<'gcc>, span: Span) {
+    fn codegen_intrinsic_call(&mut self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], llresult: RValue<'gcc>, span: Span) -> Result<(), Instance<'tcx>> {
         let tcx = self.tcx;
         let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
 
@@ -137,7 +137,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         args[2].immediate(),
                         llresult,
                     );
-                    return;
+                    return Ok(());
                 }
                 sym::breakpoint => {
                     unimplemented!();
@@ -166,12 +166,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 sym::volatile_store => {
                     let dst = args[0].deref(self.cx());
                     args[1].val.volatile_store(self, dst);
-                    return;
+                    return Ok(());
                 }
                 sym::unaligned_volatile_store => {
                     let dst = args[0].deref(self.cx());
                     args[1].val.unaligned_volatile_store(self, dst);
-                    return;
+                    return Ok(());
                 }
                 sym::prefetch_read_data
                     | sym::prefetch_write_data
@@ -269,7 +269,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                             },
                             None => {
                                 tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
-                                return;
+                                return Ok(());
                             }
                         }
                     }
@@ -339,7 +339,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     extended_asm.set_volatile_flag(true);
 
                     // We have copied the value to `result` already.
-                    return;
+                    return Ok(());
                 }
 
                 sym::ptr_mask => {
@@ -357,11 +357,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 _ if name_str.starts_with("simd_") => {
                     match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
                         Ok(llval) => llval,
-                        Err(()) => return,
+                        Err(()) => return Ok(()),
                     }
                 }
 
-                _ => bug!("unknown intrinsic '{}'", name),
+                // Fall back to default body
+                _ => return Err(Instance::new(instance.def_id(), instance.args)),
             };
 
         if !fn_abi.ret.is_ignore() {
@@ -376,6 +377,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     .store(self, result);
             }
         }
+        Ok(())
     }
 
     fn abort(&mut self) {
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index e3e48ecb3aa..4415c51acf6 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -86,7 +86,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         args: &[OperandRef<'tcx, &'ll Value>],
         llresult: &'ll Value,
         span: Span,
-    ) {
+    ) -> Result<(), ty::Instance<'tcx>> {
         let tcx = self.tcx;
         let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());
 
@@ -141,7 +141,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     args[2].immediate(),
                     llresult,
                 );
-                return;
+                return Ok(());
             }
             sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]),
             sym::va_copy => {
@@ -194,17 +194,17 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 if !result.layout.is_zst() {
                     self.store(load, result.llval, result.align);
                 }
-                return;
+                return Ok(());
             }
             sym::volatile_store => {
                 let dst = args[0].deref(self.cx());
                 args[1].val.volatile_store(self, dst);
-                return;
+                return Ok(());
             }
             sym::unaligned_volatile_store => {
                 let dst = args[0].deref(self.cx());
                 args[1].val.unaligned_volatile_store(self, dst);
-                return;
+                return Ok(());
             }
             sym::prefetch_read_data
             | sym::prefetch_write_data
@@ -305,7 +305,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             name,
                             ty,
                         });
-                        return;
+                        return Ok(());
                     }
                 }
             }
@@ -387,7 +387,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 
                 // We have copied the value to `result` already.
-                return;
+                return Ok(());
             }
 
             _ if name.as_str().starts_with("simd_") => {
@@ -395,11 +395,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span,
                 ) {
                     Ok(llval) => llval,
-                    Err(()) => return,
+                    Err(()) => return Ok(()),
                 }
             }
 
-            _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name),
+            _ => {
+                debug!("unknown intrinsic '{}' -- falling back to default body", name);
+                // Call the fallback body instead of generating the intrinsic code
+                return Err(ty::Instance::new(instance.def_id(), instance.args));
+            }
         };
 
         if !fn_abi.ret.is_ignore() {
@@ -411,6 +415,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     .store(self, result);
             }
         }
+        Ok(())
     }
 
     fn abort(&mut self) {
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index e35b4029b45..75d413dedad 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -787,7 +787,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         // Handle intrinsics old codegen wants Expr's for, ourselves.
         let intrinsic = match def {
-            Some(ty::InstanceDef::Intrinsic(def_id)) => Some(bx.tcx().item_name(def_id)),
+            Some(ty::InstanceDef::Intrinsic(def_id)) => Some(bx.tcx().intrinsic(def_id).unwrap()),
             _ => None,
         };
 
@@ -817,21 +817,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         // The arguments we'll be passing. Plus one to account for outptr, if used.
         let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
-        let mut llargs = Vec::with_capacity(arg_count);
-
-        // Prepare the return value destination
-        let ret_dest = if target.is_some() {
-            let is_intrinsic = intrinsic.is_some();
-            self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, is_intrinsic)
-        } else {
-            ReturnDest::Nothing
-        };
 
         if intrinsic == Some(sym::caller_location) {
             return if let Some(target) = target {
                 let location =
                     self.get_caller_location(bx, mir::SourceInfo { span: fn_span, ..source_info });
 
+                let mut llargs = Vec::with_capacity(arg_count);
+                let ret_dest =
+                    self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, true, true);
+                assert_eq!(llargs, []);
                 if let ReturnDest::IndirectOperand(tmp, _) = ret_dest {
                     location.val.store(bx, tmp);
                 }
@@ -842,9 +837,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             };
         }
 
-        match intrinsic {
-            None | Some(sym::drop_in_place) => {}
+        let instance = match intrinsic {
+            None | Some(sym::drop_in_place) => instance,
             Some(intrinsic) => {
+                let mut llargs = Vec::with_capacity(1);
+                let ret_dest = self.make_return_dest(
+                    bx,
+                    destination,
+                    &fn_abi.ret,
+                    &mut llargs,
+                    true,
+                    target.is_some(),
+                );
                 let dest = match ret_dest {
                     _ if fn_abi.ret.is_indirect() => llargs[0],
                     ReturnDest::Nothing => bx.const_undef(bx.type_ptr()),
@@ -878,27 +882,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     })
                     .collect();
 
-                Self::codegen_intrinsic_call(
-                    bx,
-                    *instance.as_ref().unwrap(),
-                    fn_abi,
-                    &args,
-                    dest,
-                    span,
-                );
+                let instance = *instance.as_ref().unwrap();
+                match Self::codegen_intrinsic_call(bx, instance, fn_abi, &args, dest, span) {
+                    Ok(()) => {
+                        if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
+                            self.store_return(bx, ret_dest, &fn_abi.ret, dst.llval);
+                        }
 
-                if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
-                    self.store_return(bx, ret_dest, &fn_abi.ret, dst.llval);
+                        return if let Some(target) = target {
+                            helper.funclet_br(self, bx, target, mergeable_succ)
+                        } else {
+                            bx.unreachable();
+                            MergingSucc::False
+                        };
+                    }
+                    Err(instance) => Some(instance),
                 }
-
-                return if let Some(target) = target {
-                    helper.funclet_br(self, bx, target, mergeable_succ)
-                } else {
-                    bx.unreachable();
-                    MergingSucc::False
-                };
             }
-        }
+        };
+
+        let mut llargs = Vec::with_capacity(arg_count);
+        let destination = target.as_ref().map(|&target| {
+            (self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, false, true), target)
+        });
 
         // Split the rust-call tupled arguments off.
         let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() {
@@ -1040,14 +1046,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (_, Some(llfn)) => llfn,
             _ => span_bug!(span, "no instance or llfn for call"),
         };
-
         helper.do_call(
             self,
             bx,
             fn_abi,
             fn_ptr,
             &llargs,
-            target.as_ref().map(|&target| (ret_dest, target)),
+            destination,
             unwind,
             &copied_constant_arguments,
             mergeable_succ,
@@ -1632,7 +1637,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         fn_ret: &ArgAbi<'tcx, Ty<'tcx>>,
         llargs: &mut Vec<Bx::Value>,
         is_intrinsic: bool,
+        has_target: bool,
     ) -> ReturnDest<'tcx, Bx::Value> {
+        if !has_target {
+            return ReturnDest::Nothing;
+        }
         // If the return is ignored, we can just return a do-nothing `ReturnDest`.
         if fn_ret.is_ignore() {
             return ReturnDest::Nothing;
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 8530bf9e2b3..e4633acd817 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -54,6 +54,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 }
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+    /// In the `Err` case, returns the instance that should be called instead.
     pub fn codegen_intrinsic_call(
         bx: &mut Bx,
         instance: ty::Instance<'tcx>,
@@ -61,7 +62,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         args: &[OperandRef<'tcx, Bx::Value>],
         llresult: Bx::Value,
         span: Span,
-    ) {
+    ) -> Result<(), ty::Instance<'tcx>> {
         let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all());
 
         let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else {
@@ -81,7 +82,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let llval = match name {
             sym::abort => {
                 bx.abort();
-                return;
+                return Ok(());
             }
 
             sym::va_start => bx.va_start(args[0].immediate()),
@@ -150,7 +151,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     args[0].immediate(),
                     args[2].immediate(),
                 );
-                return;
+                return Ok(());
             }
             sym::write_bytes => {
                 memset_intrinsic(
@@ -161,7 +162,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     args[1].immediate(),
                     args[2].immediate(),
                 );
-                return;
+                return Ok(());
             }
 
             sym::volatile_copy_nonoverlapping_memory => {
@@ -174,7 +175,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     args[1].immediate(),
                     args[2].immediate(),
                 );
-                return;
+                return Ok(());
             }
             sym::volatile_copy_memory => {
                 copy_intrinsic(
@@ -186,7 +187,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     args[1].immediate(),
                     args[2].immediate(),
                 );
-                return;
+                return Ok(());
             }
             sym::volatile_set_memory => {
                 memset_intrinsic(
@@ -197,17 +198,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     args[1].immediate(),
                     args[2].immediate(),
                 );
-                return;
+                return Ok(());
             }
             sym::volatile_store => {
                 let dst = args[0].deref(bx.cx());
                 args[1].val.volatile_store(bx, dst);
-                return;
+                return Ok(());
             }
             sym::unaligned_volatile_store => {
                 let dst = args[0].deref(bx.cx());
                 args[1].val.unaligned_volatile_store(bx, dst);
-                return;
+                return Ok(());
             }
             sym::exact_div => {
                 let ty = arg_tys[0];
@@ -225,7 +226,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             name,
                             ty,
                         });
-                        return;
+                        return Ok(());
                     }
                 }
             }
@@ -245,7 +246,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             name,
                             ty: arg_tys[0],
                         });
-                        return;
+                        return Ok(());
                     }
                 }
             }
@@ -256,14 +257,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         span,
                         ty: arg_tys[0],
                     });
-                    return;
+                    return Ok(());
                 }
                 let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else {
                     bx.tcx().dcx().emit_err(InvalidMonomorphization::FloatToIntUnchecked {
                         span,
                         ty: ret_ty,
                     });
-                    return;
+                    return Ok(());
                 };
                 if signed {
                     bx.fptosi(args[0].immediate(), llret_ty)
@@ -280,16 +281,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            sym::const_allocate => {
-                // returns a null pointer at runtime.
-                bx.const_null(bx.type_ptr())
-            }
-
-            sym::const_deallocate => {
-                // nop at runtime.
-                return;
-            }
-
             // This requires that atomic intrinsics follow a specific naming pattern:
             // "atomic_<operation>[_<ordering>]"
             name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
@@ -350,10 +341,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bx.store(val, dest.llval, dest.align);
                             let dest = result.project_field(bx, 1);
                             bx.store(success, dest.llval, dest.align);
-                            return;
                         } else {
-                            return invalid_monomorphization(ty);
+                            invalid_monomorphization(ty);
                         }
+                        return Ok(());
                     }
 
                     "load" => {
@@ -383,7 +374,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 )
                             }
                         } else {
-                            return invalid_monomorphization(ty);
+                            invalid_monomorphization(ty);
+                            return Ok(());
                         }
                     }
 
@@ -399,10 +391,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 val = bx.ptrtoint(val, bx.type_isize());
                             }
                             bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
-                            return;
                         } else {
-                            return invalid_monomorphization(ty);
+                            invalid_monomorphization(ty);
                         }
+                        return Ok(());
                     }
 
                     "fence" => {
@@ -410,7 +402,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             parse_ordering(bx, ordering),
                             SynchronizationScope::CrossThread,
                         );
-                        return;
+                        return Ok(());
                     }
 
                     "singlethreadfence" => {
@@ -418,7 +410,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             parse_ordering(bx, ordering),
                             SynchronizationScope::SingleThread,
                         );
-                        return;
+                        return Ok(());
                     }
 
                     // These are all AtomicRMW ops
@@ -449,7 +441,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             }
                             bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
                         } else {
-                            return invalid_monomorphization(ty);
+                            invalid_monomorphization(ty);
+                            return Ok(());
                         }
                     }
                 }
@@ -458,7 +451,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             sym::nontemporal_store => {
                 let dst = args[0].deref(bx.cx());
                 args[1].val.nontemporal_store(bx, dst);
-                return;
+                return Ok(());
             }
 
             sym::ptr_guaranteed_cmp => {
@@ -493,8 +486,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             _ => {
                 // Need to use backend-specific things in the implementation.
-                bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);
-                return;
+                return bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);
             }
         };
 
@@ -507,6 +499,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     .store(bx, result);
             }
         }
+        Ok(())
     }
 }
 
diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
index 450672fb941..502f0b3fcb5 100644
--- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
@@ -8,6 +8,8 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes {
     /// Remember to add all intrinsics here, in `compiler/rustc_hir_analysis/src/check/mod.rs`,
     /// and in `library/core/src/intrinsics.rs`; if you need access to any LLVM intrinsics,
     /// add them to `compiler/rustc_codegen_llvm/src/context.rs`.
+    /// Returns `Err` if another instance should be called instead. This is used to invoke
+    /// intrinsic default bodies in case an intrinsic is not implemented by the backend.
     fn codegen_intrinsic_call(
         &mut self,
         instance: ty::Instance<'tcx>,
@@ -15,7 +17,7 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes {
         args: &[OperandRef<'tcx, Self::Value>],
         llresult: Self::Value,
         span: Span,
-    );
+    ) -> Result<(), ty::Instance<'tcx>>;
 
     fn abort(&mut self);
     fn assume(&mut self, val: Self::Value);
diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl
index 4a6d4fe930c..c456e40d7c1 100644
--- a/compiler/rustc_const_eval/messages.ftl
+++ b/compiler/rustc_const_eval/messages.ftl
@@ -453,7 +453,7 @@ const_eval_validation_invalid_fn_ptr = {$front_matter}: encountered {$value}, bu
 const_eval_validation_invalid_ref_meta = {$front_matter}: encountered invalid reference metadata: total size is bigger than largest supported object
 const_eval_validation_invalid_ref_slice_meta = {$front_matter}: encountered invalid reference metadata: slice is bigger than largest supported object
 const_eval_validation_invalid_vtable_ptr = {$front_matter}: encountered {$value}, but expected a vtable pointer
-const_eval_validation_mutable_ref_in_const = {$front_matter}: encountered mutable reference in a `const` or `static`
+const_eval_validation_mutable_ref_in_const_or_static = {$front_matter}: encountered mutable reference in a `const` or `static`
 const_eval_validation_mutable_ref_to_immutable = {$front_matter}: encountered mutable reference or box pointing to read-only memory
 const_eval_validation_never_val = {$front_matter}: encountered a value of the never type `!`
 const_eval_validation_null_box = {$front_matter}: encountered a null box
diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
index dbc29e607ef..ddad6683afb 100644
--- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
@@ -49,7 +49,7 @@ fn constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness {
         hir::Node::ForeignItem(hir::ForeignItem { kind: hir::ForeignItemKind::Fn(..), .. }) => {
             // Intrinsics use `rustc_const_{un,}stable` attributes to indicate constness. All other
             // foreign items cannot be evaluated at compile-time.
-            let is_const = if tcx.is_intrinsic(def_id) {
+            let is_const = if tcx.intrinsic(def_id).is_some() {
                 tcx.lookup_const_stability(def_id).is_some()
             } else {
                 false
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index 11679ab77e3..2fd34b3c7fc 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -603,18 +603,18 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
             PtrToUninhabited { ptr_kind: PointerKind::Box, .. } => {
                 const_eval_validation_box_to_uninhabited
             }
-            PtrToUninhabited { ptr_kind: PointerKind::Ref, .. } => {
+            PtrToUninhabited { ptr_kind: PointerKind::Ref(_), .. } => {
                 const_eval_validation_ref_to_uninhabited
             }
 
             PtrToStatic { ptr_kind: PointerKind::Box } => const_eval_validation_box_to_static,
-            PtrToStatic { ptr_kind: PointerKind::Ref } => const_eval_validation_ref_to_static,
+            PtrToStatic { ptr_kind: PointerKind::Ref(_) } => const_eval_validation_ref_to_static,
 
             PointerAsInt { .. } => const_eval_validation_pointer_as_int,
             PartialPointer => const_eval_validation_partial_pointer,
             ConstRefToMutable => const_eval_validation_const_ref_to_mutable,
             ConstRefToExtern => const_eval_validation_const_ref_to_extern,
-            MutableRefInConst => const_eval_validation_mutable_ref_in_const,
+            MutableRefInConstOrStatic => const_eval_validation_mutable_ref_in_const_or_static,
             MutableRefToImmutable => const_eval_validation_mutable_ref_to_immutable,
             NullFnPtr => const_eval_validation_null_fn_ptr,
             NeverVal => const_eval_validation_never_val,
@@ -630,37 +630,39 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
             InvalidMetaSliceTooLarge { ptr_kind: PointerKind::Box } => {
                 const_eval_validation_invalid_box_slice_meta
             }
-            InvalidMetaSliceTooLarge { ptr_kind: PointerKind::Ref } => {
+            InvalidMetaSliceTooLarge { ptr_kind: PointerKind::Ref(_) } => {
                 const_eval_validation_invalid_ref_slice_meta
             }
 
             InvalidMetaTooLarge { ptr_kind: PointerKind::Box } => {
                 const_eval_validation_invalid_box_meta
             }
-            InvalidMetaTooLarge { ptr_kind: PointerKind::Ref } => {
+            InvalidMetaTooLarge { ptr_kind: PointerKind::Ref(_) } => {
                 const_eval_validation_invalid_ref_meta
             }
-            UnalignedPtr { ptr_kind: PointerKind::Ref, .. } => const_eval_validation_unaligned_ref,
+            UnalignedPtr { ptr_kind: PointerKind::Ref(_), .. } => {
+                const_eval_validation_unaligned_ref
+            }
             UnalignedPtr { ptr_kind: PointerKind::Box, .. } => const_eval_validation_unaligned_box,
 
             NullPtr { ptr_kind: PointerKind::Box } => const_eval_validation_null_box,
-            NullPtr { ptr_kind: PointerKind::Ref } => const_eval_validation_null_ref,
+            NullPtr { ptr_kind: PointerKind::Ref(_) } => const_eval_validation_null_ref,
             DanglingPtrNoProvenance { ptr_kind: PointerKind::Box, .. } => {
                 const_eval_validation_dangling_box_no_provenance
             }
-            DanglingPtrNoProvenance { ptr_kind: PointerKind::Ref, .. } => {
+            DanglingPtrNoProvenance { ptr_kind: PointerKind::Ref(_), .. } => {
                 const_eval_validation_dangling_ref_no_provenance
             }
             DanglingPtrOutOfBounds { ptr_kind: PointerKind::Box } => {
                 const_eval_validation_dangling_box_out_of_bounds
             }
-            DanglingPtrOutOfBounds { ptr_kind: PointerKind::Ref } => {
+            DanglingPtrOutOfBounds { ptr_kind: PointerKind::Ref(_) } => {
                 const_eval_validation_dangling_ref_out_of_bounds
             }
             DanglingPtrUseAfterFree { ptr_kind: PointerKind::Box } => {
                 const_eval_validation_dangling_box_use_after_free
             }
-            DanglingPtrUseAfterFree { ptr_kind: PointerKind::Ref } => {
+            DanglingPtrUseAfterFree { ptr_kind: PointerKind::Ref(_) } => {
                 const_eval_validation_dangling_ref_use_after_free
             }
             InvalidBool { .. } => const_eval_validation_invalid_bool,
@@ -766,7 +768,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
             }
             NullPtr { .. }
             | PtrToStatic { .. }
-            | MutableRefInConst
+            | MutableRefInConstOrStatic
             | ConstRefToMutable
             | ConstRefToExtern
             | MutableRefToImmutable
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
index b2207c3d310..e72ace8be35 100644
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -526,7 +526,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         match instance.def {
             ty::InstanceDef::Intrinsic(def_id) => {
-                assert!(self.tcx.is_intrinsic(def_id));
+                assert!(self.tcx.intrinsic(def_id).is_some());
                 // FIXME: Should `InPlace` arguments be reset to uninit?
                 M::call_intrinsic(
                     self,
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 08a2e38bfa1..792e1c9e736 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -5,7 +5,7 @@
 //! to be const-safe.
 
 use std::fmt::Write;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 
 use either::{Left, Right};
 
@@ -445,22 +445,22 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
             // Determine whether this pointer expects to be pointing to something mutable.
             let ptr_expected_mutbl = match ptr_kind {
                 PointerKind::Box => Mutability::Mut,
-                PointerKind::Ref => {
-                    let tam = value.layout.ty.builtin_deref(false).unwrap();
-                    // ZST never require mutability. We do not take into account interior mutability
-                    // here since we cannot know if there really is an `UnsafeCell` inside
-                    // `Option<UnsafeCell>` -- so we check that in the recursive descent behind this
-                    // reference.
-                    if size == Size::ZERO { Mutability::Not } else { tam.mutbl }
+                PointerKind::Ref(mutbl) => {
+                    // We do not take into account interior mutability here since we cannot know if
+                    // there really is an `UnsafeCell` inside `Option<UnsafeCell>` -- so we check
+                    // that in the recursive descent behind this reference (controlled by
+                    // `allow_immutable_unsafe_cell`).
+                    mutbl
                 }
             };
             // Proceed recursively even for ZST, no reason to skip them!
             // `!` is a ZST and we want to validate it.
             if let Ok((alloc_id, _offset, _prov)) = self.ecx.ptr_try_get_alloc_id(place.ptr()) {
+                let mut skip_recursive_check = false;
                 // Let's see what kind of memory this points to.
                 // `unwrap` since dangling pointers have already been handled.
                 let alloc_kind = self.ecx.tcx.try_get_global_alloc(alloc_id).unwrap();
-                match alloc_kind {
+                let alloc_actual_mutbl = match alloc_kind {
                     GlobalAlloc::Static(did) => {
                         // Special handling for pointers to statics (irrespective of their type).
                         assert!(!self.ecx.tcx.is_thread_local_static(did));
@@ -474,12 +474,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                                     .no_bound_vars()
                                     .expect("statics should not have generic parameters")
                                     .is_freeze(*self.ecx.tcx, ty::ParamEnv::reveal_all());
-                        // Mutability check.
-                        if ptr_expected_mutbl == Mutability::Mut {
-                            if !is_mut {
-                                throw_validation_failure!(self.path, MutableRefToImmutable);
-                            }
-                        }
                         // Mode-specific checks
                         match self.ctfe_mode {
                             Some(
@@ -494,15 +488,9 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                                 // trigger cycle errors if we try to compute the value of the other static
                                 // and that static refers back to us (potentially through a promoted).
                                 // This could miss some UB, but that's fine.
-                                return Ok(());
+                                skip_recursive_check = true;
                             }
                             Some(CtfeValidationMode::Const { .. }) => {
-                                // For consts on the other hand we have to recursively check;
-                                // pattern matching assumes a valid value. However we better make
-                                // sure this is not mutable.
-                                if is_mut {
-                                    throw_validation_failure!(self.path, ConstRefToMutable);
-                                }
                                 // We can't recursively validate `extern static`, so we better reject them.
                                 if self.ecx.tcx.is_foreign_item(did) {
                                     throw_validation_failure!(self.path, ConstRefToExtern);
@@ -510,26 +498,39 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                             }
                             None => {}
                         }
+                        // Return alloc mutability
+                        if is_mut { Mutability::Mut } else { Mutability::Not }
                     }
-                    GlobalAlloc::Memory(alloc) => {
-                        if alloc.inner().mutability == Mutability::Mut
-                            && matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. }))
-                        {
-                            throw_validation_failure!(self.path, ConstRefToMutable);
-                        }
-                        if ptr_expected_mutbl == Mutability::Mut
-                            && alloc.inner().mutability == Mutability::Not
-                        {
-                            throw_validation_failure!(self.path, MutableRefToImmutable);
-                        }
-                    }
+                    GlobalAlloc::Memory(alloc) => alloc.inner().mutability,
                     GlobalAlloc::Function(..) | GlobalAlloc::VTable(..) => {
                         // These are immutable, we better don't allow mutable pointers here.
-                        if ptr_expected_mutbl == Mutability::Mut {
-                            throw_validation_failure!(self.path, MutableRefToImmutable);
-                        }
+                        Mutability::Not
+                    }
+                };
+                // Mutability check.
+                // If this allocation has size zero, there is no actual mutability here.
+                let (size, _align, _alloc_kind) = self.ecx.get_alloc_info(alloc_id);
+                if size != Size::ZERO {
+                    if ptr_expected_mutbl == Mutability::Mut
+                        && alloc_actual_mutbl == Mutability::Not
+                    {
+                        throw_validation_failure!(self.path, MutableRefToImmutable);
+                    }
+                    if ptr_expected_mutbl == Mutability::Mut
+                        && self.ctfe_mode.is_some_and(|c| !c.may_contain_mutable_ref())
+                    {
+                        throw_validation_failure!(self.path, MutableRefInConstOrStatic);
+                    }
+                    if alloc_actual_mutbl == Mutability::Mut
+                        && matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. }))
+                    {
+                        throw_validation_failure!(self.path, ConstRefToMutable);
                     }
                 }
+                // Potentially skip recursive check.
+                if skip_recursive_check {
+                    return Ok(());
+                }
             }
             let path = &self.path;
             ref_tracking.track(place, || {
@@ -598,16 +599,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 }
                 Ok(true)
             }
-            ty::Ref(_, ty, mutbl) => {
-                if self.ctfe_mode.is_some_and(|c| !c.may_contain_mutable_ref())
-                    && *mutbl == Mutability::Mut
-                {
-                    let layout = self.ecx.layout_of(*ty)?;
-                    if !layout.is_zst() {
-                        throw_validation_failure!(self.path, MutableRefInConst);
-                    }
-                }
-                self.check_safe_pointer(value, PointerKind::Ref)?;
+            ty::Ref(_, _ty, mutbl) => {
+                self.check_safe_pointer(value, PointerKind::Ref(*mutbl))?;
                 Ok(true)
             }
             ty::FnPtr(_sig) => {
@@ -785,7 +778,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
     fn visit_union(
         &mut self,
         op: &OpTy<'tcx, M::Provenance>,
-        _fields: NonZeroUsize,
+        _fields: NonZero<usize>,
     ) -> InterpResult<'tcx> {
         // Special check for CTFE validation, preventing `UnsafeCell` inside unions in immutable memory.
         if self.ctfe_mode.is_some_and(|c| !c.allow_immutable_unsafe_cell()) {
diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs
index 340a496a689..b200ecbf73a 100644
--- a/compiler/rustc_const_eval/src/interpret/visitor.rs
+++ b/compiler/rustc_const_eval/src/interpret/visitor.rs
@@ -7,7 +7,7 @@ use rustc_middle::ty;
 use rustc_target::abi::FieldIdx;
 use rustc_target::abi::{FieldsShape, VariantIdx, Variants};
 
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 
 use super::{InterpCx, MPlaceTy, Machine, Projectable};
 
@@ -43,7 +43,7 @@ pub trait ValueVisitor<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
     }
     /// Visits the given value as a union. No automatic recursion can happen here.
     #[inline(always)]
-    fn visit_union(&mut self, _v: &Self::V, _fields: NonZeroUsize) -> InterpResult<'tcx> {
+    fn visit_union(&mut self, _v: &Self::V, _fields: NonZero<usize>) -> InterpResult<'tcx> {
         Ok(())
     }
     /// Visits the given value as the pointer of a `Box`. There is nothing to recurse into.
diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index e33f374c359..51836063945 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -11,6 +11,7 @@ Rust MIR: a lowered representation of Rust.
 #![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(decl_macro)]
+#![feature(generic_nonzero)]
 #![feature(let_chains)]
 #![feature(slice_ptr_get)]
 #![feature(never_type)]
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index 43048dc41d3..96c9e740568 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -861,7 +861,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     // We do not use `const` modifiers for intrinsic "functions", as intrinsics are
                     // `extern` functions, and these have no way to get marked `const`. So instead we
                     // use `rustc_const_(un)stable` attributes to mean that the intrinsic is `const`
-                    if self.ccx.is_const_stable_const_fn() || tcx.is_intrinsic(callee) {
+                    if self.ccx.is_const_stable_const_fn() || tcx.intrinsic(callee).is_some() {
                         self.check_op(ops::FnCallUnstable(callee, None));
                         return;
                     }
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 2b799d6f5d3..b82a9a909e6 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -20,6 +20,7 @@
 #![feature(cfg_match)]
 #![feature(core_intrinsics)]
 #![feature(extend_one)]
+#![feature(generic_nonzero)]
 #![feature(hash_raw_entry)]
 #![feature(hasher_prefixfree_extras)]
 #![feature(lazy_cell)]
diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs
index 52304c72a2f..15691804a94 100644
--- a/compiler/rustc_data_structures/src/stable_hasher.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher.rs
@@ -6,6 +6,7 @@ use std::fmt;
 use std::hash::{BuildHasher, Hash, Hasher};
 use std::marker::PhantomData;
 use std::mem;
+use std::num::NonZero;
 
 #[cfg(test)]
 mod tests;
@@ -338,14 +339,14 @@ impl<CTX, T> HashStable<CTX> for PhantomData<T> {
     fn hash_stable(&self, _ctx: &mut CTX, _hasher: &mut StableHasher) {}
 }
 
-impl<CTX> HashStable<CTX> for ::std::num::NonZeroU32 {
+impl<CTX> HashStable<CTX> for NonZero<u32> {
     #[inline]
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
         self.get().hash_stable(ctx, hasher)
     }
 }
 
-impl<CTX> HashStable<CTX> for ::std::num::NonZeroUsize {
+impl<CTX> HashStable<CTX> for NonZero<usize> {
     #[inline]
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
         self.get().hash_stable(ctx, hasher)
diff --git a/compiler/rustc_data_structures/src/sync/worker_local.rs b/compiler/rustc_data_structures/src/sync/worker_local.rs
index b34d3dd9044..50a614a1b02 100644
--- a/compiler/rustc_data_structures/src/sync/worker_local.rs
+++ b/compiler/rustc_data_structures/src/sync/worker_local.rs
@@ -1,7 +1,7 @@
 use parking_lot::Mutex;
 use std::cell::Cell;
 use std::cell::OnceCell;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::ops::Deref;
 use std::ptr;
 use std::sync::Arc;
@@ -31,7 +31,7 @@ impl RegistryId {
 }
 
 struct RegistryData {
-    thread_limit: NonZeroUsize,
+    thread_limit: NonZero<usize>,
     threads: Mutex<usize>,
 }
 
@@ -61,7 +61,7 @@ thread_local! {
 
 impl Registry {
     /// Creates a registry which can hold up to `thread_limit` threads.
-    pub fn new(thread_limit: NonZeroUsize) -> Self {
+    pub fn new(thread_limit: NonZero<usize>) -> Self {
         Registry(Arc::new(RegistryData { thread_limit, threads: Mutex::new(0) }))
     }
 
diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs
index e893a2c7813..ff4208def31 100644
--- a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs
+++ b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs
@@ -4,7 +4,7 @@ use std::fmt;
 use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
 use std::mem::ManuallyDrop;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::ops::{Deref, DerefMut};
 use std::ptr::NonNull;
 
@@ -134,7 +134,7 @@ where
 
         ptr.map_addr(|addr| {
             // Safety:
-            // - The pointer is `NonNull` => it's address is `NonZeroUsize`
+            // - The pointer is `NonNull` => it's address is `NonZero<usize>`
             // - `P::BITS` least significant bits are always zero (`Pointer` contract)
             // - `T::BITS <= P::BITS` (from `Self::ASSERTION`)
             //
@@ -143,14 +143,14 @@ where
             // `{non_zero} | packed_tag` can't make the value zero.
 
             let packed = (addr.get() >> T::BITS) | packed_tag;
-            unsafe { NonZeroUsize::new_unchecked(packed) }
+            unsafe { NonZero::new_unchecked(packed) }
         })
     }
 
     /// Retrieves the original raw pointer from `self.packed`.
     #[inline]
     pub(super) fn pointer_raw(&self) -> NonNull<P::Target> {
-        self.packed.map_addr(|addr| unsafe { NonZeroUsize::new_unchecked(addr.get() << T::BITS) })
+        self.packed.map_addr(|addr| unsafe { NonZero::new_unchecked(addr.get() << T::BITS) })
     }
 
     /// This provides a reference to the `P` pointer itself, rather than the
diff --git a/compiler/rustc_errors/src/diagnostic_impls.rs b/compiler/rustc_errors/src/diagnostic_impls.rs
index e936ebc7185..eaf75539f59 100644
--- a/compiler/rustc_errors/src/diagnostic_impls.rs
+++ b/compiler/rustc_errors/src/diagnostic_impls.rs
@@ -79,7 +79,7 @@ into_diagnostic_arg_using_display!(
     ast::ParamKindOrd,
     std::io::Error,
     Box<dyn std::error::Error>,
-    std::num::NonZeroU32,
+    std::num::NonZero<u32>,
     hir::Target,
     Edition,
     Ident,
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index b0db3545ae7..b9b257856e6 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -16,6 +16,7 @@
 #![feature(box_patterns)]
 #![feature(error_reporter)]
 #![feature(extract_if)]
+#![feature(generic_nonzero)]
 #![feature(let_chains)]
 #![feature(negative_impls)]
 #![feature(never_type)]
@@ -77,7 +78,7 @@ use std::error::Report;
 use std::fmt;
 use std::hash::Hash;
 use std::io::Write;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::ops::DerefMut;
 use std::panic;
 use std::path::{Path, PathBuf};
@@ -525,6 +526,7 @@ pub enum StashKey {
     MaybeFruTypo,
     CallAssocMethod,
     TraitMissingMethod,
+    AssociatedTypeSuggestion,
     OpaqueHiddenTypeMismatch,
     MaybeForgetReturn,
     /// Query cycle detected, stashing in favor of a better error.
@@ -546,7 +548,7 @@ pub struct DiagCtxtFlags {
     pub can_emit_warnings: bool,
     /// If Some, the Nth error-level diagnostic is upgraded to bug-level.
     /// (rustc: see `-Z treat-err-as-bug`)
-    pub treat_err_as_bug: Option<NonZeroUsize>,
+    pub treat_err_as_bug: Option<NonZero<usize>>,
     /// Eagerly emit delayed bugs as errors, so that the compiler debugger may
     /// see all of the errors being emitted at once.
     pub eagerly_emit_delayed_bugs: bool,
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 6aedd2a5e33..99875ec5405 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -788,6 +788,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_safe_intrinsic, Normal, template!(Word), WarnFollowing,
         "the `#[rustc_safe_intrinsic]` attribute is used internally to mark intrinsics as safe"
     ),
+    rustc_attr!(
+        rustc_intrinsic, Normal, template!(Word), ErrorFollowing,
+        "the `#[rustc_intrinsic]` attribute is used to declare intrinsics with function bodies",
+    ),
 
     // ==========================================================================
     // Internal attributes, Testing:
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index f1c8f2e2dde..cbc0ce8c974 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -12,6 +12,7 @@
 //! symbol to the `accepted` or `removed` modules respectively.
 
 #![allow(internal_features)]
+#![feature(generic_nonzero)]
 #![feature(rustdoc_internals)]
 #![doc(rust_logo)]
 #![feature(lazy_cell)]
@@ -25,13 +26,13 @@ mod unstable;
 mod tests;
 
 use rustc_span::symbol::Symbol;
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 #[derive(Debug, Clone)]
 pub struct Feature {
     pub name: Symbol,
     pub since: &'static str,
-    issue: Option<NonZeroU32>,
+    issue: Option<NonZero<u32>>,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -85,7 +86,7 @@ impl UnstableFeatures {
     }
 }
 
-fn find_lang_feature_issue(feature: Symbol) -> Option<NonZeroU32> {
+fn find_lang_feature_issue(feature: Symbol) -> Option<NonZero<u32>> {
     // Search in all the feature lists.
     if let Some(f) = UNSTABLE_FEATURES.iter().find(|f| f.feature.name == feature) {
         return f.feature.issue;
@@ -99,21 +100,21 @@ fn find_lang_feature_issue(feature: Symbol) -> Option<NonZeroU32> {
     panic!("feature `{feature}` is not declared anywhere");
 }
 
-const fn to_nonzero(n: Option<u32>) -> Option<NonZeroU32> {
-    // Can be replaced with `n.and_then(NonZeroU32::new)` if that is ever usable
+const fn to_nonzero(n: Option<u32>) -> Option<NonZero<u32>> {
+    // Can be replaced with `n.and_then(NonZero::new)` if that is ever usable
     // in const context. Requires https://github.com/rust-lang/rfcs/pull/2632.
     match n {
         None => None,
-        Some(n) => NonZeroU32::new(n),
+        Some(n) => NonZero::new(n),
     }
 }
 
 pub enum GateIssue {
     Language,
-    Library(Option<NonZeroU32>),
+    Library(Option<NonZero<u32>>),
 }
 
-pub fn find_feature_issue(feature: Symbol, issue: GateIssue) -> Option<NonZeroU32> {
+pub fn find_feature_issue(feature: Symbol, issue: GateIssue) -> Option<NonZero<u32>> {
     match issue {
         GateIssue::Language => find_lang_feature_issue(feature),
         GateIssue::Library(lib) => lib,
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index 488d16f18da..1410273e3bc 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -525,7 +525,18 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
         DefKind::Enum => {
             check_enum(tcx, def_id);
         }
-        DefKind::Fn => {} // entirely within check_item_body
+        DefKind::Fn => {
+            if let Some(name) = tcx.intrinsic(def_id) {
+                intrinsic::check_intrinsic_type(
+                    tcx,
+                    def_id,
+                    tcx.def_ident_span(def_id).unwrap(),
+                    name,
+                    Abi::Rust,
+                )
+            }
+            // Everything else is checked entirely within check_item_body
+        }
         DefKind::Impl { of_trait } => {
             if of_trait && let Some(impl_trait_header) = tcx.impl_trait_header(def_id) {
                 check_impl_items_against_trait(
@@ -590,15 +601,24 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
             match abi {
                 Abi::RustIntrinsic => {
                     for item in items {
-                        let item = tcx.hir().foreign_item(item.id);
-                        intrinsic::check_intrinsic_type(tcx, item);
+                        intrinsic::check_intrinsic_type(
+                            tcx,
+                            item.id.owner_id.def_id,
+                            item.span,
+                            item.ident.name,
+                            abi,
+                        );
                     }
                 }
 
                 Abi::PlatformIntrinsic => {
                     for item in items {
-                        let item = tcx.hir().foreign_item(item.id);
-                        intrinsic::check_platform_intrinsic_type(tcx, item);
+                        intrinsic::check_platform_intrinsic_type(
+                            tcx,
+                            item.id.owner_id.def_id,
+                            item.span,
+                            item.ident.name,
+                        );
                     }
                 }
 
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index dc391ab5648..f0f6bfff64a 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -7,30 +7,36 @@ use crate::errors::{
     WrongNumberOfGenericArgumentsToIntrinsic,
 };
 
-use hir::def_id::DefId;
 use rustc_errors::{codes::*, struct_span_code_err, DiagnosticMessage};
 use rustc_hir as hir;
 use rustc_middle::traits::{ObligationCause, ObligationCauseCode};
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::def_id::LocalDefId;
 use rustc_span::symbol::{kw, sym};
+use rustc_span::{Span, Symbol};
 use rustc_target::spec::abi::Abi;
 
 fn equate_intrinsic_type<'tcx>(
     tcx: TyCtxt<'tcx>,
-    it: &hir::ForeignItem<'_>,
+    span: Span,
+    def_id: LocalDefId,
     n_tps: usize,
     n_lts: usize,
     n_cts: usize,
     sig: ty::PolyFnSig<'tcx>,
 ) {
-    let (own_counts, span) = match &it.kind {
-        hir::ForeignItemKind::Fn(.., generics) => {
-            let own_counts = tcx.generics_of(it.owner_id.to_def_id()).own_counts();
+    let (own_counts, span) = match tcx.hir_node_by_def_id(def_id) {
+        hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, generics, _), .. })
+        | hir::Node::ForeignItem(hir::ForeignItem {
+            kind: hir::ForeignItemKind::Fn(.., generics),
+            ..
+        }) => {
+            let own_counts = tcx.generics_of(def_id).own_counts();
             (own_counts, generics.span)
         }
         _ => {
-            struct_span_code_err!(tcx.dcx(), it.span, E0622, "intrinsic must be a function")
-                .with_span_label(it.span, "expected a function")
+            struct_span_code_err!(tcx.dcx(), span, E0622, "intrinsic must be a function")
+                .with_span_label(span, "expected a function")
                 .emit();
             return;
         }
@@ -54,23 +60,26 @@ fn equate_intrinsic_type<'tcx>(
         && gen_count_ok(own_counts.types, n_tps, "type")
         && gen_count_ok(own_counts.consts, n_cts, "const")
     {
-        let it_def_id = it.owner_id.def_id;
         let _ = check_function_signature(
             tcx,
-            ObligationCause::new(it.span, it_def_id, ObligationCauseCode::IntrinsicType),
-            it_def_id.into(),
+            ObligationCause::new(span, def_id, ObligationCauseCode::IntrinsicType),
+            def_id.into(),
             sig,
         );
     }
 }
 
 /// Returns the unsafety of the given intrinsic.
-pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir::Unsafety {
-    let has_safe_attr = match tcx.has_attr(intrinsic_id, sym::rustc_safe_intrinsic) {
-        true => hir::Unsafety::Normal,
-        false => hir::Unsafety::Unsafe,
+pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hir::Unsafety {
+    let has_safe_attr = if tcx.has_attr(intrinsic_id, sym::rustc_intrinsic) {
+        tcx.fn_sig(intrinsic_id).skip_binder().unsafety()
+    } else {
+        match tcx.has_attr(intrinsic_id, sym::rustc_safe_intrinsic) {
+            true => hir::Unsafety::Normal,
+            false => hir::Unsafety::Unsafe,
+        }
     };
-    let is_in_list = match tcx.item_name(intrinsic_id) {
+    let is_in_list = match tcx.item_name(intrinsic_id.into()) {
         // When adding a new intrinsic to this list,
         // it's usually worth updating that intrinsic's documentation
         // to note that it's safe to call, since
@@ -112,6 +121,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
         | sym::forget
         | sym::black_box
         | sym::variant_count
+        | sym::is_val_statically_known
         | sym::ptr_mask
         | sym::debug_assertions => hir::Unsafety::Normal,
         _ => hir::Unsafety::Unsafe,
@@ -122,7 +132,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
             tcx.def_span(intrinsic_id),
             DiagnosticMessage::from(format!(
                 "intrinsic safety mismatch between list of intrinsics within the compiler and core library intrinsics for intrinsic `{}`",
-                tcx.item_name(intrinsic_id)
+                tcx.item_name(intrinsic_id.into())
             )
         )).emit();
     }
@@ -132,8 +142,14 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
 
 /// Remember to add all intrinsics here, in `compiler/rustc_codegen_llvm/src/intrinsic.rs`,
 /// and in `library/core/src/intrinsics.rs`.
-pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
-    let generics = tcx.generics_of(it.owner_id);
+pub fn check_intrinsic_type(
+    tcx: TyCtxt<'_>,
+    intrinsic_id: LocalDefId,
+    span: Span,
+    intrinsic_name: Symbol,
+    abi: Abi,
+) {
+    let generics = tcx.generics_of(intrinsic_id);
     let param = |n| {
         if let Some(&ty::GenericParamDef {
             name, kind: ty::GenericParamDefKind::Type { .. }, ..
@@ -141,11 +157,9 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
         {
             Ty::new_param(tcx, n, name)
         } else {
-            Ty::new_error_with_message(tcx, tcx.def_span(it.owner_id), "expected param")
+            Ty::new_error_with_message(tcx, span, "expected param")
         }
     };
-    let intrinsic_id = it.owner_id.to_def_id();
-    let intrinsic_name = tcx.item_name(intrinsic_id);
     let name_str = intrinsic_name.as_str();
 
     let bound_vars = tcx.mk_bound_variable_kinds(&[
@@ -169,7 +183,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
         })
     };
 
-    let (n_tps, n_lts, inputs, output, unsafety) = if name_str.starts_with("atomic_") {
+    let (n_tps, n_lts, n_cts, inputs, output, unsafety) = if name_str.starts_with("atomic_") {
         let split: Vec<&str> = name_str.split('_').collect();
         assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
 
@@ -187,49 +201,51 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             | "umin" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
             "fence" | "singlethreadfence" => (0, Vec::new(), Ty::new_unit(tcx)),
             op => {
-                tcx.dcx().emit_err(UnrecognizedAtomicOperation { span: it.span, op });
+                tcx.dcx().emit_err(UnrecognizedAtomicOperation { span, op });
                 return;
             }
         };
-        (n_tps, 0, inputs, output, hir::Unsafety::Unsafe)
+        (n_tps, 0, 0, inputs, output, hir::Unsafety::Unsafe)
     } else {
         let unsafety = intrinsic_operation_unsafety(tcx, intrinsic_id);
-        let (n_tps, inputs, output) = match intrinsic_name {
-            sym::abort => (0, Vec::new(), tcx.types.never),
-            sym::unreachable => (0, Vec::new(), tcx.types.never),
-            sym::breakpoint => (0, Vec::new(), Ty::new_unit(tcx)),
+        let (n_tps, n_cts, inputs, output) = match intrinsic_name {
+            sym::abort => (0, 0, vec![], tcx.types.never),
+            sym::unreachable => (0, 0, vec![], tcx.types.never),
+            sym::breakpoint => (0, 0, vec![], Ty::new_unit(tcx)),
             sym::size_of | sym::pref_align_of | sym::min_align_of | sym::variant_count => {
-                (1, Vec::new(), tcx.types.usize)
+                (1, 0, vec![], tcx.types.usize)
             }
             sym::size_of_val | sym::min_align_of_val => {
-                (1, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.usize)
+                (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.usize)
             }
-            sym::rustc_peek => (1, vec![param(0)], param(0)),
-            sym::caller_location => (0, vec![], tcx.caller_location_ty()),
+            sym::rustc_peek => (1, 0, vec![param(0)], param(0)),
+            sym::caller_location => (0, 0, vec![], tcx.caller_location_ty()),
             sym::assert_inhabited
             | sym::assert_zero_valid
-            | sym::assert_mem_uninitialized_valid => (1, Vec::new(), Ty::new_unit(tcx)),
-            sym::forget => (1, vec![param(0)], Ty::new_unit(tcx)),
-            sym::transmute | sym::transmute_unchecked => (2, vec![param(0)], param(1)),
+            | sym::assert_mem_uninitialized_valid => (1, 0, vec![], Ty::new_unit(tcx)),
+            sym::forget => (1, 0, vec![param(0)], Ty::new_unit(tcx)),
+            sym::transmute | sym::transmute_unchecked => (2, 0, vec![param(0)], param(1)),
             sym::prefetch_read_data
             | sym::prefetch_write_data
             | sym::prefetch_read_instruction
             | sym::prefetch_write_instruction => (
                 1,
+                0,
                 vec![
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
                     tcx.types.i32,
                 ],
                 Ty::new_unit(tcx),
             ),
-            sym::drop_in_place => (1, vec![Ty::new_mut_ptr(tcx, param(0))], Ty::new_unit(tcx)),
-            sym::needs_drop => (1, Vec::new(), tcx.types.bool),
+            sym::drop_in_place => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0))], Ty::new_unit(tcx)),
+            sym::needs_drop => (1, 0, vec![], tcx.types.bool),
 
-            sym::type_name => (1, Vec::new(), Ty::new_static_str(tcx)),
-            sym::type_id => (1, Vec::new(), tcx.types.u128),
-            sym::offset => (2, vec![param(0), param(1)], param(0)),
+            sym::type_name => (1, 0, vec![], Ty::new_static_str(tcx)),
+            sym::type_id => (1, 0, vec![], tcx.types.u128),
+            sym::offset => (2, 0, vec![param(0), param(1)], param(0)),
             sym::arith_offset => (
                 1,
+                0,
                 vec![
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
                     tcx.types.isize,
@@ -238,6 +254,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             ),
             sym::ptr_mask => (
                 1,
+                0,
                 vec![
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
                     tcx.types.usize,
@@ -247,6 +264,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
 
             sym::copy | sym::copy_nonoverlapping => (
                 1,
+                0,
                 vec![
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Mut }),
@@ -256,6 +274,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             ),
             sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => (
                 1,
+                0,
                 vec![
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Mut }),
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
@@ -265,10 +284,11 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             ),
             sym::compare_bytes => {
                 let byte_ptr = Ty::new_imm_ptr(tcx, tcx.types.u8);
-                (0, vec![byte_ptr, byte_ptr, tcx.types.usize], tcx.types.i32)
+                (0, 0, vec![byte_ptr, byte_ptr, tcx.types.usize], tcx.types.i32)
             }
             sym::write_bytes | sym::volatile_set_memory => (
                 1,
+                0,
                 vec![
                     Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Mut }),
                     tcx.types.u8,
@@ -276,56 +296,56 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 ],
                 Ty::new_unit(tcx),
             ),
-            sym::sqrtf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::sqrtf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::powif32 => (0, vec![tcx.types.f32, tcx.types.i32], tcx.types.f32),
-            sym::powif64 => (0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64),
-            sym::sinf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::sinf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::cosf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::cosf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::powf32 => (0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::powf64 => (0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::expf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::expf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::exp2f32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::exp2f64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::logf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::logf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::log10f32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::log10f64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::log2f32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::log2f64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::fmaf32 => (0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::fmaf64 => (0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::fabsf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::fabsf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::minnumf32 => (0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::minnumf64 => (0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::maxnumf32 => (0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::maxnumf64 => (0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::copysignf32 => (0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::copysignf64 => (0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::floorf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::floorf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::ceilf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::ceilf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::truncf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::truncf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::rintf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::rintf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::nearbyintf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::nearbyintf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::roundf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::roundf64 => (0, vec![tcx.types.f64], tcx.types.f64),
-            sym::roundevenf32 => (0, vec![tcx.types.f32], tcx.types.f32),
-            sym::roundevenf64 => (0, vec![tcx.types.f64], tcx.types.f64),
+            sym::sqrtf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::sqrtf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::powif32 => (0, 0, vec![tcx.types.f32, tcx.types.i32], tcx.types.f32),
+            sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64),
+            sym::sinf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::sinf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::cosf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::cosf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::powf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+            sym::powf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+            sym::expf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::expf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::exp2f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::exp2f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::logf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::logf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::log10f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::log10f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::log2f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::log2f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::fmaf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32),
+            sym::fmaf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64),
+            sym::fabsf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::fabsf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::minnumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+            sym::minnumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+            sym::maxnumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+            sym::maxnumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+            sym::copysignf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+            sym::copysignf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+            sym::floorf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::floorf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::ceilf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::ceilf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::truncf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::truncf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::rintf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::rintf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::nearbyintf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::nearbyintf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::roundf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::roundf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+            sym::roundevenf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+            sym::roundevenf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
 
             sym::volatile_load | sym::unaligned_volatile_load => {
-                (1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0))
+                (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0))
             }
             sym::volatile_store | sym::unaligned_volatile_store => {
-                (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
+                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
             }
 
             sym::ctpop
@@ -334,62 +354,66 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             | sym::cttz
             | sym::cttz_nonzero
             | sym::bswap
-            | sym::bitreverse => (1, vec![param(0)], param(0)),
+            | sym::bitreverse => (1, 0, vec![param(0)], param(0)),
 
             sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-                (1, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
+                (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
             }
 
             sym::ptr_guaranteed_cmp => (
                 1,
+                0,
                 vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
                 tcx.types.u8,
             ),
 
             sym::const_allocate => {
-                (0, vec![tcx.types.usize, tcx.types.usize], Ty::new_mut_ptr(tcx, tcx.types.u8))
+                (0, 1, vec![tcx.types.usize, tcx.types.usize], Ty::new_mut_ptr(tcx, tcx.types.u8))
             }
             sym::const_deallocate => (
                 0,
+                1,
                 vec![Ty::new_mut_ptr(tcx, tcx.types.u8), tcx.types.usize, tcx.types.usize],
                 Ty::new_unit(tcx),
             ),
 
             sym::ptr_offset_from => (
                 1,
+                0,
                 vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
                 tcx.types.isize,
             ),
             sym::ptr_offset_from_unsigned => (
                 1,
+                0,
                 vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
                 tcx.types.usize,
             ),
             sym::unchecked_div | sym::unchecked_rem | sym::exact_div => {
-                (1, vec![param(0), param(0)], param(0))
+                (1, 0, vec![param(0), param(0)], param(0))
             }
             sym::unchecked_shl | sym::unchecked_shr | sym::rotate_left | sym::rotate_right => {
-                (1, vec![param(0), param(0)], param(0))
+                (1, 0, vec![param(0), param(0)], param(0))
             }
             sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => {
-                (1, vec![param(0), param(0)], param(0))
+                (1, 0, vec![param(0), param(0)], param(0))
             }
             sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
-                (1, vec![param(0), param(0)], param(0))
+                (1, 0, vec![param(0), param(0)], param(0))
             }
-            sym::saturating_add | sym::saturating_sub => (1, vec![param(0), param(0)], param(0)),
+            sym::saturating_add | sym::saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
             sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
-                (1, vec![param(0), param(0)], param(0))
+                (1, 0, vec![param(0), param(0)], param(0))
             }
-            sym::float_to_int_unchecked => (2, vec![param(0)], param(1)),
+            sym::float_to_int_unchecked => (2, 0, vec![param(0)], param(1)),
 
-            sym::assume => (0, vec![tcx.types.bool], Ty::new_unit(tcx)),
-            sym::likely => (0, vec![tcx.types.bool], tcx.types.bool),
-            sym::unlikely => (0, vec![tcx.types.bool], tcx.types.bool),
+            sym::assume => (0, 0, vec![tcx.types.bool], Ty::new_unit(tcx)),
+            sym::likely => (0, 0, vec![tcx.types.bool], tcx.types.bool),
+            sym::unlikely => (0, 0, vec![tcx.types.bool], tcx.types.bool),
 
-            sym::read_via_copy => (1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
+            sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
             sym::write_via_move => {
-                (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
+                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
             }
 
             sym::discriminant_value => {
@@ -401,6 +425,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon };
                 (
                     1,
+                    0,
                     vec![Ty::new_imm_ref(
                         tcx,
                         ty::Region::new_bound(tcx, ty::INNERMOST, br),
@@ -428,67 +453,73 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 ));
                 (
                     0,
+                    0,
                     vec![Ty::new_fn_ptr(tcx, try_fn_ty), mut_u8, Ty::new_fn_ptr(tcx, catch_fn_ty)],
                     tcx.types.i32,
                 )
             }
 
             sym::va_start | sym::va_end => match mk_va_list_ty(hir::Mutability::Mut) {
-                Some((va_list_ref_ty, _)) => (0, vec![va_list_ref_ty], Ty::new_unit(tcx)),
+                Some((va_list_ref_ty, _)) => (0, 0, vec![va_list_ref_ty], Ty::new_unit(tcx)),
                 None => bug!("`va_list` language item needed for C-variadic intrinsics"),
             },
 
             sym::va_copy => match mk_va_list_ty(hir::Mutability::Not) {
                 Some((va_list_ref_ty, va_list_ty)) => {
                     let va_list_ptr_ty = Ty::new_mut_ptr(tcx, va_list_ty);
-                    (0, vec![va_list_ptr_ty, va_list_ref_ty], Ty::new_unit(tcx))
+                    (0, 0, vec![va_list_ptr_ty, va_list_ref_ty], Ty::new_unit(tcx))
                 }
                 None => bug!("`va_list` language item needed for C-variadic intrinsics"),
             },
 
             sym::va_arg => match mk_va_list_ty(hir::Mutability::Mut) {
-                Some((va_list_ref_ty, _)) => (1, vec![va_list_ref_ty], param(0)),
+                Some((va_list_ref_ty, _)) => (1, 0, vec![va_list_ref_ty], param(0)),
                 None => bug!("`va_list` language item needed for C-variadic intrinsics"),
             },
 
             sym::nontemporal_store => {
-                (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
+                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx))
             }
 
             sym::raw_eq => {
                 let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon };
                 let param_ty =
                     Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0));
-                (1, vec![param_ty; 2], tcx.types.bool)
+                (1, 0, vec![param_ty; 2], tcx.types.bool)
             }
 
-            sym::black_box => (1, vec![param(0)], param(0)),
+            sym::black_box => (1, 0, vec![param(0)], param(0)),
 
-            sym::is_val_statically_known => (1, vec![param(0)], tcx.types.bool),
+            sym::is_val_statically_known => (1, 1, vec![param(0)], tcx.types.bool),
 
-            sym::const_eval_select => (4, vec![param(0), param(1), param(2)], param(3)),
+            sym::const_eval_select => (4, 0, vec![param(0), param(1), param(2)], param(3)),
 
             sym::vtable_size | sym::vtable_align => {
-                (0, vec![Ty::new_imm_ptr(tcx, Ty::new_unit(tcx))], tcx.types.usize)
+                (0, 0, vec![Ty::new_imm_ptr(tcx, Ty::new_unit(tcx))], tcx.types.usize)
             }
 
-            sym::debug_assertions => (0, Vec::new(), tcx.types.bool),
+            sym::debug_assertions => (0, 1, Vec::new(), tcx.types.bool),
 
             other => {
-                tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span: it.span, name: other });
+                tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span, name: other });
                 return;
             }
         };
-        (n_tps, 0, inputs, output, unsafety)
+        (n_tps, 0, n_cts, inputs, output, unsafety)
     };
-    let sig = tcx.mk_fn_sig(inputs, output, false, unsafety, Abi::RustIntrinsic);
+    let sig = tcx.mk_fn_sig(inputs, output, false, unsafety, abi);
     let sig = ty::Binder::bind_with_vars(sig, bound_vars);
-    equate_intrinsic_type(tcx, it, n_tps, n_lts, 0, sig)
+    equate_intrinsic_type(tcx, span, intrinsic_id, n_tps, n_lts, n_cts, sig)
 }
 
 /// Type-check `extern "platform-intrinsic" { ... }` functions.
-pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
-    let generics = tcx.generics_of(it.owner_id);
+pub fn check_platform_intrinsic_type(
+    tcx: TyCtxt<'_>,
+    intrinsic_id: LocalDefId,
+    span: Span,
+    name: Symbol,
+) {
+    let generics = tcx.generics_of(intrinsic_id);
     let param = |n| {
         if let Some(&ty::GenericParamDef {
             name, kind: ty::GenericParamDefKind::Type { .. }, ..
@@ -496,12 +527,10 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         {
             Ty::new_param(tcx, n, name)
         } else {
-            Ty::new_error_with_message(tcx, tcx.def_span(it.owner_id), "expected param")
+            Ty::new_error_with_message(tcx, span, "expected param")
         }
     };
 
-    let name = it.ident.name;
-
     let (n_tps, n_cts, inputs, output) = match name {
         sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
             (2, 0, vec![param(0), param(0)], param(1))
@@ -574,12 +603,12 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         sym::simd_shuffle_generic => (2, 1, vec![param(0), param(0)], param(1)),
         _ => {
             let msg = format!("unrecognized platform-specific intrinsic function: `{name}`");
-            tcx.dcx().span_err(it.span, msg);
+            tcx.dcx().span_err(span, msg);
             return;
         }
     };
 
     let sig = tcx.mk_fn_sig(inputs, output, false, hir::Unsafety::Unsafe, Abi::PlatformIntrinsic);
     let sig = ty::Binder::dummy(sig);
-    equate_intrinsic_type(tcx, it, n_tps, 0, n_cts, sig)
+    equate_intrinsic_type(tcx, span, intrinsic_id, n_tps, 0, n_cts, sig)
 }
diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs
index 8884c218d05..b9052672a26 100644
--- a/compiler/rustc_hir_analysis/src/check/mod.rs
+++ b/compiler/rustc_hir_analysis/src/check/mod.rs
@@ -74,7 +74,7 @@ pub mod wfcheck;
 
 pub use check::check_abi;
 
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_errors::ErrorGuaranteed;
@@ -270,7 +270,7 @@ fn default_body_is_unstable(
     item_did: DefId,
     feature: Symbol,
     reason: Option<Symbol>,
-    issue: Option<NonZeroU32>,
+    issue: Option<NonZero<u32>>,
 ) {
     let missing_item_name = tcx.associated_item(item_did).name;
     let (mut some_note, mut none_note, mut reason_str) = (false, false, String::new());
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index 41420b9caec..43f0af5bd1d 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -1651,7 +1651,7 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
     abi: abi::Abi,
 ) -> ty::PolyFnSig<'tcx> {
     let unsafety = if abi == abi::Abi::RustIntrinsic {
-        intrinsic_operation_unsafety(tcx, def_id.to_def_id())
+        intrinsic_operation_unsafety(tcx, def_id)
     } else {
         hir::Unsafety::Unsafe
     };
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index 33092825e89..e53f922ad10 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -63,6 +63,7 @@ This API is completely unstable and subject to change.
 #![feature(rustdoc_internals)]
 #![allow(internal_features)]
 #![feature(control_flow_enum)]
+#![feature(generic_nonzero)]
 #![feature(if_let_guard)]
 #![feature(is_sorted)]
 #![feature(iter_intersperse)]
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index c4271c66e1c..27614634c6b 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -540,8 +540,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         if let Some(def_id) = def_id
             && self.tcx.def_kind(def_id) == hir::def::DefKind::Fn
-            && self.tcx.is_intrinsic(def_id)
-            && self.tcx.item_name(def_id) == sym::const_eval_select
+            && matches!(self.tcx.intrinsic(def_id), Some(sym::const_eval_select))
         {
             let fn_sig = self.resolve_vars_if_possible(fn_sig);
             for idx in 0..=1 {
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index e6319747e78..6bab8f75d24 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -867,7 +867,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
                 let a_sig = a.fn_sig(self.tcx);
                 if let ty::FnDef(def_id, _) = *a.kind() {
                     // Intrinsics are not coercible to function pointers
-                    if self.tcx.is_intrinsic(def_id) {
+                    if self.tcx.intrinsic(def_id).is_some() {
                         return Err(TypeError::IntrinsicCast);
                     }
 
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index 729ce1f00cd..f39b496154b 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -109,6 +109,93 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.autoderef(span, ty).any(|(ty, _)| matches!(ty.kind(), ty::Slice(..) | ty::Array(..)))
     }
 
+    fn impl_into_iterator_should_be_iterator(
+        &self,
+        ty: Ty<'tcx>,
+        span: Span,
+        unsatisfied_predicates: &Vec<(
+            ty::Predicate<'_>,
+            Option<ty::Predicate<'_>>,
+            Option<ObligationCause<'_>>,
+        )>,
+    ) -> bool {
+        fn predicate_bounds_generic_param<'tcx>(
+            predicate: ty::Predicate<'_>,
+            generics: &'tcx ty::Generics,
+            generic_param: &ty::GenericParamDef,
+            tcx: TyCtxt<'tcx>,
+        ) -> bool {
+            if let ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_pred)) =
+                predicate.kind().as_ref().skip_binder()
+            {
+                let ty::TraitPredicate { trait_ref: ty::TraitRef { args, .. }, .. } = trait_pred;
+                if args.is_empty() {
+                    return false;
+                }
+                let Some(arg_ty) = args[0].as_type() else {
+                    return false;
+                };
+                let ty::Param(param) = arg_ty.kind() else {
+                    return false;
+                };
+                // Is `generic_param` the same as the arg for this trait predicate?
+                generic_param.index == generics.type_param(&param, tcx).index
+            } else {
+                false
+            }
+        }
+
+        fn is_iterator_predicate(predicate: ty::Predicate<'_>, tcx: TyCtxt<'_>) -> bool {
+            if let ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_pred)) =
+                predicate.kind().as_ref().skip_binder()
+            {
+                tcx.is_diagnostic_item(sym::Iterator, trait_pred.trait_ref.def_id)
+            } else {
+                false
+            }
+        }
+
+        // Does the `ty` implement `IntoIterator`?
+        let Some(into_iterator_trait) = self.tcx.get_diagnostic_item(sym::IntoIterator) else {
+            return false;
+        };
+        let trait_ref = ty::TraitRef::new(self.tcx, into_iterator_trait, [ty]);
+        let cause = ObligationCause::new(span, self.body_id, ObligationCauseCode::MiscObligation);
+        let obligation = Obligation::new(self.tcx, cause, self.param_env, trait_ref);
+        if !self.predicate_must_hold_modulo_regions(&obligation) {
+            return false;
+        }
+
+        match ty.kind() {
+            ty::Param(param) => {
+                let generics = self.tcx.generics_of(self.body_id);
+                let generic_param = generics.type_param(&param, self.tcx);
+                for unsatisfied in unsatisfied_predicates.iter() {
+                    // The parameter implements `IntoIterator`
+                    // but it has called a method that requires it to implement `Iterator`
+                    if predicate_bounds_generic_param(
+                        unsatisfied.0,
+                        generics,
+                        generic_param,
+                        self.tcx,
+                    ) && is_iterator_predicate(unsatisfied.0, self.tcx)
+                    {
+                        return true;
+                    }
+                }
+            }
+            ty::Alias(ty::AliasKind::Opaque, _) => {
+                for unsatisfied in unsatisfied_predicates.iter() {
+                    if is_iterator_predicate(unsatisfied.0, self.tcx) {
+                        return true;
+                    }
+                }
+            }
+            _ => return false,
+        }
+        false
+    }
+
     #[instrument(level = "debug", skip(self))]
     pub fn report_method_error(
         &self,
@@ -555,6 +642,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     "`count` is defined on `{iterator_trait}`, which `{rcvr_ty}` does not implement"
                 ));
             }
+        } else if self.impl_into_iterator_should_be_iterator(rcvr_ty, span, unsatisfied_predicates)
+        {
+            err.span_label(span, format!("`{rcvr_ty}` is not an iterator"));
+            err.multipart_suggestion_verbose(
+                "call `.into_iter()` first",
+                vec![(span.shrink_to_lo(), format!("into_iter()."))],
+                Applicability::MaybeIncorrect,
+            );
+            return Some(err);
         } else if !unsatisfied_predicates.is_empty() && matches!(rcvr_ty.kind(), ty::Param(_)) {
             // We special case the situation where we are looking for `_` in
             // `<TypeParam as _>::method` because otherwise the machinery will look for blanket
diff --git a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
index c6f6c32fe60..f49369c5a23 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
@@ -316,7 +316,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
                 if !self.same_type_modulo_infer(*found_sig, *expected_sig)
                     || !sig.is_suggestable(self.tcx, true)
-                    || self.tcx.is_intrinsic(*did)
+                    || self.tcx.intrinsic(*did).is_some()
                 {
                     return;
                 }
@@ -348,8 +348,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 if !self.same_type_modulo_infer(*found_sig, *expected_sig)
                     || !found_sig.is_suggestable(self.tcx, true)
                     || !expected_sig.is_suggestable(self.tcx, true)
-                    || self.tcx.is_intrinsic(*did1)
-                    || self.tcx.is_intrinsic(*did2)
+                    || self.tcx.intrinsic(*did1).is_some()
+                    || self.tcx.intrinsic(*did2).is_some()
                 {
                     return;
                 }
diff --git a/compiler/rustc_interface/src/lib.rs b/compiler/rustc_interface/src/lib.rs
index 7d69e49b209..24c2e290534 100644
--- a/compiler/rustc_interface/src/lib.rs
+++ b/compiler/rustc_interface/src/lib.rs
@@ -1,5 +1,6 @@
 #![feature(decl_macro)]
 #![feature(error_iter)]
+#![feature(generic_nonzero)]
 #![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(thread_spawn_unchecked)]
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index bfc4fc07d4c..112553b2f70 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -20,7 +20,7 @@ use rustc_span::{FileName, SourceFileHashAlgorithm};
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel};
 use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel};
 use std::collections::{BTreeMap, BTreeSet};
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -827,7 +827,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(tls_model, Some(TlsModel::GeneralDynamic));
     tracked!(translate_remapped_path_to_local_path, false);
     tracked!(trap_unreachable, Some(false));
-    tracked!(treat_err_as_bug, NonZeroUsize::new(1));
+    tracked!(treat_err_as_bug, NonZero::new(1));
     tracked!(tune_cpu, Some(String::from("abc")));
     tracked!(uninit_const_chunk_threshold, 123);
     tracked!(unleash_the_miri_inside_of_you, true);
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 76b9e8de75f..087c43075f1 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -107,7 +107,7 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
     use rustc_query_impl::QueryCtxt;
     use rustc_query_system::query::{deadlock, QueryContext};
 
-    let registry = sync::Registry::new(std::num::NonZeroUsize::new(threads).unwrap());
+    let registry = sync::Registry::new(std::num::NonZero::new(threads).unwrap());
 
     if !sync::is_dyn_thread_safe() {
         return run_in_thread_with_globals(edition, || {
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 6ee1d1ca924..faa35f51cd4 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -1227,7 +1227,7 @@ impl<'tcx> LateLintPass<'tcx> for MutableTransmutes {
         }
 
         fn def_id_is_transmute(cx: &LateContext<'_>, def_id: DefId) -> bool {
-            cx.tcx.is_intrinsic(def_id) && cx.tcx.item_name(def_id) == sym::transmute
+            matches!(cx.tcx.intrinsic(def_id), Some(sym::transmute))
         }
     }
 }
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index 5f769e9ad8a..85f9d3bd63e 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -31,6 +31,7 @@
 #![feature(array_windows)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
+#![feature(generic_nonzero)]
 #![feature(if_let_guard)]
 #![feature(iter_order_by)]
 #![feature(let_chains)]
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 7445e2e80b4..da59ffebdc5 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -1,7 +1,6 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
-
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 use crate::errors::RequestedLevel;
 use crate::fluent_generated as fluent;
@@ -402,7 +401,7 @@ pub struct BuiltinIncompleteFeaturesHelp;
 #[derive(Subdiagnostic)]
 #[note(lint_note)]
 pub struct BuiltinFeatureIssueNote {
-    pub n: NonZeroU32,
+    pub n: NonZero<u32>,
 }
 
 pub struct BuiltinUnpermittedTypeInit<'a> {
diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs
index 2e7130f3565..70ad8598957 100644
--- a/compiler/rustc_metadata/src/lib.rs
+++ b/compiler/rustc_metadata/src/lib.rs
@@ -5,6 +5,7 @@
 #![feature(decl_macro)]
 #![feature(extract_if)]
 #![feature(coroutines)]
+#![feature(generic_nonzero)]
 #![feature(iter_from_coroutine)]
 #![feature(let_chains)]
 #![feature(if_let_guard)]
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index 72e9744295b..2cfbaff35ef 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -327,7 +327,7 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
     }
 
     #[inline]
-    fn read_lazy_offset_then<T>(&mut self, f: impl Fn(NonZeroUsize) -> T) -> T {
+    fn read_lazy_offset_then<T>(&mut self, f: impl Fn(NonZero<usize>) -> T) -> T {
         let distance = self.read_usize();
         let position = match self.lazy_state {
             LazyState::NoNode => bug!("read_lazy_with_meta: outside of a metadata node"),
@@ -338,7 +338,7 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
             }
             LazyState::Previous(last_pos) => last_pos.get() + distance,
         };
-        let position = NonZeroUsize::new(position).unwrap();
+        let position = NonZero::new(position).unwrap();
         self.lazy_state = LazyState::Previous(position);
         f(position)
     }
@@ -685,15 +685,15 @@ impl MetadataBlob {
     }
 
     pub(crate) fn get_rustc_version(&self) -> String {
-        LazyValue::<String>::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 8).unwrap())
+        LazyValue::<String>::from_position(NonZero::new(METADATA_HEADER.len() + 8).unwrap())
             .decode(self)
     }
 
-    fn root_pos(&self) -> NonZeroUsize {
+    fn root_pos(&self) -> NonZero<usize> {
         let offset = METADATA_HEADER.len();
         let pos_bytes = self.blob()[offset..][..8].try_into().unwrap();
         let pos = u64::from_le_bytes(pos_bytes);
-        NonZeroUsize::new(pos as usize).unwrap()
+        NonZero::new(pos as usize).unwrap()
     }
 
     pub(crate) fn get_header(&self) -> CrateHeader {
@@ -1749,8 +1749,8 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.root.tables.attr_flags.get(self, index)
     }
 
-    fn get_is_intrinsic(self, index: DefIndex) -> bool {
-        self.root.tables.is_intrinsic.get(self, index)
+    fn get_intrinsic(self, index: DefIndex) -> Option<Symbol> {
+        self.root.tables.intrinsic.get(self, index).map(|d| d.decode(self))
     }
 
     fn get_doc_link_resolutions(self, index: DefIndex) -> DocLinkResMap {
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 178bfc3a380..9df28799f4f 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -356,7 +356,7 @@ provide! { tcx, def_id, other, cdata,
         cdata.get_stability_implications(tcx).iter().copied().collect()
     }
     stripped_cfg_items => { cdata.get_stripped_cfg_items(cdata.cnum, tcx) }
-    is_intrinsic => { cdata.get_is_intrinsic(def_id.index) }
+    intrinsic => { cdata.get_intrinsic(def_id.index) }
     defined_lang_items => { cdata.get_lang_items(tcx) }
     diagnostic_items => { cdata.get_diagnostic_items() }
     missing_lang_items => { cdata.get_missing_lang_items(tcx) }
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 6f908f7752a..fdb2b4f2024 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -421,7 +421,7 @@ macro_rules! record_defaulted_array {
 }
 
 impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
-    fn emit_lazy_distance(&mut self, position: NonZeroUsize) {
+    fn emit_lazy_distance(&mut self, position: NonZero<usize>) {
         let pos = position.get();
         let distance = match self.lazy_state {
             LazyState::NoNode => bug!("emit_lazy_distance: outside of a metadata node"),
@@ -439,7 +439,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 position.get() - last_pos.get()
             }
         };
-        self.lazy_state = LazyState::Previous(NonZeroUsize::new(pos).unwrap());
+        self.lazy_state = LazyState::Previous(NonZero::new(pos).unwrap());
         self.emit_usize(distance);
     }
 
@@ -447,7 +447,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     where
         T::Value<'tcx>: Encodable<EncodeContext<'a, 'tcx>>,
     {
-        let pos = NonZeroUsize::new(self.position()).unwrap();
+        let pos = NonZero::new(self.position()).unwrap();
 
         assert_eq!(self.lazy_state, LazyState::NoNode);
         self.lazy_state = LazyState::NodeStart(pos);
@@ -466,7 +466,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     where
         T::Value<'tcx>: Encodable<EncodeContext<'a, 'tcx>>,
     {
-        let pos = NonZeroUsize::new(self.position()).unwrap();
+        let pos = NonZero::new(self.position()).unwrap();
 
         assert_eq!(self.lazy_state, LazyState::NoNode);
         self.lazy_state = LazyState::NodeStart(pos);
@@ -1409,7 +1409,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             if let DefKind::Fn | DefKind::AssocFn = def_kind {
                 self.tables.asyncness.set_some(def_id.index, tcx.asyncness(def_id));
                 record_array!(self.tables.fn_arg_names[def_id] <- tcx.fn_arg_names(def_id));
-                self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
+                if let Some(name) = tcx.intrinsic(def_id) {
+                    record!(self.tables.intrinsic[def_id] <- name);
+                }
             }
             if let DefKind::TyParam = def_kind {
                 let default = self.tcx.object_lifetime_default(def_id);
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index eac78a3cd7c..9c0e8029571 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -37,7 +37,7 @@ use rustc_target::abi::{FieldIdx, VariantIdx};
 use rustc_target::spec::{PanicStrategy, TargetTriple};
 
 use std::marker::PhantomData;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 
 use decoder::DecodeContext;
 pub(crate) use decoder::{CrateMetadata, CrateNumMap, MetadataBlob};
@@ -83,7 +83,7 @@ pub const METADATA_HEADER: &[u8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_V
 /// order than they were encoded in.
 #[must_use]
 struct LazyValue<T> {
-    position: NonZeroUsize,
+    position: NonZero<usize>,
     _marker: PhantomData<fn() -> T>,
 }
 
@@ -92,7 +92,7 @@ impl<T: ParameterizedOverTcx> ParameterizedOverTcx for LazyValue<T> {
 }
 
 impl<T> LazyValue<T> {
-    fn from_position(position: NonZeroUsize) -> LazyValue<T> {
+    fn from_position(position: NonZero<usize>) -> LazyValue<T> {
         LazyValue { position, _marker: PhantomData }
     }
 }
@@ -108,7 +108,7 @@ impl<T> LazyValue<T> {
 /// the minimal distance the length of the sequence, i.e.
 /// it's assumed there's no 0-byte element in the sequence.
 struct LazyArray<T> {
-    position: NonZeroUsize,
+    position: NonZero<usize>,
     num_elems: usize,
     _marker: PhantomData<fn() -> T>,
 }
@@ -119,12 +119,12 @@ impl<T: ParameterizedOverTcx> ParameterizedOverTcx for LazyArray<T> {
 
 impl<T> Default for LazyArray<T> {
     fn default() -> LazyArray<T> {
-        LazyArray::from_position_and_num_elems(NonZeroUsize::new(1).unwrap(), 0)
+        LazyArray::from_position_and_num_elems(NonZero::new(1).unwrap(), 0)
     }
 }
 
 impl<T> LazyArray<T> {
-    fn from_position_and_num_elems(position: NonZeroUsize, num_elems: usize) -> LazyArray<T> {
+    fn from_position_and_num_elems(position: NonZero<usize>, num_elems: usize) -> LazyArray<T> {
         LazyArray { position, num_elems, _marker: PhantomData }
     }
 }
@@ -135,7 +135,7 @@ impl<T> LazyArray<T> {
 /// `LazyArray<T>`, but without requiring encoding or decoding all the values
 /// eagerly and in-order.
 struct LazyTable<I, T> {
-    position: NonZeroUsize,
+    position: NonZero<usize>,
     /// The encoded size of the elements of a table is selected at runtime to drop
     /// trailing zeroes. This is the number of bytes used for each table element.
     width: usize,
@@ -150,7 +150,7 @@ impl<I: 'static, T: ParameterizedOverTcx> ParameterizedOverTcx for LazyTable<I,
 
 impl<I, T> LazyTable<I, T> {
     fn from_position_and_encoded_size(
-        position: NonZeroUsize,
+        position: NonZero<usize>,
         width: usize,
         len: usize,
     ) -> LazyTable<I, T> {
@@ -187,11 +187,11 @@ enum LazyState {
 
     /// Inside a metadata node, and before any `Lazy`s.
     /// The position is that of the node itself.
-    NodeStart(NonZeroUsize),
+    NodeStart(NonZero<usize>),
 
     /// Inside a metadata node, with a previous `Lazy`s.
     /// The position is where that previous `Lazy` would start.
-    Previous(NonZeroUsize),
+    Previous(NonZero<usize>),
 }
 
 type SyntaxContextTable = LazyTable<u32, Option<LazyValue<SyntaxContextData>>>;
@@ -375,7 +375,7 @@ macro_rules! define_tables {
 
 define_tables! {
 - defaulted:
-    is_intrinsic: Table<DefIndex, bool>,
+    intrinsic: Table<DefIndex, Option<LazyValue<Symbol>>>,
     is_macro_rules: Table<DefIndex, bool>,
     is_type_alias_impl_trait: Table<DefIndex, bool>,
     type_alias_is_lazy: Table<DefIndex, bool>,
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
index 306bf07a976..c5f281964df 100644
--- a/compiler/rustc_metadata/src/rmeta/table.rs
+++ b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -339,7 +339,7 @@ impl<T> FixedSizeEncoding for Option<LazyValue<T>> {
 
     #[inline]
     fn from_bytes(b: &[u8; 8]) -> Self {
-        let position = NonZeroUsize::new(u64::from_bytes(b) as usize)?;
+        let position = NonZero::new(u64::from_bytes(b) as usize)?;
         Some(LazyValue::from_position(position))
     }
 
@@ -366,7 +366,7 @@ impl<T> LazyArray<T> {
     }
 
     fn from_bytes_impl(position: &[u8; 8], meta: &[u8; 8]) -> Option<LazyArray<T>> {
-        let position = NonZeroUsize::new(u64::from_bytes(position) as usize)?;
+        let position = NonZero::new(u64::from_bytes(position) as usize)?;
         let len = u64::from_bytes(meta) as usize;
         Some(LazyArray::from_position_and_num_elems(position, len))
     }
@@ -497,7 +497,7 @@ impl<I: Idx, const N: usize, T: FixedSizeEncoding<ByteArray = [u8; N]>> TableBui
         }
 
         LazyTable::from_position_and_encoded_size(
-            NonZeroUsize::new(pos).unwrap(),
+            NonZero::new(pos).unwrap(),
             width,
             self.blocks.len(),
         )
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 2aaece1060a..9c0846e9fb1 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -34,6 +34,7 @@
 #![feature(discriminant_kind)]
 #![feature(exhaustive_patterns)]
 #![feature(coroutines)]
+#![feature(generic_nonzero)]
 #![feature(if_let_guard)]
 #![feature(inline_const)]
 #![feature(iter_from_coroutine)]
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index afb6937b43b..2f624ab0527 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -21,7 +21,7 @@ use rustc_session::parse::feature_err_issue;
 use rustc_session::Session;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 #[derive(PartialEq, Clone, Copy, Debug)]
 pub enum StabilityLevel {
@@ -102,7 +102,7 @@ pub fn report_unstable(
     sess: &Session,
     feature: Symbol,
     reason: Option<Symbol>,
-    issue: Option<NonZeroU32>,
+    issue: Option<NonZero<u32>>,
     suggestion: Option<(Span, String, String, Applicability)>,
     is_soft: bool,
     span: Span,
@@ -235,7 +235,7 @@ pub enum EvalResult {
     Deny {
         feature: Symbol,
         reason: Option<Symbol>,
-        issue: Option<NonZeroU32>,
+        issue: Option<NonZero<u32>>,
         suggestion: Option<(Span, String, String, Applicability)>,
         is_soft: bool,
     },
@@ -433,7 +433,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 // the `-Z force-unstable-if-unmarked` flag present (we're
                 // compiling a compiler crate), then let this missing feature
                 // annotation slide.
-                if feature == sym::rustc_private && issue == NonZeroU32::new(27812) {
+                if feature == sym::rustc_private && issue == NonZero::new(27812) {
                     if self.sess.opts.unstable_opts.force_unstable_if_unmarked {
                         return EvalResult::Allow;
                     }
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index 9d4ec7d25bb..125fac48df8 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -12,6 +12,7 @@ use rustc_macros::HashStable;
 use rustc_session::CtfeBacktrace;
 use rustc_span::{def_id::DefId, Span, DUMMY_SP};
 use rustc_target::abi::{call, Align, Size, VariantIdx, WrappingRange};
+use rustc_type_ir::Mutability;
 
 use std::borrow::Cow;
 use std::{any::Any, backtrace::Backtrace, fmt};
@@ -367,7 +368,7 @@ pub enum UndefinedBehaviorInfo<'tcx> {
 
 #[derive(Debug, Clone, Copy)]
 pub enum PointerKind {
-    Ref,
+    Ref(Mutability),
     Box,
 }
 
@@ -375,7 +376,7 @@ impl IntoDiagnosticArg for PointerKind {
     fn into_diagnostic_arg(self) -> DiagnosticArgValue {
         DiagnosticArgValue::Str(
             match self {
-                Self::Ref => "ref",
+                Self::Ref(_) => "ref",
                 Self::Box => "box",
             }
             .into(),
@@ -408,7 +409,7 @@ impl From<PointerKind> for ExpectedKind {
     fn from(x: PointerKind) -> ExpectedKind {
         match x {
             PointerKind::Box => ExpectedKind::Box,
-            PointerKind::Ref => ExpectedKind::Reference,
+            PointerKind::Ref(_) => ExpectedKind::Reference,
         }
     }
 }
@@ -419,7 +420,7 @@ pub enum ValidationErrorKind<'tcx> {
     PartialPointer,
     PtrToUninhabited { ptr_kind: PointerKind, ty: Ty<'tcx> },
     PtrToStatic { ptr_kind: PointerKind },
-    MutableRefInConst,
+    MutableRefInConstOrStatic,
     ConstRefToMutable,
     ConstRefToExtern,
     MutableRefToImmutable,
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index ec2af393639..5be09b06d9e 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -122,7 +122,7 @@ mod value;
 use std::fmt;
 use std::io;
 use std::io::{Read, Write};
-use std::num::{NonZeroU32, NonZeroU64};
+use std::num::NonZero;
 use std::sync::atomic::{AtomicU32, Ordering};
 
 use rustc_ast::LitKind;
@@ -206,7 +206,7 @@ pub enum LitToConstError {
 }
 
 #[derive(Copy, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
-pub struct AllocId(pub NonZeroU64);
+pub struct AllocId(pub NonZero<u64>);
 
 // We want the `Debug` output to be readable as it is used by `derive(Debug)` for
 // all the Miri types.
@@ -261,7 +261,7 @@ pub fn specialized_encode_alloc_id<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>>(
 }
 
 // Used to avoid infinite recursion when decoding cyclic allocations.
-type DecodingSessionId = NonZeroU32;
+type DecodingSessionId = NonZero<u32>;
 
 #[derive(Clone)]
 enum State {
@@ -501,7 +501,7 @@ impl<'tcx> AllocMap<'tcx> {
         AllocMap {
             alloc_map: Default::default(),
             dedup: Default::default(),
-            next_id: AllocId(NonZeroU64::new(1).unwrap()),
+            next_id: AllocId(NonZero::new(1).unwrap()),
         }
     }
     fn reserve(&mut self) -> AllocId {
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index dabf6297aa9..e2767ee2989 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -3,7 +3,7 @@ use super::{AllocId, InterpResult};
 use rustc_macros::HashStable;
 use rustc_target::abi::{HasDataLayout, Size};
 
-use std::{fmt, num::NonZeroU64};
+use std::{fmt, num::NonZero};
 
 ////////////////////////////////////////////////////////////////////////////////
 // Pointer arithmetic
@@ -129,7 +129,7 @@ pub trait Provenance: Copy + fmt::Debug + 'static {
 /// The type of provenance in the compile-time interpreter.
 /// This is a packed representation of an `AllocId` and an `immutable: bool`.
 #[derive(Copy, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
-pub struct CtfeProvenance(NonZeroU64);
+pub struct CtfeProvenance(NonZero<u64>);
 
 impl From<AllocId> for CtfeProvenance {
     fn from(value: AllocId) -> Self {
@@ -155,7 +155,7 @@ impl CtfeProvenance {
     /// Returns the `AllocId` of this provenance.
     #[inline(always)]
     pub fn alloc_id(self) -> AllocId {
-        AllocId(NonZeroU64::new(self.0.get() & !IMMUTABLE_MASK).unwrap())
+        AllocId(NonZero::new(self.0.get() & !IMMUTABLE_MASK).unwrap())
     }
 
     /// Returns whether this provenance is immutable.
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index 7ac7fa0ac33..2cdcdcb1492 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -241,6 +241,7 @@ trivial! {
     Option<rustc_target::abi::FieldIdx>,
     Option<rustc_target::spec::PanicStrategy>,
     Option<usize>,
+    Option<rustc_span::Symbol>,
     Result<(), rustc_errors::ErrorGuaranteed>,
     Result<(), rustc_middle::traits::query::NoSolution>,
     Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index a7f4e75e214..5be45c33e11 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -1760,8 +1760,8 @@ rustc_queries! {
         separate_provide_extern
     }
     /// Whether the function is an intrinsic
-    query is_intrinsic(def_id: DefId) -> bool {
-        desc { |tcx| "checking whether `{}` is an intrinsic", tcx.def_path_str(def_id) }
+    query intrinsic(def_id: DefId) -> Option<Symbol> {
+        desc { |tcx| "fetch intrinsic name if `{}` is an intrinsic", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
     /// Returns the lang items defined in another crate by loading it from metadata.
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index 515d564e81d..5d50510338c 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -4,7 +4,7 @@ use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use rustc_target::abi::Size;
 use std::fmt;
-use std::num::NonZeroU8;
+use std::num::NonZero;
 
 use crate::ty::TyCtxt;
 
@@ -132,7 +132,7 @@ pub struct ScalarInt {
     /// The first `size` bytes of `data` are the value.
     /// Do not try to read less or more bytes than that. The remaining bytes must be 0.
     data: u128,
-    size: NonZeroU8,
+    size: NonZero<u8>,
 }
 
 // Cannot derive these, as the derives take references to the fields, and we
@@ -161,14 +161,14 @@ impl<D: Decoder> Decodable<D> for ScalarInt {
         let mut data = [0u8; 16];
         let size = d.read_u8();
         data[..size as usize].copy_from_slice(d.read_raw_bytes(size as usize));
-        ScalarInt { data: u128::from_le_bytes(data), size: NonZeroU8::new(size).unwrap() }
+        ScalarInt { data: u128::from_le_bytes(data), size: NonZero::new(size).unwrap() }
     }
 }
 
 impl ScalarInt {
-    pub const TRUE: ScalarInt = ScalarInt { data: 1_u128, size: NonZeroU8::new(1).unwrap() };
+    pub const TRUE: ScalarInt = ScalarInt { data: 1_u128, size: NonZero::new(1).unwrap() };
 
-    pub const FALSE: ScalarInt = ScalarInt { data: 0_u128, size: NonZeroU8::new(1).unwrap() };
+    pub const FALSE: ScalarInt = ScalarInt { data: 0_u128, size: NonZero::new(1).unwrap() };
 
     #[inline]
     pub fn size(self) -> Size {
@@ -196,7 +196,7 @@ impl ScalarInt {
 
     #[inline]
     pub fn null(size: Size) -> Self {
-        Self { data: 0, size: NonZeroU8::new(size.bytes() as u8).unwrap() }
+        Self { data: 0, size: NonZero::new(size.bytes() as u8).unwrap() }
     }
 
     #[inline]
@@ -208,7 +208,7 @@ impl ScalarInt {
     pub fn try_from_uint(i: impl Into<u128>, size: Size) -> Option<Self> {
         let data = i.into();
         if size.truncate(data) == data {
-            Some(Self { data, size: NonZeroU8::new(size.bytes() as u8).unwrap() })
+            Some(Self { data, size: NonZero::new(size.bytes() as u8).unwrap() })
         } else {
             None
         }
@@ -220,7 +220,7 @@ impl ScalarInt {
         // `into` performed sign extension, we have to truncate
         let truncated = size.truncate(i as u128);
         if size.sign_extend(truncated) as i128 == i {
-            Some(Self { data: truncated, size: NonZeroU8::new(size.bytes() as u8).unwrap() })
+            Some(Self { data: truncated, size: NonZero::new(size.bytes() as u8).unwrap() })
         } else {
             None
         }
@@ -388,7 +388,7 @@ macro_rules! from {
                 fn from(u: $ty) -> Self {
                     Self {
                         data: u128::from(u),
-                        size: NonZeroU8::new(std::mem::size_of::<$ty>() as u8).unwrap(),
+                        size: NonZero::new(std::mem::size_of::<$ty>() as u8).unwrap(),
                     }
                 }
             }
@@ -427,7 +427,7 @@ impl TryFrom<ScalarInt> for bool {
 impl From<char> for ScalarInt {
     #[inline]
     fn from(c: char) -> Self {
-        Self { data: c as u128, size: NonZeroU8::new(std::mem::size_of::<char>() as u8).unwrap() }
+        Self { data: c as u128, size: NonZero::new(std::mem::size_of::<char>() as u8).unwrap() }
     }
 }
 
@@ -454,7 +454,7 @@ impl From<Single> for ScalarInt {
     #[inline]
     fn from(f: Single) -> Self {
         // We trust apfloat to give us properly truncated data.
-        Self { data: f.to_bits(), size: NonZeroU8::new((Single::BITS / 8) as u8).unwrap() }
+        Self { data: f.to_bits(), size: NonZero::new((Single::BITS / 8) as u8).unwrap() }
     }
 }
 
@@ -470,7 +470,7 @@ impl From<Double> for ScalarInt {
     #[inline]
     fn from(f: Double) -> Self {
         // We trust apfloat to give us properly truncated data.
-        Self { data: f.to_bits(), size: NonZeroU8::new((Double::BITS / 8) as u8).unwrap() }
+        Self { data: f.to_bits(), size: NonZero::new((Double::BITS / 8) as u8).unwrap() }
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs
index 84de12b23a0..813a7a64daf 100644
--- a/compiler/rustc_middle/src/ty/generic_args.rs
+++ b/compiler/rustc_middle/src/ty/generic_args.rs
@@ -18,7 +18,7 @@ use core::intrinsics;
 use std::cmp::Ordering;
 use std::marker::PhantomData;
 use std::mem;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::ops::{ControlFlow, Deref};
 use std::ptr::NonNull;
 
@@ -143,9 +143,8 @@ impl<'tcx> From<ty::Term<'tcx>> for GenericArg<'tcx> {
 impl<'tcx> GenericArg<'tcx> {
     #[inline]
     pub fn unpack(self) -> GenericArgKind<'tcx> {
-        let ptr = unsafe {
-            self.ptr.map_addr(|addr| NonZeroUsize::new_unchecked(addr.get() & !TAG_MASK))
-        };
+        let ptr =
+            unsafe { self.ptr.map_addr(|addr| NonZero::new_unchecked(addr.get() & !TAG_MASK)) };
         // SAFETY: use of `Interned::new_unchecked` here is ok because these
         // pointers were originally created from `Interned` types in `pack()`,
         // and this is just going in the other direction.
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 8d8d06b7c0b..2b34f5daaf6 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -20,7 +20,7 @@ use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Targ
 
 use std::cmp;
 use std::fmt;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::ops::Bound;
 
 pub trait IntegerExt {
@@ -761,7 +761,7 @@ where
                 };
                 tcx.mk_layout(LayoutS {
                     variants: Variants::Single { index: variant_index },
-                    fields: match NonZeroUsize::new(fields) {
+                    fields: match NonZero::new(fields) {
                         Some(fields) => FieldsShape::Union(fields),
                         None => FieldsShape::Arbitrary { offsets: IndexVec::new(), memory_index: IndexVec::new() },
                     },
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 15bddb2a64f..eea3624898c 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -61,7 +61,7 @@ use std::fmt::Debug;
 use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
 use std::mem;
-use std::num::NonZeroUsize;
+use std::num::NonZero;
 use std::ops::ControlFlow;
 use std::ptr::NonNull;
 use std::{fmt, str};
@@ -617,9 +617,8 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for Term<'tcx> {
 impl<'tcx> Term<'tcx> {
     #[inline]
     pub fn unpack(self) -> TermKind<'tcx> {
-        let ptr = unsafe {
-            self.ptr.map_addr(|addr| NonZeroUsize::new_unchecked(addr.get() & !TAG_MASK))
-        };
+        let ptr =
+            unsafe { self.ptr.map_addr(|addr| NonZero::new_unchecked(addr.get() & !TAG_MASK)) };
         // SAFETY: use of `Interned::new_unchecked` here is ok because these
         // pointers were originally created from `Interned` types in `pack()`,
         // and this is just going in the other direction.
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 09bb06de483..2addfa37f8b 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -18,7 +18,7 @@ use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_macros::HashStable;
 use rustc_session::Limit;
-use rustc_span::sym;
+use rustc_span::{sym, Symbol};
 use rustc_target::abi::{Integer, IntegerType, Primitive, Size};
 use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
@@ -1552,9 +1552,15 @@ pub fn is_doc_notable_trait(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
         .any(|items| items.iter().any(|item| item.has_name(sym::notable_trait)))
 }
 
-/// Determines whether an item is an intrinsic by Abi.
-pub fn is_intrinsic(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
-    matches!(tcx.fn_sig(def_id).skip_binder().abi(), Abi::RustIntrinsic | Abi::PlatformIntrinsic)
+/// Determines whether an item is an intrinsic by Abi. or by whether it has a `rustc_intrinsic` attribute
+pub fn intrinsic(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<Symbol> {
+    if matches!(tcx.fn_sig(def_id).skip_binder().abi(), Abi::RustIntrinsic | Abi::PlatformIntrinsic)
+        || tcx.has_attr(def_id, sym::rustc_intrinsic)
+    {
+        Some(tcx.item_name(def_id.into()))
+    } else {
+        None
+    }
 }
 
 pub fn provide(providers: &mut Providers) {
@@ -1562,7 +1568,7 @@ pub fn provide(providers: &mut Providers) {
         reveal_opaque_types_in_bounds,
         is_doc_hidden,
         is_doc_notable_trait,
-        is_intrinsic,
+        intrinsic,
         ..*providers
     }
 }
diff --git a/compiler/rustc_mir_dataflow/src/rustc_peek.rs b/compiler/rustc_mir_dataflow/src/rustc_peek.rs
index cbbf3548c07..1575f31e75e 100644
--- a/compiler/rustc_mir_dataflow/src/rustc_peek.rs
+++ b/compiler/rustc_mir_dataflow/src/rustc_peek.rs
@@ -202,8 +202,7 @@ impl PeekCall {
             &terminator.kind
         {
             if let ty::FnDef(def_id, fn_args) = *func.const_.ty().kind() {
-                let name = tcx.item_name(def_id);
-                if !tcx.is_intrinsic(def_id) || name != sym::rustc_peek {
+                if tcx.intrinsic(def_id)? != sym::rustc_peek {
                     return None;
                 }
 
diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs
index 79bed960b95..2c692c95003 100644
--- a/compiler/rustc_mir_transform/src/cost_checker.rs
+++ b/compiler/rustc_mir_transform/src/cost_checker.rs
@@ -70,7 +70,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
             TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => {
                 let fn_ty = self.instantiate_ty(f.const_.ty());
                 self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind()
-                    && tcx.is_intrinsic(def_id)
+                    && tcx.intrinsic(def_id).is_some()
                 {
                     // Don't give intrinsics the extra penalty for calls
                     INSTR_COST
diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs
index d936c64c55c..73102a5f026 100644
--- a/compiler/rustc_mir_transform/src/instsimplify.rs
+++ b/compiler/rustc_mir_transform/src/instsimplify.rs
@@ -289,9 +289,9 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
         if args.is_empty() {
             return;
         }
-        let ty = args.type_at(0);
 
-        let known_is_valid = intrinsic_assert_panics(self.tcx, self.param_env, ty, intrinsic_name);
+        let known_is_valid =
+            intrinsic_assert_panics(self.tcx, self.param_env, args[0], intrinsic_name);
         match known_is_valid {
             // We don't know the layout or it's not validity assertion at all, don't touch it
             None => {}
@@ -310,10 +310,11 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
 fn intrinsic_assert_panics<'tcx>(
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    ty: Ty<'tcx>,
+    arg: ty::GenericArg<'tcx>,
     intrinsic_name: Symbol,
 ) -> Option<bool> {
     let requirement = ValidityRequirement::from_intrinsic(intrinsic_name)?;
+    let ty = arg.expect_ty();
     Some(!tcx.check_validity_requirement((requirement, param_env.and(ty))).ok()?)
 }
 
@@ -322,9 +323,8 @@ fn resolve_rust_intrinsic<'tcx>(
     func_ty: Ty<'tcx>,
 ) -> Option<(Symbol, GenericArgsRef<'tcx>)> {
     if let ty::FnDef(def_id, args) = *func_ty.kind() {
-        if tcx.is_intrinsic(def_id) {
-            return Some((tcx.item_name(def_id), args));
-        }
+        let name = tcx.intrinsic(def_id)?;
+        return Some((name, args));
     }
     None
 }
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index fb174192b84..7f0e6f90dbb 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -161,8 +161,7 @@ fn remap_mir_for_const_eval_select<'tcx>(
                 fn_span,
                 ..
             } if let ty::FnDef(def_id, _) = *const_.ty().kind()
-                && tcx.item_name(def_id) == sym::const_eval_select
-                && tcx.is_intrinsic(def_id) =>
+                && matches!(tcx.intrinsic(def_id), Some(sym::const_eval_select)) =>
             {
                 let [tupled_args, called_in_const, called_at_rt]: [_; 3] =
                     std::mem::take(args).try_into().unwrap();
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index f43b85173d4..a0af902c4e1 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -14,9 +14,8 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
             if let TerminatorKind::Call { func, args, destination, target, .. } =
                 &mut terminator.kind
                 && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
-                && tcx.is_intrinsic(def_id)
+                && let Some(intrinsic_name) = tcx.intrinsic(def_id)
             {
-                let intrinsic_name = tcx.item_name(def_id);
                 match intrinsic_name {
                     sym::unreachable => {
                         terminator.kind = TerminatorKind::Unreachable;
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 1c9913fa203..5593de60784 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -956,19 +956,24 @@ fn visit_instance_use<'tcx>(
     if !should_codegen_locally(tcx, &instance) {
         return;
     }
-
-    // The intrinsics assert_inhabited, assert_zero_valid, and assert_mem_uninitialized_valid will
-    // be lowered in codegen to nothing or a call to panic_nounwind. So if we encounter any
-    // of those intrinsics, we need to include a mono item for panic_nounwind, else we may try to
-    // codegen a call to that function without generating code for the function itself.
     if let ty::InstanceDef::Intrinsic(def_id) = instance.def {
         let name = tcx.item_name(def_id);
         if let Some(_requirement) = ValidityRequirement::from_intrinsic(name) {
+            // The intrinsics assert_inhabited, assert_zero_valid, and assert_mem_uninitialized_valid will
+            // be lowered in codegen to nothing or a call to panic_nounwind. So if we encounter any
+            // of those intrinsics, we need to include a mono item for panic_nounwind, else we may try to
+            // codegen a call to that function without generating code for the function itself.
             let def_id = tcx.lang_items().get(LangItem::PanicNounwind).unwrap();
             let panic_instance = Instance::mono(tcx, def_id);
             if should_codegen_locally(tcx, &panic_instance) {
                 output.push(create_fn_mono_item(tcx, panic_instance, source));
             }
+        } else if tcx.has_attr(def_id, sym::rustc_intrinsic) {
+            // Codegen the fallback body of intrinsics with fallback bodies
+            let instance = ty::Instance::new(def_id, instance.args);
+            if should_codegen_locally(tcx, &instance) {
+                output.push(create_fn_mono_item(tcx, instance, source));
+            }
         }
     }
 
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
index e795537e84a..7227b185f4d 100644
--- a/compiler/rustc_passes/src/lib.rs
+++ b/compiler/rustc_passes/src/lib.rs
@@ -8,6 +8,7 @@
 #![doc(rust_logo)]
 #![feature(rustdoc_internals)]
 #![allow(internal_features)]
+#![feature(generic_nonzero)]
 #![feature(let_chains)]
 #![feature(map_try_insert)]
 #![feature(try_blocks)]
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 17ad08b0569..19272b52b32 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -27,7 +27,7 @@ use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
 
 use std::mem::replace;
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 #[derive(PartialEq)]
 enum AnnotationKind {
@@ -645,7 +645,7 @@ fn stability_index(tcx: TyCtxt<'_>, (): ()) -> Index {
             let stability = Stability {
                 level: attr::StabilityLevel::Unstable {
                     reason: UnstableReason::Default,
-                    issue: NonZeroU32::new(27812),
+                    issue: NonZero::new(27812),
                     is_soft: false,
                     implied_by: None,
                 },
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 0fe5b9c664a..33116737a42 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -3,6 +3,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(rustdoc_internals)]
+#![feature(generic_nonzero)]
 #![feature(min_specialization)]
 #![feature(rustc_attrs)]
 #![allow(rustc::potential_query_instability, unused_parens)]
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index a827717d9bb..5917d79983d 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -30,7 +30,7 @@ use rustc_serialize::Decodable;
 use rustc_serialize::Encodable;
 use rustc_session::Limit;
 use rustc_span::def_id::LOCAL_CRATE;
-use std::num::NonZeroU64;
+use std::num::NonZero;
 use thin_vec::ThinVec;
 
 #[derive(Copy, Clone)]
@@ -68,10 +68,8 @@ impl QueryContext for QueryCtxt<'_> {
     #[inline]
     fn next_job_id(self) -> QueryJobId {
         QueryJobId(
-            NonZeroU64::new(
-                self.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
-            )
-            .unwrap(),
+            NonZero::new(self.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed))
+                .unwrap(),
         )
     }
 
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index 416f556f57d..6a959a99e5d 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -1,5 +1,6 @@
 #![feature(assert_matches)]
 #![feature(core_intrinsics)]
+#![feature(generic_nonzero)]
 #![feature(hash_raw_entry)]
 #![feature(min_specialization)]
 #![feature(let_chains)]
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index 8d7c0ca0144..bf89bc7f7c3 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -11,7 +11,7 @@ use rustc_span::Span;
 
 use std::hash::Hash;
 use std::io::Write;
-use std::num::NonZeroU64;
+use std::num::NonZero;
 
 #[cfg(parallel_compiler)]
 use {
@@ -36,7 +36,7 @@ pub type QueryMap = FxHashMap<QueryJobId, QueryJobInfo>;
 
 /// A value uniquely identifying an active query job.
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
-pub struct QueryJobId(pub NonZeroU64);
+pub struct QueryJobId(pub NonZero<u64>);
 
 impl QueryJobId {
     fn query(self, map: &QueryMap) -> QueryStackFrame {
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 6bd221ff058..3ea4df1d2a4 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -18,6 +18,7 @@ use rustc_ast::*;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_errors::{
     codes::*, struct_span_code_err, Applicability, DiagnosticArgValue, ErrCode, IntoDiagnosticArg,
+    StashKey,
 };
 use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::{self, CtorKind, DefKind, LifetimeRes, NonMacroAttrKind, PartialRes, PerNS};
@@ -3890,6 +3891,23 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             finalize,
         ) {
             Ok(Some(partial_res)) if let Some(res) = partial_res.full_res() => {
+                // if we also have an associated type that matches the ident, stash a suggestion
+                if let Some(items) = self.diagnostic_metadata.current_trait_assoc_items
+                    && let [Segment { ident, .. }] = path
+                    && items.iter().any(|item| {
+                        item.ident == *ident && matches!(item.kind, AssocItemKind::Type(_))
+                    })
+                {
+                    let mut diag = self.r.tcx.dcx().struct_allow("");
+                    diag.span_suggestion_verbose(
+                        path_span.shrink_to_lo(),
+                        "there is an associated type with the same name",
+                        "Self::",
+                        Applicability::MaybeIncorrect,
+                    );
+                    diag.stash(path_span, StashKey::AssociatedTypeSuggestion);
+                }
+
                 if source.is_expected(res) || res == Res::Err {
                     partial_res
                 } else {
diff --git a/compiler/rustc_serialize/src/lib.rs b/compiler/rustc_serialize/src/lib.rs
index 95833f532f4..bb822c611a1 100644
--- a/compiler/rustc_serialize/src/lib.rs
+++ b/compiler/rustc_serialize/src/lib.rs
@@ -11,6 +11,7 @@
 #![feature(associated_type_bounds)]
 #![feature(const_option)]
 #![feature(core_intrinsics)]
+#![feature(generic_nonzero)]
 #![feature(inline_const)]
 #![feature(min_specialization)]
 #![feature(never_type)]
diff --git a/compiler/rustc_serialize/src/serialize.rs b/compiler/rustc_serialize/src/serialize.rs
index 287e317b10f..412f7eced43 100644
--- a/compiler/rustc_serialize/src/serialize.rs
+++ b/compiler/rustc_serialize/src/serialize.rs
@@ -6,6 +6,7 @@ use std::cell::{Cell, RefCell};
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
 use std::hash::{BuildHasher, Hash};
 use std::marker::PhantomData;
+use std::num::NonZero;
 use std::path;
 use std::rc::Rc;
 use std::sync::Arc;
@@ -216,15 +217,15 @@ impl<D: Decoder> Decodable<D> for ! {
     }
 }
 
-impl<S: Encoder> Encodable<S> for ::std::num::NonZeroU32 {
+impl<S: Encoder> Encodable<S> for NonZero<u32> {
     fn encode(&self, s: &mut S) {
         s.emit_u32(self.get());
     }
 }
 
-impl<D: Decoder> Decodable<D> for ::std::num::NonZeroU32 {
+impl<D: Decoder> Decodable<D> for NonZero<u32> {
     fn decode(d: &mut D) -> Self {
-        ::std::num::NonZeroU32::new(d.read_u32()).unwrap()
+        NonZero::new(d.read_u32()).unwrap()
     }
 }
 
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index a8b9ff056c2..ef0512bf686 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -3230,7 +3230,7 @@ pub(crate) mod dep_tracking {
     };
     use std::collections::BTreeMap;
     use std::hash::{DefaultHasher, Hash};
-    use std::num::NonZeroUsize;
+    use std::num::NonZero;
     use std::path::PathBuf;
 
     pub trait DepTrackingHash {
@@ -3272,7 +3272,7 @@ pub(crate) mod dep_tracking {
     impl_dep_tracking_hash_via_hash!(
         bool,
         usize,
-        NonZeroUsize,
+        NonZero<usize>,
         u64,
         Hash64,
         String,
diff --git a/compiler/rustc_session/src/errors.rs b/compiler/rustc_session/src/errors.rs
index 82846e3b4e8..de7e04ba00f 100644
--- a/compiler/rustc_session/src/errors.rs
+++ b/compiler/rustc_session/src/errors.rs
@@ -1,4 +1,4 @@
-use std::num::NonZeroU32;
+use std::num::NonZero;
 
 use rustc_ast::token;
 use rustc_ast::util::literal::LitError;
@@ -27,7 +27,7 @@ impl<'a> IntoDiagnostic<'a> for FeatureGateError {
 #[derive(Subdiagnostic)]
 #[note(session_feature_diagnostic_for_issue)]
 pub struct FeatureDiagnosticForIssue {
-    pub n: NonZeroU32,
+    pub n: NonZero<u32>,
 }
 
 #[derive(Subdiagnostic)]
diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs
index 58e1394c090..c63af90a7f3 100644
--- a/compiler/rustc_session/src/lib.rs
+++ b/compiler/rustc_session/src/lib.rs
@@ -1,3 +1,4 @@
+#![feature(generic_nonzero)]
 #![feature(let_chains)]
 #![feature(lazy_cell)]
 #![feature(option_get_or_insert_default)]
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index ea93ac5841f..743f4760339 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -21,7 +21,7 @@ use rustc_span::SourceFileHashAlgorithm;
 use std::collections::BTreeMap;
 
 use std::hash::{DefaultHasher, Hasher};
-use std::num::{IntErrorKind, NonZeroUsize};
+use std::num::{IntErrorKind, NonZero};
 use std::path::PathBuf;
 use std::str;
 
@@ -617,7 +617,7 @@ mod parse {
     pub(crate) fn parse_threads(slot: &mut usize, v: Option<&str>) -> bool {
         match v.and_then(|s| s.parse().ok()) {
             Some(0) => {
-                *slot = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get);
+                *slot = std::thread::available_parallelism().map_or(1, NonZero::<usize>::get);
                 true
             }
             Some(i) => {
@@ -991,7 +991,10 @@ mod parse {
         true
     }
 
-    pub(crate) fn parse_treat_err_as_bug(slot: &mut Option<NonZeroUsize>, v: Option<&str>) -> bool {
+    pub(crate) fn parse_treat_err_as_bug(
+        slot: &mut Option<NonZero<usize>>,
+        v: Option<&str>,
+    ) -> bool {
         match v {
             Some(s) => match s.parse() {
                 Ok(val) => {
@@ -1004,7 +1007,7 @@ mod parse {
                 }
             },
             None => {
-                *slot = NonZeroUsize::new(1);
+                *slot = NonZero::new(1);
                 true
             }
         }
@@ -1950,7 +1953,7 @@ written to standard error output)"),
         "translate remapped paths into local paths when possible (default: yes)"),
     trap_unreachable: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "generate trap instructions for unreachable intrinsics (default: use target setting, usually yes)"),
-    treat_err_as_bug: Option<NonZeroUsize> = (None, parse_treat_err_as_bug, [TRACKED],
+    treat_err_as_bug: Option<NonZero<usize>> = (None, parse_treat_err_as_bug, [TRACKED],
         "treat the `val`th error that occurs as bug (default if not specified: 0 - don't treat errors as bugs. \
         default if specified without a value: 1 - treat the first error as bug)"),
     trim_diagnostic_paths: bool = (true, parse_bool, [UNTRACKED],
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 134aa9efc01..29c88783357 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1422,6 +1422,7 @@ symbols! {
         rustc_if_this_changed,
         rustc_inherit_overflow_checks,
         rustc_insignificant_dtor,
+        rustc_intrinsic,
         rustc_layout,
         rustc_layout_scalar_valid_range_end,
         rustc_layout_scalar_valid_range_start,
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index 68b1a0d4e61..661a444d049 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -2993,6 +2993,15 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 &mut Default::default(),
             );
             self.suggest_unsized_bound_if_applicable(err, obligation);
+            if let Some(span) = err.span.primary_span()
+                && let Some(mut diag) =
+                    self.tcx.dcx().steal_diagnostic(span, StashKey::AssociatedTypeSuggestion)
+                && let Ok(ref mut s1) = err.suggestions
+                && let Ok(ref mut s2) = diag.suggestions
+            {
+                s1.append(s2);
+                diag.cancel()
+            }
         }
     }
 
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 7fa416197b3..5fc93d666ab 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -28,7 +28,8 @@ fn resolve_instance<'tcx>(
             tcx.normalize_erasing_regions(param_env, args),
         )
     } else {
-        let def = if matches!(tcx.def_kind(def_id), DefKind::Fn) && tcx.is_intrinsic(def_id) {
+        let def = if matches!(tcx.def_kind(def_id), DefKind::Fn) && tcx.intrinsic(def_id).is_some()
+        {
             debug!(" => intrinsic");
             ty::InstanceDef::Intrinsic(def_id)
         } else if Some(def_id) == tcx.lang_items().drop_in_place_fn() {