Simplify implementation of Rust intrinsics by using type parameters in the cache

1 files changed, 128 insertions, 334 deletions

diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 8d6e1d8941b..f36f3b2b16b 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -137,7 +137,8 @@ pub(crate) struct FullCx<'ll, 'tcx> {
     eh_catch_typeinfo: Cell<Option<&'ll Value>>,
     pub rust_try_fn: Cell<Option<(&'ll Type, &'ll Value)>>,
 
-    intrinsics: RefCell<FxHashMap<&'static str, (&'ll Type, &'ll Value)>>,
+    intrinsics:
+        RefCell<FxHashMap<(&'static str, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
 
     /// A counter that is used for generating local symbol names
     local_gen_sym_counter: Cell<usize>,
@@ -842,17 +843,24 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 }
 
 impl<'ll> CodegenCx<'ll, '_> {
-    pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) {
-        if let Some(v) = self.intrinsics.borrow().get(key).cloned() {
-            return v;
+    pub(crate) fn get_intrinsic(
+        &self,
+        base_name: &str,
+        type_params: &[&'ll Type],
+    ) -> (&'ll Type, &'ll Value) {
+        if let Some(v) =
+            self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params)))
+        {
+            return *v;
         }
 
-        self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key))
+        self.declare_intrinsic(base_name, type_params)
     }
 
     fn insert_intrinsic(
         &self,
-        name: &'static str,
+        base_name: &'static str,
+        type_params: &[&'ll Type],
         args: Option<&[&'ll llvm::Type]>,
         ret: &'ll llvm::Type,
     ) -> (&'ll llvm::Type, &'ll llvm::Value) {
@@ -861,372 +869,153 @@ impl<'ll> CodegenCx<'ll, '_> {
         } else {
             self.type_variadic_func(&[], ret)
         };
-        let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty);
-        self.intrinsics.borrow_mut().insert(name, (fn_ty, f));
+
+        let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
+            .expect("Unknown LLVM intrinsic `{base_name}`");
+
+        let full_name = if intrinsic.is_overloaded() {
+            &intrinsic.overloaded_name(self.llmod, type_params)
+        } else {
+            base_name
+        };
+
+        let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty);
+        self.intrinsics
+            .borrow_mut()
+            .insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f));
         (fn_ty, f)
     }
 
-    fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> {
+    fn declare_intrinsic(
+        &self,
+        base_name: &str,
+        type_params: &[&'ll Type],
+    ) -> (&'ll Type, &'ll Value) {
+        macro_rules! param {
+            ($index:literal) => {
+                type_params[$index]
+            };
+            ($other:expr) => {
+                $other
+            };
+        }
         macro_rules! ifn {
-            ($name:expr, fn() -> $ret:expr) => (
-                if key == $name {
-                    return Some(self.insert_intrinsic($name, Some(&[]), $ret));
-                }
-            );
             ($name:expr, fn(...) -> $ret:expr) => (
-                if key == $name {
-                    return Some(self.insert_intrinsic($name, None, $ret));
+                if base_name == $name {
+                    return self.insert_intrinsic($name, type_params, None, param!($ret));
                 }
             );
             ($name:expr, fn($($arg:expr),*) -> $ret:expr) => (
-                if key == $name {
-                    return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret));
+                if base_name == $name {
+                    return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret));
                 }
             );
         }
         macro_rules! mk_struct {
-            ($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false))
+            ($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false))
         }
 
+        let same_width_vector = |index, element_ty| {
+            self.type_vector(element_ty, self.vector_length(type_params[index]) as u64)
+        };
+
         let ptr = self.type_ptr();
         let void = self.type_void();
         let i1 = self.type_i1();
-        let t_i8 = self.type_i8();
-        let t_i16 = self.type_i16();
         let t_i32 = self.type_i32();
         let t_i64 = self.type_i64();
-        let t_i128 = self.type_i128();
         let t_isize = self.type_isize();
-        let t_f16 = self.type_f16();
-        let t_f32 = self.type_f32();
-        let t_f64 = self.type_f64();
-        let t_f128 = self.type_f128();
         let t_metadata = self.type_metadata();
         let t_token = self.type_token();
 
         ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr);
         ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32);
 
-        ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64);
-        ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64);
-
-        ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8);
-        ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16);
-        ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128);
-        ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8);
-        ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16);
-        ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64);
-        ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128);
-
-        ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8);
-        ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16);
-        ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32);
-        ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64);
-        ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128);
-        ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8);
-        ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16);
-        ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32);
-        ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64);
-        ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128);
+        ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0);
+        ifn!("llvm.wasm.trunc.signed", fn(1) -> 0);
+        ifn!("llvm.fptosi.sat", fn(1) -> 0);
+        ifn!("llvm.fptoui.sat", fn(1) -> 0);
 
         ifn!("llvm.trap", fn() -> void);
         ifn!("llvm.debugtrap", fn() -> void);
         ifn!("llvm.frameaddress", fn(t_i32) -> ptr);
 
-        ifn!("llvm.powi.f16.i32", fn(t_f16, t_i32) -> t_f16);
-        ifn!("llvm.powi.f32.i32", fn(t_f32, t_i32) -> t_f32);
-        ifn!("llvm.powi.f64.i32", fn(t_f64, t_i32) -> t_f64);
-        ifn!("llvm.powi.f128.i32", fn(t_f128, t_i32) -> t_f128);
-
-        ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16);
-        ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32);
-        ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64);
-        ifn!("llvm.pow.f128", fn(t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.sqrt.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.sqrt.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.sin.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.sin.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.sin.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.sin.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.cos.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.cos.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.cos.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.cos.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.exp.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.exp.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.exp.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.exp.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.exp2.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.exp2.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.log.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.log.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.log.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.log.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.log10.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.log10.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.log10.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.log10.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.log2.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.log2.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.log2.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.log2.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16);
-        ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32);
-        ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64);
-        ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16);
-        ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32);
-        ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64);
-        ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.fabs.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16);
-        ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32);
-        ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64);
-        ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.minimum.f16", fn(t_f16, t_f16) -> t_f16);
-        ifn!("llvm.minimum.f32", fn(t_f32, t_f32) -> t_f32);
-        ifn!("llvm.minimum.f64", fn(t_f64, t_f64) -> t_f64);
-        // There are issues on x86_64 and aarch64 with the f128 variant.
-        //  - https://github.com/llvm/llvm-project/issues/139380
-        //  - https://github.com/llvm/llvm-project/issues/139381
-        // ifn!("llvm.minimum.f128", fn(t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16);
-        ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32);
-        ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64);
-        ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.maximum.f16", fn(t_f16, t_f16) -> t_f16);
-        ifn!("llvm.maximum.f32", fn(t_f32, t_f32) -> t_f32);
-        ifn!("llvm.maximum.f64", fn(t_f64, t_f64) -> t_f64);
-        // There are issues on x86_64 and aarch64 with the f128 variant.
-        //  - https://github.com/llvm/llvm-project/issues/139380
-        //  - https://github.com/llvm/llvm-project/issues/139381
-        // ifn!("llvm.maximum.f128", fn(t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.floor.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.floor.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.floor.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.floor.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.ceil.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.ceil.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.trunc.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.trunc.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16);
-        ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32);
-        ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64);
-        ifn!("llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128);
-
-        ifn!("llvm.round.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.round.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.round.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.round.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.roundeven.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.roundeven.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.rint.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.rint.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.rint.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.rint.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.nearbyint.f16", fn(t_f16) -> t_f16);
-        ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32);
-        ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64);
-        ifn!("llvm.nearbyint.f128", fn(t_f128) -> t_f128);
-
-        ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8);
-        ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16);
-        ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32);
-        ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64);
-        ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128);
-
-        ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8);
-        ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16);
-        ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32);
-        ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64);
-        ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128);
-
-        ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8);
-        ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16);
-        ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32);
-        ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64);
-        ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128);
-
-        ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16);
-        ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32);
-        ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64);
-        ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128);
-
-        ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8);
-        ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16);
-        ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32);
-        ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64);
-        ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128);
-
-        ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8);
-        ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16);
-        ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32);
-        ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64);
-        ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128);
-
-        ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8);
-        ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16);
-        ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32);
-        ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64);
-        ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128);
-
-        ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
-        ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
-        ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
-        ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
-        ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
-
-        ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
-        ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
-        ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
-        ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
-        ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
-
-        ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
-        ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
-        ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
-        ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
-        ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
-
-        ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
-        ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
-        ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
-        ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
-        ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
-
-        ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
-        ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
-        ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
-        ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
-        ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
-
-        ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
-        ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
-        ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
-        ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
-        ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
-
-        ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
-        ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
-        ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
-        ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
-        ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
-
-        ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
-        ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
-        ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
-        ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
-        ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
-
-        ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8);
-        ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16);
-        ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32);
-        ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64);
-        ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128);
-
-        ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8);
-        ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16);
-        ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32);
-        ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64);
-        ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128);
-
-        ifn!("llvm.scmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
-        ifn!("llvm.scmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
-        ifn!("llvm.scmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
-        ifn!("llvm.scmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
-        ifn!("llvm.scmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
-
-        ifn!("llvm.ucmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
-        ifn!("llvm.ucmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
-        ifn!("llvm.ucmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
-        ifn!("llvm.ucmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
-        ifn!("llvm.ucmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
-
-        ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void);
-        ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void);
-
-        // FIXME: This is an infinitesimally small portion of the types you can
-        // pass to this intrinsic, if we can ever lazily register intrinsics we
-        // should register these when they're used, that way any type can be
-        // passed.
-        ifn!("llvm.is.constant.i1", fn(i1) -> i1);
-        ifn!("llvm.is.constant.i8", fn(t_i8) -> i1);
-        ifn!("llvm.is.constant.i16", fn(t_i16) -> i1);
-        ifn!("llvm.is.constant.i32", fn(t_i32) -> i1);
-        ifn!("llvm.is.constant.i64", fn(t_i64) -> i1);
-        ifn!("llvm.is.constant.i128", fn(t_i128) -> i1);
-        ifn!("llvm.is.constant.isize", fn(t_isize) -> i1);
-        ifn!("llvm.is.constant.f16", fn(t_f16) -> i1);
-        ifn!("llvm.is.constant.f32", fn(t_f32) -> i1);
-        ifn!("llvm.is.constant.f64", fn(t_f64) -> i1);
-        ifn!("llvm.is.constant.f128", fn(t_f128) -> i1);
-        ifn!("llvm.is.constant.ptr", fn(ptr) -> i1);
-
-        ifn!("llvm.expect.i1", fn(i1, i1) -> i1);
-        ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32);
+        ifn!("llvm.powi", fn(0, 1) -> 0);
+        ifn!("llvm.pow", fn(0, 0) -> 0);
+        ifn!("llvm.sqrt", fn(0) -> 0);
+        ifn!("llvm.sin", fn(0) -> 0);
+        ifn!("llvm.cos", fn(0) -> 0);
+        ifn!("llvm.exp", fn(0) -> 0);
+        ifn!("llvm.exp2", fn(0) -> 0);
+        ifn!("llvm.log", fn(0) -> 0);
+        ifn!("llvm.log10", fn(0) -> 0);
+        ifn!("llvm.log2", fn(0) -> 0);
+        ifn!("llvm.fma", fn(0, 0, 0) -> 0);
+        ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0);
+        ifn!("llvm.fabs", fn(0) -> 0);
+        ifn!("llvm.minnum", fn(0, 0) -> 0);
+        ifn!("llvm.minimum", fn(0, 0) -> 0);
+        ifn!("llvm.maxnum", fn(0, 0) -> 0);
+        ifn!("llvm.maximum", fn(0, 0) -> 0);
+        ifn!("llvm.floor", fn(0) -> 0);
+        ifn!("llvm.ceil", fn(0) -> 0);
+        ifn!("llvm.trunc", fn(0) -> 0);
+        ifn!("llvm.copysign", fn(0, 0) -> 0);
+        ifn!("llvm.round", fn(0) -> 0);
+        ifn!("llvm.rint", fn(0) -> 0);
+        ifn!("llvm.nearbyint", fn(0) -> 0);
+
+        ifn!("llvm.ctpop", fn(0) -> 0);
+        ifn!("llvm.ctlz", fn(0, i1) -> 0);
+        ifn!("llvm.cttz", fn(0, i1) -> 0);
+        ifn!("llvm.bswap", fn(0) -> 0);
+        ifn!("llvm.bitreverse", fn(0) -> 0);
+        ifn!("llvm.fshl", fn(0, 0, 0) -> 0);
+        ifn!("llvm.fshr", fn(0, 0, 0) -> 0);
+
+        ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
+        ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
+        ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
+        ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
+        ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
+        ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
+
+        ifn!("llvm.sadd.sat", fn(0, 0) -> 0);
+        ifn!("llvm.uadd.sat", fn(0, 0) -> 0);
+        ifn!("llvm.ssub.sat", fn(0, 0) -> 0);
+        ifn!("llvm.usub.sat", fn(0, 0) -> 0);
+
+        ifn!("llvm.scmp", fn(1, 1) -> 0);
+        ifn!("llvm.ucmp", fn(1, 1) -> 0);
+
+        ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void);
+        ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void);
+
+        ifn!("llvm.is.constant", fn(0) -> i1);
+        ifn!("llvm.expect", fn(0, 0) -> 0);
+
+        ifn!("llvm.eh.typeid.for", fn(0) -> t_i32);
         ifn!("llvm.localescape", fn(...) -> void);
         ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr);
-        ifn!("llvm.x86.seh.recoverfp", fn(ptr, ptr) -> ptr);
 
         ifn!("llvm.assume", fn(i1) -> void);
-        ifn!("llvm.prefetch", fn(ptr, t_i32, t_i32, t_i32) -> void);
+        ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void);
 
         // This isn't an "LLVM intrinsic", but LLVM's optimization passes
         // recognize it like one (including turning it into `bcmp` sometimes)
         // and we use it to implement intrinsics like `raw_eq` and `compare_bytes`
-        match self.sess().target.arch.as_ref() {
-            "avr" | "msp430" => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i16),
-            _ => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i32),
+        if base_name == "memcmp" {
+            let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int());
+            let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty);
+            self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f));
+
+            return (fn_ty, f);
         }
 
         // variadic intrinsics
-        ifn!("llvm.va_start", fn(ptr) -> void);
-        ifn!("llvm.va_end", fn(ptr) -> void);
-        ifn!("llvm.va_copy", fn(ptr, ptr) -> void);
+        ifn!("llvm.va_start", fn(0) -> void);
+        ifn!("llvm.va_end", fn(0) -> void);
+        ifn!("llvm.va_copy", fn(0, 0) -> void);
 
         if self.sess().instrument_coverage() {
             ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
@@ -1238,14 +1027,19 @@ impl<'ll> CodegenCx<'ll, '_> {
         ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1});
 
         if self.sess().opts.debuginfo != DebugInfo::None {
-            ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void);
-            ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void);
+            ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void);
+            ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void);
         }
 
-        ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr);
+        ifn!("llvm.ptrmask", fn(0, 1) -> 0);
         ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);
 
-        None
+        ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
+        ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
+        ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
+        ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
+
+        bug!("Unknown intrinsic: `{base_name}`")
     }
 
     pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value {