Auto merge of #18183 - lnicola:sync-from-rust, r=lnicola

internal: Sync from downstream

35 files changed, 999 insertions, 1260 deletions

diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index dea574a53cd..3d75393bf06 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -1,19 +1,19 @@
 use std::cmp;
 
 use libc::c_uint;
+use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
 use rustc_codegen_ssa::traits::*;
-use rustc_codegen_ssa::MemFlags;
+use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::LayoutOf;
 pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
-use rustc_middle::ty::Ty;
 use rustc_middle::{bug, ty};
 use rustc_session::config;
 pub(crate) use rustc_target::abi::call::*;
 use rustc_target::abi::{self, HasDataLayout, Int, Size};
-pub(crate) use rustc_target::spec::abi::Abi;
 use rustc_target::spec::SanitizerSet;
+pub(crate) use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
 
 use crate::attributes::llfn_attrs_from_instance;
@@ -285,7 +285,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
     }
 }
 
-impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> {
+impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     fn store_fn_arg(
         &mut self,
         arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
@@ -422,6 +422,9 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         if let Conv::RiscvInterrupt { kind } = self.conv {
             func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", kind.as_str()));
         }
+        if let Conv::CCmseNonSecureEntry = self.conv {
+            func_attrs.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry"))
+        }
         attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs });
 
         let mut i = 0;
@@ -442,11 +445,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 // LLVM also rejects full range.
                 && !scalar.is_always_valid(cx)
             {
-                attributes::apply_to_llfn(
-                    llfn,
-                    idx,
-                    &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))],
-                );
+                attributes::apply_to_llfn(llfn, idx, &[llvm::CreateRangeAttr(
+                    cx.llcx,
+                    scalar.size(cx),
+                    scalar.valid_range(cx),
+                )]);
             }
         };
 
@@ -465,17 +468,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()),
                 );
                 attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]);
-                if cx.sess().opts.optimize != config::OptLevel::No
-                    && llvm_util::get_version() >= (18, 0, 0)
-                {
-                    attributes::apply_to_llfn(
-                        llfn,
-                        llvm::AttributePlace::Argument(i),
-                        &[
-                            llvm::AttributeKind::Writable.create_attr(cx.llcx),
-                            llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx),
-                        ],
-                    );
+                if cx.sess().opts.optimize != config::OptLevel::No {
+                    attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[
+                        llvm::AttributeKind::Writable.create_attr(cx.llcx),
+                        llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx),
+                    ]);
                 }
             }
             PassMode::Cast { cast, pad_i32: _ } => {
@@ -591,11 +588,9 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         bx.cx.llcx,
                         bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()),
                     );
-                    attributes::apply_to_callsite(
-                        callsite,
-                        llvm::AttributePlace::Argument(i),
-                        &[byval],
-                    );
+                    attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[
+                        byval,
+                    ]);
                 }
                 PassMode::Direct(attrs)
                 | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
@@ -627,11 +622,9 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             // This will probably get ignored on all targets but those supporting the TrustZone-M
             // extension (thumbv8m targets).
             let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call");
-            attributes::apply_to_callsite(
-                callsite,
-                llvm::AttributePlace::Function,
-                &[cmse_nonsecure_call],
-            );
+            attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &[
+                cmse_nonsecure_call,
+            ]);
         }
 
         // Some intrinsics require that an elementtype attribute (with the pointee type of a
@@ -661,9 +654,11 @@ impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
 impl From<Conv> for llvm::CallConv {
     fn from(conv: Conv) -> Self {
         match conv {
-            Conv::C | Conv::Rust | Conv::CCmseNonSecureCall | Conv::RiscvInterrupt { .. } => {
-                llvm::CCallConv
-            }
+            Conv::C
+            | Conv::Rust
+            | Conv::CCmseNonSecureCall
+            | Conv::CCmseNonSecureEntry
+            | Conv::RiscvInterrupt { .. } => llvm::CCallConv,
             Conv::Cold => llvm::ColdCallConv,
             Conv::PreserveMost => llvm::PreserveMost,
             Conv::PreserveAll => llvm::PreserveAll,
diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs
index b4f3784a31a..2adac278c62 100644
--- a/compiler/rustc_codegen_llvm/src/allocator.rs
+++ b/compiler/rustc_codegen_llvm/src/allocator.rs
@@ -1,14 +1,14 @@
 use libc::c_uint;
 use rustc_ast::expand::allocator::{
-    alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy,
-    ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE,
+    ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE,
+    alloc_error_handler_name, default_fn_name, global_fn_name,
 };
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{DebugInfo, OomStrategy};
 
 use crate::llvm::{self, Context, False, Module, True, Type};
-use crate::{attributes, debuginfo, ModuleLlvm};
+use crate::{ModuleLlvm, attributes, debuginfo};
 
 pub(crate) unsafe fn codegen(
     tcx: TyCtxt<'_>,
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index 1d91c3fb17d..7621f111fe2 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -5,10 +5,10 @@ use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_codegen_ssa::mir::operand::OperandValue;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::Instance;
+use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::{bug, span_bug};
-use rustc_span::{sym, Pos, Span, Symbol};
+use rustc_span::{Pos, Span, Symbol, sym};
 use rustc_target::abi::*;
 use rustc_target::asm::*;
 use smallvec::SmallVec;
@@ -356,7 +356,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     }
 }
 
-impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> {
+impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
     fn codegen_global_asm(
         &self,
         template: &[InlineAsmTemplatePiece],
@@ -504,10 +504,15 @@ pub(crate) fn inline_asm_call<'ll>(
                 // due to the asm template string coming from a macro. LLVM will
                 // default to the first srcloc for lines that don't have an
                 // associated srcloc.
-                srcloc.push(bx.const_i32(0));
+                srcloc.push(llvm::LLVMValueAsMetadata(bx.const_i32(0)));
             }
-            srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32)));
-            let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32);
+            srcloc.extend(
+                line_spans
+                    .iter()
+                    .map(|span| llvm::LLVMValueAsMetadata(bx.const_i32(span.lo().to_u32() as i32))),
+            );
+            let md = llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len());
+            let md = llvm::LLVMMetadataAsValue(&bx.llcx, md);
             llvm::LLVMSetMetadata(call, kind, md);
 
             Some(call)
@@ -520,24 +525,16 @@ pub(crate) fn inline_asm_call<'ll>(
 
 /// If the register is an xmm/ymm/zmm register then return its index.
 fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
+    use X86InlineAsmReg::*;
     match reg {
-        InlineAsmReg::X86(reg)
-            if reg as u32 >= X86InlineAsmReg::xmm0 as u32
-                && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
-        {
-            Some(reg as u32 - X86InlineAsmReg::xmm0 as u32)
+        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
+            Some(reg as u32 - xmm0 as u32)
         }
-        InlineAsmReg::X86(reg)
-            if reg as u32 >= X86InlineAsmReg::ymm0 as u32
-                && reg as u32 <= X86InlineAsmReg::ymm15 as u32 =>
-        {
-            Some(reg as u32 - X86InlineAsmReg::ymm0 as u32)
+        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
+            Some(reg as u32 - ymm0 as u32)
         }
-        InlineAsmReg::X86(reg)
-            if reg as u32 >= X86InlineAsmReg::zmm0 as u32
-                && reg as u32 <= X86InlineAsmReg::zmm31 as u32 =>
-        {
-            Some(reg as u32 - X86InlineAsmReg::zmm0 as u32)
+        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
+            Some(reg as u32 - zmm0 as u32)
         }
         _ => None,
     }
@@ -545,50 +542,56 @@ fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
 
 /// If the register is an AArch64 integer register then return its index.
 fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
-    match reg {
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x0) => Some(0),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x1) => Some(1),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x2) => Some(2),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x3) => Some(3),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x4) => Some(4),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x5) => Some(5),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x6) => Some(6),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x7) => Some(7),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x8) => Some(8),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x9) => Some(9),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x10) => Some(10),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x11) => Some(11),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x12) => Some(12),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x13) => Some(13),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x14) => Some(14),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x15) => Some(15),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x16) => Some(16),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x17) => Some(17),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x18) => Some(18),
-        // x19 is reserved
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x20) => Some(20),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x21) => Some(21),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x22) => Some(22),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x23) => Some(23),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x24) => Some(24),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x25) => Some(25),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x26) => Some(26),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x27) => Some(27),
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x28) => Some(28),
-        // x29 is reserved
-        InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) => Some(30),
-        _ => None,
-    }
+    use AArch64InlineAsmReg::*;
+    // Unlike `a64_vreg_index`, we can't subtract `x0` to get the u32 because
+    // `x19` and `x29` are missing and the integer constants for the
+    // `x0`..`x30` enum variants don't all match the register number. E.g. the
+    // integer constant for `x18` is 18, but the constant for `x20` is 19.
+    Some(match reg {
+        InlineAsmReg::AArch64(r) => match r {
+            x0 => 0,
+            x1 => 1,
+            x2 => 2,
+            x3 => 3,
+            x4 => 4,
+            x5 => 5,
+            x6 => 6,
+            x7 => 7,
+            x8 => 8,
+            x9 => 9,
+            x10 => 10,
+            x11 => 11,
+            x12 => 12,
+            x13 => 13,
+            x14 => 14,
+            x15 => 15,
+            x16 => 16,
+            x17 => 17,
+            x18 => 18,
+            // x19 is reserved
+            x20 => 20,
+            x21 => 21,
+            x22 => 22,
+            x23 => 23,
+            x24 => 24,
+            x25 => 25,
+            x26 => 26,
+            x27 => 27,
+            x28 => 28,
+            // x29 is reserved
+            x30 => 30,
+            _ => return None,
+        },
+        _ => return None,
+    })
 }
 
 /// If the register is an AArch64 vector register then return its index.
 fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
+    use AArch64InlineAsmReg::*;
     match reg {
-        InlineAsmReg::AArch64(reg)
-            if reg as u32 >= AArch64InlineAsmReg::v0 as u32
-                && reg as u32 <= AArch64InlineAsmReg::v31 as u32 =>
-        {
-            Some(reg as u32 - AArch64InlineAsmReg::v0 as u32)
+        InlineAsmReg::AArch64(reg) if reg as u32 >= v0 as u32 && reg as u32 <= v31 as u32 => {
+            Some(reg as u32 - v0 as u32)
         }
         _ => None,
     }
@@ -596,6 +599,7 @@ fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
 
 /// Converts a register class to an LLVM constraint code.
 fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
+    use InlineAsmRegClass::*;
     match reg {
         // For vector registers LLVM wants the register name to match the type size.
         InlineAsmRegOrRegClass::Reg(reg) => {
@@ -652,75 +656,66 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
         // The constraints can be retrieved from
         // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
         InlineAsmRegOrRegClass::RegClass(reg) => match reg {
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
-                unreachable!("clobber-only")
-            }
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
-            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w",
-            InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
-            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
-            InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
-            | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
+            AArch64(AArch64InlineAsmRegClass::reg) => "r",
+            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
+            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
+            AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
+            Arm(ArmInlineAsmRegClass::reg) => "r",
+            Arm(ArmInlineAsmRegClass::sreg)
+            | Arm(ArmInlineAsmRegClass::dreg_low16)
+            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
+            Arm(ArmInlineAsmRegClass::sreg_low16)
+            | Arm(ArmInlineAsmRegClass::dreg_low8)
+            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
+            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
+            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
+            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
+            LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
+            Mips(MipsInlineAsmRegClass::reg) => "r",
+            Mips(MipsInlineAsmRegClass::freg) => "f",
+            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
+            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
+            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
+            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
+            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
+            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
+            PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
                 unreachable!("clobber-only")
             }
-            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
-                unreachable!("clobber-only")
-            }
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q",
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
-            | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk",
-            InlineAsmRegClass::X86(
+            RiscV(RiscVInlineAsmRegClass::reg) => "r",
+            RiscV(RiscVInlineAsmRegClass::freg) => "f",
+            RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
+            X86(X86InlineAsmRegClass::reg) => "r",
+            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
+            X86(X86InlineAsmRegClass::reg_byte) => "q",
+            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
+            X86(X86InlineAsmRegClass::zmm_reg) => "v",
+            X86(X86InlineAsmRegClass::kreg) => "^Yk",
+            X86(
                 X86InlineAsmRegClass::x87_reg
                 | X86InlineAsmRegClass::mmx_reg
                 | X86InlineAsmRegClass::kreg0
                 | X86InlineAsmRegClass::tmm_reg,
             ) => unreachable!("clobber-only"),
-            InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r",
-            InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w",
-            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d",
-            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r",
-            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w",
-            InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
-            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
-            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a",
-            InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d",
-            InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::reg) => "r",
-            InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
-                bug!("LLVM backend does not support SPIR-V")
-            }
-            InlineAsmRegClass::Err => unreachable!(),
+            Wasm(WasmInlineAsmRegClass::local) => "r",
+            Bpf(BpfInlineAsmRegClass::reg) => "r",
+            Bpf(BpfInlineAsmRegClass::wreg) => "w",
+            Avr(AvrInlineAsmRegClass::reg) => "r",
+            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
+            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
+            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
+            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
+            S390x(S390xInlineAsmRegClass::reg) => "r",
+            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
+            S390x(S390xInlineAsmRegClass::freg) => "f",
+            Msp430(Msp430InlineAsmRegClass::reg) => "r",
+            M68k(M68kInlineAsmRegClass::reg) => "r",
+            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
+            M68k(M68kInlineAsmRegClass::reg_data) => "d",
+            CSKY(CSKYInlineAsmRegClass::reg) => "r",
+            CSKY(CSKYInlineAsmRegClass::freg) => "f",
+            SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
+            Err => unreachable!(),
         }
         .to_string(),
     }
@@ -732,44 +727,41 @@ fn modifier_to_llvm(
     reg: InlineAsmRegClass,
     modifier: Option<char>,
 ) -> Option<char> {
+    use InlineAsmRegClass::*;
     // The modifiers can be retrieved from
     // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
     match reg {
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
-        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
-            if modifier == Some('v') { None } else { modifier }
-        }
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
-            unreachable!("clobber-only")
+        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
+        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
+            if modifier == Some('v') {
+                None
+            } else {
+                modifier
+            }
         }
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None,
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
+        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
+        Arm(ArmInlineAsmRegClass::reg) => None,
+        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
+        Arm(ArmInlineAsmRegClass::dreg)
+        | Arm(ArmInlineAsmRegClass::dreg_low16)
+        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
+        Arm(ArmInlineAsmRegClass::qreg)
+        | Arm(ArmInlineAsmRegClass::qreg_low8)
+        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
             if modifier.is_none() {
                 Some('q')
             } else {
                 modifier
             }
         }
-        InlineAsmRegClass::Hexagon(_) => None,
-        InlineAsmRegClass::LoongArch(_) => None,
-        InlineAsmRegClass::Mips(_) => None,
-        InlineAsmRegClass::Nvptx(_) => None,
-        InlineAsmRegClass::PowerPC(_) => None,
-        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
-        | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
-        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
-            unreachable!("clobber-only")
-        }
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
-        | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
+        Hexagon(_) => None,
+        LoongArch(_) => None,
+        Mips(_) => None,
+        Nvptx(_) => None,
+        PowerPC(_) => None,
+        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
+        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
+        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
             None if arch == InlineAsmArch::X86_64 => Some('q'),
             None => Some('k'),
             Some('l') => Some('b'),
@@ -779,10 +771,10 @@ fn modifier_to_llvm(
             Some('r') => Some('q'),
             _ => unreachable!(),
         },
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None,
-        InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg)
-        | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg)
-        | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
+        X86(X86InlineAsmRegClass::reg_byte) => None,
+        X86(reg @ X86InlineAsmRegClass::xmm_reg)
+        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
+        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
             (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
             (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
             (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
@@ -791,116 +783,97 @@ fn modifier_to_llvm(
             (_, Some('z')) => Some('g'),
             _ => unreachable!(),
         },
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
-        InlineAsmRegClass::X86(
+        X86(X86InlineAsmRegClass::kreg) => None,
+        X86(
             X86InlineAsmRegClass::x87_reg
             | X86InlineAsmRegClass::mmx_reg
             | X86InlineAsmRegClass::kreg0
             | X86InlineAsmRegClass::tmm_reg,
-        ) => {
-            unreachable!("clobber-only")
-        }
-        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None,
-        InlineAsmRegClass::Bpf(_) => None,
-        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair)
-        | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw)
-        | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
+        ) => unreachable!("clobber-only"),
+        Wasm(WasmInlineAsmRegClass::local) => None,
+        Bpf(_) => None,
+        Avr(AvrInlineAsmRegClass::reg_pair)
+        | Avr(AvrInlineAsmRegClass::reg_iw)
+        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
             Some('h') => Some('B'),
             Some('l') => Some('A'),
             _ => None,
         },
-        InlineAsmRegClass::Avr(_) => None,
-        InlineAsmRegClass::S390x(_) => None,
-        InlineAsmRegClass::Msp430(_) => None,
-        InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
-            bug!("LLVM backend does not support SPIR-V")
-        }
-        InlineAsmRegClass::M68k(_) => None,
-        InlineAsmRegClass::CSKY(_) => None,
-        InlineAsmRegClass::Err => unreachable!(),
+        Avr(_) => None,
+        S390x(_) => None,
+        Msp430(_) => None,
+        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
+        M68k(_) => None,
+        CSKY(_) => None,
+        Err => unreachable!(),
     }
 }
 
 /// Type to use for outputs that are discarded. It doesn't really matter what
 /// the type is, as long as it is valid for the constraint code.
 fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
+    use InlineAsmRegClass::*;
     match reg {
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
-        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
+        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
+        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
             cx.type_vector(cx.type_i64(), 2)
         }
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
-            unreachable!("clobber-only")
-        }
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
-        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
-            cx.type_vector(cx.type_i64(), 2)
-        }
-        InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
-        InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
-        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
-        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
-        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
-        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
-        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
-        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
-        | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
-            unreachable!("clobber-only")
-        }
-        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
-        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
+        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
+        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
+        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
+        Arm(ArmInlineAsmRegClass::dreg)
+        | Arm(ArmInlineAsmRegClass::dreg_low16)
+        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
+        Arm(ArmInlineAsmRegClass::qreg)
+        | Arm(ArmInlineAsmRegClass::qreg_low8)
+        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
+        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
+        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
+        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
+        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
+        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
+        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
+        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
+        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
+        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
+        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
+        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
+        PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
             unreachable!("clobber-only")
         }
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
-        | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
-        | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg)
-        | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
-        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
-        InlineAsmRegClass::X86(
+        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
+        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
+        RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"),
+        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
+        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
+        X86(X86InlineAsmRegClass::xmm_reg)
+        | X86(X86InlineAsmRegClass::ymm_reg)
+        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
+        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
+        X86(
             X86InlineAsmRegClass::x87_reg
             | X86InlineAsmRegClass::mmx_reg
             | X86InlineAsmRegClass::kreg0
             | X86InlineAsmRegClass::tmm_reg,
-        ) => {
-            unreachable!("clobber-only")
-        }
-        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
-        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
-        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
-        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
-        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
-        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
-        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
-        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
-        InlineAsmRegClass::S390x(
-            S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
-        ) => cx.type_i32(),
-        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
-        InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
-        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
-        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
-        InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
-        InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
-        InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
-            bug!("LLVM backend does not support SPIR-V")
-        }
-        InlineAsmRegClass::Err => unreachable!(),
+        ) => unreachable!("clobber-only"),
+        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
+        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
+        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
+        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
+        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
+        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
+        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
+        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
+        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
+        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
+        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
+        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
+        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
+        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
+        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
+        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
+        SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"),
+        Err => unreachable!(),
     }
 }
 
@@ -940,9 +913,10 @@ fn llvm_fixup_input<'ll, 'tcx>(
     layout: &TyAndLayout<'tcx>,
     instance: Instance<'_>,
 ) -> &'ll Value {
+    use InlineAsmRegClass::*;
     let dl = &bx.tcx.data_layout;
     match (reg, layout.abi) {
-        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
+        (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I8, _) = s.primitive() {
                 let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
                 bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
@@ -950,7 +924,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
                 value
             }
         }
-        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
+        (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
             if s.primitive() != Primitive::Float(Float::F128) =>
         {
             let elem_ty = llvm_asm_scalar_type(bx.cx, s);
@@ -963,26 +937,25 @@ fn llvm_fixup_input<'ll, 'tcx>(
             }
             bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
         }
-        (
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
-            Abi::Vector { element, count },
-        ) if layout.size.bytes() == 8 => {
+        (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count })
+            if layout.size.bytes() == 8 =>
+        {
             let elem_ty = llvm_asm_scalar_type(bx.cx, element);
             let vec_ty = bx.cx.type_vector(elem_ty, count);
             let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
             bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
         }
-        (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
+        (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
             if s.primitive() == Primitive::Float(Float::F64) =>
         {
             bx.bitcast(value, bx.cx.type_i64())
         }
         (
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
+            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
             Abi::Vector { .. },
         ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -994,7 +967,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
             bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
         }
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1009,7 +982,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
             bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
         }
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1018,10 +991,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
         ) if element.primitive() == Primitive::Float(Float::F16) => {
             bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
         }
-        (
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
-            Abi::Scalar(s),
-        ) => {
+        (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I32, _) = s.primitive() {
                 bx.bitcast(value, bx.cx.type_f32())
             } else {
@@ -1029,7 +999,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
             }
         }
         (
-            InlineAsmRegClass::Arm(
+            Arm(
                 ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
                 | ArmInlineAsmRegClass::dreg_low16,
@@ -1043,7 +1013,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
             }
         }
         (
-            InlineAsmRegClass::Arm(
+            Arm(
                 ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
                 | ArmInlineAsmRegClass::dreg_low16
@@ -1055,7 +1025,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
         ) if element.primitive() == Primitive::Float(Float::F16) => {
             bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
         }
-        (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
+        (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
             match s.primitive() {
                 // MIPS only supports register-length arithmetics.
                 Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
@@ -1064,7 +1034,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
                 _ => value,
             }
         }
-        (InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
+        (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
             if s.primitive() == Primitive::Float(Float::F16)
                 && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
         {
@@ -1086,15 +1056,16 @@ fn llvm_fixup_output<'ll, 'tcx>(
     layout: &TyAndLayout<'tcx>,
     instance: Instance<'_>,
 ) -> &'ll Value {
+    use InlineAsmRegClass::*;
     match (reg, layout.abi) {
-        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
+        (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I8, _) = s.primitive() {
                 bx.extract_element(value, bx.const_i32(0))
             } else {
                 value
             }
         }
-        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
+        (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
             if s.primitive() != Primitive::Float(Float::F128) =>
         {
             value = bx.extract_element(value, bx.const_i32(0));
@@ -1103,26 +1074,25 @@ fn llvm_fixup_output<'ll, 'tcx>(
             }
             value
         }
-        (
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
-            Abi::Vector { element, count },
-        ) if layout.size.bytes() == 8 => {
+        (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count })
+            if layout.size.bytes() == 8 =>
+        {
             let elem_ty = llvm_asm_scalar_type(bx.cx, element);
             let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
             let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
             bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
         }
-        (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
+        (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
             if s.primitive() == Primitive::Float(Float::F64) =>
         {
             bx.bitcast(value, bx.cx.type_f64())
         }
         (
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
+            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
             Abi::Vector { .. },
         ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1134,7 +1104,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
             bx.bitcast(value, bx.type_f128())
         }
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1145,7 +1115,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
             bx.extract_element(value, bx.const_usize(0))
         }
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1154,10 +1124,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
         ) if element.primitive() == Primitive::Float(Float::F16) => {
             bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
         }
-        (
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
-            Abi::Scalar(s),
-        ) => {
+        (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I32, _) = s.primitive() {
                 bx.bitcast(value, bx.cx.type_i32())
             } else {
@@ -1165,7 +1132,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
             }
         }
         (
-            InlineAsmRegClass::Arm(
+            Arm(
                 ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
                 | ArmInlineAsmRegClass::dreg_low16,
@@ -1179,7 +1146,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
             }
         }
         (
-            InlineAsmRegClass::Arm(
+            Arm(
                 ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
                 | ArmInlineAsmRegClass::dreg_low16
@@ -1191,7 +1158,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
         ) if element.primitive() == Primitive::Float(Float::F16) => {
             bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
         }
-        (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
+        (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
             match s.primitive() {
                 // MIPS only supports register-length arithmetics.
                 Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
@@ -1201,7 +1168,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
                 _ => value,
             }
         }
-        (InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
+        (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
             if s.primitive() == Primitive::Float(Float::F16)
                 && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
         {
@@ -1220,39 +1187,39 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
     layout: &TyAndLayout<'tcx>,
     instance: Instance<'_>,
 ) -> &'ll Type {
+    use InlineAsmRegClass::*;
     match (reg, layout.abi) {
-        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
+        (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I8, _) = s.primitive() {
                 cx.type_vector(cx.type_i8(), 8)
             } else {
                 layout.llvm_type(cx)
             }
         }
-        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
+        (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
             if s.primitive() != Primitive::Float(Float::F128) =>
         {
             let elem_ty = llvm_asm_scalar_type(cx, s);
             let count = 16 / layout.size.bytes();
             cx.type_vector(elem_ty, count)
         }
-        (
-            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
-            Abi::Vector { element, count },
-        ) if layout.size.bytes() == 8 => {
+        (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count })
+            if layout.size.bytes() == 8 =>
+        {
             let elem_ty = llvm_asm_scalar_type(cx, element);
             cx.type_vector(elem_ty, count * 2)
         }
-        (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
+        (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
             if s.primitive() == Primitive::Float(Float::F64) =>
         {
             cx.type_i64()
         }
         (
-            InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
+            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
             Abi::Vector { .. },
         ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1264,7 +1231,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
             cx.type_vector(cx.type_i32(), 4)
         }
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1272,7 +1239,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
             Abi::Scalar(s),
         ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
         (
-            InlineAsmRegClass::X86(
+            X86(
                 X86InlineAsmRegClass::xmm_reg
                 | X86InlineAsmRegClass::ymm_reg
                 | X86InlineAsmRegClass::zmm_reg,
@@ -1281,10 +1248,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
         ) if element.primitive() == Primitive::Float(Float::F16) => {
             cx.type_vector(cx.type_i16(), count)
         }
-        (
-            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
-            Abi::Scalar(s),
-        ) => {
+        (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I32, _) = s.primitive() {
                 cx.type_f32()
             } else {
@@ -1292,7 +1256,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
             }
         }
         (
-            InlineAsmRegClass::Arm(
+            Arm(
                 ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
                 | ArmInlineAsmRegClass::dreg_low16,
@@ -1306,7 +1270,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
             }
         }
         (
-            InlineAsmRegClass::Arm(
+            Arm(
                 ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
                 | ArmInlineAsmRegClass::dreg_low16
@@ -1318,7 +1282,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
         ) if element.primitive() == Primitive::Float(Float::F16) => {
             cx.type_vector(cx.type_i16(), count)
         }
-        (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
+        (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
             match s.primitive() {
                 // MIPS only supports register-length arithmetics.
                 Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
@@ -1327,7 +1291,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
                 _ => layout.llvm_type(cx),
             }
         }
-        (InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
+        (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
             if s.primitive() == Primitive::Float(Float::F16)
                 && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
         {
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 92a857c2adc..489259da856 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -403,34 +403,40 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) {
         to_add.push(AttributeKind::Naked.create_attr(cx.llcx));
         // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions.
-        // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules.
-        // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768
+        // And it is a module-level attribute, so the alternative is pulling naked functions into
+        // new LLVM modules. Otherwise LLVM's "naked" functions come with endbr prefixes per
+        // https://github.com/rust-lang/rust/issues/98768
         to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx));
         if llvm_util::get_version() < (19, 0, 0) {
             // Prior to LLVM 19, branch-target-enforcement was disabled by setting the attribute to
             // the string "false". Now it is disabled by absence of the attribute.
             to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false"));
         }
-    } else if llvm_util::get_version() >= (19, 0, 0) {
-        // For non-naked functions, set branch protection attributes on aarch64.
-        if let Some(BranchProtection { bti, pac_ret }) =
-            cx.sess().opts.unstable_opts.branch_protection
-        {
-            assert!(cx.sess().target.arch == "aarch64");
-            if bti {
-                to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement"));
-            }
-            if let Some(PacRet { leaf, key }) = pac_ret {
-                to_add.push(llvm::CreateAttrStringValue(
-                    cx.llcx,
-                    "sign-return-address",
-                    if leaf { "all" } else { "non-leaf" },
-                ));
-                to_add.push(llvm::CreateAttrStringValue(
-                    cx.llcx,
-                    "sign-return-address-key",
-                    if key == PAuthKey::A { "a_key" } else { "b_key" },
-                ));
+    } else {
+        // Do not set sanitizer attributes for naked functions.
+        to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize));
+
+        if llvm_util::get_version() >= (19, 0, 0) {
+            // For non-naked functions, set branch protection attributes on aarch64.
+            if let Some(BranchProtection { bti, pac_ret }) =
+                cx.sess().opts.unstable_opts.branch_protection
+            {
+                assert!(cx.sess().target.arch == "aarch64");
+                if bti {
+                    to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement"));
+                }
+                if let Some(PacRet { leaf, key }) = pac_ret {
+                    to_add.push(llvm::CreateAttrStringValue(
+                        cx.llcx,
+                        "sign-return-address",
+                        if leaf { "all" } else { "non-leaf" },
+                    ));
+                    to_add.push(llvm::CreateAttrStringValue(
+                        cx.llcx,
+                        "sign-return-address-key",
+                        if key == PAuthKey::A { "a_key" } else { "b_key" },
+                    ));
+                }
             }
         }
     }
@@ -449,7 +455,8 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
             flags |= AllocKindFlags::Zeroed;
         }
         to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags));
-        // apply to return place instead of function (unlike all other attributes applied in this function)
+        // apply to return place instead of function (unlike all other attributes applied in this
+        // function)
         let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx);
         attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]);
     }
@@ -476,16 +483,12 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx);
         attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]);
     }
-    if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) {
-        to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry"));
-    }
     if let Some(align) = codegen_fn_attrs.alignment {
         llvm::set_alignment(llfn, align);
     }
     if let Some(backchain) = backchain_attr(cx) {
         to_add.push(backchain);
     }
-    to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize));
     to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry));
 
     // Always annotate functions with the target-cpu they are compiled for.
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs
index 4f2c83634a8..1b4f6682af0 100644
--- a/compiler/rustc_codegen_llvm/src/back/archive.rs
+++ b/compiler/rustc_codegen_llvm/src/back/archive.rs
@@ -1,12 +1,12 @@
 //! A helper class for dealing with static archives
 
-use std::ffi::{c_char, c_void, CStr, CString};
+use std::ffi::{CStr, CString, c_char, c_void};
 use std::path::{Path, PathBuf};
 use std::{io, mem, ptr, str};
 
 use rustc_codegen_ssa::back::archive::{
-    try_extract_macho_fat_archive, ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder,
-    ArchiveBuilderBuilder, ObjectReader, UnknownArchiveKind, DEFAULT_OBJECT_READER,
+    ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder,
+    DEFAULT_OBJECT_READER, ObjectReader, UnknownArchiveKind, try_extract_macho_fat_archive,
 };
 use rustc_session::Session;
 
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index aa6842c75ce..5e510177818 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -11,7 +11,7 @@ use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModul
 use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, TargetMachineFactoryConfig};
 use rustc_codegen_ssa::traits::*;
-use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
+use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_errors::{DiagCtxtHandle, FatalError};
@@ -23,7 +23,7 @@ use rustc_session::config::{self, CrateType, Lto};
 use tracing::{debug, info};
 
 use crate::back::write::{
-    self, bitcode_section_name, save_temp_bitcode, CodegenDiagnosticsStage, DiagnosticHandlers,
+    self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode,
 };
 use crate::errors::{
     DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro,
@@ -92,11 +92,9 @@ fn prepare_lto(
                     dcx.emit_err(LtoDylib);
                     return Err(FatalError);
                 }
-            } else if *crate_type == CrateType::ProcMacro {
-                if !cgcx.opts.unstable_opts.dylib_lto {
-                    dcx.emit_err(LtoProcMacro);
-                    return Err(FatalError);
-                }
+            } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
+                dcx.emit_err(LtoProcMacro);
+                return Err(FatalError);
             }
         }
 
@@ -158,15 +156,15 @@ fn get_bitcode_slice_from_object_data<'a>(
     obj: &'a [u8],
     cgcx: &CodegenContext<LlvmCodegenBackend>,
 ) -> Result<&'a [u8], LtoBitcodeFromRlib> {
-    // We're about to assume the data here is an object file with sections, but if it's raw LLVM IR that
-    // won't work. Fortunately, if that's what we have we can just return the object directly, so we sniff
-    // the relevant magic strings here and return.
+    // We're about to assume the data here is an object file with sections, but if it's raw LLVM IR
+    // that won't work. Fortunately, if that's what we have we can just return the object directly,
+    // so we sniff the relevant magic strings here and return.
     if obj.starts_with(b"\xDE\xC0\x17\x0B") || obj.starts_with(b"BC\xC0\xDE") {
         return Ok(obj);
     }
-    // We drop the "__LLVM," prefix here because on Apple platforms there's a notion of "segment name"
-    // which in the public API for sections gets treated as part of the section name, but internally
-    // in MachOObjectFile.cpp gets treated separately.
+    // We drop the "__LLVM," prefix here because on Apple platforms there's a notion of "segment
+    // name" which in the public API for sections gets treated as part of the section name, but
+    // internally in MachOObjectFile.cpp gets treated separately.
     let section_name = bitcode_section_name(cgcx).trim_start_matches("__LLVM,");
     let mut len = 0;
     let data = unsafe {
@@ -314,7 +312,6 @@ fn fat_lto(
             }
         }
     };
-    let mut serialized_bitcode = Vec::new();
     {
         let (llcx, llmod) = {
             let llvm = &module.module_llvm;
@@ -342,9 +339,7 @@ fn fat_lto(
         serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
 
         // For all serialized bitcode files we parse them and link them in as we did
-        // above, this is all mostly handled in C++. Like above, though, we don't
-        // know much about the memory management here so we err on the side of being
-        // save and persist everything with the original module.
+        // above, this is all mostly handled in C++.
         let mut linker = Linker::new(llmod);
         for (bc_decoded, name) in serialized_modules {
             let _timer = cgcx
@@ -355,7 +350,6 @@ fn fat_lto(
             info!("linking {:?}", name);
             let data = bc_decoded.data();
             linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?;
-            serialized_bitcode.push(bc_decoded);
         }
         drop(linker);
         save_temp_bitcode(cgcx, &module, "lto.input");
@@ -372,7 +366,7 @@ fn fat_lto(
         }
     }
 
-    Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode })
+    Ok(LtoModuleCodegen::Fat(module))
 }
 
 pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>);
@@ -850,7 +844,7 @@ impl ThinLTOKeysMap {
                     llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0);
                 })
                 .expect("Invalid ThinLTO module key");
-                (name.clone().into_string().unwrap(), key)
+                (module_name_to_str(name).to_string(), key)
             })
             .collect();
         Self { keys }
diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
index 681ac75c877..44c30d22a9e 100644
--- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
+++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
@@ -1,4 +1,4 @@
-use std::ffi::{c_char, CStr};
+use std::ffi::{CStr, c_char};
 use std::marker::PhantomData;
 use std::ops::Deref;
 use std::ptr::NonNull;
@@ -30,7 +30,7 @@ impl OwnedTargetMachine {
         data_sections: bool,
         unique_section_names: bool,
         trap_unreachable: bool,
-        singletree: bool,
+        singlethread: bool,
         verbose_asm: bool,
         emit_stack_size_section: bool,
         relax_elf_relocations: bool,
@@ -62,7 +62,7 @@ impl OwnedTargetMachine {
                 data_sections,
                 unique_section_names,
                 trap_unreachable,
-                singletree,
+                singlethread,
                 verbose_asm,
                 emit_stack_size_section,
                 relax_elf_relocations,
@@ -86,15 +86,17 @@ impl Deref for OwnedTargetMachine {
     type Target = llvm::TargetMachine;
 
     fn deref(&self) -> &Self::Target {
-        // SAFETY: constructing ensures we have a valid pointer created by llvm::LLVMRustCreateTargetMachine
+        // SAFETY: constructing ensures we have a valid pointer created by
+        // llvm::LLVMRustCreateTargetMachine.
         unsafe { self.tm_unique.as_ref() }
     }
 }
 
 impl Drop for OwnedTargetMachine {
     fn drop(&mut self) {
-        // SAFETY: constructing ensures we have a valid pointer created by llvm::LLVMRustCreateTargetMachine
-        // OwnedTargetMachine is not copyable so there is no double free or use after free
+        // SAFETY: constructing ensures we have a valid pointer created by
+        // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no
+        // double free or use after free.
         unsafe {
             llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut());
         }
diff --git a/compiler/rustc_codegen_llvm/src/back/profiling.rs b/compiler/rustc_codegen_llvm/src/back/profiling.rs
index 79794775b7b..73ae0072c42 100644
--- a/compiler/rustc_codegen_llvm/src/back/profiling.rs
+++ b/compiler/rustc_codegen_llvm/src/back/profiling.rs
@@ -1,4 +1,4 @@
-use std::ffi::{c_void, CStr};
+use std::ffi::{CStr, c_void};
 use std::os::raw::c_char;
 use std::sync::Arc;
 
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index adf7e933f43..afdd2b581b8 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -20,28 +20,28 @@ use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_errors::{DiagCtxtHandle, FatalError, Level};
 use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_middle::ty::TyCtxt;
+use rustc_session::Session;
 use rustc_session::config::{
     self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath,
 };
-use rustc_session::Session;
-use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
+use rustc_span::symbol::sym;
 use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
 use tracing::debug;
 
 use crate::back::lto::ThinBuffer;
 use crate::back::owned_target_machine::OwnedTargetMachine;
 use crate::back::profiling::{
-    selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler,
+    LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback,
 };
 use crate::errors::{
     CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
     WithLlvmError, WriteBytecode,
 };
-use crate::llvm::diagnostic::OptimizationDiagnosticKind;
+use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
 use crate::llvm::{self, DiagnosticInfo, PassManager};
 use crate::type_::Type;
-use crate::{base, common, llvm_util, LlvmCodegenBackend, ModuleLlvm};
+use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
 
 pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError {
     match llvm::last_error() {
@@ -157,7 +157,8 @@ fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel
 fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
     match relocation_model {
         RelocModel::Static => llvm::RelocModel::Static,
-        // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute.
+        // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra
+        // attribute.
         RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
         RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
         RelocModel::Ropi => llvm::RelocModel::ROPI,
@@ -185,7 +186,13 @@ pub(crate) fn target_machine_factory(
     let reloc_model = to_llvm_relocation_model(sess.relocation_model());
 
     let (opt_level, _) = to_llvm_opt_settings(optlvl);
-    let use_softfp = sess.opts.cg.soft_float;
+    let use_softfp = if sess.target.arch == "arm" && sess.target.abi == "eabihf" {
+        sess.opts.cg.soft_float
+    } else {
+        // `validate_commandline_args_with_session_available` has already warned about this being
+        // ignored. Let's make sure LLVM doesn't suddenly start using this flag on more targets.
+        false
+    };
 
     let ffunction_sections =
         sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections);
@@ -440,13 +447,12 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void
                 column: opt.column,
                 pass_name: &opt.pass_name,
                 kind: match opt.kind {
-                    OptimizationDiagnosticKind::OptimizationRemark => "success",
-                    OptimizationDiagnosticKind::OptimizationMissed
-                    | OptimizationDiagnosticKind::OptimizationFailure => "missed",
-                    OptimizationDiagnosticKind::OptimizationAnalysis
-                    | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute
-                    | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis",
-                    OptimizationDiagnosticKind::OptimizationRemarkOther => "other",
+                    OptimizationRemark => "success",
+                    OptimizationMissed | OptimizationFailure => "missed",
+                    OptimizationAnalysis
+                    | OptimizationAnalysisFPCommute
+                    | OptimizationAnalysisAliasing => "analysis",
+                    OptimizationRemarkOther => "other",
                 },
                 message: &opt.message,
             });
@@ -571,6 +577,7 @@ pub(crate) unsafe fn llvm_optimize(
             cgcx.opts.cg.linker_plugin_lto.enabled(),
             config.no_prepopulate_passes,
             config.verify_llvm_ir,
+            config.lint_llvm_ir,
             using_thin_buffers,
             config.merge_functions,
             unroll_loops,
@@ -938,11 +945,12 @@ fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data:
 }
 
 fn target_is_apple(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool {
-    cgcx.opts.target_triple.triple().contains("-ios")
-        || cgcx.opts.target_triple.triple().contains("-darwin")
-        || cgcx.opts.target_triple.triple().contains("-tvos")
-        || cgcx.opts.target_triple.triple().contains("-watchos")
-        || cgcx.opts.target_triple.triple().contains("-visionos")
+    let triple = cgcx.opts.target_triple.triple();
+    triple.contains("-ios")
+        || triple.contains("-darwin")
+        || triple.contains("-tvos")
+        || triple.contains("-watchos")
+        || triple.contains("-visionos")
 }
 
 fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool {
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 40783825cae..4e0c62c8bf8 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -3,11 +3,11 @@ use std::ops::Deref;
 use std::{iter, ptr};
 
 use libc::{c_char, c_uint};
+use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
-use rustc_codegen_ssa::MemFlags;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
@@ -26,13 +26,13 @@ use smallvec::SmallVec;
 use tracing::{debug, instrument};
 
 use crate::abi::FnAbiLlvmExt;
+use crate::attributes;
 use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True};
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
-use crate::{attributes, llvm_util};
 
 // All Builders must have an llfn associated with them
 #[must_use]
@@ -56,6 +56,7 @@ const UNNAMED: *const c_char = c"".as_ptr();
 
 impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> {
     type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value;
+    type Metadata = <CodegenCx<'ll, 'tcx> as BackendTypes>::Metadata;
     type Function = <CodegenCx<'ll, 'tcx> as BackendTypes>::Function;
     type BasicBlock = <CodegenCx<'ll, 'tcx> as BackendTypes>::BasicBlock;
     type Type = <CodegenCx<'ll, 'tcx> as BackendTypes>::Type;
@@ -93,8 +94,6 @@ impl HasTargetSpec for Builder<'_, '_, '_> {
 }
 
 impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
-    type LayoutOfResult = TyAndLayout<'tcx>;
-
     #[inline]
     fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
         self.cx.handle_layout_err(err, span, ty)
@@ -102,8 +101,6 @@ impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
 }
 
 impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
-    type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
-
     #[inline]
     fn handle_fn_abi_err(
         &self,
@@ -124,11 +121,7 @@ impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> {
-    type CodegenCx = CodegenCx<'ll, 'tcx>;
-}
-
-macro_rules! builder_methods_for_value_instructions {
+macro_rules! math_builder_methods {
     ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
         $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
             unsafe {
@@ -138,7 +131,21 @@ macro_rules! builder_methods_for_value_instructions {
     }
 }
 
+macro_rules! set_math_builder_methods {
+    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
+        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
+            unsafe {
+                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
+                llvm::$llvm_set_math(instr);
+                instr
+            }
+        })+
+    }
+}
+
 impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
+    type CodegenCx = CodegenCx<'ll, 'tcx>;
+
     fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
         let bx = Builder::with_cx(cx);
         unsafe {
@@ -273,7 +280,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
-    builder_methods_for_value_instructions! {
+    math_builder_methods! {
         add(a, b) => LLVMBuildAdd,
         fadd(a, b) => LLVMBuildFAdd,
         sub(a, b) => LLVMBuildSub,
@@ -305,84 +312,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unchecked_umul(x, y) => LLVMBuildNUWMul,
     }
 
-    fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetFastMath(instr);
-            instr
-        }
-    }
-
-    fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetAlgebraicMath(instr);
-            instr
-        }
-    }
-
-    fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetAlgebraicMath(instr);
-            instr
-        }
-    }
-
-    fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetAlgebraicMath(instr);
-            instr
-        }
-    }
-
-    fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetAlgebraicMath(instr);
-            instr
-        }
-    }
-
-    fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        unsafe {
-            let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
-            llvm::LLVMRustSetAlgebraicMath(instr);
-            instr
-        }
+    set_math_builder_methods! {
+        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
+        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
+        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
+        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
+        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
+        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
+        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
+        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
+        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
+        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
     }
 
     fn checked_binop(
@@ -465,6 +405,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             val
         }
     }
+
     fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
         if scalar.is_bool() {
             return self.trunc(val, self.cx().type_i1());
@@ -683,26 +624,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
         unsafe {
             let llty = self.cx.val_ty(load);
-            let v = [
-                self.cx.const_uint_big(llty, range.start),
-                self.cx.const_uint_big(llty, range.end.wrapping_add(1)),
+            let md = [
+                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
+                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
             ];
-
-            llvm::LLVMSetMetadata(
-                load,
-                llvm::MD_range as c_uint,
-                llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
-            );
+            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
+            self.set_metadata(load, llvm::MD_range, md);
         }
     }
 
     fn nonnull_metadata(&mut self, load: &'ll Value) {
         unsafe {
-            llvm::LLVMSetMetadata(
-                load,
-                llvm::MD_nonnull as c_uint,
-                llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
-            );
+            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
+            self.set_metadata(load, llvm::MD_nonnull, md);
         }
     }
 
@@ -733,11 +667,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 // for performance. LLVM doesn't seem to care about this, and will happily treat
                 // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
                 // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
-                // regular stores but require special fences.
-                // So we keep a list of architectures where `!nontemporal` is known to be truly just
-                // a hint, and use regular stores everywhere else.
-                // (In the future, we could alternatively ensure that an sfence gets emitted after a sequence of movnt
-                // before any kind of synchronizing operation. But it's not clear how to do that with LLVM.)
+                // regular stores but require special fences. So we keep a list of architectures
+                // where `!nontemporal` is known to be truly just a hint, and use regular stores
+                // everywhere else. (In the future, we could alternatively ensure that an sfence
+                // gets emitted after a sequence of movnt before any kind of synchronizing
+                // operation. But it's not clear how to do that with LLVM.)
                 // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
                 // <https://github.com/llvm/llvm-project/issues/64521>.
                 const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
@@ -750,9 +684,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                     // *always* point to a metadata value of the integer 1.
                     //
                     // [1]: https://llvm.org/docs/LangRef.html#store-instruction
-                    let one = self.cx.const_i32(1);
-                    let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1);
-                    llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node);
+                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
+                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
+                    self.set_metadata(store, llvm::MD_nontemporal, md);
                 }
             }
             store
@@ -1166,6 +1100,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             (val, success)
         }
     }
+
     fn atomic_rmw(
         &mut self,
         op: rustc_codegen_ssa::common::AtomicRmwBinOp,
@@ -1216,11 +1151,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn set_invariant_load(&mut self, load: &'ll Value) {
         unsafe {
-            llvm::LLVMSetMetadata(
-                load,
-                llvm::MD_invariant_load as c_uint,
-                llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
-            );
+            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
+            self.set_metadata(load, llvm::MD_invariant_load, md);
         }
     }
 
@@ -1317,15 +1249,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
-        if llvm_util::get_version() < (17, 0, 2) {
-            // Work around https://github.com/llvm/llvm-project/issues/66984.
-            let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx);
-            attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]);
-        } else {
-            // Cleanup is always the cold path.
-            let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
-            attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
-        }
+        // Cleanup is always the cold path.
+        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
+        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
     }
 }
 
@@ -1355,33 +1281,23 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
 
     fn align_metadata(&mut self, load: &'ll Value, align: Align) {
         unsafe {
-            let v = [self.cx.const_u64(align.bytes())];
-
-            llvm::LLVMSetMetadata(
-                load,
-                llvm::MD_align as c_uint,
-                llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
-            );
+            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
+            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
+            self.set_metadata(load, llvm::MD_align, md);
         }
     }
 
     fn noundef_metadata(&mut self, load: &'ll Value) {
         unsafe {
-            llvm::LLVMSetMetadata(
-                load,
-                llvm::MD_noundef as c_uint,
-                llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
-            );
+            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
+            self.set_metadata(load, llvm::MD_noundef, md);
         }
     }
 
     pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
         unsafe {
-            llvm::LLVMSetMetadata(
-                inst,
-                llvm::MD_unpredictable as c_uint,
-                llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
-            );
+            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
+            self.set_metadata(inst, llvm::MD_unpredictable, md);
         }
     }
 
@@ -1767,8 +1683,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     ) {
         debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes);
 
-        assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later");
-
         let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) };
         let llty = self.cx.type_func(
             &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()],
@@ -1802,7 +1716,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
             fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp
         );
-        assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later");
 
         let llfn =
             unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) };
@@ -1844,7 +1757,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
             fn_name, hash, cond_loc, mcdc_temp, bool_value
         );
-        assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later");
         let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) };
         let llty = self.cx.type_func(
             &[
diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs
index 949fd1bc124..206a7069792 100644
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@@ -15,11 +15,6 @@ use crate::value::Value;
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
-///
-/// # Parameters
-///
-/// - `cx`: the crate context
-/// - `instance`: the instance to be instantiated
 pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value {
     let tcx = cx.tcx();
 
@@ -106,62 +101,42 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t
             let is_generic =
                 instance.args.non_erasable_generics(tcx, instance.def_id()).next().is_some();
 
-            if is_generic {
-                // This is a monomorphization. Its expected visibility depends
-                // on whether we are in share-generics mode.
-
-                if cx.tcx.sess.opts.share_generics() {
-                    // We are in share_generics mode.
-
+            let is_hidden = if is_generic {
+                // This is a monomorphization of a generic function.
+                if !cx.tcx.sess.opts.share_generics() {
+                    // When not sharing generics, all instances are in the same
+                    // crate and have hidden visibility.
+                    true
+                } else {
                     if let Some(instance_def_id) = instance_def_id.as_local() {
-                        // This is a definition from the current crate. If the
-                        // definition is unreachable for downstream crates or
-                        // the current crate does not re-export generics, the
-                        // definition of the instance will have been declared
-                        // as `hidden`.
-                        if cx.tcx.is_unreachable_local_definition(instance_def_id)
+                        // This is a monomorphization of a generic function
+                        // defined in the current crate. It is hidden if:
+                        // - the definition is unreachable for downstream
+                        //   crates, or
+                        // - the current crate does not re-export generics
+                        //   (because the crate is a C library or executable)
+                        cx.tcx.is_unreachable_local_definition(instance_def_id)
                             || !cx.tcx.local_crate_exports_generics()
-                        {
-                            llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
-                        }
                     } else {
                         // This is a monomorphization of a generic function
-                        // defined in an upstream crate.
-                        if instance.upstream_monomorphization(tcx).is_some() {
-                            // This is instantiated in another crate. It cannot
-                            // be `hidden`.
-                        } else {
-                            // This is a local instantiation of an upstream definition.
-                            // If the current crate does not re-export it
-                            // (because it is a C library or an executable), it
-                            // will have been declared `hidden`.
-                            if !cx.tcx.local_crate_exports_generics() {
-                                llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
-                            }
-                        }
+                        // defined in an upstream crate. It is hidden if:
+                        // - it is instantiated in this crate, and
+                        // - the current crate does not re-export generics
+                        instance.upstream_monomorphization(tcx).is_none()
+                            && !cx.tcx.local_crate_exports_generics()
                     }
-                } else {
-                    // When not sharing generics, all instances are in the same
-                    // crate and have hidden visibility
-                    llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
                 }
             } else {
-                // This is a non-generic function
-                if cx.tcx.is_codegened_item(instance_def_id) {
-                    // This is a function that is instantiated in the local crate
-
-                    if instance_def_id.is_local() {
-                        // This is function that is defined in the local crate.
-                        // If it is not reachable, it is hidden.
-                        if !cx.tcx.is_reachable_non_generic(instance_def_id) {
-                            llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
-                        }
-                    } else {
-                        // This is a function from an upstream crate that has
-                        // been instantiated here. These are always hidden.
-                        llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
-                    }
-                }
+                // This is a non-generic function. It is hidden if:
+                // - it is instantiated in the local crate, and
+                //   - it is defined an upstream crate (non-local), or
+                //   - it is not reachable
+                cx.tcx.is_codegened_item(instance_def_id)
+                    && (!instance_def_id.is_local()
+                        || !cx.tcx.is_reachable_non_generic(instance_def_id))
+            };
+            if is_hidden {
+                llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
             }
 
             // MinGW: For backward compatibility we rely on the linker to decide whether it
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index ef6560ecbe5..31d59905446 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -14,7 +14,7 @@ use tracing::debug;
 
 use crate::consts::const_alloc_to_llvm;
 pub(crate) use crate::context::CodegenCx;
-use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True};
+use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, OperandBundleDef, True};
 use crate::type_::Type;
 use crate::value::Value;
 
@@ -79,6 +79,7 @@ impl<'ll> Funclet<'ll> {
 
 impl<'ll> BackendTypes for CodegenCx<'ll, '_> {
     type Value = &'ll Value;
+    type Metadata = &'ll Metadata;
     // FIXME(eddyb) replace this with a `Function` "subclass" of `Value`.
     type Function = &'ll Value;
 
@@ -113,7 +114,7 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 }
 
-impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn const_null(&self, t: &'ll Type) -> &'ll Value {
         unsafe { llvm::LLVMConstNull(t) }
     }
@@ -126,25 +127,14 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         unsafe { llvm::LLVMGetPoison(t) }
     }
 
-    fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value {
-        unsafe { llvm::LLVMConstInt(t, i as u64, True) }
-    }
-
-    fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value {
-        unsafe { llvm::LLVMConstInt(t, i, False) }
-    }
-
-    fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value {
-        unsafe {
-            let words = [u as u64, (u >> 64) as u64];
-            llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr())
-        }
-    }
-
     fn const_bool(&self, val: bool) -> &'ll Value {
         self.const_uint(self.type_i1(), val as u64)
     }
 
+    fn const_i8(&self, i: i8) -> &'ll Value {
+        self.const_int(self.type_i8(), i as i64)
+    }
+
     fn const_i16(&self, i: i16) -> &'ll Value {
         self.const_int(self.type_i16(), i as i64)
     }
@@ -153,8 +143,12 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         self.const_int(self.type_i32(), i as i64)
     }
 
-    fn const_i8(&self, i: i8) -> &'ll Value {
-        self.const_int(self.type_i8(), i as i64)
+    fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value {
+        unsafe { llvm::LLVMConstInt(t, i as u64, True) }
+    }
+
+    fn const_u8(&self, i: u8) -> &'ll Value {
+        self.const_uint(self.type_i8(), i as u64)
     }
 
     fn const_u32(&self, i: u32) -> &'ll Value {
@@ -179,8 +173,15 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         self.const_uint(self.isize_ty, i)
     }
 
-    fn const_u8(&self, i: u8) -> &'ll Value {
-        self.const_uint(self.type_i8(), i as u64)
+    fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value {
+        unsafe { llvm::LLVMConstInt(t, i, False) }
+    }
+
+    fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value {
+        unsafe {
+            let words = [u as u64, (u >> 64) as u64];
+            llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr())
+        }
     }
 
     fn const_real(&self, t: &'ll Type, val: f64) -> &'ll Value {
@@ -290,10 +291,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                         self.get_fn_addr(instance.polymorphize(self.tcx)),
                         self.data_layout().instruction_address_space,
                     ),
-                    GlobalAlloc::VTable(ty, trait_ref) => {
+                    GlobalAlloc::VTable(ty, dyn_ty) => {
                         let alloc = self
                             .tcx
-                            .global_alloc(self.tcx.vtable_allocation((ty, trait_ref)))
+                            .global_alloc(self.tcx.vtable_allocation((ty, dyn_ty.principal())))
                             .unwrap_memory();
                         let init = const_alloc_to_llvm(self, alloc, /*static*/ false);
                         let value = self.static_addr_of(init, alloc.inner().align, None);
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index b143e28c5f9..33a85adeb87 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -6,8 +6,8 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::interpret::{
-    read_target_uint, Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer,
-    Scalar as InterpScalar,
+    Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar,
+    read_target_uint,
 };
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::layout::LayoutOf;
@@ -73,8 +73,8 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
 
         // Generating partially-uninit consts is limited to small numbers of chunks,
         // to avoid the cost of generating large complex const expressions.
-        // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element,
-        // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`.
+        // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, and
+        // would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`.
         let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold;
         let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max;
 
@@ -249,8 +249,8 @@ impl<'ll> CodegenCx<'ll, '_> {
         trace!(?instance);
 
         let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
-        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
-        // the llvm type from the actual evaluated initializer.
+        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure
+        // out the llvm type from the actual evaluated initializer.
         let llty = if nested {
             self.type_i8()
         } else {
@@ -262,7 +262,7 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 
     #[instrument(level = "debug", skip(self, llty))]
-    pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value {
+    fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value {
         let instance = Instance::mono(self.tcx, def_id);
         if let Some(&g) = self.instances.borrow().get(&instance) {
             trace!("used cached value");
@@ -320,15 +320,16 @@ impl<'ll> CodegenCx<'ll, '_> {
         }
 
         if !def_id.is_local() {
-            let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) &&
+            let needs_dll_storage_attr = self.use_dll_storage_attrs
+                && !self.tcx.is_foreign_item(def_id)
                 // Local definitions can never be imported, so we must not apply
                 // the DLLImport annotation.
-                !dso_local &&
+                && !dso_local
                 // ThinLTO can't handle this workaround in all cases, so we don't
                 // emit the attrs. Instead we make them unnecessary by disallowing
                 // dynamic linking when linker plugin based LTO is enabled.
-                !self.tcx.sess.opts.cg.linker_plugin_lto.enabled() &&
-                self.tcx.sess.lto() != Lto::Thin;
+                && !self.tcx.sess.opts.cg.linker_plugin_lto.enabled()
+                && self.tcx.sess.lto() != Lto::Thin;
 
             // If this assertion triggers, there's something wrong with commandline
             // argument validation.
@@ -390,7 +391,7 @@ impl<'ll> CodegenCx<'ll, '_> {
             let val_llty = self.val_ty(v);
 
             let g = self.get_static_inner(def_id, val_llty);
-            let llty = self.val_ty(g);
+            let llty = llvm::LLVMGlobalGetValueType(g);
 
             let g = if val_llty == llty {
                 g
@@ -442,58 +443,6 @@ impl<'ll> CodegenCx<'ll, '_> {
 
             if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
                 llvm::set_thread_local_mode(g, self.tls_model);
-
-                // Do not allow LLVM to change the alignment of a TLS on macOS.
-                //
-                // By default a global's alignment can be freely increased.
-                // This allows LLVM to generate more performant instructions
-                // e.g., using load-aligned into a SIMD register.
-                //
-                // However, on macOS 10.10 or below, the dynamic linker does not
-                // respect any alignment given on the TLS (radar 24221680).
-                // This will violate the alignment assumption, and causing segfault at runtime.
-                //
-                // This bug is very easy to trigger. In `println!` and `panic!`,
-                // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS,
-                // which the values would be `mem::replace`d on initialization.
-                // The implementation of `mem::replace` will use SIMD
-                // whenever the size is 32 bytes or higher. LLVM notices SIMD is used
-                // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary,
-                // which macOS's dyld disregarded and causing crashes
-                // (see issues #51794, #51758, #50867, #48866 and #44056).
-                //
-                // To workaround the bug, we trick LLVM into not increasing
-                // the global's alignment by explicitly assigning a section to it
-                // (equivalent to automatically generating a `#[link_section]` attribute).
-                // See the comment in the `GlobalValue::canIncreaseAlignment()` function
-                // of `lib/IR/Globals.cpp` for why this works.
-                //
-                // When the alignment is not increased, the optimized `mem::replace`
-                // will use load-unaligned instructions instead, and thus avoiding the crash.
-                //
-                // We could remove this hack whenever we decide to drop macOS 10.10 support.
-                if self.tcx.sess.target.is_like_osx {
-                    // The `inspect` method is okay here because we checked for provenance, and
-                    // because we are doing this access to inspect the final interpreter state
-                    // (not as part of the interpreter execution).
-                    //
-                    // FIXME: This check requires that the (arbitrary) value of undefined bytes
-                    // happens to be zero. Instead, we should only check the value of defined bytes
-                    // and set all undefined bytes to zero if this allocation is headed for the
-                    // BSS.
-                    let all_bytes_are_zero = alloc.provenance().ptrs().is_empty()
-                        && alloc
-                            .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len())
-                            .iter()
-                            .all(|&byte| byte == 0);
-
-                    let sect_name = if all_bytes_are_zero {
-                        c"__DATA,__thread_bss"
-                    } else {
-                        c"__DATA,__thread_data"
-                    };
-                    llvm::LLVMSetSection(g, sect_name.as_ptr());
-                }
             }
 
             // Wasm statics with custom link sections get special treatment as they
@@ -551,8 +500,8 @@ impl<'ll> CodegenCx<'ll, '_> {
                 // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage
                 // on other targets, in particular MachO targets have *their* static constructor
                 // lists broken if `llvm.compiler.used` is emitted rather than `llvm.used`. However,
-                // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_hir_analysis`,
-                // so we don't need to take care of it here.
+                // that check happens when assigning the `CodegenFnAttrFlags` in
+                // `rustc_hir_analysis`, so we don't need to take care of it here.
                 self.add_compiler_used_global(g);
             }
             if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) {
@@ -565,7 +514,7 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 }
 
-impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
+impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
     fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
         if let Some(&gv) = self.const_globals.borrow().get(&cv) {
             unsafe {
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 01aae24ab56..2b8912d1db2 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -1,13 +1,12 @@
 use std::borrow::Borrow;
 use std::cell::{Cell, RefCell};
-use std::ffi::CStr;
+use std::ffi::{CStr, c_uint};
 use std::str;
 
-use libc::c_uint;
 use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::errors as ssa_errors;
 use rustc_codegen_ssa::traits::*;
-use rustc_data_structures::base_n::{ToBaseN, ALPHANUMERIC_ONLY};
+use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_hir::def_id::DefId;
@@ -15,30 +14,29 @@ use rustc_middle::middle::codegen_fn_attrs::PatchableFunctionEntry;
 use rustc_middle::mir::mono::CodegenUnit;
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers,
-    TyAndLayout,
 };
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
+use rustc_session::Session;
 use rustc_session::config::{
     BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, PAuthKey, PacRet,
 };
-use rustc_session::Session;
 use rustc_span::source_map::Spanned;
-use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::call::FnAbi;
+use rustc_span::{DUMMY_SP, Span};
 use rustc_target::abi::{HasDataLayout, TargetDataLayout, VariantIdx};
-use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel};
+use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel};
 use smallvec::SmallVec;
 
 use crate::back::write::to_llvm_code_model;
 use crate::callee::get_fn;
 use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
+use crate::llvm::{Metadata, MetadataType};
 use crate::type_::Type;
 use crate::value::Value;
 use crate::{attributes, coverageinfo, debuginfo, llvm, llvm_util};
 
-/// There is one `CodegenCx` per compilation unit. Each one has its own LLVM
-/// `llvm::Context` so that several compilation units may be optimized in parallel.
+/// There is one `CodegenCx` per codegen unit. Each one has its own LLVM
+/// `llvm::Context` so that several codegen units may be processed in parallel.
 /// All other LLVM data structures in the `CodegenCx` are tied to that `llvm::Context`.
 pub(crate) struct CodegenCx<'ll, 'tcx> {
     pub tcx: TyCtxt<'tcx>,
@@ -122,14 +120,6 @@ pub(crate) unsafe fn create_module<'ll>(
 
     let mut target_data_layout = sess.target.data_layout.to_string();
     let llvm_version = llvm_util::get_version();
-    if llvm_version < (18, 0, 0) {
-        if sess.target.arch == "x86" || sess.target.arch == "x86_64" {
-            // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants.
-            // Earlier LLVMs leave this as default alignment, so remove it.
-            // See https://reviews.llvm.org/D86310
-            target_data_layout = target_data_layout.replace("-i128:128", "");
-        }
-    }
 
     if llvm_version < (19, 0, 0) {
         if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") {
@@ -241,7 +231,8 @@ pub(crate) unsafe fn create_module<'ll>(
         }
     }
 
-    // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.)
+    // Enable LTO unit splitting if specified or if CFI is enabled. (See
+    // https://reviews.llvm.org/D53891.)
     if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() {
         let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr();
         unsafe {
@@ -387,6 +378,24 @@ pub(crate) unsafe fn create_module<'ll>(
         }
     }
 
+    match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support())
+    {
+        // Set up the small-data optimization limit for architectures that use
+        // an LLVM module flag to control this.
+        (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => {
+            let flag = SmallCStr::new(flag.as_ref());
+            unsafe {
+                llvm::LLVMRustAddModuleFlagU32(
+                    llmod,
+                    llvm::LLVMModFlagBehavior::Error,
+                    flag.as_c_str().as_ptr(),
+                    threshold as u32,
+                )
+            }
+        }
+        _ => (),
+    };
+
     // Insert `llvm.ident` metadata.
     //
     // On the wasm targets it will get hooked up to the "producer" sections
@@ -395,17 +404,17 @@ pub(crate) unsafe fn create_module<'ll>(
     let rustc_producer =
         format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"));
     let name_metadata = unsafe {
-        llvm::LLVMMDStringInContext(
+        llvm::LLVMMDStringInContext2(
             llcx,
             rustc_producer.as_ptr().cast(),
-            rustc_producer.as_bytes().len() as c_uint,
+            rustc_producer.as_bytes().len(),
         )
     };
     unsafe {
         llvm::LLVMAddNamedMetadataOperand(
             llmod,
             c"llvm.ident".as_ptr(),
-            llvm::LLVMMDNodeInContext(llcx, &name_metadata, 1),
+            &llvm::LLVMMetadataAsValue(llcx, llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)),
         );
     }
 
@@ -580,7 +589,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn vtables(
         &self,
     ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>
@@ -1110,6 +1119,14 @@ impl CodegenCx<'_, '_> {
         name.push_str(&(idx as u64).to_base(ALPHANUMERIC_ONLY));
         name
     }
+
+    /// A wrapper for [`llvm::LLVMSetMetadata`], but it takes `Metadata` as a parameter instead of `Value`.
+    pub(crate) fn set_metadata<'a>(&self, val: &'a Value, kind_id: MetadataType, md: &'a Metadata) {
+        unsafe {
+            let node = llvm::LLVMMetadataAsValue(&self.llcx, md);
+            llvm::LLVMSetMetadata(val, kind_id as c_uint, node);
+        }
+    }
 }
 
 impl HasDataLayout for CodegenCx<'_, '_> {
@@ -1140,8 +1157,6 @@ impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> {
 }
 
 impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
-    type LayoutOfResult = TyAndLayout<'tcx>;
-
     #[inline]
     fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
         if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
@@ -1153,8 +1168,6 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
 }
 
 impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
-    type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
-
     #[inline]
     fn handle_fn_abi_err(
         &self,
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
index d7a4f105f3c..77821ca89bc 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
@@ -121,7 +121,8 @@ mod mcdc {
         num_conditions: u16,
     }
 
-    // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at [19](https://github.com/llvm/llvm-project/pull/81257)
+    // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at
+    // [19](https://github.com/llvm/llvm-project/pull/81257).
     type LLVMConditionId = i16;
 
     /// Must match the layout of `LLVMRustMCDCBranchParameters`.
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index 069b62af5e7..267a2244916 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -1,12 +1,12 @@
 use itertools::Itertools as _;
-use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods};
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_index::IndexVec;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_middle::{bug, mir};
-use rustc_span::def_id::DefIdSet;
 use rustc_span::Symbol;
+use rustc_span::def_id::DefIdSet;
 use tracing::debug;
 
 use crate::common::CodegenCx;
@@ -183,8 +183,8 @@ impl GlobalFileTable {
         // Since rustc generates coverage maps with relative paths, the
         // compilation directory can be combined with the relative paths
         // to get absolute paths, if needed.
-        use rustc_session::config::RemapPathScopeComponents;
         use rustc_session::RemapFileNameExt;
+        use rustc_session::config::RemapPathScopeComponents;
         let working_dir: &str = &tcx
             .sess
             .opts
@@ -422,7 +422,7 @@ fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> {
             (instance.def_id(), body)
         });
 
-    // Functions whose coverage statments were found inlined into other functions.
+    // Functions whose coverage statements were found inlined into other functions.
     let mut used_via_inlining = FxHashSet::default();
     // Functions that were instrumented, but had all of their coverage statements
     // removed by later MIR transforms (e.g. UnreachablePropagation).
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index 69babc7c9cf..3a80d216f47 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -2,15 +2,15 @@ use std::cell::RefCell;
 
 use libc::c_uint;
 use rustc_codegen_ssa::traits::{
-    BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods,
-    StaticMethods,
+    BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods,
+    MiscCodegenMethods, StaticCodegenMethods,
 };
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_llvm::RustString;
 use rustc_middle::bug;
 use rustc_middle::mir::coverage::CoverageKind;
-use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::Instance;
+use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_target::abi::{Align, Size};
 use tracing::{debug, instrument};
 
@@ -48,11 +48,10 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
         self.function_coverage_map.replace(FxIndexMap::default())
     }
 
-    /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is called condition bitmap.
-    /// In order to handle nested decisions, several condition bitmaps can be
-    /// allocated for a function body.
-    /// These values are named `mcdc.addr.{i}` and are a 32-bit integers.
-    /// They respectively hold the condition bitmaps for decisions with a depth of `i`.
+    /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is
+    /// called condition bitmap. In order to handle nested decisions, several condition bitmaps can
+    /// be allocated for a function body. These values are named `mcdc.addr.{i}` and are a 32-bit
+    /// integers. They respectively hold the condition bitmaps for decisions with a depth of `i`.
     fn try_get_mcdc_condition_bitmap(
         &self,
         instance: &Instance<'tcx>,
@@ -157,8 +156,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
             ),
             CoverageKind::CounterIncrement { id } => {
                 func_coverage.mark_counter_id_seen(id);
-                // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`,
-                // as that needs an exclusive borrow.
+                // We need to explicitly drop the `RefMut` before calling into
+                // `instrprof_increment`, as that needs an exclusive borrow.
                 drop(coverage_map);
 
                 // The number of counters passed to `llvm.instrprof.increment` might
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
index c3087d8ec30..ac6c2fb1b83 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
@@ -1,7 +1,7 @@
 use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext};
 use rustc_codegen_ssa::traits::*;
-use rustc_index::bit_set::BitSet;
 use rustc_index::Idx;
+use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::{Body, SourceScope};
 use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::{self, Instance};
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
index dc228e94811..f93d3e40b20 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
@@ -44,7 +44,8 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>(
         // Add the pretty printers for the standard library first.
         section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0");
 
-        // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute.
+        // Next, add the pretty printers that were specified via the `#[debugger_visualizer]`
+        // attribute.
         let visualizers = collect_debugger_visualizers_transitive(
             cx.tcx,
             DebuggerVisualizerType::GdbPrettyPrinter,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index dc59e9349b0..964b83c0fa0 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
 use std::{iter, ptr};
 
 use libc::{c_char, c_longlong, c_uint};
-use rustc_codegen_ssa::debuginfo::type_names::{cpp_like_debuginfo, VTableNameKind};
+use rustc_codegen_ssa::debuginfo::type_names::{VTableNameKind, cpp_like_debuginfo};
 use rustc_codegen_ssa::traits::*;
 use rustc_fs_util::path_to_c_string;
 use rustc_hir::def::{CtorKind, DefKind};
@@ -18,7 +18,7 @@ use rustc_middle::ty::{
 };
 use rustc_session::config::{self, DebugInfo, Lto};
 use rustc_span::symbol::Symbol;
-use rustc_span::{hygiene, FileName, FileNameDisplayPreference, SourceFile, DUMMY_SP};
+use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, hygiene};
 use rustc_symbol_mangling::typeid_for_trait_ref;
 use rustc_target::abi::{Align, Size};
 use rustc_target::spec::DebuginfoKind;
@@ -26,15 +26,15 @@ use smallvec::smallvec;
 use tracing::{debug, instrument};
 
 use self::type_map::{DINodeCreationResult, Stub, UniqueTypeId};
+use super::CodegenUnitDebugContext;
 use super::namespace::mangled_name_of_instance;
 use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name};
 use super::utils::{
-    create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, DIB,
+    DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit,
 };
-use super::CodegenUnitDebugContext;
 use crate::common::CodegenCx;
 use crate::debuginfo::metadata::type_map::build_type_with_children;
-use crate::debuginfo::utils::{fat_pointer_kind, FatPtrKind};
+use crate::debuginfo::utils::{FatPtrKind, fat_pointer_kind};
 use crate::llvm::debuginfo::{
     DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind,
     DebugNameTableKind,
@@ -125,7 +125,9 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>(
 
     let (size, align) = cx.size_and_align_of(array_type);
 
-    let upper_bound = len.eval_target_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong;
+    let upper_bound = len
+        .try_to_target_usize(cx.tcx)
+        .expect("expected monomorphic const in codegen") as c_longlong;
 
     let subrange =
         unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) };
@@ -216,6 +218,9 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>(
                     //        need to make sure that we don't break existing debuginfo consumers
                     //        by doing that (at least not without a warning period).
                     let layout_type = if ptr_type.is_box() {
+                        // The assertion at the start of this function ensures we have a ZST
+                        // allocator. We'll make debuginfo "skip" all ZST allocators, not just the
+                        // default allocator.
                         Ty::new_mut_ptr(cx.tcx, pointee_type)
                     } else {
                         ptr_type
@@ -278,8 +283,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     unique_type_id: UniqueTypeId<'tcx>,
 ) -> DINodeCreationResult<'ll> {
-    // It's possible to create a self-referential
-    // type in Rust by using 'impl trait':
+    // It's possible to create a self-referential type in Rust by using 'impl trait':
     //
     // fn foo() -> impl Copy { foo }
     //
@@ -454,7 +458,7 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) ->
             if def.is_box()
                 && args.get(1).map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) =>
         {
-            build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id)
+            build_pointer_or_reference_di_node(cx, t, t.expect_boxed_ty(), unique_type_id)
         }
         ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id),
         ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id),
@@ -571,14 +575,14 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi
                     {
                         // If the compiler's working directory (which also is the DW_AT_comp_dir of
                         // the compilation unit) is a prefix of the path we are about to emit, then
-                        // only emit the part relative to the working directory.
-                        // Because of path remapping we sometimes see strange things here: `abs_path`
-                        // might actually look like a relative path
-                        // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without
-                        // taking the working directory into account, downstream tooling will
-                        // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`,
-                        // which makes no sense. Usually in such cases the working directory will also
-                        // be remapped to `<crate-name-and-version>` or some other prefix of the path
+                        // only emit the part relative to the working directory. Because of path
+                        // remapping we sometimes see strange things here: `abs_path` might
+                        // actually look like a relative path (e.g.
+                        // `<crate-name-and-version>/src/lib.rs`), so if we emit it without taking
+                        // the working directory into account, downstream tooling will interpret it
+                        // as `<working-directory>/<crate-name-and-version>/src/lib.rs`, which
+                        // makes no sense. Usually in such cases the working directory will also be
+                        // remapped to `<crate-name-and-version>` or some other prefix of the path
                         // we are remapping, so we end up with
                         // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`.
                         // By moving the working directory portion into the `directory` part of the
@@ -871,8 +875,8 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>(
     codegen_unit_name: &str,
     debug_context: &CodegenUnitDebugContext<'ll, 'tcx>,
 ) -> &'ll DIDescriptor {
-    use rustc_session::config::RemapPathScopeComponents;
     use rustc_session::RemapFileNameExt;
+    use rustc_session::config::RemapPathScopeComponents;
     let mut name_in_debuginfo = tcx
         .sess
         .local_crate_source_file()
@@ -1543,20 +1547,16 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>(
     let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref);
 
     unsafe {
-        let typeid = llvm::LLVMMDStringInContext(
+        let typeid = llvm::LLVMMDStringInContext2(
             cx.llcx,
             trait_ref_typeid.as_ptr() as *const c_char,
-            trait_ref_typeid.as_bytes().len() as c_uint,
+            trait_ref_typeid.as_bytes().len(),
         );
-        let v = [cx.const_usize(0), typeid];
+        let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid];
         llvm::LLVMRustGlobalAddMetadata(
             vtable,
             llvm::MD_type as c_uint,
-            llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext(
-                cx.llcx,
-                v.as_ptr(),
-                v.len() as c_uint,
-            )),
+            llvm::LLVMMDNodeInContext2(cx.llcx, v.as_ptr(), v.len()),
         );
         let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64));
         let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1);
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
index 181022087f3..966788cf32f 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
@@ -3,7 +3,7 @@ use std::borrow::Cow;
 use libc::c_uint;
 use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name;
 use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo};
-use rustc_codegen_ssa::traits::ConstMethods;
+use rustc_codegen_ssa::traits::ConstCodegenMethods;
 use rustc_index::IndexVec;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
@@ -15,9 +15,9 @@ use crate::common::CodegenCx;
 use crate::debuginfo::metadata::enums::DiscrResult;
 use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId};
 use crate::debuginfo::metadata::{
+    DINodeCreationResult, NO_GENERICS, NO_SCOPE_METADATA, SmallVec, UNKNOWN_LINE_NUMBER,
     build_field_di_node, file_metadata, size_and_align_of, type_di_node, unknown_file_metadata,
-    visibility_di_flags, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA,
-    UNKNOWN_LINE_NUMBER,
+    visibility_di_flags,
 };
 use crate::debuginfo::utils::DIB;
 use crate::llvm::debuginfo::{DIFile, DIFlags, DIType};
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
index edaf73b74a2..fe1634146ff 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
@@ -12,14 +12,14 @@ use rustc_span::Symbol;
 use rustc_target::abi::{FieldIdx, TagEncoding, VariantIdx, Variants};
 
 use super::type_map::{DINodeCreationResult, UniqueTypeId};
-use super::{size_and_align_of, SmallVec};
+use super::{SmallVec, size_and_align_of};
 use crate::common::CodegenCx;
 use crate::debuginfo::metadata::type_map::{self, Stub};
 use crate::debuginfo::metadata::{
-    build_field_di_node, build_generic_type_param_di_nodes, type_di_node, unknown_file_metadata,
-    UNKNOWN_LINE_NUMBER,
+    UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, type_di_node,
+    unknown_file_metadata,
 };
-use crate::debuginfo::utils::{create_DIArray, get_namespace_for_item, DIB};
+use crate::debuginfo::utils::{DIB, create_DIArray, get_namespace_for_item};
 use crate::llvm::debuginfo::{DIFlags, DIType};
 use crate::llvm::{self};
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
index 238fbad4dfd..5e7dbdd921a 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
@@ -3,7 +3,7 @@ use std::borrow::Cow;
 use libc::c_uint;
 use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name;
 use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo};
-use rustc_codegen_ssa::traits::ConstMethods;
+use rustc_codegen_ssa::traits::ConstCodegenMethods;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self};
@@ -13,10 +13,10 @@ use smallvec::smallvec;
 use crate::common::CodegenCx;
 use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId};
 use crate::debuginfo::metadata::{
-    file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, visibility_di_flags,
-    DINodeCreationResult, SmallVec, NO_GENERICS, UNKNOWN_LINE_NUMBER,
+    DINodeCreationResult, NO_GENERICS, SmallVec, UNKNOWN_LINE_NUMBER, file_metadata,
+    size_and_align_of, type_di_node, unknown_file_metadata, visibility_di_flags,
 };
-use crate::debuginfo::utils::{create_DIArray, get_namespace_for_item, DIB};
+use crate::debuginfo::utils::{DIB, create_DIArray, get_namespace_for_item};
 use crate::llvm::debuginfo::{DIFile, DIFlags, DIType};
 use crate::llvm::{self};
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
index 6d21f4204e3..714e3c0b145 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs
@@ -8,9 +8,9 @@ use rustc_middle::bug;
 use rustc_middle::ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt};
 use rustc_target::abi::{Align, Size, VariantIdx};
 
-use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER};
+use super::{SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata};
 use crate::common::CodegenCx;
-use crate::debuginfo::utils::{create_DIArray, debug_context, DIB};
+use crate::debuginfo::utils::{DIB, create_DIArray, debug_context};
 use crate::llvm::debuginfo::{DIFlags, DIScope, DIType};
 use crate::llvm::{self};
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index f5d6fc6f080..1a8153a54e8 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -1,8 +1,8 @@
 #![doc = include_str!("doc.md")]
 
 use std::cell::{OnceCell, RefCell};
-use std::iter;
 use std::ops::Range;
+use std::{iter, ptr};
 
 use libc::c_uint;
 use rustc_codegen_ssa::debuginfo::type_names;
@@ -16,8 +16,8 @@ use rustc_index::IndexVec;
 use rustc_middle::mir;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, GenericArgsRef, Instance, ParamEnv, Ty, TypeVisitableExt};
-use rustc_session::config::{self, DebugInfo};
 use rustc_session::Session;
+use rustc_session::config::{self, DebugInfo};
 use rustc_span::symbol::Symbol;
 use rustc_span::{
     BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId,
@@ -26,9 +26,9 @@ use rustc_target::abi::Size;
 use smallvec::SmallVec;
 use tracing::debug;
 
-use self::metadata::{file_metadata, type_di_node, UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER};
+use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, type_di_node};
 use self::namespace::mangled_name_of_instance;
-use self::utils::{create_DIArray, is_node_local_to_unit, DIB};
+use self::utils::{DIB, create_DIArray, is_node_local_to_unit};
 use crate::abi::FnAbi;
 use crate::builder::Builder;
 use crate::common::CodegenCx;
@@ -209,6 +209,12 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
         }
     }
 
+    fn clear_dbg_loc(&mut self) {
+        unsafe {
+            llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, ptr::null());
+        }
+    }
+
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) {
         gdb::insert_reference_to_gdb_debug_scripts_section_global(self)
     }
@@ -280,7 +286,7 @@ impl CodegenCx<'_, '_> {
     }
 }
 
-impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn create_function_debug_context(
         &self,
         instance: Instance<'tcx>,
@@ -549,17 +555,14 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 }
             }
 
-            let scope = namespace::item_namespace(
-                cx,
-                DefId {
-                    krate: instance.def_id().krate,
-                    index: cx
-                        .tcx
-                        .def_key(instance.def_id())
-                        .parent
-                        .expect("get_containing_scope: missing parent?"),
-                },
-            );
+            let scope = namespace::item_namespace(cx, DefId {
+                krate: instance.def_id().krate,
+                index: cx
+                    .tcx
+                    .def_key(instance.def_id())
+                    .parent
+                    .expect("get_containing_scope: missing parent?"),
+            });
             (scope, false)
         }
     }
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs
index 83d7a82dadc..3578755aae0 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs
@@ -4,7 +4,7 @@ use rustc_codegen_ssa::debuginfo::type_names;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::{self, Instance};
 
-use super::utils::{debug_context, DIB};
+use super::utils::{DIB, debug_context};
 use crate::common::CodegenCx;
 use crate::llvm;
 use crate::llvm::debuginfo::DIScope;
@@ -13,8 +13,7 @@ pub(crate) fn mangled_name_of_instance<'a, 'tcx>(
     cx: &CodegenCx<'a, 'tcx>,
     instance: Instance<'tcx>,
 ) -> ty::SymbolName<'tcx> {
-    let tcx = cx.tcx;
-    tcx.symbol_name(instance)
+    cx.tcx.symbol_name(instance)
 }
 
 pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope {
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
index 321553a3df0..acb15449ce3 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
@@ -5,8 +5,8 @@ use rustc_middle::ty::layout::{HasParamEnv, LayoutOf};
 use rustc_middle::ty::{self, Ty};
 use tracing::trace;
 
-use super::namespace::item_namespace;
 use super::CodegenUnitDebugContext;
+use super::namespace::item_namespace;
 use crate::common::CodegenCx;
 use crate::llvm;
 use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope};
diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs
index 4e4500b6373..b0b29ca1280 100644
--- a/compiler/rustc_codegen_llvm/src/declare.rs
+++ b/compiler/rustc_codegen_llvm/src/declare.rs
@@ -12,7 +12,7 @@
 //! * When in doubt, define.
 
 use itertools::Itertools;
-use rustc_codegen_ssa::traits::TypeMembershipMethods;
+use rustc_codegen_ssa::traits::TypeMembershipCodegenMethods;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::ty::{Instance, Ty};
 use rustc_sanitizers::{cfi, kcfi};
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index a300f5f707a..c66c80da9fc 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -12,7 +12,7 @@ use rustc_middle::mir::BinOp;
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf};
 use rustc_middle::ty::{self, GenericArgsRef, Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_span::{sym, Span, Symbol};
+use rustc_span::{Span, Symbol, sym};
 use rustc_target::abi::{self, Align, Float, HasDataLayout, Primitive, Size};
 use rustc_target::spec::{HasTargetSpec, PanicStrategy};
 use tracing::debug;
@@ -20,7 +20,7 @@ use tracing::debug;
 use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode};
 use crate::builder::Builder;
 use crate::context::CodegenCx;
-use crate::llvm;
+use crate::llvm::{self, Metadata};
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::va_arg::emit_va_arg;
@@ -148,7 +148,7 @@ fn get_simple_intrinsic<'ll>(
     Some(cx.get_intrinsic(llvm_name))
 }
 
-impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
+impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     fn codegen_intrinsic_call(
         &mut self,
         instance: ty::Instance<'tcx>,
@@ -187,9 +187,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Some(instance),
                 )
             }
-            sym::likely => {
-                self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)])
-            }
+            sym::likely => self.expect(args[0].immediate(), true),
             sym::is_val_statically_known => {
                 let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
                 let kind = self.type_kind(intrinsic_type);
@@ -210,8 +208,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     self.const_bool(false)
                 }
             }
-            sym::unlikely => self
-                .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]),
+            sym::unlikely => self.expect(args[0].immediate(), false),
             sym::select_unpredictable => {
                 let cond = args[0].immediate();
                 assert_eq!(args[1].layout, args[2].layout);
@@ -333,15 +330,12 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     sym::prefetch_write_instruction => (1, 0),
                     _ => bug!(),
                 };
-                self.call_intrinsic(
-                    "llvm.prefetch",
-                    &[
-                        args[0].immediate(),
-                        self.const_i32(rw),
-                        args[1].immediate(),
-                        self.const_i32(cache_type),
-                    ],
-                )
+                self.call_intrinsic("llvm.prefetch", &[
+                    args[0].immediate(),
+                    self.const_i32(rw),
+                    args[1].immediate(),
+                    self.const_i32(cache_type),
+                ])
             }
             sym::ctlz
             | sym::ctlz_nonzero
@@ -359,10 +353,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Some((width, signed)) => match name {
                         sym::ctlz | sym::cttz => {
                             let y = self.const_bool(false);
-                            let ret = self.call_intrinsic(
-                                &format!("llvm.{name}.i{width}"),
-                                &[args[0].immediate(), y],
-                            );
+                            let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[
+                                args[0].immediate(),
+                                y,
+                            ]);
 
                             self.intcast(ret, llret_ty, false)
                         }
@@ -379,26 +373,24 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             self.intcast(ret, llret_ty, false)
                         }
                         sym::ctpop => {
-                            let ret = self.call_intrinsic(
-                                &format!("llvm.ctpop.i{width}"),
-                                &[args[0].immediate()],
-                            );
+                            let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[args
+                                [0]
+                            .immediate()]);
                             self.intcast(ret, llret_ty, false)
                         }
                         sym::bswap => {
                             if width == 8 {
                                 args[0].immediate() // byte swap a u8/i8 is just a no-op
                             } else {
-                                self.call_intrinsic(
-                                    &format!("llvm.bswap.i{width}"),
-                                    &[args[0].immediate()],
-                                )
+                                self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[
+                                    args[0].immediate()
+                                ])
                             }
                         }
-                        sym::bitreverse => self.call_intrinsic(
-                            &format!("llvm.bitreverse.i{width}"),
-                            &[args[0].immediate()],
-                        ),
+                        sym::bitreverse => self
+                            .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[
+                                args[0].immediate()
+                            ]),
                         sym::rotate_left | sym::rotate_right => {
                             let is_left = name == sym::rotate_left;
                             let val = args[0].immediate();
@@ -407,7 +399,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             let llvm_name =
                                 &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
 
-                            // llvm expects shift to be the same type as the values, but rust always uses `u32`
+                            // llvm expects shift to be the same type as the values, but rust
+                            // always uses `u32`.
                             let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
 
                             self.call_intrinsic(llvm_name, &[val, val, raw_shift])
@@ -473,10 +466,11 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
 
             sym::compare_bytes => {
                 // Here we assume that the `memcmp` provided by the target is a NOP for size 0.
-                let cmp = self.call_intrinsic(
-                    "memcmp",
-                    &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
-                );
+                let cmp = self.call_intrinsic("memcmp", &[
+                    args[0].immediate(),
+                    args[1].immediate(),
+                    args[2].immediate(),
+                ]);
                 // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`.
                 self.sext(cmp, self.type_ix(32))
             }
@@ -576,6 +570,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     span,
                 ) {
                     Ok(llval) => llval,
+                    // If there was an error, just skip this invocation... we'll abort compilation
+                    // anyway, but we can keep codegen'ing to find more errors.
                     Err(()) => return Ok(()),
                 }
             }
@@ -604,16 +600,23 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     }
 
     fn assume(&mut self, val: Self::Value) {
-        self.call_intrinsic("llvm.assume", &[val]);
+        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
+            self.call_intrinsic("llvm.assume", &[val]);
+        }
     }
 
     fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
-        self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
+        if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
+            self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
+        } else {
+            cond
+        }
     }
 
-    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value {
+    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Metadata) -> Self::Value {
         // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time
         // optimization pass replaces calls to this intrinsic with code to test type membership.
+        let typeid = unsafe { llvm::LLVMMetadataAsValue(&self.llcx, typeid) };
         self.call_intrinsic("llvm.type.test", &[pointer, typeid])
     }
 
@@ -621,8 +624,9 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         &mut self,
         llvtable: &'ll Value,
         vtable_byte_offset: u64,
-        typeid: &'ll Value,
+        typeid: &'ll Metadata,
     ) -> Self::Value {
+        let typeid = unsafe { llvm::LLVMMetadataAsValue(&self.llcx, typeid) };
         let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
         let type_checked_load =
             self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]);
@@ -1210,17 +1214,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     if let Some(cmp_op) = comparison {
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
 
-        require!(
-            in_len == out_len,
-            InvalidMonomorphization::ReturnLengthInputType {
-                span,
-                name,
-                in_len,
-                in_ty,
-                ret_ty,
-                out_len
-            }
-        );
+        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
+            span,
+            name,
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        });
         require!(
             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
             InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
@@ -1246,14 +1247,21 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let n = idx.len() as u64;
 
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
-        require!(
-            out_len == n,
-            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
-        );
-        require!(
-            in_elem == out_ty,
-            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
-        );
+        require!(out_len == n, InvalidMonomorphization::ReturnLength {
+            span,
+            name,
+            in_len: n,
+            ret_ty,
+            out_len
+        });
+        require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement {
+            span,
+            name,
+            in_elem,
+            in_ty,
+            ret_ty,
+            out_ty
+        });
 
         let total_len = in_len * 2;
 
@@ -1287,94 +1295,73 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_shuffle {
-        // Make sure this is actually an array or SIMD vector, since typeck only checks the length-suffixed
-        // version of this intrinsic.
+        // Make sure this is actually a SIMD vector.
         let idx_ty = args[2].layout.ty;
-        let n: u64 = match idx_ty.kind() {
-            ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
-                len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
-                    || span_bug!(span, "could not evaluate shuffle index array length"),
-                )
-            }
-            _ if idx_ty.is_simd()
-                && matches!(
-                    idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(),
-                    ty::Uint(ty::UintTy::U32)
-                ) =>
-            {
-                idx_ty.simd_size_and_type(bx.cx.tcx).0
-            }
-            _ => return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }),
+        let n: u64 = if idx_ty.is_simd()
+            && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
+        {
+            idx_ty.simd_size_and_type(bx.cx.tcx).0
+        } else {
+            return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
         };
 
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
-        require!(
-            out_len == n,
-            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
-        );
-        require!(
-            in_elem == out_ty,
-            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
-        );
+        require!(out_len == n, InvalidMonomorphization::ReturnLength {
+            span,
+            name,
+            in_len: n,
+            ret_ty,
+            out_len
+        });
+        require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement {
+            span,
+            name,
+            in_elem,
+            in_ty,
+            ret_ty,
+            out_ty
+        });
 
         let total_len = u128::from(in_len) * 2;
 
-        let vector = args[2].immediate();
-
-        let indices: Option<Vec<_>> = (0..n)
-            .map(|i| {
-                let arg_idx = i;
-                let val = bx.const_get_elt(vector, i as u64);
-                match bx.const_to_opt_u128(val, true) {
-                    None => {
-                        bug!("typeck should have already ensured that these are const")
-                    }
-                    Some(idx) if idx >= total_len => {
-                        bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
-                            span,
-                            name,
-                            arg_idx,
-                            total_len,
-                        });
-                        None
-                    }
-                    Some(idx) => Some(bx.const_i32(idx as i32)),
-                }
-            })
-            .collect();
-        let Some(indices) = indices else {
-            return Ok(bx.const_null(llret_ty));
-        };
+        // Check that the indices are in-bounds.
+        let indices = args[2].immediate();
+        for i in 0..n {
+            let val = bx.const_get_elt(indices, i as u64);
+            let idx = bx
+                .const_to_opt_u128(val, true)
+                .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
+            if idx >= total_len {
+                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
+                    span,
+                    name,
+                    arg_idx: i,
+                    total_len,
+                });
+            }
+        }
 
-        return Ok(bx.shuffle_vector(
-            args[0].immediate(),
-            args[1].immediate(),
-            bx.const_vector(&indices),
-        ));
+        return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
     }
 
     if name == sym::simd_insert {
-        require!(
-            in_elem == arg_tys[2],
-            InvalidMonomorphization::InsertedType {
-                span,
-                name,
-                in_elem,
-                in_ty,
-                out_ty: arg_tys[2]
-            }
-        );
+        require!(in_elem == arg_tys[2], InvalidMonomorphization::InsertedType {
+            span,
+            name,
+            in_elem,
+            in_ty,
+            out_ty: arg_tys[2]
+        });
         let idx = bx
             .const_to_opt_u128(args[1].immediate(), false)
             .expect("typeck should have ensure that this is a const");
         if idx >= in_len.into() {
-            bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
+            return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
                 span,
                 name,
                 arg_idx: 1,
                 total_len: in_len.into(),
             });
-            return Ok(bx.const_null(llret_ty));
         }
         return Ok(bx.insert_element(
             args[0].immediate(),
@@ -1383,21 +1370,23 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         ));
     }
     if name == sym::simd_extract {
-        require!(
-            ret_ty == in_elem,
-            InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
-        );
+        require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
+            span,
+            name,
+            in_elem,
+            in_ty,
+            ret_ty
+        });
         let idx = bx
             .const_to_opt_u128(args[1].immediate(), false)
             .expect("typeck should have ensure that this is a const");
         if idx >= in_len.into() {
-            bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
+            return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
                 span,
                 name,
                 arg_idx: 1,
                 total_len: in_len.into(),
             });
-            return Ok(bx.const_null(llret_ty));
         }
         return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32)));
     }
@@ -1406,10 +1395,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let m_elem_ty = in_elem;
         let m_len = in_len;
         let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
-        require!(
-            m_len == v_len,
-            InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
-        );
+        require!(m_len == v_len, InvalidMonomorphization::MismatchedLengths {
+            span,
+            name,
+            m_len,
+            v_len
+        });
         match m_elem_ty.kind() {
             ty::Int(_) => {}
             _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }),
@@ -1636,34 +1627,30 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
-        require!(
-            in_len == out_len,
-            InvalidMonomorphization::SecondArgumentLength {
-                span,
-                name,
-                in_len,
-                in_ty,
-                arg_ty: arg_tys[1],
-                out_len
-            }
-        );
-        require!(
-            in_len == out_len2,
-            InvalidMonomorphization::ThirdArgumentLength {
-                span,
-                name,
-                in_len,
-                in_ty,
-                arg_ty: arg_tys[2],
-                out_len: out_len2
-            }
-        );
+        require!(in_len == out_len, InvalidMonomorphization::SecondArgumentLength {
+            span,
+            name,
+            in_len,
+            in_ty,
+            arg_ty: arg_tys[1],
+            out_len
+        });
+        require!(in_len == out_len2, InvalidMonomorphization::ThirdArgumentLength {
+            span,
+            name,
+            in_len,
+            in_ty,
+            arg_ty: arg_tys[2],
+            out_len: out_len2
+        });
 
         // The return type must match the first argument type
-        require!(
-            ret_ty == in_ty,
-            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
-        );
+        require!(ret_ty == in_ty, InvalidMonomorphization::ExpectedReturnType {
+            span,
+            name,
+            in_ty,
+            ret_ty
+        });
 
         require!(
             matches!(
@@ -1754,23 +1741,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
-        require!(
-            values_len == mask_len,
-            InvalidMonomorphization::ThirdArgumentLength {
-                span,
-                name,
-                in_len: mask_len,
-                in_ty: mask_ty,
-                arg_ty: values_ty,
-                out_len: values_len
-            }
-        );
+        require!(values_len == mask_len, InvalidMonomorphization::ThirdArgumentLength {
+            span,
+            name,
+            in_len: mask_len,
+            in_ty: mask_ty,
+            arg_ty: values_ty,
+            out_len: values_len
+        });
 
         // The return type must match the last argument type
-        require!(
-            ret_ty == values_ty,
-            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
-        );
+        require!(ret_ty == values_ty, InvalidMonomorphization::ExpectedReturnType {
+            span,
+            name,
+            in_ty: values_ty,
+            ret_ty
+        });
 
         require!(
             matches!(
@@ -1852,23 +1838,21 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
 
         // Of the same length:
-        require!(
-            values_len == mask_len,
-            InvalidMonomorphization::ThirdArgumentLength {
-                span,
-                name,
-                in_len: mask_len,
-                in_ty: mask_ty,
-                arg_ty: values_ty,
-                out_len: values_len
-            }
-        );
+        require!(values_len == mask_len, InvalidMonomorphization::ThirdArgumentLength {
+            span,
+            name,
+            in_len: mask_len,
+            in_ty: mask_ty,
+            arg_ty: values_ty,
+            out_len: values_len
+        });
 
         // The second argument must be a mutable pointer type matching the element type
         require!(
             matches!(
                 *pointer_ty.kind(),
-                ty::RawPtr(p_ty, p_mutbl) if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
+                ty::RawPtr(p_ty, p_mutbl)
+                    if p_ty == values_elem && p_ty.kind() == values_elem.kind() && p_mutbl.is_mut()
             ),
             InvalidMonomorphization::ExpectedElementType {
                 span,
@@ -1940,28 +1924,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
 
         // Of the same length:
-        require!(
-            in_len == element_len1,
-            InvalidMonomorphization::SecondArgumentLength {
-                span,
-                name,
-                in_len,
-                in_ty,
-                arg_ty: arg_tys[1],
-                out_len: element_len1
-            }
-        );
-        require!(
-            in_len == element_len2,
-            InvalidMonomorphization::ThirdArgumentLength {
-                span,
-                name,
-                in_len,
-                in_ty,
-                arg_ty: arg_tys[2],
-                out_len: element_len2
-            }
-        );
+        require!(in_len == element_len1, InvalidMonomorphization::SecondArgumentLength {
+            span,
+            name,
+            in_len,
+            in_ty,
+            arg_ty: arg_tys[1],
+            out_len: element_len1
+        });
+        require!(in_len == element_len2, InvalidMonomorphization::ThirdArgumentLength {
+            span,
+            name,
+            in_len,
+            in_ty,
+            arg_ty: arg_tys[2],
+            out_len: element_len2
+        });
 
         require!(
             matches!(
@@ -2035,10 +2013,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
          $identity:expr) => {
             if name == sym::$name {
-                require!(
-                    ret_ty == in_elem,
-                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
-                );
+                require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
+                    span,
+                    name,
+                    in_elem,
+                    in_ty,
+                    ret_ty
+                });
                 return match in_elem.kind() {
                     ty::Int(_) | ty::Uint(_) => {
                         let r = bx.$integer_reduce(args[0].immediate());
@@ -2087,14 +2068,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         };
     }
 
-    arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0);
+    arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0);
     arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0);
     arith_red!(
         simd_reduce_add_unordered: vector_reduce_add,
         vector_reduce_fadd_reassoc,
         false,
         add,
-        0.0
+        -0.0
     );
     arith_red!(
         simd_reduce_mul_unordered: vector_reduce_mul,
@@ -2107,10 +2088,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     macro_rules! minmax_red {
         ($name:ident: $int_red:ident, $float_red:ident) => {
             if name == sym::$name {
-                require!(
-                    ret_ty == in_elem,
-                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
-                );
+                require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
+                    span,
+                    name,
+                    in_elem,
+                    in_ty,
+                    ret_ty
+                });
                 return match in_elem.kind() {
                     ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
                     ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
@@ -2135,10 +2119,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         ($name:ident : $red:ident, $boolean:expr) => {
             if name == sym::$name {
                 let input = if !$boolean {
-                    require!(
-                        ret_ty == in_elem,
-                        InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
-                    );
+                    require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
+                        span,
+                        name,
+                        in_elem,
+                        in_ty,
+                        ret_ty
+                    });
                     args[0].immediate()
                 } else {
                     match in_elem.kind() {
@@ -2184,27 +2171,25 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_cast_ptr {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(
-            in_len == out_len,
-            InvalidMonomorphization::ReturnLengthInputType {
-                span,
-                name,
-                in_len,
-                in_ty,
-                ret_ty,
-                out_len
-            }
-        );
+        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
+            span,
+            name,
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        });
 
         match in_elem.kind() {
             ty::RawPtr(p_ty, _) => {
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
                 });
-                require!(
-                    metadata.is_unit(),
-                    InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem }
-                );
+                require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer {
+                    span,
+                    name,
+                    ty: in_elem
+                });
             }
             _ => {
                 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
@@ -2215,10 +2200,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
                 });
-                require!(
-                    metadata.is_unit(),
-                    InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem }
-                );
+                require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer {
+                    span,
+                    name,
+                    ty: out_elem
+                });
             }
             _ => {
                 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
@@ -2230,17 +2216,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_expose_provenance {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(
-            in_len == out_len,
-            InvalidMonomorphization::ReturnLengthInputType {
-                span,
-                name,
-                in_len,
-                in_ty,
-                ret_ty,
-                out_len
-            }
-        );
+        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
+            span,
+            name,
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        });
 
         match in_elem.kind() {
             ty::RawPtr(_, _) => {}
@@ -2258,17 +2241,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_with_exposed_provenance {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(
-            in_len == out_len,
-            InvalidMonomorphization::ReturnLengthInputType {
-                span,
-                name,
-                in_len,
-                in_ty,
-                ret_ty,
-                out_len
-            }
-        );
+        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
+            span,
+            name,
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        });
 
         match in_elem.kind() {
             ty::Uint(ty::UintTy::Usize) => {}
@@ -2286,17 +2266,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_cast || name == sym::simd_as {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(
-            in_len == out_len,
-            InvalidMonomorphization::ReturnLengthInputType {
-                span,
-                name,
-                in_len,
-                in_ty,
-                ret_ty,
-                out_len
-            }
-        );
+        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
+            span,
+            name,
+            in_len,
+            in_ty,
+            ret_ty,
+            out_len
+        });
         // casting cares about nominal type, not just structural type
         if in_elem == out_elem {
             return Ok(args[0].immediate());
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 6a303e1e602..d69112612ba 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -6,7 +6,6 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
-#![cfg_attr(bootstrap, feature(unsafe_extern_blocks))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]
@@ -42,8 +41,8 @@ use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::util::Providers;
-use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest};
 use rustc_session::Session;
+use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest};
 use rustc_span::symbol::Symbol;
 
 mod back {
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 5e0d7418993..9aabfd794ba 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -5,13 +5,13 @@ use std::marker::PhantomData;
 
 use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t};
 
+use super::RustString;
 use super::debuginfo::{
     DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator,
     DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace,
     DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable,
     DebugEmissionKind, DebugNameTableKind,
 };
-use super::RustString;
 
 pub type Bool = c_uint;
 
@@ -220,17 +220,18 @@ pub enum IntPredicate {
 
 impl IntPredicate {
     pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self {
+        use rustc_codegen_ssa::common::IntPredicate as Common;
         match intpre {
-            rustc_codegen_ssa::common::IntPredicate::IntEQ => IntPredicate::IntEQ,
-            rustc_codegen_ssa::common::IntPredicate::IntNE => IntPredicate::IntNE,
-            rustc_codegen_ssa::common::IntPredicate::IntUGT => IntPredicate::IntUGT,
-            rustc_codegen_ssa::common::IntPredicate::IntUGE => IntPredicate::IntUGE,
-            rustc_codegen_ssa::common::IntPredicate::IntULT => IntPredicate::IntULT,
-            rustc_codegen_ssa::common::IntPredicate::IntULE => IntPredicate::IntULE,
-            rustc_codegen_ssa::common::IntPredicate::IntSGT => IntPredicate::IntSGT,
-            rustc_codegen_ssa::common::IntPredicate::IntSGE => IntPredicate::IntSGE,
-            rustc_codegen_ssa::common::IntPredicate::IntSLT => IntPredicate::IntSLT,
-            rustc_codegen_ssa::common::IntPredicate::IntSLE => IntPredicate::IntSLE,
+            Common::IntEQ => Self::IntEQ,
+            Common::IntNE => Self::IntNE,
+            Common::IntUGT => Self::IntUGT,
+            Common::IntUGE => Self::IntUGE,
+            Common::IntULT => Self::IntULT,
+            Common::IntULE => Self::IntULE,
+            Common::IntSGT => Self::IntSGT,
+            Common::IntSGE => Self::IntSGE,
+            Common::IntSLT => Self::IntSLT,
+            Common::IntSLE => Self::IntSLE,
         }
     }
 }
@@ -259,27 +260,24 @@ pub enum RealPredicate {
 
 impl RealPredicate {
     pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self {
+        use rustc_codegen_ssa::common::RealPredicate as Common;
         match realp {
-            rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => {
-                RealPredicate::RealPredicateFalse
-            }
-            rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ,
-            rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT,
-            rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE,
-            rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT,
-            rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE,
-            rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE,
-            rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD,
-            rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO,
-            rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ,
-            rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT,
-            rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE,
-            rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT,
-            rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE,
-            rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE,
-            rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => {
-                RealPredicate::RealPredicateTrue
-            }
+            Common::RealPredicateFalse => Self::RealPredicateFalse,
+            Common::RealOEQ => Self::RealOEQ,
+            Common::RealOGT => Self::RealOGT,
+            Common::RealOGE => Self::RealOGE,
+            Common::RealOLT => Self::RealOLT,
+            Common::RealOLE => Self::RealOLE,
+            Common::RealONE => Self::RealONE,
+            Common::RealORD => Self::RealORD,
+            Common::RealUNO => Self::RealUNO,
+            Common::RealUEQ => Self::RealUEQ,
+            Common::RealUGT => Self::RealUGT,
+            Common::RealUGE => Self::RealUGE,
+            Common::RealULT => Self::RealULT,
+            Common::RealULE => Self::RealULE,
+            Common::RealUNE => Self::RealUNE,
+            Common::RealPredicateTrue => Self::RealPredicateTrue,
         }
     }
 }
@@ -311,26 +309,27 @@ pub enum TypeKind {
 
 impl TypeKind {
     pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind {
+        use rustc_codegen_ssa::common::TypeKind as Common;
         match self {
-            TypeKind::Void => rustc_codegen_ssa::common::TypeKind::Void,
-            TypeKind::Half => rustc_codegen_ssa::common::TypeKind::Half,
-            TypeKind::Float => rustc_codegen_ssa::common::TypeKind::Float,
-            TypeKind::Double => rustc_codegen_ssa::common::TypeKind::Double,
-            TypeKind::X86_FP80 => rustc_codegen_ssa::common::TypeKind::X86_FP80,
-            TypeKind::FP128 => rustc_codegen_ssa::common::TypeKind::FP128,
-            TypeKind::PPC_FP128 => rustc_codegen_ssa::common::TypeKind::PPC_FP128,
-            TypeKind::Label => rustc_codegen_ssa::common::TypeKind::Label,
-            TypeKind::Integer => rustc_codegen_ssa::common::TypeKind::Integer,
-            TypeKind::Function => rustc_codegen_ssa::common::TypeKind::Function,
-            TypeKind::Struct => rustc_codegen_ssa::common::TypeKind::Struct,
-            TypeKind::Array => rustc_codegen_ssa::common::TypeKind::Array,
-            TypeKind::Pointer => rustc_codegen_ssa::common::TypeKind::Pointer,
-            TypeKind::Vector => rustc_codegen_ssa::common::TypeKind::Vector,
-            TypeKind::Metadata => rustc_codegen_ssa::common::TypeKind::Metadata,
-            TypeKind::Token => rustc_codegen_ssa::common::TypeKind::Token,
-            TypeKind::ScalableVector => rustc_codegen_ssa::common::TypeKind::ScalableVector,
-            TypeKind::BFloat => rustc_codegen_ssa::common::TypeKind::BFloat,
-            TypeKind::X86_AMX => rustc_codegen_ssa::common::TypeKind::X86_AMX,
+            Self::Void => Common::Void,
+            Self::Half => Common::Half,
+            Self::Float => Common::Float,
+            Self::Double => Common::Double,
+            Self::X86_FP80 => Common::X86_FP80,
+            Self::FP128 => Common::FP128,
+            Self::PPC_FP128 => Common::PPC_FP128,
+            Self::Label => Common::Label,
+            Self::Integer => Common::Integer,
+            Self::Function => Common::Function,
+            Self::Struct => Common::Struct,
+            Self::Array => Common::Array,
+            Self::Pointer => Common::Pointer,
+            Self::Vector => Common::Vector,
+            Self::Metadata => Common::Metadata,
+            Self::Token => Common::Token,
+            Self::ScalableVector => Common::ScalableVector,
+            Self::BFloat => Common::BFloat,
+            Self::X86_AMX => Common::X86_AMX,
         }
     }
 }
@@ -354,18 +353,19 @@ pub enum AtomicRmwBinOp {
 
 impl AtomicRmwBinOp {
     pub fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self {
+        use rustc_codegen_ssa::common::AtomicRmwBinOp as Common;
         match op {
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg => AtomicRmwBinOp::AtomicXchg,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAdd => AtomicRmwBinOp::AtomicAdd,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicSub => AtomicRmwBinOp::AtomicSub,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAnd => AtomicRmwBinOp::AtomicAnd,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicNand => AtomicRmwBinOp::AtomicNand,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicOr => AtomicRmwBinOp::AtomicOr,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXor => AtomicRmwBinOp::AtomicXor,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMax => AtomicRmwBinOp::AtomicMax,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMin => AtomicRmwBinOp::AtomicMin,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMax => AtomicRmwBinOp::AtomicUMax,
-            rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMin => AtomicRmwBinOp::AtomicUMin,
+            Common::AtomicXchg => Self::AtomicXchg,
+            Common::AtomicAdd => Self::AtomicAdd,
+            Common::AtomicSub => Self::AtomicSub,
+            Common::AtomicAnd => Self::AtomicAnd,
+            Common::AtomicNand => Self::AtomicNand,
+            Common::AtomicOr => Self::AtomicOr,
+            Common::AtomicXor => Self::AtomicXor,
+            Common::AtomicMax => Self::AtomicMax,
+            Common::AtomicMin => Self::AtomicMin,
+            Common::AtomicUMax => Self::AtomicUMax,
+            Common::AtomicUMin => Self::AtomicUMin,
         }
     }
 }
@@ -387,17 +387,14 @@ pub enum AtomicOrdering {
 
 impl AtomicOrdering {
     pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self {
+        use rustc_codegen_ssa::common::AtomicOrdering as Common;
         match ao {
-            rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered,
-            rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic,
-            rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire,
-            rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release,
-            rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => {
-                AtomicOrdering::AcquireRelease
-            }
-            rustc_codegen_ssa::common::AtomicOrdering::SequentiallyConsistent => {
-                AtomicOrdering::SequentiallyConsistent
-            }
+            Common::Unordered => Self::Unordered,
+            Common::Relaxed => Self::Monotonic,
+            Common::Acquire => Self::Acquire,
+            Common::Release => Self::Release,
+            Common::AcquireRelease => Self::AcquireRelease,
+            Common::SequentiallyConsistent => Self::SequentiallyConsistent,
         }
     }
 }
@@ -563,13 +560,11 @@ pub enum ArchiveKind {
     K_AIXBIG,
 }
 
-// LLVMRustThinLTOData
 unsafe extern "C" {
+    // LLVMRustThinLTOData
     pub type ThinLTOData;
-}
 
-// LLVMRustThinLTOBuffer
-unsafe extern "C" {
+    // LLVMRustThinLTOBuffer
     pub type ThinLTOBuffer;
 }
 
@@ -633,26 +628,12 @@ struct InvariantOpaque<'a> {
 // Opaque pointer types
 unsafe extern "C" {
     pub type Module;
-}
-unsafe extern "C" {
     pub type Context;
-}
-unsafe extern "C" {
     pub type Type;
-}
-unsafe extern "C" {
     pub type Value;
-}
-unsafe extern "C" {
     pub type ConstantInt;
-}
-unsafe extern "C" {
     pub type Attribute;
-}
-unsafe extern "C" {
     pub type Metadata;
-}
-unsafe extern "C" {
     pub type BasicBlock;
 }
 #[repr(C)]
@@ -661,11 +642,7 @@ pub struct Builder<'a>(InvariantOpaque<'a>);
 pub struct PassManager<'a>(InvariantOpaque<'a>);
 unsafe extern "C" {
     pub type Pass;
-}
-unsafe extern "C" {
     pub type TargetMachine;
-}
-unsafe extern "C" {
     pub type Archive;
 }
 #[repr(C)]
@@ -674,11 +651,7 @@ pub struct ArchiveIterator<'a>(InvariantOpaque<'a>);
 pub struct ArchiveChild<'a>(InvariantOpaque<'a>);
 unsafe extern "C" {
     pub type Twine;
-}
-unsafe extern "C" {
     pub type DiagnosticInfo;
-}
-unsafe extern "C" {
     pub type SMDiagnostic;
 }
 #[repr(C)]
@@ -912,17 +885,7 @@ unsafe extern "C" {
     pub fn LLVMGetPoison(Ty: &Type) -> &Value;
 
     // Operations on metadata
-    // FIXME: deprecated, replace with LLVMMDStringInContext2
-    pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value;
-
     pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata;
-
-    // FIXME: deprecated, replace with LLVMMDNodeInContext2
-    pub fn LLVMMDNodeInContext<'a>(
-        C: &'a Context,
-        Vals: *const &'a Value,
-        Count: c_uint,
-    ) -> &'a Value;
     pub fn LLVMMDNodeInContext2<'a>(
         C: &'a Context,
         Vals: *const &'a Metadata,
@@ -974,6 +937,7 @@ unsafe extern "C" {
     pub fn LLVMGetAlignment(Global: &Value) -> c_uint;
     pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint);
     pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass);
+    pub fn LLVMGlobalGetValueType(Global: &Value) -> &Type;
 
     // Operations on global variables
     pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>;
@@ -1040,7 +1004,7 @@ unsafe extern "C" {
     pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>);
 
     // Metadata
-    pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata);
+    pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata);
 
     // Terminators
     pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value;
@@ -2176,7 +2140,8 @@ unsafe extern "C" {
 
     pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char;
 
-    // This function makes copies of pointed to data, so the data's lifetime may end after this function returns
+    // This function makes copies of pointed to data, so the data's lifetime may end after this
+    // function returns.
     pub fn LLVMRustCreateTargetMachine(
         Triple: *const c_char,
         CPU: *const c_char,
@@ -2225,6 +2190,7 @@ unsafe extern "C" {
         IsLinkerPluginLTO: bool,
         NoPrepopulatePasses: bool,
         VerifyIR: bool,
+        LintIR: bool,
         UseThinLTOBuffers: bool,
         MergeFunctions: bool,
         UnrollLoops: bool,
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 618602ed70f..71fd7afb148 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -1,4 +1,4 @@
-use std::ffi::{c_char, c_void, CStr, CString};
+use std::ffi::{CStr, CString, c_char, c_void};
 use std::fmt::Write;
 use std::path::Path;
 use std::sync::Once;
@@ -11,10 +11,10 @@ use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_data_structures::unord::UnordSet;
 use rustc_fs_util::path_to_c_string;
 use rustc_middle::bug;
-use rustc_session::config::{PrintKind, PrintRequest};
 use rustc_session::Session;
+use rustc_session::config::{PrintKind, PrintRequest};
 use rustc_span::symbol::Symbol;
-use rustc_target::spec::{MergeFunctions, PanicStrategy};
+use rustc_target::spec::{MergeFunctions, PanicStrategy, SmallDataThresholdSupport};
 use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES};
 
 use crate::back::write::create_informational_target_machine;
@@ -125,6 +125,18 @@ unsafe fn configure_llvm(sess: &Session) {
         for arg in sess_args {
             add(&(*arg), true);
         }
+
+        match (
+            sess.opts.unstable_opts.small_data_threshold,
+            sess.target.small_data_threshold_support(),
+        ) {
+            // Set up the small-data optimization limit for architectures that use
+            // an LLVM argument to control this.
+            (Some(threshold), SmallDataThresholdSupport::LlvmArg(arg)) => {
+                add(&format!("--{arg}={threshold}"), false)
+            }
+            _ => (),
+        };
     }
 
     if sess.opts.unstable_opts.llvm_time_trace {
@@ -205,10 +217,10 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
 // where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`.
 //
 // Check the current rustc fork of LLVM in the repo at https://github.com/rust-lang/llvm-project/.
-// The commit in use can be found via the `llvm-project` submodule in https://github.com/rust-lang/rust/tree/master/src
-// Though note that Rust can also be build with an external precompiled version of LLVM
-// which might lead to failures if the oldest tested / supported LLVM version
-// doesn't yet support the relevant intrinsics
+// The commit in use can be found via the `llvm-project` submodule in
+// https://github.com/rust-lang/rust/tree/master/src Though note that Rust can also be build with
+// an external precompiled version of LLVM which might lead to failures if the oldest tested /
+// supported LLVM version doesn't yet support the relevant intrinsics.
 pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> {
     let arch = if sess.target.arch == "x86_64" {
         "x86"
@@ -246,28 +258,14 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea
         ("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")),
         ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")),
         // Filter out features that are not supported by the current LLVM version
-        ("aarch64", "faminmax") if get_version().0 < 18 => None,
-        ("aarch64", "fp8") if get_version().0 < 18 => None,
-        ("aarch64", "fp8dot2") if get_version().0 < 18 => None,
-        ("aarch64", "fp8dot4") if get_version().0 < 18 => None,
-        ("aarch64", "fp8fma") if get_version().0 < 18 => None,
         ("aarch64", "fpmr") if get_version().0 != 18 => None,
-        ("aarch64", "lut") if get_version().0 < 18 => None,
-        ("aarch64", "sme-f8f16") if get_version().0 < 18 => None,
-        ("aarch64", "sme-f8f32") if get_version().0 < 18 => None,
-        ("aarch64", "sme-fa64") if get_version().0 < 18 => None,
-        ("aarch64", "sme-lutv2") if get_version().0 < 18 => None,
-        ("aarch64", "ssve-fp8dot2") if get_version().0 < 18 => None,
-        ("aarch64", "ssve-fp8dot4") if get_version().0 < 18 => None,
-        ("aarch64", "ssve-fp8fma") if get_version().0 < 18 => None,
-        ("aarch64", "v9.5a") if get_version().0 < 18 => None,
-        // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called
-        // `fast-unaligned-access`. In LLVM 19, it was split back out.
+        // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single
+        // feature called `fast-unaligned-access`. In LLVM 19, it was split back out.
         ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => {
             Some(LLVMFeature::new("fast-unaligned-access"))
         }
-        // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled.
-        ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => {
+        // Enable the evex512 target feature if an avx512 target feature is enabled.
+        ("x86", s) if s.starts_with("avx512") => {
             Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")))
         }
         (_, s) => Some(LLVMFeature::new(s)),
@@ -290,7 +288,7 @@ pub(crate) fn check_tied_features(
             }
         }
     }
-    return None;
+    None
 }
 
 /// Used to generate cfg variables and apply features
@@ -353,9 +351,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
                 None
             }
         })
-        .filter(|feature| {
-            RUSTC_SPECIAL_FEATURES.contains(feature) || features.contains(&Symbol::intern(feature))
-        })
+        .filter(|feature| features.contains(&Symbol::intern(feature)))
         .map(|feature| Symbol::intern(feature))
         .collect()
 }
@@ -410,7 +406,8 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach
         .supported_target_features()
         .iter()
         .filter_map(|(feature, _gate, _implied)| {
-            // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
+            // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these
+            // strings.
             let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name;
             let desc =
                 match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() {
@@ -577,7 +574,6 @@ pub(crate) fn global_llvm_features(
     // -Ctarget-features
     if !only_base_features {
         let supported_features = sess.target.supported_target_features();
-        let (llvm_major, _, _) = get_version();
         let mut featsmap = FxHashMap::default();
 
         // insert implied features
@@ -654,12 +650,6 @@ pub(crate) fn global_llvm_features(
                     return None;
                 }
 
-                // if the target-feature is "backchain" and LLVM version is greater than 18
-                // then we also need to add "+backchain" to the target-features attribute.
-                // otherwise, we will only add the naked `backchain` attribute to the attribute-group.
-                if feature == "backchain" && llvm_major < 18 {
-                    return None;
-                }
                 // ... otherwise though we run through `to_llvm_features` when
                 // passing requests down to LLVM. This means that all in-language
                 // features also work on the command line instead of having two
diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs
index f1ef359594b..02e1995620b 100644
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@@ -14,7 +14,7 @@ use crate::errors::SymbolAlreadyDefined;
 use crate::type_of::LayoutLlvmExt;
 use crate::{base, llvm};
 
-impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> {
+impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
     fn predefine_static(
         &self,
         def_id: DefId,
@@ -24,8 +24,8 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> {
     ) {
         let instance = Instance::mono(self.tcx, def_id);
         let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
-        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
-        // the llvm type from the actual evaluated initializer.
+        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure
+        // out the llvm type from the actual evaluated initializer.
         let ty = if nested {
             self.tcx.types.unit
         } else {
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
index ec1e2cb8094..f1efc7a3dac 100644
--- a/compiler/rustc_codegen_llvm/src/type_.rs
+++ b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -13,7 +13,7 @@ use rustc_target::abi::{AddressSpace, Align, Integer, Size};
 use crate::abi::{FnAbiLlvmExt, LlvmType};
 use crate::context::CodegenCx;
 pub(crate) use crate::llvm::Type;
-use crate::llvm::{Bool, False, True};
+use crate::llvm::{Bool, False, Metadata, True};
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 use crate::{common, llvm};
@@ -141,7 +141,7 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 }
 
-impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> BaseTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn type_i8(&self) -> &'ll Type {
         unsafe { llvm::LLVMInt8TypeInContext(self.llcx) }
     }
@@ -245,7 +245,7 @@ impl Type {
     }
 }
 
-impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type {
         layout.llvm_type(self)
     }
@@ -280,46 +280,34 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     }
 }
 
-impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> {
+impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn add_type_metadata(&self, function: &'ll Value, typeid: String) {
         let typeid_metadata = self.typeid_metadata(typeid).unwrap();
-        let v = [self.const_usize(0), typeid_metadata];
         unsafe {
+            let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
             llvm::LLVMRustGlobalAddMetadata(
                 function,
                 llvm::MD_type as c_uint,
-                llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext(
-                    self.llcx,
-                    v.as_ptr(),
-                    v.len() as c_uint,
-                )),
+                llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()),
             )
         }
     }
 
     fn set_type_metadata(&self, function: &'ll Value, typeid: String) {
         let typeid_metadata = self.typeid_metadata(typeid).unwrap();
-        let v = [self.const_usize(0), typeid_metadata];
         unsafe {
+            let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
             llvm::LLVMGlobalSetMetadata(
                 function,
                 llvm::MD_type as c_uint,
-                llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext(
-                    self.llcx,
-                    v.as_ptr(),
-                    v.len() as c_uint,
-                )),
+                llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()),
             )
         }
     }
 
-    fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> {
+    fn typeid_metadata(&self, typeid: String) -> Option<&'ll Metadata> {
         Some(unsafe {
-            llvm::LLVMMDStringInContext(
-                self.llcx,
-                typeid.as_ptr() as *const c_char,
-                typeid.len() as c_uint,
-            )
+            llvm::LLVMMDStringInContext2(self.llcx, typeid.as_ptr() as *const c_char, typeid.len())
         })
     }
 
diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index 94e77c5bd70..f12b94d5887 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -1,8 +1,8 @@
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
-use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods, ConstMethods};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods};
 use rustc_middle::ty::Ty;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_target::abi::{Align, Endian, HasDataLayout, Size};
 
 use crate::builder::Builder;