6 files changed, 72 insertions, 62 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index 09896b89ebf..2ebe0be53aa 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -92,11 +92,9 @@ fn prepare_lto(
                     dcx.emit_err(LtoDylib);
                     return Err(FatalError);
                 }
-            } else if *crate_type == CrateType::ProcMacro {
-                if !cgcx.opts.unstable_opts.dylib_lto {
-                    dcx.emit_err(LtoProcMacro);
-                    return Err(FatalError);
-                }
+            } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
+                dcx.emit_err(LtoProcMacro);
+                return Err(FatalError);
             }
         }
 
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 01aae24ab56..73c2c15717f 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -27,7 +27,7 @@ use rustc_span::source_map::Spanned;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{HasDataLayout, TargetDataLayout, VariantIdx};
-use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel};
+use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel};
 use smallvec::SmallVec;
 
 use crate::back::write::to_llvm_code_model;
@@ -387,6 +387,24 @@ pub(crate) unsafe fn create_module<'ll>(
         }
     }
 
+    match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support())
+    {
+        // Set up the small-data optimization limit for architectures that use
+        // an LLVM module flag to control this.
+        (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => {
+            let flag = SmallCStr::new(flag.as_ref());
+            unsafe {
+                llvm::LLVMRustAddModuleFlagU32(
+                    llmod,
+                    llvm::LLVMModFlagBehavior::Error,
+                    flag.as_c_str().as_ptr(),
+                    threshold as u32,
+                )
+            }
+        }
+        _ => (),
+    };
+
     // Insert `llvm.ident` metadata.
     //
     // On the wasm targets it will get hooked up to the "producer" sections
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index f5d6fc6f080..842212ac05d 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -1,8 +1,8 @@
 #![doc = include_str!("doc.md")]
 
 use std::cell::{OnceCell, RefCell};
-use std::iter;
 use std::ops::Range;
+use std::{iter, ptr};
 
 use libc::c_uint;
 use rustc_codegen_ssa::debuginfo::type_names;
@@ -209,6 +209,12 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
         }
     }
 
+    fn clear_dbg_loc(&mut self) {
+        unsafe {
+            llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, ptr::null());
+        }
+    }
+
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) {
         gdb::insert_reference_to_gdb_debug_scripts_section_global(self)
     }
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 05fb77a193a..9705dd506b9 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -573,6 +573,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     span,
                 ) {
                     Ok(llval) => llval,
+                    // If there was an error, just skip this invocation... we'll abort compilation anyway,
+                    // but we can keep codegen'ing to find more errors.
                     Err(()) => return Ok(()),
                 }
             }
@@ -1290,24 +1292,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_shuffle {
-        // Make sure this is actually an array or SIMD vector, since typeck only checks the length-suffixed
-        // version of this intrinsic.
+        // Make sure this is actually a SIMD vector.
         let idx_ty = args[2].layout.ty;
-        let n: u64 = match idx_ty.kind() {
-            ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
-                len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
-                    || span_bug!(span, "could not evaluate shuffle index array length"),
-                )
-            }
-            _ if idx_ty.is_simd()
-                && matches!(
-                    idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(),
-                    ty::Uint(ty::UintTy::U32)
-                ) =>
-            {
-                idx_ty.simd_size_and_type(bx.cx.tcx).0
-            }
-            _ => return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }),
+        let n: u64 = if idx_ty.is_simd()
+            && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32))
+        {
+            idx_ty.simd_size_and_type(bx.cx.tcx).0
+        } else {
+            return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty })
         };
 
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
@@ -1322,38 +1314,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
         let total_len = u128::from(in_len) * 2;
 
-        let vector = args[2].immediate();
-
-        let indices: Option<Vec<_>> = (0..n)
-            .map(|i| {
-                let arg_idx = i;
-                let val = bx.const_get_elt(vector, i as u64);
-                match bx.const_to_opt_u128(val, true) {
-                    None => {
-                        bug!("typeck should have already ensured that these are const")
-                    }
-                    Some(idx) if idx >= total_len => {
-                        bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
-                            span,
-                            name,
-                            arg_idx,
-                            total_len,
-                        });
-                        None
-                    }
-                    Some(idx) => Some(bx.const_i32(idx as i32)),
-                }
-            })
-            .collect();
-        let Some(indices) = indices else {
-            return Ok(bx.const_null(llret_ty));
-        };
+        // Check that the indices are in-bounds.
+        let indices = args[2].immediate();
+        for i in 0..n {
+            let val = bx.const_get_elt(indices, i as u64);
+            let idx = bx
+                .const_to_opt_u128(val, true)
+                .unwrap_or_else(|| bug!("typeck should have already ensured that these are const"));
+            if idx >= total_len {
+                return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
+                    span,
+                    name,
+                    arg_idx: i,
+                    total_len,
+                });
+            }
+        }
 
-        return Ok(bx.shuffle_vector(
-            args[0].immediate(),
-            args[1].immediate(),
-            bx.const_vector(&indices),
-        ));
+        return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices));
     }
 
     if name == sym::simd_insert {
@@ -1371,13 +1349,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             .const_to_opt_u128(args[1].immediate(), false)
             .expect("typeck should have ensure that this is a const");
         if idx >= in_len.into() {
-            bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
+            return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
                 span,
                 name,
                 arg_idx: 1,
                 total_len: in_len.into(),
             });
-            return Ok(bx.const_null(llret_ty));
         }
         return Ok(bx.insert_element(
             args[0].immediate(),
@@ -1394,13 +1371,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             .const_to_opt_u128(args[1].immediate(), false)
             .expect("typeck should have ensure that this is a const");
         if idx >= in_len.into() {
-            bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds {
+            return_error!(InvalidMonomorphization::SimdIndexOutOfBounds {
                 span,
                 name,
                 arg_idx: 1,
                 total_len: in_len.into(),
             });
-            return Ok(bx.const_null(llret_ty));
         }
         return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32)));
     }
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 3bf4d496408..e84ab0aa538 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -1041,7 +1041,7 @@ unsafe extern "C" {
     pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>);
 
     // Metadata
-    pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata);
+    pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata);
 
     // Terminators
     pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value;
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index d55220ba5c3..29afe6f6bfc 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -14,7 +14,7 @@ use rustc_middle::bug;
 use rustc_session::config::{PrintKind, PrintRequest};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
-use rustc_target::spec::{MergeFunctions, PanicStrategy};
+use rustc_target::spec::{MergeFunctions, PanicStrategy, SmallDataThresholdSupport};
 use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES};
 
 use crate::back::write::create_informational_target_machine;
@@ -125,6 +125,18 @@ unsafe fn configure_llvm(sess: &Session) {
         for arg in sess_args {
             add(&(*arg), true);
         }
+
+        match (
+            sess.opts.unstable_opts.small_data_threshold,
+            sess.target.small_data_threshold_support(),
+        ) {
+            // Set up the small-data optimization limit for architectures that use
+            // an LLVM argument to control this.
+            (Some(threshold), SmallDataThresholdSupport::LlvmArg(arg)) => {
+                add(&format!("--{arg}={threshold}"), false)
+            }
+            _ => (),
+        };
     }
 
     if sess.opts.unstable_opts.llvm_time_trace {
@@ -290,7 +302,7 @@ pub(crate) fn check_tied_features(
             }
         }
     }
-    return None;
+    None
 }
 
 /// Used to generate cfg variables and apply features