diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
20 files changed, 110 insertions, 151 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index dea574a53cd..26718792f5f 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -285,7 +285,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } } -impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { +impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn store_fn_arg( &mut self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, @@ -465,9 +465,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()), ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); - if cx.sess().opts.optimize != config::OptLevel::No - && llvm_util::get_version() >= (18, 0, 0) - { + if cx.sess().opts.optimize != config::OptLevel::No { attributes::apply_to_llfn( llfn, llvm::AttributePlace::Argument(i), diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 1d91c3fb17d..775266fec97 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -356,7 +356,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { +impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { fn codegen_global_asm( &self, template: &[InlineAsmTemplatePiece], diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index aa6842c75ce..2ebe0be53aa 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -92,11 +92,9 @@ fn prepare_lto( dcx.emit_err(LtoDylib); return Err(FatalError); } - } else if *crate_type == CrateType::ProcMacro { - if !cgcx.opts.unstable_opts.dylib_lto { - dcx.emit_err(LtoProcMacro); - return Err(FatalError); - } + } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto { + dcx.emit_err(LtoProcMacro); + return Err(FatalError); } } @@ -314,7 +312,6 @@ fn fat_lto( } } }; - let mut serialized_bitcode = Vec::new(); { let (llcx, llmod) = { let llvm = &module.module_llvm; @@ -342,9 +339,7 @@ fn fat_lto( serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1)); // For all serialized bitcode files we parse them and link them in as we did - // above, this is all mostly handled in C++. Like above, though, we don't - // know much about the memory management here so we err on the side of being - // save and persist everything with the original module. + // above, this is all mostly handled in C++. let mut linker = Linker::new(llmod); for (bc_decoded, name) in serialized_modules { let _timer = cgcx @@ -355,7 +350,6 @@ fn fat_lto( info!("linking {:?}", name); let data = bc_decoded.data(); linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?; - serialized_bitcode.push(bc_decoded); } drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); @@ -372,7 +366,7 @@ fn fat_lto( } } - Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) + Ok(LtoModuleCodegen::Fat(module)) } pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index a5c27d2282e..b1b692cc027 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -185,7 +185,13 @@ pub(crate) fn target_machine_factory( let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); - let use_softfp = sess.opts.cg.soft_float; + let use_softfp = if sess.target.arch == "arm" && sess.target.abi == "eabihf" { + sess.opts.cg.soft_float + } else { + // `validate_commandline_args_with_session_available` has already warned about this being ignored. + // Let's make sure LLVM doesn't suddenly start using this flag on more targets. + false + }; let ffunction_sections = sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 40783825cae..c806d0c5499 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -26,13 +26,13 @@ use smallvec::SmallVec; use tracing::{debug, instrument}; use crate::abi::FnAbiLlvmExt; +use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use crate::{attributes, llvm_util}; // All Builders must have an llfn associated with them #[must_use] @@ -93,8 +93,6 @@ impl HasTargetSpec for Builder<'_, '_, '_> { } impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { - type LayoutOfResult = TyAndLayout<'tcx>; - #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { self.cx.handle_layout_err(err, span, ty) @@ -102,8 +100,6 @@ impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { } impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { - type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; - #[inline] fn handle_fn_abi_err( &self, @@ -124,10 +120,6 @@ impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { } } -impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> { - type CodegenCx = CodegenCx<'ll, 'tcx>; -} - macro_rules! builder_methods_for_value_instructions { ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { @@ -139,6 +131,8 @@ macro_rules! builder_methods_for_value_instructions { } impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { + type CodegenCx = CodegenCx<'ll, 'tcx>; + fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { let bx = Builder::with_cx(cx); unsafe { @@ -1317,15 +1311,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { - if llvm_util::get_version() < (17, 0, 2) { - // Work around https://github.com/llvm/llvm-project/issues/66984. - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); - } else { - // Cleanup is always the cold path. - let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); - } + // Cleanup is always the cold path. + let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); } } @@ -1767,8 +1755,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) { debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); - assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], @@ -1802,7 +1788,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp ); - assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; @@ -1844,7 +1829,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", fn_name, hash, cond_loc, mcdc_temp, bool_value ); - assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( &[ diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index ef6560ecbe5..203c63f0ae7 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -113,7 +113,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } -impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { +impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn const_null(&self, t: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMConstNull(t) } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index d60122fccee..c3b1676f77e 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -565,7 +565,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } -impl<'ll> StaticMethods for CodegenCx<'ll, '_> { +impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> { fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { if let Some(&gv) = self.const_globals.borrow().get(&cv) { unsafe { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 01aae24ab56..03af31d36fb 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -15,7 +15,6 @@ use rustc_middle::middle::codegen_fn_attrs::PatchableFunctionEntry; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, - TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; @@ -25,9 +24,8 @@ use rustc_session::config::{ use rustc_session::Session; use rustc_span::source_map::Spanned; use rustc_span::{Span, DUMMY_SP}; -use rustc_target::abi::call::FnAbi; use rustc_target::abi::{HasDataLayout, TargetDataLayout, VariantIdx}; -use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; +use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; @@ -122,14 +120,6 @@ pub(crate) unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (18, 0, 0) { - if sess.target.arch == "x86" || sess.target.arch == "x86_64" { - // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants. - // Earlier LLVMs leave this as default alignment, so remove it. - // See https://reviews.llvm.org/D86310 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - } if llvm_version < (19, 0, 0) { if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { @@ -387,6 +377,24 @@ pub(crate) unsafe fn create_module<'ll>( } } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) + { + // Set up the small-data optimization limit for architectures that use + // an LLVM module flag to control this. + (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => { + let flag = SmallCStr::new(flag.as_ref()); + unsafe { + llvm::LLVMRustAddModuleFlagU32( + llmod, + llvm::LLVMModFlagBehavior::Error, + flag.as_c_str().as_ptr(), + threshold as u32, + ) + } + } + _ => (), + }; + // Insert `llvm.ident` metadata. // // On the wasm targets it will get hooked up to the "producer" sections @@ -580,7 +588,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } } -impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { +impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn vtables( &self, ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>> @@ -1140,8 +1148,6 @@ impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { - type LayoutOfResult = TyAndLayout<'tcx>; - #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { @@ -1153,8 +1159,6 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { } impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { - type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; - #[inline] fn handle_fn_abi_err( &self, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index b5acfabfde2..a9f65ee8a93 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,5 +1,5 @@ use itertools::Itertools as _; -use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 69babc7c9cf..2ca5fc3300b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -2,8 +2,8 @@ use std::cell::RefCell; use libc::c_uint; use rustc_codegen_ssa::traits::{ - BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, - StaticMethods, + BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, + MiscCodegenMethods, StaticCodegenMethods, }; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 181022087f3..8a132f89aa3 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; -use rustc_codegen_ssa::traits::ConstMethods; +use rustc_codegen_ssa::traits::ConstCodegenMethods; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 238fbad4dfd..0b3140cc91f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; -use rustc_codegen_ssa::traits::ConstMethods; +use rustc_codegen_ssa::traits::ConstCodegenMethods; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self}; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index f5d6fc6f080..920c9e06be4 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -1,8 +1,8 @@ #![doc = include_str!("doc.md")] use std::cell::{OnceCell, RefCell}; -use std::iter; use std::ops::Range; +use std::{iter, ptr}; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names; @@ -209,6 +209,12 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { } } + fn clear_dbg_loc(&mut self) { + unsafe { + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, ptr::null()); + } + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { gdb::insert_reference_to_gdb_debug_scripts_section_global(self) } @@ -280,7 +286,7 @@ impl CodegenCx<'_, '_> { } } -impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { +impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn create_function_debug_context( &self, instance: Instance<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 4e4500b6373..b0b29ca1280 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -12,7 +12,7 @@ //! * When in doubt, define. use itertools::Itertools; -use rustc_codegen_ssa::traits::TypeMembershipMethods; +use rustc_codegen_ssa::traits::TypeMembershipCodegenMethods; use rustc_data_structures::fx::FxIndexSet; use rustc_middle::ty::{Instance, Ty}; use rustc_sanitizers::{cfi, kcfi}; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 05fb77a193a..cc921aa87bc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -148,7 +148,7 @@ fn get_simple_intrinsic<'ll>( Some(cx.get_intrinsic(llvm_name)) } -impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { +impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, @@ -573,6 +573,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { span, ) { Ok(llval) => llval, + // If there was an error, just skip this invocation... we'll abort compilation anyway, + // but we can keep codegen'ing to find more errors. Err(()) => return Ok(()), } } @@ -1290,24 +1292,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle { - // Make sure this is actually an array or SIMD vector, since typeck only checks the length-suffixed - // version of this intrinsic. + // Make sure this is actually a SIMD vector. let idx_ty = args[2].layout.ty; - let n: u64 = match idx_ty.kind() { - ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( - || span_bug!(span, "could not evaluate shuffle index array length"), - ) - } - _ if idx_ty.is_simd() - && matches!( - idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), - ty::Uint(ty::UintTy::U32) - ) => - { - idx_ty.simd_size_and_type(bx.cx.tcx).0 - } - _ => return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }), + let n: u64 = if idx_ty.is_simd() + && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32)) + { + idx_ty.simd_size_and_type(bx.cx.tcx).0 + } else { + return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }) }; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); @@ -1322,38 +1314,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let total_len = u128::from(in_len) * 2; - let vector = args[2].immediate(); - - let indices: Option<Vec<_>> = (0..n) - .map(|i| { - let arg_idx = i; - let val = bx.const_get_elt(vector, i as u64); - match bx.const_to_opt_u128(val, true) { - None => { - bug!("typeck should have already ensured that these are const") - } - Some(idx) if idx >= total_len => { - bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { - span, - name, - arg_idx, - total_len, - }); - None - } - Some(idx) => Some(bx.const_i32(idx as i32)), - } - }) - .collect(); - let Some(indices) = indices else { - return Ok(bx.const_null(llret_ty)); - }; + // Check that the indices are in-bounds. + let indices = args[2].immediate(); + for i in 0..n { + let val = bx.const_get_elt(indices, i as u64); + let idx = bx + .const_to_opt_u128(val, true) + .unwrap_or_else(|| bug!("typeck should have already ensured that these are const")); + if idx >= total_len { + return_error!(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: i, + total_len, + }); + } + } - return Ok(bx.shuffle_vector( - args[0].immediate(), - args[1].immediate(), - bx.const_vector(&indices), - )); + return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices)); } if name == sym::simd_insert { @@ -1371,13 +1349,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .const_to_opt_u128(args[1].immediate(), false) .expect("typeck should have ensure that this is a const"); if idx >= in_len.into() { - bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + return_error!(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: 1, total_len: in_len.into(), }); - return Ok(bx.const_null(llret_ty)); } return Ok(bx.insert_element( args[0].immediate(), @@ -1394,13 +1371,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .const_to_opt_u128(args[1].immediate(), false) .expect("typeck should have ensure that this is a const"); if idx >= in_len.into() { - bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + return_error!(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: 1, total_len: in_len.into(), }); - return Ok(bx.const_null(llret_ty)); } return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32))); } @@ -2090,14 +2066,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; } - arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0); + arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0); arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, vector_reduce_fadd_reassoc, false, add, - 0.0 + -0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 3bf4d496408..e84ab0aa538 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1041,7 +1041,7 @@ unsafe extern "C" { pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata); + pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index d55220ba5c3..358bfcb1573 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -14,7 +14,7 @@ use rustc_middle::bug; use rustc_session::config::{PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_target::spec::{MergeFunctions, PanicStrategy}; +use rustc_target::spec::{MergeFunctions, PanicStrategy, SmallDataThresholdSupport}; use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES}; use crate::back::write::create_informational_target_machine; @@ -125,6 +125,18 @@ unsafe fn configure_llvm(sess: &Session) { for arg in sess_args { add(&(*arg), true); } + + match ( + sess.opts.unstable_opts.small_data_threshold, + sess.target.small_data_threshold_support(), + ) { + // Set up the small-data optimization limit for architectures that use + // an LLVM argument to control this. + (Some(threshold), SmallDataThresholdSupport::LlvmArg(arg)) => { + add(&format!("--{arg}={threshold}"), false) + } + _ => (), + }; } if sess.opts.unstable_opts.llvm_time_trace { @@ -246,28 +258,14 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")), ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")), // Filter out features that are not supported by the current LLVM version - ("aarch64", "faminmax") if get_version().0 < 18 => None, - ("aarch64", "fp8") if get_version().0 < 18 => None, - ("aarch64", "fp8dot2") if get_version().0 < 18 => None, - ("aarch64", "fp8dot4") if get_version().0 < 18 => None, - ("aarch64", "fp8fma") if get_version().0 < 18 => None, ("aarch64", "fpmr") if get_version().0 != 18 => None, - ("aarch64", "lut") if get_version().0 < 18 => None, - ("aarch64", "sme-f8f16") if get_version().0 < 18 => None, - ("aarch64", "sme-f8f32") if get_version().0 < 18 => None, - ("aarch64", "sme-fa64") if get_version().0 < 18 => None, - ("aarch64", "sme-lutv2") if get_version().0 < 18 => None, - ("aarch64", "ssve-fp8dot2") if get_version().0 < 18 => None, - ("aarch64", "ssve-fp8dot4") if get_version().0 < 18 => None, - ("aarch64", "ssve-fp8fma") if get_version().0 < 18 => None, - ("aarch64", "v9.5a") if get_version().0 < 18 => None, // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called // `fast-unaligned-access`. In LLVM 19, it was split back out. ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { Some(LLVMFeature::new("fast-unaligned-access")) } - // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled. - ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => { + // Enable the evex512 target feature if an avx512 target feature is enabled. + ("x86", s) if s.starts_with("avx512") => { Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))) } (_, s) => Some(LLVMFeature::new(s)), @@ -290,7 +288,7 @@ pub(crate) fn check_tied_features( } } } - return None; + None } /// Used to generate cfg variables and apply features @@ -575,7 +573,6 @@ pub(crate) fn global_llvm_features( // -Ctarget-features if !only_base_features { let supported_features = sess.target.supported_target_features(); - let (llvm_major, _, _) = get_version(); let mut featsmap = FxHashMap::default(); // insert implied features @@ -652,12 +649,6 @@ pub(crate) fn global_llvm_features( return None; } - // if the target-feature is "backchain" and LLVM version is greater than 18 - // then we also need to add "+backchain" to the target-features attribute. - // otherwise, we will only add the naked `backchain` attribute to the attribute-group. - if feature == "backchain" && llvm_major < 18 { - return None; - } // ... otherwise though we run through `to_llvm_features` when // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f1ef359594b..9bd491664d8 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -14,7 +14,7 @@ use crate::errors::SymbolAlreadyDefined; use crate::type_of::LayoutLlvmExt; use crate::{base, llvm}; -impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { +impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( &self, def_id: DefId, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index ec1e2cb8094..2c2b9030b7c 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -141,7 +141,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } -impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { +impl<'ll, 'tcx> BaseTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn type_i8(&self) -> &'ll Type { unsafe { llvm::LLVMInt8TypeInContext(self.llcx) } } @@ -245,7 +245,7 @@ impl Type { } } -impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { +impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { layout.llvm_type(self) } @@ -280,7 +280,7 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } -impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { +impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn add_type_metadata(&self, function: &'ll Value, typeid: String) { let typeid_metadata = self.typeid_metadata(typeid).unwrap(); let v = [self.const_usize(0), typeid_metadata]; diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 94e77c5bd70..781cee81180 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -1,6 +1,6 @@ use rustc_codegen_ssa::common::IntPredicate; use rustc_codegen_ssa::mir::operand::OperandRef; -use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods, ConstMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::Ty; use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; |