diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 64 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 61 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 48 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/mono_item.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 2 |
13 files changed, 164 insertions, 106 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 0718bebb31b..b5b4f894e4d 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -590,7 +590,6 @@ impl From<Conv> for llvm::CallConv { Conv::Cold => llvm::ColdCallConv, Conv::PreserveMost => llvm::PreserveMost, Conv::PreserveAll => llvm::PreserveAll, - Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, Conv::ArmAapcs => llvm::ArmAapcsCallConv, diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 0a7ea599431..f9eaa0d94cb 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -356,9 +356,6 @@ pub fn from_fn_attrs<'ll, 'tcx>( if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { - to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); - } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { to_add.push(MemoryEffects::ReadOnly.create_attr(cx.llcx)); } @@ -482,7 +479,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // `+multivalue` feature because the purpose of the wasm abi is to match // the WebAssembly specification, which has this feature. This won't be // needed when LLVM enables this `multivalue` feature by default. - if !cx.tcx.is_closure_or_coroutine(instance.def_id()) { + if !cx.tcx.is_closure_like(instance.def_id()) { let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); if abi == Abi::Wasm { function_features.push("+multivalue".to_string()); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 42bd8687042..06a681c24e6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -21,6 +21,7 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::config::{self, CrateType, Lto}; +use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; @@ -787,7 +788,7 @@ pub unsafe fn optimize_thin_module( #[derive(Debug, Default)] pub struct ThinLTOKeysMap { // key = llvm name of importing module, value = LLVM cache key - keys: FxHashMap<String, String>, + keys: BTreeMap<String, String>, } impl ThinLTOKeysMap { @@ -797,7 +798,6 @@ impl ThinLTOKeysMap { let mut writer = io::BufWriter::new(file); // The entries are loaded back into a hash map in `load_from_file()`, so // the order in which we write them to file here does not matter. - #[allow(rustc::potential_query_instability)] for (module, key) in &self.keys { writeln!(writer, "{module} {key}")?; } @@ -806,7 +806,7 @@ impl ThinLTOKeysMap { fn load_from_file(path: &Path) -> io::Result<Self> { use std::io::BufRead; - let mut keys = FxHashMap::default(); + let mut keys = BTreeMap::default(); let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7ed27b33dce..8cab2a3f27c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + fn checked_binop( &mut self, oop: OverflowOp, @@ -1327,17 +1367,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } @@ -1366,22 +1406,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1d1b6e6148d..7dfcf1ab50e 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -34,6 +34,7 @@ use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; use libc::c_uint; +use std::borrow::Borrow; use std::cell::{Cell, RefCell}; use std::ffi::CStr; use std::str; @@ -145,10 +146,17 @@ pub unsafe fn create_module<'ll>( .replace("-Fi64", ""); } } + if llvm_version < (18, 0, 0) { + if sess.target.arch == "x86" || sess.target.arch == "x86_64" { + // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants. + // Earlier LLVMs leave this as default alignment, so remove it. + // See https://reviews.llvm.org/D86310 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. - if sess.target.is_builtin { - // tm is disposed by its drop impl + { let tm = crate::back::write::create_informational_target_machine(tcx.sess); llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); @@ -156,33 +164,13 @@ pub unsafe fn create_module<'ll>( let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) .expect("got a non-UTF8 data-layout from LLVM"); - // Unfortunately LLVM target specs change over time, and right now we - // don't have proper support to work with any more than one - // `data_layout` than the one that is in the rust-lang/rust repo. If - // this compiler is configured against a custom LLVM, we may have a - // differing data layout, even though we should update our own to use - // that one. - // - // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we - // disable this check entirely as we may be configured with something - // that has a different target layout. - // - // Unsure if this will actually cause breakage when rustc is configured - // as such. - // - // FIXME(#34960) - let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = !cfg_llvm_root.trim().is_empty(); - - if !custom_llvm_used && target_data_layout != llvm_data_layout { - bug!( - "data-layout for target `{rustc_target}`, `{rustc_layout}`, \ - differs from LLVM target's `{llvm_target}` default layout, `{llvm_layout}`", - rustc_target = sess.opts.target_triple, - rustc_layout = target_data_layout, - llvm_target = sess.target.llvm_target, - llvm_layout = llvm_data_layout - ); + if target_data_layout != llvm_data_layout { + tcx.dcx().emit_err(crate::errors::MismatchedDataLayout { + rustc_target: sess.opts.target_triple.to_string().as_str(), + rustc_layout: target_data_layout.as_str(), + llvm_target: sess.target.llvm_target.borrow(), + llvm_layout: llvm_data_layout, + }); } } @@ -908,6 +896,21 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void); ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void); + // FIXME: This is an infinitesimally small portion of the types you can + // pass to this intrinsic, if we can ever lazily register intrinsics we + // should register these when they're used, that way any type can be + // passed. + ifn!("llvm.is.constant.i1", fn(i1) -> i1); + ifn!("llvm.is.constant.i8", fn(t_i8) -> i1); + ifn!("llvm.is.constant.i16", fn(t_i16) -> i1); + ifn!("llvm.is.constant.i32", fn(t_i32) -> i1); + ifn!("llvm.is.constant.i64", fn(t_i64) -> i1); + ifn!("llvm.is.constant.i128", fn(t_i128) -> i1); + ifn!("llvm.is.constant.isize", fn(t_isize) -> i1); + ifn!("llvm.is.constant.f32", fn(t_f32) -> i1); + ifn!("llvm.is.constant.f64", fn(t_f64) -> i1); + ifn!("llvm.is.constant.ptr", fn(ptr) -> i1); + ifn!("llvm.expect.i1", fn(i1, i1) -> i1); ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32); ifn!("llvm.localescape", fn(...) -> void); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 76c9ac6614a..f961cd2d00b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -461,6 +461,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), @@ -1068,6 +1069,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), + ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { bug!( "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}", @@ -1153,7 +1155,8 @@ fn build_closure_env_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let closure_env_type = unique_type_id.expect_ty(); - let &ty::Closure(def_id, _args) = closure_env_type.kind() else { + let &(ty::Closure(def_id, _) | ty::CoroutineClosure(def_id, _)) = closure_env_type.kind() + else { bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) }; let containing_scope = get_namespace_for_item(cx, def_id); diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 697ce602298..87e3774068b 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -102,7 +102,7 @@ pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_> { fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { let diag: DiagnosticBuilder<'_, G> = self.0.into_diagnostic(dcx, level); - let (message, _) = diag.messages().first().expect("`LlvmError` with no message"); + let (message, _) = diag.messages.first().expect("`LlvmError` with no message"); let message = dcx.eagerly_translate_to_string(message.clone(), diag.args()); DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) @@ -131,7 +131,7 @@ impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for TargetFeatureDisableOrEnabl diag.span(span); }; if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); + diag.subdiagnostic(dcx, missing_features); } diag.arg("features", self.features.join(", ")); diag @@ -244,3 +244,18 @@ pub(crate) struct CopyBitcode { pub struct UnknownCompression { pub algorithm: &'static str, } + +#[derive(Diagnostic)] +#[diag(codegen_llvm_mismatch_data_layout)] +pub struct MismatchedDataLayout<'a> { + pub rustc_target: &'a str, + pub rustc_layout: &'a str, + pub llvm_target: &'a str, + pub llvm_layout: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_target_feature_prefix)] +pub(crate) struct InvalidTargetFeaturePrefix<'a> { + pub feature: &'a str, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index a0f9d5cf7cd..23e6f054a7c 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -86,7 +86,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, &'ll Value>], llresult: &'ll Value, span: Span, - ) { + ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); @@ -119,6 +119,18 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::likely => { self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) } + sym::is_val_statically_known => { + let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); + match self.type_kind(intrinsic_type) { + TypeKind::Pointer | TypeKind::Integer | TypeKind::Float | TypeKind::Double => { + self.call_intrinsic( + &format!("llvm.is.constant.{:?}", intrinsic_type), + &[args[0].immediate()], + ) + } + _ => self.const_bool(false), + } + } sym::unlikely => self .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), kw::Try => { @@ -129,7 +141,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args[2].immediate(), llresult, ); - return; + return Ok(()); } sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), sym::va_copy => { @@ -182,17 +194,17 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { if !result.layout.is_zst() { self.store(load, result.llval, result.align); } - return; + return Ok(()); } sym::volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.volatile_store(self, dst); - return; + return Ok(()); } sym::unaligned_volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.unaligned_volatile_store(self, dst); - return; + return Ok(()); } sym::prefetch_read_data | sym::prefetch_write_data @@ -293,7 +305,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { name, ty, }); - return; + return Ok(()); } } } @@ -375,7 +387,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); // We have copied the value to `result` already. - return; + return Ok(()); } _ if name.as_str().starts_with("simd_") => { @@ -383,11 +395,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, ) { Ok(llval) => llval, - Err(()) => return, + Err(()) => return Ok(()), } } - _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), + _ => { + debug!("unknown intrinsic '{}' -- falling back to default body", name); + // Call the fallback body instead of generating the intrinsic code + return Err(ty::Instance::new(instance.def_id(), instance.args)); + } }; if !fn_abi.ret.is_ignore() { @@ -399,6 +415,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .store(self, result); } } + Ok(()) } fn abort(&mut self) { @@ -1863,14 +1880,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, + vector_reduce_fadd_reassoc, false, add, 0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, + vector_reduce_fmul_reassoc, false, mul, 1.0 @@ -1903,9 +1920,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); - minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); - minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); - macro_rules! bitwise_red { ($name:ident : $red:ident, $boolean:expr) => { if name == sym::$name { @@ -1973,10 +1987,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match in_elem.kind() { ty::RawPtr(p) => { - let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - assert!(!check_sized); // we are in codegen, so we shouldn't see these types require!( metadata.is_unit(), InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } @@ -1988,10 +2001,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } match out_elem.kind() { ty::RawPtr(p) => { - let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - assert!(!check_sized); // we are in codegen, so we shouldn't see these types require!( metadata.is_unit(), InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index a81056ed3ad..35210b0b2e8 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -13,12 +13,7 @@ #![feature(hash_raw_entry)] #![feature(iter_intersperse)] #![feature(let_chains)] -#![feature(min_specialization)] -#![feature(never_type)] #![feature(impl_trait_in_assoc_type)] -#![recursion_limit = "256"] -#![deny(rustc::untranslatable_diagnostic)] -#![deny(rustc::diagnostic_outside_of_impl)] #[macro_use] extern crate rustc_macros; @@ -374,7 +369,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") @@ -387,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend { }); } - Ok((codegen_results, work_products)) + (codegen_results, work_products) } fn link( diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index ee73c6b4756..dbf35e5f499 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -106,7 +106,6 @@ pub enum CallConv { X86_Intr = 83, AvrNonBlockingInterrupt = 84, AvrInterrupt = 85, - AmdGpuKernel = 91, } /// LLVMRustLinkage @@ -185,7 +184,6 @@ pub enum AttributeKind { SanitizeMemory = 22, NonLazyBind = 23, OptimizeNone = 24, - ReturnsTwice = 25, ReadNone = 26, SanitizeHWAddress = 28, WillReturn = 29, @@ -1620,6 +1618,8 @@ extern "C" { ) -> &'a Value; pub fn LLVMRustSetFastMath(Instr: &Value); + pub fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 99f4488ac0f..54e8ed85e32 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,7 +1,7 @@ use crate::back::write::create_informational_target_machine; use crate::errors::{ - PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, - UnknownCTargetFeaturePrefix, UnstableCTargetFeature, + InvalidTargetFeaturePrefix, PossibleFeature, TargetFeatureDisableOrEnable, + UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; use libc::c_int; @@ -213,6 +213,7 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("x86", "rdrand") => LLVMFeature::new("rdrnd"), ("x86", "bmi1") => LLVMFeature::new("bmi"), ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("x86", "lahfsahf") => LLVMFeature::new("sahf"), ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), ("aarch64", "dpb") => LLVMFeature::new("ccpp"), ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), @@ -265,6 +266,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { LLVMFeature::new("unaligned-scalar-mem") } + // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled. + ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => { + LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")) + } (_, s) => LLVMFeature::new(s), } } @@ -511,7 +516,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str sess.target .features .split(',') - .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + .filter(|v| !v.is_empty() && backend_feature_name(sess, v).is_some()) .map(String::from), ); @@ -535,7 +540,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } }; - let feature = backend_feature_name(s)?; + let feature = backend_feature_name(sess, s)?; // Warn against use of LLVM specific feature names and unstable features on the CLI. if diagnostics { let feature_state = supported_features.iter().find(|&&(v, _)| v == feature); @@ -611,11 +616,11 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str /// Returns a feature name for the given `+feature` or `-feature` string. /// /// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) -fn backend_feature_name(s: &str) -> Option<&str> { +fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { // features must start with a `+` or `-`. - let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { - bug!("target feature `{}` must begin with a `+` or `-`", s); - }); + let feature = s + .strip_prefix(&['+', '-'][..]) + .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. if RUSTC_SPECIFIC_FEATURES.contains(&feature) { diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f796ce0990f..f7630719368 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -123,6 +123,17 @@ impl CodegenCx<'_, '_> { return false; } + // Match clang by only supporting COFF and ELF for now. + if self.tcx.sess.target.is_like_osx { + return false; + } + + // With pie relocation model calls of functions defined in the translation + // unit can use copy relocations. + if self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration { + return true; + } + // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); @@ -130,18 +141,12 @@ impl CodegenCx<'_, '_> { return false; } - // Match clang by only supporting COFF and ELF for now. - if self.tcx.sess.target.is_like_osx { - return false; + // Respect the direct-access-external-data to override default behavior if present. + if let Some(direct) = self.tcx.sess.direct_access_external_data() { + return direct; } // Static relocation model should force copy relocations everywhere. - if self.tcx.sess.relocation_model() == RelocModel::Static { - return true; - } - - // With pie relocation model calls of functions defined in the translation - // unit can use copy relocations. - self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration + self.tcx.sess.relocation_model() == RelocModel::Static } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index e88f4217c9d..219c7025311 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -33,7 +33,7 @@ fn uncached_llvm_type<'a, 'tcx>( // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str // For performance reasons we use names only when emitting LLVM IR. if !cx.sess().fewer_names() => { |