diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
20 files changed, 280 insertions, 188 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 97dc401251c..b5b4f894e4d 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -6,7 +6,7 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; @@ -253,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { bx.lifetime_end(llscratch, scratch_size); } } else { - OperandValue::Immediate(val).store(bx, dst); + OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst); } } @@ -590,7 +590,6 @@ impl From<Conv> for llvm::CallConv { Conv::Cold => llvm::ColdCallConv, Conv::PreserveMost => llvm::PreserveMost, Conv::PreserveAll => llvm::PreserveAll, - Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, Conv::ArmAapcs => llvm::ArmAapcsCallConv, diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 58b3aa438c5..ca376029735 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -134,7 +134,8 @@ fn create_wrapper_function( llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); } if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); + let uwtable = + attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 1323261ae92..a413466093b 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -690,6 +690,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", @@ -867,7 +868,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::S390x( + S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr, + ) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 3cc33b83434..f9eaa0d94cb 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -95,11 +95,12 @@ pub fn sanitize_attrs<'ll>( /// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. #[inline] -pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { +pub fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) -> &Attribute { // NOTE: We should determine if we even need async unwind tables, as they // take have more overhead and if we can use sync unwind tables we // probably should. - llvm::CreateUWTableAttr(llcx, true) + let async_unwind = !use_sync_unwind.unwrap_or(false); + llvm::CreateUWTableAttr(llcx, async_unwind) } pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { @@ -333,7 +334,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // You can also find more info on why Windows always requires uwtables here: // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx)); + to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); } if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { @@ -355,9 +356,6 @@ pub fn from_fn_attrs<'ll, 'tcx>( if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { - to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); - } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { to_add.push(MemoryEffects::ReadOnly.create_attr(cx.llcx)); } @@ -481,7 +479,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // `+multivalue` feature because the purpose of the wasm abi is to match // the WebAssembly specification, which has this feature. This won't be // needed when LLVM enables this `multivalue` feature by default. - if !cx.tcx.is_closure(instance.def_id()) { + if !cx.tcx.is_closure_like(instance.def_id()) { let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); if abi == Abi::Wasm { function_features.push("+multivalue".to_string()); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index e9e8ade09b7..06a681c24e6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -21,6 +21,7 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::config::{self, CrateType, Lto}; +use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; @@ -60,7 +61,7 @@ fn prepare_lto( }; let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { - if info.level.is_below_threshold(export_threshold) || info.used || info.used_compiler { + if info.level.is_below_threshold(export_threshold) || info.used { Some(CString::new(name.as_str()).unwrap()) } else { None @@ -787,7 +788,7 @@ pub unsafe fn optimize_thin_module( #[derive(Debug, Default)] pub struct ThinLTOKeysMap { // key = llvm name of importing module, value = LLVM cache key - keys: FxHashMap<String, String>, + keys: BTreeMap<String, String>, } impl ThinLTOKeysMap { @@ -797,7 +798,6 @@ impl ThinLTOKeysMap { let mut writer = io::BufWriter::new(file); // The entries are loaded back into a hash map in `load_from_file()`, so // the order in which we write them to file here does not matter. - #[allow(rustc::potential_query_instability)] for (module, key) in &self.keys { writeln!(writer, "{module} {key}")?; } @@ -806,7 +806,7 @@ impl ThinLTOKeysMap { fn load_from_file(path: &Path) -> io::Result<Self> { use std::io::BufRead; - let mut keys = FxHashMap::default(); + let mut keys = BTreeMap::default(); let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c607533a08e..27cb0366f17 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -245,12 +245,12 @@ pub fn target_machine_factory( match sess.opts.debuginfo_compression { rustc_session::config::DebugInfoCompression::Zlib => { if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { - sess.dcx().emit_warning(UnknownCompression { algorithm: "zlib" }); + sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" }); } } rustc_session::config::DebugInfoCompression::Zstd => { if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { - sess.dcx().emit_warning(UnknownCompression { algorithm: "zstd" }); + sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" }); } } rustc_session::config::DebugInfoCompression::None => {} @@ -416,8 +416,8 @@ fn report_inline_asm( cookie = 0; } let level = match level { - llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning(None), + llvm::DiagnosticLevel::Error => Level::Error, + llvm::DiagnosticLevel::Warning => Level::Warning, llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); @@ -457,7 +457,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - dcx.emit_warning(FromLlvmDiag { message }); + dcx.emit_warn(FromLlvmDiag { message }); } llvm::diagnostic::Unsupported(diagnostic_ref) => { let message = llvm::build_string(|s| { @@ -569,6 +569,7 @@ pub(crate) unsafe fn llvm_optimize( unroll_loops, config.vectorize_slp, config.vectorize_loop, + config.no_builtins, config.emit_lifetime_markers, sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), @@ -677,6 +678,7 @@ pub(crate) unsafe fn codegen( unsafe fn with_codegen<'ll, F, R>( tm: &'ll llvm::TargetMachine, llmod: &'ll llvm::Module, + no_builtins: bool, f: F, ) -> R where @@ -684,7 +686,7 @@ pub(crate) unsafe fn codegen( { let cpm = llvm::LLVMCreatePassManager(); llvm::LLVMAddAnalysisPasses(tm, cpm); - llvm::LLVMRustAddLibraryInfo(cpm, llmod); + llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); f(cpm) } @@ -785,7 +787,7 @@ pub(crate) unsafe fn codegen( } else { llmod }; - with_codegen(tm, llmod, |cpm| { + with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file( dcx, tm, @@ -820,7 +822,7 @@ pub(crate) unsafe fn codegen( (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; - with_codegen(tm, llmod, |cpm| { + with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file( dcx, tm, diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 4e5fe290bb1..8cab2a3f27c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + fn checked_binop( &mut self, oop: OverflowOp, @@ -558,10 +598,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Immediate(self.to_immediate(llval, place.layout)) } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { let b_offset = a.size(self).align_to(b.align(self).abi); - let pair_ty = place.layout.llvm_type(self); let mut load = |i, scalar: abi::Scalar, layout, align, offset| { - let llptr = self.struct_gep(pair_ty, place.llval, i as u64); + let llptr = if i == 0 { + place.llval + } else { + self.inbounds_gep( + self.type_i8(), + place.llval, + &[self.const_usize(b_offset.bytes())], + ) + }; let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); let load = self.load(llty, llptr, align); scalar_load_metadata(self, load, scalar, layout, offset); @@ -1320,17 +1367,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } @@ -1359,22 +1406,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1d1b6e6148d..7dfcf1ab50e 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -34,6 +34,7 @@ use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; use libc::c_uint; +use std::borrow::Borrow; use std::cell::{Cell, RefCell}; use std::ffi::CStr; use std::str; @@ -145,10 +146,17 @@ pub unsafe fn create_module<'ll>( .replace("-Fi64", ""); } } + if llvm_version < (18, 0, 0) { + if sess.target.arch == "x86" || sess.target.arch == "x86_64" { + // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants. + // Earlier LLVMs leave this as default alignment, so remove it. + // See https://reviews.llvm.org/D86310 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. - if sess.target.is_builtin { - // tm is disposed by its drop impl + { let tm = crate::back::write::create_informational_target_machine(tcx.sess); llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); @@ -156,33 +164,13 @@ pub unsafe fn create_module<'ll>( let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) .expect("got a non-UTF8 data-layout from LLVM"); - // Unfortunately LLVM target specs change over time, and right now we - // don't have proper support to work with any more than one - // `data_layout` than the one that is in the rust-lang/rust repo. If - // this compiler is configured against a custom LLVM, we may have a - // differing data layout, even though we should update our own to use - // that one. - // - // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we - // disable this check entirely as we may be configured with something - // that has a different target layout. - // - // Unsure if this will actually cause breakage when rustc is configured - // as such. - // - // FIXME(#34960) - let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = !cfg_llvm_root.trim().is_empty(); - - if !custom_llvm_used && target_data_layout != llvm_data_layout { - bug!( - "data-layout for target `{rustc_target}`, `{rustc_layout}`, \ - differs from LLVM target's `{llvm_target}` default layout, `{llvm_layout}`", - rustc_target = sess.opts.target_triple, - rustc_layout = target_data_layout, - llvm_target = sess.target.llvm_target, - llvm_layout = llvm_data_layout - ); + if target_data_layout != llvm_data_layout { + tcx.dcx().emit_err(crate::errors::MismatchedDataLayout { + rustc_target: sess.opts.target_triple.to_string().as_str(), + rustc_layout: target_data_layout.as_str(), + llvm_target: sess.target.llvm_target.borrow(), + llvm_layout: llvm_data_layout, + }); } } @@ -908,6 +896,21 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void); ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void); + // FIXME: This is an infinitesimally small portion of the types you can + // pass to this intrinsic, if we can ever lazily register intrinsics we + // should register these when they're used, that way any type can be + // passed. + ifn!("llvm.is.constant.i1", fn(i1) -> i1); + ifn!("llvm.is.constant.i8", fn(t_i8) -> i1); + ifn!("llvm.is.constant.i16", fn(t_i16) -> i1); + ifn!("llvm.is.constant.i32", fn(t_i32) -> i1); + ifn!("llvm.is.constant.i64", fn(t_i64) -> i1); + ifn!("llvm.is.constant.i128", fn(t_i128) -> i1); + ifn!("llvm.is.constant.isize", fn(t_isize) -> i1); + ifn!("llvm.is.constant.f32", fn(t_f32) -> i1); + ifn!("llvm.is.constant.f64", fn(t_f64) -> i1); + ifn!("llvm.is.constant.ptr", fn(ptr) -> i1); + ifn!("llvm.expect.i1", fn(i1, i1) -> i1); ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32); ifn!("llvm.localescape", fn(...) -> void); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 7ad2d03a5ed..017843c7e7d 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,4 +1,4 @@ -use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId}; +use rustc_middle::mir::coverage::{CodeRegion, CounterId, CovTerm, ExpressionId, MappingKind}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] @@ -149,6 +149,24 @@ pub struct CounterMappingRegion { } impl CounterMappingRegion { + pub(crate) fn from_mapping( + mapping_kind: &MappingKind, + local_file_id: u32, + code_region: &CodeRegion, + ) -> Self { + let &CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + match *mapping_kind { + MappingKind::Code(term) => Self::code_region( + Counter::from_term(term), + local_file_id, + start_line, + start_col, + end_line, + end_col, + ), + } + } + pub(crate) fn code_region( counter: Counter, file_id: u32, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index cd67fafb8e4..d85d9411f03 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -4,7 +4,8 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxIndexSet; use rustc_index::bit_set::BitSet; use rustc_middle::mir::coverage::{ - CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, Op, + CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, + MappingKind, Op, }; use rustc_middle::ty::Instance; use rustc_span::Symbol; @@ -64,8 +65,8 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { // For each expression ID that is directly used by one or more mappings, // mark it as not-yet-seen. This indicates that we expect to see a // corresponding `ExpressionUsed` statement during MIR traversal. - for Mapping { term, .. } in &function_coverage_info.mappings { - if let &CovTerm::Expression(id) = term { + for term in function_coverage_info.mappings.iter().flat_map(|m| m.kind.terms()) { + if let CovTerm::Expression(id) = term { expressions_seen.remove(id); } } @@ -221,20 +222,21 @@ impl<'tcx> FunctionCoverage<'tcx> { /// that will be used by `mapgen` when preparing for FFI. pub(crate) fn counter_regions( &self, - ) -> impl Iterator<Item = (Counter, &CodeRegion)> + ExactSizeIterator { + ) -> impl Iterator<Item = (MappingKind, &CodeRegion)> + ExactSizeIterator { self.function_coverage_info.mappings.iter().map(move |mapping| { - let &Mapping { term, ref code_region } = mapping; - let counter = self.counter_for_term(term); - (counter, code_region) + let Mapping { kind, code_region } = mapping; + let kind = + kind.map_terms(|term| if self.is_zero_term(term) { CovTerm::Zero } else { term }); + (kind, code_region) }) } fn counter_for_term(&self, term: CovTerm) -> Counter { - if is_zero_term(&self.counters_seen, &self.zero_expressions, term) { - Counter::ZERO - } else { - Counter::from_term(term) - } + if self.is_zero_term(term) { Counter::ZERO } else { Counter::from_term(term) } + } + + fn is_zero_term(&self, term: CovTerm) -> bool { + is_zero_term(&self.counters_seen, &self.zero_expressions, term) } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 33bfde03a31..6116a6fd222 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -12,7 +12,6 @@ use rustc_hir::def_id::DefId; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::mir; -use rustc_middle::mir::coverage::CodeRegion; use rustc_middle::ty::{self, TyCtxt}; use rustc_span::def_id::DefIdSet; use rustc_span::Symbol; @@ -58,11 +57,6 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { return; } - // The entries of the map are only used to get a list of all files with - // coverage info. In the end the list of files is passed into - // `GlobalFileTable::new()` which internally do `.sort_unstable_by()`, so - // the iteration order here does not matter. - #[allow(rustc::potential_query_instability)] let function_coverage_entries = function_coverage_map .into_iter() .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) @@ -242,7 +236,7 @@ fn encode_mappings_for_function( // Prepare file IDs for each filename, and prepare the mapping data so that // we can pass it through FFI to LLVM. for (file_name, counter_regions_for_file) in - &counter_regions.group_by(|(_counter, region)| region.file_name) + &counter_regions.group_by(|(_, region)| region.file_name) { // Look up the global file ID for this filename. let global_file_id = global_file_table.global_file_id_for_file_name(file_name); @@ -253,17 +247,12 @@ fn encode_mappings_for_function( // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - for (counter, region) in counter_regions_for_file { - let CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = *region; - - debug!("Adding counter {counter:?} to map for {region:?}"); - mapping_regions.push(CounterMappingRegion::code_region( - counter, + for (mapping_kind, region) in counter_regions_for_file { + debug!("Adding counter {mapping_kind:?} to map for {region:?}"); + mapping_regions.push(CounterMappingRegion::from_mapping( + &mapping_kind, local_file_id.as_u32(), - start_line, - start_col, - end_line, - end_col, + region, )); } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 0befbb5a39b..733a77d24c2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -10,7 +10,7 @@ use rustc_codegen_ssa::traits::{ BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, StaticMethods, }; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; @@ -30,7 +30,7 @@ const VAR_ALIGN_BYTES: usize = 8; pub struct CrateCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: - RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, + RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -44,8 +44,8 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { pub fn take_function_coverage_map( &self, - ) -> FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { - self.function_coverage_map.replace(FxHashMap::default()) + ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { + self.function_coverage_map.replace(FxIndexMap::default()) } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 76c9ac6614a..f961cd2d00b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -461,6 +461,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), @@ -1068,6 +1069,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), + ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { bug!( "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}", @@ -1153,7 +1155,8 @@ fn build_closure_env_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let closure_env_type = unique_type_id.expect_ty(); - let &ty::Closure(def_id, _args) = closure_env_type.kind() else { + let &(ty::Closure(def_id, _) | ty::CoroutineClosure(def_id, _)) = closure_env_type.kind() + else { bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) }; let containing_scope = get_namespace_for_item(cx, def_id); diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 8db97d577ca..24b3aa4223a 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -102,13 +102,10 @@ pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_> { fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { let diag: DiagnosticBuilder<'_, G> = self.0.into_diagnostic(dcx, level); - let (message, _) = diag.messages().first().expect("`LlvmError` with no message"); - let message = dcx.eagerly_translate_to_string(message.clone(), diag.args()); - - let mut diag = - DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config); - diag.set_arg("error", message); - diag + let (message, _) = diag.messages.first().expect("`LlvmError` with no message"); + let message = dcx.eagerly_translate_to_string(message.clone(), diag.args.iter()); + DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) + .with_arg("error", message) } } @@ -130,12 +127,12 @@ impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for TargetFeatureDisableOrEnabl fluent::codegen_llvm_target_feature_disable_or_enable, ); if let Some(span) = self.span { - diag.set_span(span); + diag.span(span); }; if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); + diag.subdiagnostic(dcx, missing_features); } - diag.set_arg("features", self.features.join(", ")); + diag.arg("features", self.features.join(", ")); diag } } @@ -204,10 +201,10 @@ impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, }; - let mut diag = self.0.into_diagnostic(dcx, level); - diag.set_primary_message(msg_with_llvm_err); - diag.set_arg("llvm_err", self.1); - diag + self.0 + .into_diagnostic(dcx, level) + .with_primary_message(msg_with_llvm_err) + .with_arg("llvm_err", self.1) } } @@ -246,3 +243,18 @@ pub(crate) struct CopyBitcode { pub struct UnknownCompression { pub algorithm: &'static str, } + +#[derive(Diagnostic)] +#[diag(codegen_llvm_mismatch_data_layout)] +pub struct MismatchedDataLayout<'a> { + pub rustc_target: &'a str, + pub rustc_layout: &'a str, + pub llvm_target: &'a str, + pub llvm_layout: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_target_feature_prefix)] +pub(crate) struct InvalidTargetFeaturePrefix<'a> { + pub feature: &'a str, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 6043a8ebded..23e6f054a7c 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -86,7 +86,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, &'ll Value>], llresult: &'ll Value, span: Span, - ) { + ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); @@ -119,6 +119,18 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::likely => { self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) } + sym::is_val_statically_known => { + let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); + match self.type_kind(intrinsic_type) { + TypeKind::Pointer | TypeKind::Integer | TypeKind::Float | TypeKind::Double => { + self.call_intrinsic( + &format!("llvm.is.constant.{:?}", intrinsic_type), + &[args[0].immediate()], + ) + } + _ => self.const_bool(false), + } + } sym::unlikely => self .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), kw::Try => { @@ -129,7 +141,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args[2].immediate(), llresult, ); - return; + return Ok(()); } sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), sym::va_copy => { @@ -179,17 +191,20 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { unsafe { llvm::LLVMSetAlignment(load, align); } - self.to_immediate(load, self.layout_of(tp_ty)) + if !result.layout.is_zst() { + self.store(load, result.llval, result.align); + } + return Ok(()); } sym::volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.volatile_store(self, dst); - return; + return Ok(()); } sym::unaligned_volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.unaligned_volatile_store(self, dst); - return; + return Ok(()); } sym::prefetch_read_data | sym::prefetch_write_data @@ -290,7 +305,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { name, ty, }); - return; + return Ok(()); } } } @@ -372,7 +387,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); // We have copied the value to `result` already. - return; + return Ok(()); } _ if name.as_str().starts_with("simd_") => { @@ -380,11 +395,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, ) { Ok(llval) => llval, - Err(()) => return, + Err(()) => return Ok(()), } } - _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), + _ => { + debug!("unknown intrinsic '{}' -- falling back to default body", name); + // Call the fallback body instead of generating the intrinsic code + return Err(ty::Instance::new(instance.def_id(), instance.args)); + } }; if !fn_abi.ret.is_ignore() { @@ -396,6 +415,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .store(self, result); } } + Ok(()) } fn abort(&mut self) { @@ -1860,14 +1880,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, + vector_reduce_fadd_reassoc, false, add, 0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, + vector_reduce_fmul_reassoc, false, mul, 1.0 @@ -1900,9 +1920,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); - minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); - minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); - macro_rules! bitwise_red { ($name:ident : $red:ident, $boolean:expr) => { if name == sym::$name { @@ -1970,10 +1987,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match in_elem.kind() { ty::RawPtr(p) => { - let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - assert!(!check_sized); // we are in codegen, so we shouldn't see these types require!( metadata.is_unit(), InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } @@ -1985,10 +2001,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } match out_elem.kind() { ty::RawPtr(p) => { - let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - assert!(!check_sized); // we are in codegen, so we shouldn't see these types require!( metadata.is_unit(), InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index a81056ed3ad..35210b0b2e8 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -13,12 +13,7 @@ #![feature(hash_raw_entry)] #![feature(iter_intersperse)] #![feature(let_chains)] -#![feature(min_specialization)] -#![feature(never_type)] #![feature(impl_trait_in_assoc_type)] -#![recursion_limit = "256"] -#![deny(rustc::untranslatable_diagnostic)] -#![deny(rustc::diagnostic_outside_of_impl)] #[macro_use] extern crate rustc_macros; @@ -374,7 +369,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") @@ -387,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend { }); } - Ok((codegen_results, work_products)) + (codegen_results, work_products) } fn link( diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index aefca6b34f5..dbf35e5f499 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -106,7 +106,6 @@ pub enum CallConv { X86_Intr = 83, AvrNonBlockingInterrupt = 84, AvrInterrupt = 85, - AmdGpuKernel = 91, } /// LLVMRustLinkage @@ -185,7 +184,6 @@ pub enum AttributeKind { SanitizeMemory = 22, NonLazyBind = 23, OptimizeNone = 24, - ReturnsTwice = 25, ReadNone = 26, SanitizeHWAddress = 28, WillReturn = 29, @@ -1620,6 +1618,8 @@ extern "C" { ) -> &'a Value; pub fn LLVMRustSetFastMath(Instr: &Value); + pub fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; @@ -2173,8 +2173,13 @@ extern "C" { ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; + pub fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); - pub fn LLVMRustAddLibraryInfo<'a>(PM: &PassManager<'a>, M: &'a Module); + pub fn LLVMRustAddLibraryInfo<'a>( + PM: &PassManager<'a>, + M: &'a Module, + DisableSimplifyLibCalls: bool, + ); pub fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, PM: &PassManager<'a>, @@ -2196,6 +2201,7 @@ extern "C" { UnrollLoops: bool, SLPVectorize: bool, LoopVectorize: bool, + DisableSimplifyLibCalls: bool, EmitLifetimeMarkers: bool, SanitizerOptions: Option<&SanitizerOptions>, PGOGenPath: *const c_char, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 03b79a143cc..54e8ed85e32 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,7 +1,7 @@ use crate::back::write::create_informational_target_machine; use crate::errors::{ - PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, - UnknownCTargetFeaturePrefix, UnstableCTargetFeature, + InvalidTargetFeaturePrefix, PossibleFeature, TargetFeatureDisableOrEnable, + UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; use libc::c_int; @@ -213,6 +213,7 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("x86", "rdrand") => LLVMFeature::new("rdrnd"), ("x86", "bmi1") => LLVMFeature::new("bmi"), ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("x86", "lahfsahf") => LLVMFeature::new("sahf"), ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), ("aarch64", "dpb") => LLVMFeature::new("ccpp"), ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), @@ -265,6 +266,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { LLVMFeature::new("unaligned-scalar-mem") } + // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled. + ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => { + LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")) + } (_, s) => LLVMFeature::new(s), } } @@ -511,7 +516,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str sess.target .features .split(',') - .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + .filter(|v| !v.is_empty() && backend_feature_name(sess, v).is_some()) .map(String::from), ); @@ -529,13 +534,13 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str Some(c @ ('+' | '-')) => c, Some(_) => { if diagnostics { - sess.dcx().emit_warning(UnknownCTargetFeaturePrefix { feature: s }); + sess.dcx().emit_warn(UnknownCTargetFeaturePrefix { feature: s }); } return None; } }; - let feature = backend_feature_name(s)?; + let feature = backend_feature_name(sess, s)?; // Warn against use of LLVM specific feature names and unstable features on the CLI. if diagnostics { let feature_state = supported_features.iter().find(|&&(v, _)| v == feature); @@ -557,12 +562,12 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } else { UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } }; - sess.dcx().emit_warning(unknown_feature); + sess.dcx().emit_warn(unknown_feature); } else if feature_state .is_some_and(|(_name, feature_gate)| !feature_gate.is_stable()) { // An unstable feature. Warn about using it. - sess.dcx().emit_warning(UnstableCTargetFeature { feature }); + sess.dcx().emit_warn(UnstableCTargetFeature { feature }); } } @@ -611,11 +616,11 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str /// Returns a feature name for the given `+feature` or `-feature` string. /// /// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) -fn backend_feature_name(s: &str) -> Option<&str> { +fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { // features must start with a `+` or `-`. - let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { - bug!("target feature `{}` must begin with a `+` or `-`", s); - }); + let feature = s + .strip_prefix(&['+', '-'][..]) + .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. if RUSTC_SPECIFIC_FEATURES.contains(&feature) { diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f796ce0990f..f7630719368 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -123,6 +123,17 @@ impl CodegenCx<'_, '_> { return false; } + // Match clang by only supporting COFF and ELF for now. + if self.tcx.sess.target.is_like_osx { + return false; + } + + // With pie relocation model calls of functions defined in the translation + // unit can use copy relocations. + if self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration { + return true; + } + // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); @@ -130,18 +141,12 @@ impl CodegenCx<'_, '_> { return false; } - // Match clang by only supporting COFF and ELF for now. - if self.tcx.sess.target.is_like_osx { - return false; + // Respect the direct-access-external-data to override default behavior if present. + if let Some(direct) = self.tcx.sess.direct_access_external_data() { + return direct; } // Static relocation model should force copy relocations everywhere. - if self.tcx.sess.relocation_model() == RelocModel::Static { - return true; - } - - // With pie relocation model calls of functions defined in the translation - // unit can use copy relocations. - self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration + self.tcx.sess.relocation_model() == RelocModel::Static } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 57b46382c96..219c7025311 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -26,23 +26,14 @@ fn uncached_llvm_type<'a, 'tcx>( let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } - Abi::ScalarPair(..) => { - return cx.type_struct( - &[ - layout.scalar_pair_element_llvm_type(cx, 0, false), - layout.scalar_pair_element_llvm_type(cx, 1, false), - ], - false, - ); - } - Abi::Uninhabited | Abi::Aggregate { .. } => {} + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalarPair(..) => {} } let name = match layout.ty.kind() { // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str // For performance reasons we use names only when emitting LLVM IR. if !cx.sess().fewer_names() => { @@ -275,11 +266,25 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(scalar) = self.abi { - if scalar.is_bool() { - return cx.type_i1(); + match self.abi { + Abi::Scalar(scalar) => { + if scalar.is_bool() { + return cx.type_i1(); + } } - } + Abi::ScalarPair(..) => { + // An immediate pair always contains just the two elements, without any padding + // filler, as it should never be stored to memory. + return cx.type_struct( + &[ + self.scalar_pair_element_llvm_type(cx, 0, true), + self.scalar_pair_element_llvm_type(cx, 1, true), + ], + false, + ); + } + _ => {} + }; self.llvm_type(cx) } |