diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
23 files changed, 181 insertions, 235 deletions
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index a93baf88413..03a871297c4 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -14,6 +14,7 @@ libc = "0.2" measureme = "11" object = { version = "0.36.3", default-features = false, features = ["std", "read"] } rustc-demangle = "0.1.21" +rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_attr = { path = "../rustc_attr" } rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index df2198df14b..0950e4bb26b 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -33,9 +33,6 @@ codegen_llvm_lto_proc_macro = lto cannot be used for `proc-macro` crate type wit codegen_llvm_mismatch_data_layout = data-layout for target `{$rustc_target}`, `{$rustc_layout}`, differs from LLVM target's `{$llvm_target}` default layout, `{$llvm_layout}` -codegen_llvm_missing_features = - add the missing features in a `target_feature` attribute - codegen_llvm_multiple_source_dicompileunit = multiple source DICompileUnits found codegen_llvm_multiple_source_dicompileunit_with_llvm_err = multiple source DICompileUnits found: {$llvm_err} @@ -63,9 +60,6 @@ codegen_llvm_serialize_module_with_llvm_err = failed to serialize module {$name} codegen_llvm_symbol_already_defined = symbol `{$symbol_name}` is already defined -codegen_llvm_target_feature_disable_or_enable = - the target features {$features} must all be either enabled or disabled together - codegen_llvm_target_machine = could not create LLVM TargetMachine for triple: {$triple} codegen_llvm_target_machine_with_llvm_err = could not create LLVM TargetMachine for triple: {$triple}: {$llvm_err} diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 3d75393bf06..2fe5ed32daa 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,17 +1,19 @@ use std::cmp; use libc::c_uint; +use rustc_abi as abi; +use rustc_abi::Primitive::Int; +use rustc_abi::{HasDataLayout, Size}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::LayoutOf; -pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; +pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA}; use rustc_middle::{bug, ty}; use rustc_session::config; pub(crate) use rustc_target::abi::call::*; -use rustc_target::abi::{self, HasDataLayout, Int, Size}; use rustc_target::spec::SanitizerSet; pub(crate) use rustc_target::spec::abi::Abi; use smallvec::SmallVec; diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 2adac278c62..89f5305840b 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -77,18 +77,20 @@ pub(crate) unsafe fn codegen( // __rust_alloc_error_handler_should_panic let name = OomStrategy::SYMBOL; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.default_hidden_visibility() { - llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); - } + llvm::LLVMRustSetVisibility( + ll_g, + llvm::Visibility::from_generic(tcx.sess.default_visibility()), + ); let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); let name = NO_ALLOC_SHIM_IS_UNSTABLE; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.default_hidden_visibility() { - llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); - } + llvm::LLVMRustSetVisibility( + ll_g, + llvm::Visibility::from_generic(tcx.sess.default_visibility()), + ); let llval = llvm::LLVMConstInt(i8, 0, False); llvm::LLVMSetInitializer(ll_g, llval); } @@ -132,9 +134,11 @@ fn create_wrapper_function( None }; - if tcx.sess.default_hidden_visibility() { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } + llvm::LLVMRustSetVisibility( + llfn, + llvm::Visibility::from_generic(tcx.sess.default_visibility()), + ); + if tcx.sess.must_emit_unwind_tables() { let uwtable = attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 7621f111fe2..82ca3f519f7 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -708,6 +708,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> S390x(S390xInlineAsmRegClass::reg) => "r", S390x(S390xInlineAsmRegClass::reg_addr) => "a", S390x(S390xInlineAsmRegClass::freg) => "f", + S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => { + unreachable!("clobber-only") + } Msp430(Msp430InlineAsmRegClass::reg) => "r", M68k(M68kInlineAsmRegClass::reg) => "r", M68k(M68kInlineAsmRegClass::reg_addr) => "a", @@ -866,6 +869,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(), S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), + S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => { + unreachable!("clobber-only") + } Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 489259da856..2c5ec9dad59 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -6,12 +6,11 @@ use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; -use rustc_span::symbol::sym; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; use crate::context::CodegenCx; -use crate::errors::{MissingFeatures, SanitizerMemtagRequiresMte, TargetFeatureDisableOrEnable}; +use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; @@ -502,26 +501,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); - if let Some(f) = llvm_util::check_tied_features( - cx.tcx.sess, - &function_features.iter().map(|f| (*f, true)).collect(), - ) { - let span = cx - .tcx - .get_attrs(instance.def_id(), sym::target_feature) - .next() - .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); - cx.tcx - .dcx() - .create_err(TargetFeatureDisableOrEnable { - features: f, - span: Some(span), - missing_features: Some(MissingFeatures), - }) - .emit(); - return; - } - let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 1b4f6682af0..33a956e552f 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -123,7 +123,7 @@ fn get_llvm_object_symbols( llvm::LLVMRustGetSymbols( buf.as_ptr(), buf.len(), - std::ptr::addr_of_mut!(*state) as *mut c_void, + (&raw mut *state) as *mut c_void, callback, error_callback, ) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 5e510177818..1f7a923dd2c 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -808,8 +808,7 @@ struct ThinLTOKeysMap { impl ThinLTOKeysMap { fn save_to_file(&self, path: &Path) -> io::Result<()> { use std::io::Write; - let file = File::create(path)?; - let mut writer = io::BufWriter::new(file); + let mut writer = File::create_buffered(path)?; // The entries are loaded back into a hash map in `load_from_file()`, so // the order in which we write them to file here does not matter. for (module, key) in &self.keys { @@ -821,8 +820,8 @@ impl ThinLTOKeysMap { fn load_from_file(path: &Path) -> io::Result<Self> { use std::io::BufRead; let mut keys = BTreeMap::default(); - let file = File::open(path)?; - for line in io::BufReader::new(file).lines() { + let file = File::open_buffered(path)?; + for line in file.lines() { let line = line?; let mut split = line.split(' '); let module = split.next().unwrap(); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 4e0c62c8bf8..dbf5298d64b 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,6 +3,8 @@ use std::ops::Deref; use std::{iter, ptr}; use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::{Align, Size, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -20,7 +22,6 @@ use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_target::abi::call::FnAbi; -use rustc_target::abi::{self, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -505,12 +506,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } match scalar.primitive() { - abi::Int(..) => { + abi::Primitive::Int(..) => { if !scalar.is_always_valid(bx) { bx.range_metadata(load, scalar.valid_range(bx)); } } - abi::Pointer(_) => { + abi::Primitive::Pointer(_) => { if !scalar.valid_range(bx).contains(0) { bx.nonnull_metadata(load); } @@ -521,7 +522,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } } - abi::Float(_) => {} + abi::Primitive::Float(_) => {} } } @@ -1679,16 +1680,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { &mut self, fn_name: &'ll Value, hash: &'ll Value, - bitmap_bytes: &'ll Value, + bitmap_bits: &'ll Value, ) { - debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); + debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits); + + assert!( + crate::llvm_util::get_version() >= (19, 0, 0), + "MCDC intrinsics require LLVM 19 or later" + ); let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], self.cx.type_void(), ); - let args = &[fn_name, hash, bitmap_bytes]; + let args = &[fn_name, hash, bitmap_bits]; let args = self.check_call("call", llty, llfn, args); unsafe { @@ -1708,28 +1714,25 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { &mut self, fn_name: &'ll Value, hash: &'ll Value, - bitmap_bytes: &'ll Value, bitmap_index: &'ll Value, mcdc_temp: &'ll Value, ) { debug!( - "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", - fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp + "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})", + fn_name, hash, bitmap_index, mcdc_temp + ); + assert!( + crate::llvm_util::get_version() >= (19, 0, 0), + "MCDC intrinsics require LLVM 19 or later" ); let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( - &[ - self.cx.type_ptr(), - self.cx.type_i64(), - self.cx.type_i32(), - self.cx.type_i32(), - self.cx.type_ptr(), - ], + &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()], self.cx.type_void(), ); - let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp]; + let args = &[fn_name, hash, bitmap_index, mcdc_temp]; let args = self.check_call("call", llty, llfn, args); unsafe { let _ = llvm::LLVMRustBuildCall( @@ -1745,41 +1748,15 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } - pub(crate) fn mcdc_condbitmap_update( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - cond_loc: &'ll Value, - mcdc_temp: &'ll Value, - bool_value: &'ll Value, - ) { - debug!( - "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", - fn_name, hash, cond_loc, mcdc_temp, bool_value - ); - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[ - self.cx.type_ptr(), - self.cx.type_i64(), - self.cx.type_i32(), - self.cx.type_ptr(), - self.cx.type_i1(), - ], - self.cx.type_void(), + pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { + debug!("mcdc_condbitmap_update() with args ({:?}, {:?})", cond_index, mcdc_temp); + assert!( + crate::llvm_util::get_version() >= (19, 0, 0), + "MCDC intrinsics require LLVM 19 or later" ); - let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value]; - self.check_call("call", llty, llfn, args); - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + let align = self.tcx.data_layout.i32_align.abi; + let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align); + let new_tv_index = self.add(current_tv_index, cond_index); + self.store(new_tv_index, mcdc_temp, align); } } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 31d59905446..4ca19ab2f12 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -1,6 +1,9 @@ //! Code that is useful in various codegen modules. use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::Primitive::Pointer; +use rustc_abi::{AddressSpace, HasDataLayout}; use rustc_ast::Mutability; use rustc_codegen_ssa::traits::*; use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; @@ -9,7 +12,6 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::DllImport; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use tracing::debug; use crate::consts::const_alloc_to_llvm; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 2b8912d1db2..c836dd5473f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -19,7 +19,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::Session; use rustc_session::config::{ - BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, PAuthKey, PacRet, + BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet, }; use rustc_span::source_map::Spanned; use rustc_span::{DUMMY_SP, Span}; @@ -378,6 +378,18 @@ pub(crate) unsafe fn create_module<'ll>( } } + match sess.opts.unstable_opts.function_return { + FunctionReturn::Keep => {} + FunctionReturn::ThunkExtern => unsafe { + llvm::LLVMRustAddModuleFlagU32( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"function_return_thunk_extern".as_ptr(), + 1, + ) + }, + } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) { // Set up the small-data optimization limit for architectures that use @@ -872,6 +884,11 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16); + ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32); + ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16); ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); @@ -1175,10 +1192,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { span: Span, fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { - if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.tcx.dcx().emit_fatal(Spanned { span, node: err }) - } else { - match fn_abi_request { + match err { + FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + self.tcx.dcx().emit_fatal(Spanned { span, node: err }); + } + _ => match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } @@ -1188,7 +1206,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } - } + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 77821ca89bc..90f7dd733ca 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -111,7 +111,7 @@ enum RegionKind { } mod mcdc { - use rustc_middle::mir::coverage::{ConditionInfo, DecisionInfo}; + use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; /// Must match the layout of `LLVMRustMCDCDecisionParameters`. #[repr(C)] @@ -167,12 +167,13 @@ mod mcdc { impl From<ConditionInfo> for BranchParameters { fn from(value: ConditionInfo) -> Self { + let to_llvm_cond_id = |cond_id: Option<ConditionId>| { + cond_id.and_then(|id| LLVMConditionId::try_from(id.as_usize()).ok()).unwrap_or(-1) + }; + let ConditionInfo { condition_id, true_next_id, false_next_id } = value; Self { - condition_id: value.condition_id.as_u32() as LLVMConditionId, - condition_ids: [ - value.false_next_id.as_u32() as LLVMConditionId, - value.true_next_id.as_u32() as LLVMConditionId, - ], + condition_id: to_llvm_cond_id(Some(condition_id)), + condition_ids: [to_llvm_cond_id(false_next_id), to_llvm_cond_id(true_next_id)], } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 267a2244916..c2c261da79b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -14,29 +14,20 @@ use crate::coverageinfo::ffi::CounterMappingRegion; use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; use crate::{coverageinfo, llvm}; -/// Generates and exports the Coverage Map. +/// Generates and exports the coverage map, which is embedded in special +/// linker sections in the final binary. /// -/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions -/// 6 and 7 (encoded as 5 and 6 respectively), as described at -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/18.0-2024-02-13/llvm/docs/CoverageMappingFormat.rst). -/// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) -/// distributed in the `llvm-tools-preview` rustup component. -/// -/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with -/// the same version. Clang's implementation of Coverage Map generation was referenced when -/// implementing this Rust version, and though the format documentation is very explicit and -/// detailed, some undocumented details in Clang's implementation (that may or may not be important) -/// were also replicated for Rust's Coverage Map. +/// Those sections are then read and understood by LLVM's `llvm-cov` tool, +/// which is distributed in the `llvm-tools` rustup component. pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure that LLVM is using a version of the coverage mapping format that // agrees with our Rust-side code. Expected versions (encoded as n-1) are: - // - `CovMapVersion::Version6` (5) used by LLVM 13-17 - // - `CovMapVersion::Version7` (6) used by LLVM 18 + // - `CovMapVersion::Version7` (6) used by LLVM 18-19 let covmap_version = { let llvm_covmap_version = coverageinfo::mapping_version(); - let expected_versions = 5..=6; + let expected_versions = 6..=6; assert!( expected_versions.contains(&llvm_covmap_version), "Coverage mapping version exposed by `llvm-wrapper` is out of sync; \ diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 3a80d216f47..d7d29eebf85 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -98,14 +98,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { }; // If there are no MC/DC bitmaps to set up, return immediately. - if function_coverage_info.mcdc_bitmap_bytes == 0 { + if function_coverage_info.mcdc_bitmap_bits == 0 { return; } let fn_name = self.get_pgo_func_name_var(instance); let hash = self.const_u64(function_coverage_info.function_source_hash); - let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes); - self.mcdc_parameters(fn_name, hash, bitmap_bytes); + let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32); + self.mcdc_parameters(fn_name, hash, bitmap_bits); // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps. let mut cond_bitmaps = vec![]; @@ -185,35 +185,28 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::ExpressionUsed { id } => { func_coverage.mark_expression_id_seen(id); } - CoverageKind::CondBitmapUpdate { id, value, decision_depth } => { + CoverageKind::CondBitmapUpdate { index, decision_depth } => { drop(coverage_map); - assert_ne!( - id.as_u32(), - 0, - "ConditionId of evaluated conditions should never be zero" - ); let cond_bitmap = coverage_context .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); - let cond_loc = bx.const_i32(id.as_u32() as i32 - 1); - let bool_value = bx.const_bool(value); - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_coverage_info.function_source_hash); - bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value); + let cond_index = bx.const_i32(index as i32); + bx.mcdc_condbitmap_update(cond_index, cond_bitmap); } CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { drop(coverage_map); let cond_bitmap = coverage_context .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); - let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes; - assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range"); + assert!( + bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits, + "bitmap index of the decision out of range" + ); let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); - let bitmap_bytes = bx.const_u32(bitmap_bytes); let bitmap_index = bx.const_u32(bitmap_idx); - bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap); + bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 964b83c0fa0..15d441a986d 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -34,7 +34,7 @@ use super::utils::{ }; use crate::common::CodegenCx; use crate::debuginfo::metadata::type_map::build_type_with_children; -use crate::debuginfo::utils::{FatPtrKind, fat_pointer_kind}; +use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind}; use crate::llvm::debuginfo::{ DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, DebugNameTableKind, @@ -161,7 +161,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { // The debuginfo generated by this function is only valid if `ptr_type` is really just - // a (fat) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. + // a (wide) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. assert_eq!( cx.size_and_align_of(ptr_type), cx.size_and_align_of(Ty::new_mut_ptr(cx.tcx, pointee_type)) @@ -174,7 +174,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( let data_layout = &cx.tcx.data_layout; let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); - match fat_pointer_kind(cx, pointee_type) { + match wide_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. assert_eq!( @@ -197,7 +197,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( DINodeCreationResult { di_node, already_stored_in_typemap: false } } - Some(fat_pointer_kind) => { + Some(wide_pointer_kind) => { type_map::build_type_with_children( cx, type_map::stub( @@ -210,7 +210,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( DIFlags::FlagZero, ), |cx, owner| { - // FIXME: If this fat pointer is a `Box` then we don't want to use its + // FIXME: If this wide pointer is a `Box` then we don't want to use its // type layout and instead use the layout of the raw pointer inside // of it. // The proper way to handle this is to not treat Box as a pointer @@ -227,16 +227,16 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( }; let layout = cx.layout_of(layout_type); - let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); - let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); + let addr_field = layout.field(cx, abi::WIDE_PTR_ADDR); + let extra_field = layout.field(cx, abi::WIDE_PTR_EXTRA); - let (addr_field_name, extra_field_name) = match fat_pointer_kind { - FatPtrKind::Dyn => ("pointer", "vtable"), - FatPtrKind::Slice => ("data_ptr", "length"), + let (addr_field_name, extra_field_name) = match wide_pointer_kind { + WidePtrKind::Dyn => ("pointer", "vtable"), + WidePtrKind::Slice => ("data_ptr", "length"), }; - assert_eq!(abi::FAT_PTR_ADDR, 0); - assert_eq!(abi::FAT_PTR_EXTRA, 1); + assert_eq!(abi::WIDE_PTR_ADDR, 0); + assert_eq!(abi::WIDE_PTR_EXTRA, 1); // The data pointer type is a regular, thin pointer, regardless of whether this // is a slice or a trait object. @@ -258,7 +258,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( owner, addr_field_name, (addr_field.size, addr_field.align.abi), - layout.fields.offset(abi::FAT_PTR_ADDR), + layout.fields.offset(abi::WIDE_PTR_ADDR), DIFlags::FlagZero, data_ptr_type_di_node, ), @@ -267,7 +267,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( owner, extra_field_name, (extra_field.size, extra_field.align.abi), - layout.fields.offset(abi::FAT_PTR_EXTRA), + layout.fields.offset(abi::WIDE_PTR_EXTRA), DIFlags::FlagZero, type_di_node(cx, extra_field.ty), ), @@ -391,7 +391,7 @@ fn build_dyn_type_di_node<'ll, 'tcx>( /// /// NOTE: We currently emit just emit the debuginfo for the element type here /// (i.e. `T` for slices and `u8` for `str`), so that we end up with -/// `*const T` for the `data_ptr` field of the corresponding fat-pointer +/// `*const T` for the `data_ptr` field of the corresponding wide-pointer /// debuginfo of `&[T]`. /// /// It would be preferable and more accurate if we emitted a DIArray of T @@ -630,6 +630,7 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, + rustc_span::SourceFileHashAlgorithm::Blake3 => llvm::ChecksumKind::None, }; let hash_value = hex_encode(source_file.src_hash.hash_bytes()); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index acb15449ce3..960487ada16 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -49,23 +49,23 @@ pub(crate) fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId } #[derive(Debug, PartialEq, Eq)] -pub(crate) enum FatPtrKind { +pub(crate) enum WidePtrKind { Slice, Dyn, } /// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. -/// if the second field of the fat pointer is a length or a vtable-pointer. -/// If `pointee_ty` does not require a fat pointer (because it is Sized) then +/// if the second field of the wide pointer is a length or a vtable-pointer. +/// If `pointee_ty` does not require a wide pointer (because it is Sized) then /// the function returns `None`. -pub(crate) fn fat_pointer_kind<'ll, 'tcx>( +pub(crate) fn wide_pointer_kind<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, pointee_ty: Ty<'tcx>, -) -> Option<FatPtrKind> { +) -> Option<WidePtrKind> { let pointee_tail_ty = cx.tcx.struct_tail_for_codegen(pointee_ty, cx.param_env()); let layout = cx.layout_of(pointee_tail_ty); trace!( - "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", + "wide_pointer_kind: {:?} has layout {:?} (is_unsized? {})", pointee_tail_ty, layout, layout.is_unsized() @@ -76,8 +76,8 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( } match *pointee_tail_ty.kind() { - ty::Str | ty::Slice(_) => Some(FatPtrKind::Slice), - ty::Dynamic(..) => Some(FatPtrKind::Dyn), + ty::Str | ty::Slice(_) => Some(WidePtrKind::Slice), + ty::Dynamic(..) => Some(WidePtrKind::Dyn), ty::Foreign(_) => { // Assert that pointers to foreign types really are thin: assert_eq!( @@ -90,7 +90,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( // For all other pointee types we should already have returned None // at the beginning of the function. panic!( - "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}" + "wide_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}" ) } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index b0b29ca1280..33258cb46fa 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -22,6 +22,7 @@ use tracing::debug; use crate::abi::{FnAbi, FnAbiLlvmExt}; use crate::context::CodegenCx; use crate::llvm::AttributePlace::Function; +use crate::llvm::Visibility; use crate::type_::Type; use crate::value::Value; use crate::{attributes, llvm}; @@ -83,14 +84,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - // Declare C ABI functions with the visibility used by C by default. - let visibility = if self.tcx.sess.default_hidden_visibility() { - llvm::Visibility::Hidden - } else { - llvm::Visibility::Default - }; - - declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type) + // Visibility should always be default for declarations, otherwise the linker may report an + // error. + declare_raw_fn(self, name, llvm::CCallConv, unnamed, Visibility::Default, fn_type) } /// Declare an entry Function @@ -107,11 +103,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - let visibility = if self.tcx.sess.default_hidden_visibility() { - llvm::Visibility::Hidden - } else { - llvm::Visibility::Default - }; + let visibility = Visibility::from_generic(self.tcx.sess.default_visibility()); declare_raw_fn(self, name, callconv, unnamed, visibility, fn_type) } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index bb481d2a308..0d436e1891e 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -80,30 +80,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { } } -pub(crate) struct TargetFeatureDisableOrEnable<'a> { - pub features: &'a [&'a str], - pub span: Option<Span>, - pub missing_features: Option<MissingFeatures>, -} - -#[derive(Subdiagnostic)] -#[help(codegen_llvm_missing_features)] -pub(crate) struct MissingFeatures; - -impl<G: EmissionGuarantee> Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { - fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> { - let mut diag = Diag::new(dcx, level, fluent::codegen_llvm_target_feature_disable_or_enable); - if let Some(span) = self.span { - diag.span(span); - }; - if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); - } - diag.arg("features", self.features.join(", ")); - diag - } -} - #[derive(Diagnostic)] #[diag(codegen_llvm_lto_disallowed)] pub(crate) struct LtoDisallowed; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c66c80da9fc..bfe623e7fc3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -86,6 +86,11 @@ fn get_simple_intrinsic<'ll>( sym::fmaf64 => "llvm.fma.f64", sym::fmaf128 => "llvm.fma.f128", + sym::fmuladdf16 => "llvm.fmuladd.f16", + sym::fmuladdf32 => "llvm.fmuladd.f32", + sym::fmuladdf64 => "llvm.fmuladd.f64", + sym::fmuladdf128 => "llvm.fmuladd.f128", + sym::fabsf16 => "llvm.fabs.f16", sym::fabsf32 => "llvm.fabs.f32", sym::fabsf64 => "llvm.fabs.f64", @@ -2185,7 +2190,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer { + require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem @@ -2200,7 +2205,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer { + require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index d69112612ba..b85d28a2f1f 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -11,6 +11,7 @@ #![feature(assert_matches)] #![feature(exact_size_is_empty)] #![feature(extern_types)] +#![feature(file_buffered)] #![feature(hash_raw_entry)] #![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] @@ -166,7 +167,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_pass_timings(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintPassTimings(std::ptr::addr_of_mut!(size)); + let cstr = llvm::LLVMRustPrintPassTimings(&raw mut size); if cstr.is_null() { println!("failed to get pass timings"); } else { @@ -179,7 +180,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_statistics(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintStatistics(std::ptr::addr_of_mut!(size)); + let cstr = llvm::LLVMRustPrintStatistics(&raw mut size); if cstr.is_null() { println!("failed to get pass stats"); } else { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 9aabfd794ba..661debbb9f1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -4,6 +4,7 @@ use std::marker::PhantomData; use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; +use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ @@ -133,6 +134,16 @@ pub enum Visibility { Protected = 2, } +impl Visibility { + pub fn from_generic(visibility: SymbolVisibility) -> Self { + match visibility { + SymbolVisibility::Hidden => Visibility::Hidden, + SymbolVisibility::Protected => Visibility::Protected, + SymbolVisibility::Interposable => Visibility::Default, + } + } +} + /// LLVMUnnamedAddr #[repr(C)] pub enum UnnamedAddr { @@ -1603,7 +1614,6 @@ unsafe extern "C" { pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value; pub fn LLVMRustBuildCall<'a>( B: &Builder<'a>, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 71fd7afb148..57936215ff1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -6,6 +6,7 @@ use std::{ptr, slice, str}; use libc::c_int; use rustc_codegen_ssa::base::wants_wasm_eh; +use rustc_codegen_ssa::codegen_attrs::check_tied_features; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_data_structures::unord::UnordSet; @@ -19,8 +20,8 @@ use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATU use crate::back::write::create_informational_target_machine; use crate::errors::{ - FixedX18InvalidArch, InvalidTargetFeaturePrefix, PossibleFeature, TargetFeatureDisableOrEnable, - UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, + FixedX18InvalidArch, InvalidTargetFeaturePrefix, PossibleFeature, UnknownCTargetFeature, + UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; @@ -247,7 +248,9 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")), + // Before LLVM 20 those two features were packaged together as b16b16 + ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), + ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => { @@ -264,6 +267,10 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { Some(LLVMFeature::new("fast-unaligned-access")) } + // Filter out features that are not supported by the current LLVM version + ("riscv32" | "riscv64", "zaamo") if get_version().0 < 19 => None, + ("riscv32" | "riscv64", "zabha") if get_version().0 < 19 => None, + ("riscv32" | "riscv64", "zalrsc") if get_version().0 < 19 => None, // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => { Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))) @@ -272,25 +279,6 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea } } -/// Given a map from target_features to whether they are enabled or disabled, -/// ensure only valid combinations are allowed. -pub(crate) fn check_tied_features( - sess: &Session, - features: &FxHashMap<&str, bool>, -) -> Option<&'static [&'static str]> { - if !features.is_empty() { - for tied in sess.target.tied_target_features() { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); - } - } - } - None -} - /// Used to generate cfg variables and apply features /// Must express features in the way Rust understands them pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { @@ -478,7 +466,7 @@ pub(crate) fn print(req: &PrintRequest, mut out: &mut String, sess: &Session) { &tm, cpu_cstring.as_ptr(), callback, - std::ptr::addr_of_mut!(out) as *mut c_void, + (&raw mut out) as *mut c_void, ); } } @@ -536,6 +524,11 @@ pub(crate) fn global_llvm_features( // -Ctarget-cpu=native match sess.opts.cg.target_cpu { Some(ref s) if s == "native" => { + // We have already figured out the actual CPU name with `LLVMRustGetHostCPUName` and set + // that for LLVM, so the features implied by that CPU name will be available everywhere. + // However, that is not sufficient: e.g. `skylake` alone is not sufficient to tell if + // some of the instructions are available or not. So we have to also explicitly ask for + // the exact set of features available on the host, and enable all of them. let features_string = unsafe { let ptr = llvm::LLVMGetHostCPUFeatures(); let features_string = if !ptr.is_null() { @@ -676,7 +669,7 @@ pub(crate) fn global_llvm_features( features.extend(feats); if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.dcx().emit_err(TargetFeatureDisableOrEnable { + sess.dcx().emit_err(rustc_codegen_ssa::errors::TargetFeatureDisableOrEnable { features: f, span: None, missing_features: None, diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 6e429a1674a..1af666f818b 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,11 +1,12 @@ use std::fmt::Write; +use rustc_abi::Primitive::{Float, Int, Pointer}; +use rustc_abi::{Abi, Align, FieldsShape, Scalar, Size, Variants}; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; -use rustc_target::abi::{Abi, Align, FieldsShape, Float, Int, Pointer, Scalar, Size, Variants}; use tracing::debug; use crate::common::*; @@ -199,7 +200,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // layout. if let Abi::Scalar(scalar) = self.abi { // Use a different cache for scalars because pointers to DSTs - // can be either fat or thin (data pointers of fat pointers). + // can be either wide or thin (data pointers of wide pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { return llty; } |