diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 53 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 26 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs | 390 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 75 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 49 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/declare.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 24 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 28 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 31 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/mono_item.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 5 |
17 files changed, 260 insertions, 540 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 6a29eb5fa04..2fe5ed32daa 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,6 +1,9 @@ use std::cmp; use libc::c_uint; +use rustc_abi as abi; +use rustc_abi::Primitive::Int; +use rustc_abi::{HasDataLayout, Size}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; @@ -11,7 +14,6 @@ pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA}; use rustc_middle::{bug, ty}; use rustc_session::config; pub(crate) use rustc_target::abi::call::*; -use rustc_target::abi::{self, HasDataLayout, Int, Size}; use rustc_target::spec::SanitizerSet; pub(crate) use rustc_target::spec::abi::Abi; use smallvec::SmallVec; diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 82ca3f519f7..d1d7d0cf4ce 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -154,7 +154,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // We prefer the latter because it matches the behavior of // Clang. if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) { - constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string()); + constraints.push(reg_to_llvm(reg, Some(&in_value.layout))); } else { constraints.push(format!("{}", op_idx[&idx])); } @@ -542,57 +542,16 @@ fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { /// If the register is an AArch64 integer register then return its index. fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> { - use AArch64InlineAsmReg::*; - // Unlike `a64_vreg_index`, we can't subtract `x0` to get the u32 because - // `x19` and `x29` are missing and the integer constants for the - // `x0`..`x30` enum variants don't all match the register number. E.g. the - // integer constant for `x18` is 18, but the constant for `x20` is 19. - Some(match reg { - InlineAsmReg::AArch64(r) => match r { - x0 => 0, - x1 => 1, - x2 => 2, - x3 => 3, - x4 => 4, - x5 => 5, - x6 => 6, - x7 => 7, - x8 => 8, - x9 => 9, - x10 => 10, - x11 => 11, - x12 => 12, - x13 => 13, - x14 => 14, - x15 => 15, - x16 => 16, - x17 => 17, - x18 => 18, - // x19 is reserved - x20 => 20, - x21 => 21, - x22 => 22, - x23 => 23, - x24 => 24, - x25 => 25, - x26 => 26, - x27 => 27, - x28 => 28, - // x29 is reserved - x30 => 30, - _ => return None, - }, - _ => return None, - }) + match reg { + InlineAsmReg::AArch64(r) => r.reg_index(), + _ => None, + } } /// If the register is an AArch64 vector register then return its index. fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { - use AArch64InlineAsmReg::*; match reg { - InlineAsmReg::AArch64(reg) if reg as u32 >= v0 as u32 && reg as u32 <= v31 as u32 => { - Some(reg as u32 - v0 as u32) - } + InlineAsmReg::AArch64(reg) => reg.vreg_index(), _ => None, } } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 489259da856..2c5ec9dad59 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -6,12 +6,11 @@ use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; -use rustc_span::symbol::sym; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; use crate::context::CodegenCx; -use crate::errors::{MissingFeatures, SanitizerMemtagRequiresMte, TargetFeatureDisableOrEnable}; +use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; @@ -502,26 +501,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); - if let Some(f) = llvm_util::check_tied_features( - cx.tcx.sess, - &function_features.iter().map(|f| (*f, true)).collect(), - ) { - let span = cx - .tcx - .get_attrs(instance.def_id(), sym::target_feature) - .next() - .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); - cx.tcx - .dcx() - .create_err(TargetFeatureDisableOrEnable { - features: f, - span: Some(span), - missing_features: Some(MissingFeatures), - }) - .emit(); - return; - } - let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 30d7ba4421b..dbf5298d64b 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,6 +3,8 @@ use std::ops::Deref; use std::{iter, ptr}; use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::{Align, Size, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -20,7 +22,6 @@ use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_target::abi::call::FnAbi; -use rustc_target::abi::{self, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -505,12 +506,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } match scalar.primitive() { - abi::Int(..) => { + abi::Primitive::Int(..) => { if !scalar.is_always_valid(bx) { bx.range_metadata(load, scalar.valid_range(bx)); } } - abi::Pointer(_) => { + abi::Primitive::Pointer(_) => { if !scalar.valid_range(bx).contains(0) { bx.nonnull_metadata(load); } @@ -521,7 +522,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } } - abi::Float(_) => {} + abi::Primitive::Float(_) => {} } } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 31d59905446..0ced37b53a8 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -1,7 +1,11 @@ //! Code that is useful in various codegen modules. use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::Primitive::Pointer; +use rustc_abi::{AddressSpace, HasDataLayout}; use rustc_ast::Mutability; +use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; use rustc_hir::def_id::DefId; @@ -9,7 +13,6 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::DllImport; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use tracing::debug; use crate::consts::const_alloc_to_llvm; @@ -144,6 +147,10 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_int" + ); unsafe { llvm::LLVMConstInt(t, i as u64, True) } } @@ -174,10 +181,18 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_uint" + ); unsafe { llvm::LLVMConstInt(t, i, False) } } fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_uint_big" + ); unsafe { let words = [u as u64, (u >> 64) as u64]; llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr()) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 81b82840472..2f830d6f941 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -138,6 +138,16 @@ pub(crate) unsafe fn create_module<'ll>( } } + if llvm_version < (20, 0, 0) { + if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { + // LLVM 20 defines three additional address spaces for alternate + // pointer kinds used in Windows. + // See https://github.com/llvm/llvm-project/pull/111879 + target_data_layout = + target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); + } + } + // Ensure the data-layout values hardcoded remain the defaults. { let tm = crate::back::write::create_informational_target_machine(tcx.sess, false); @@ -884,6 +894,11 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16); + ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32); + ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16); ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); @@ -1187,10 +1202,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { span: Span, fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { - if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.tcx.dcx().emit_fatal(Spanned { span, node: err }) - } else { - match fn_abi_request { + match err { + FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + self.tcx.dcx().emit_fatal(Spanned { span, node: err }); + } + _ => match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } @@ -1200,7 +1216,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } - } + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 90f7dd733ca..feac97f3e2b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,11 +1,9 @@ -use rustc_middle::mir::coverage::{ - ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, SourceRegion, -}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum CounterKind { +pub(crate) enum CounterKind { Zero = 0, CounterValueReference = 1, Expression = 2, @@ -25,9 +23,9 @@ pub enum CounterKind { /// Must match the layout of `LLVMRustCounter`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct Counter { +pub(crate) struct Counter { // Important: The layout (order and types of fields) must match its C++ counterpart. - pub kind: CounterKind, + pub(crate) kind: CounterKind, id: u32, } @@ -36,7 +34,7 @@ impl Counter { pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; /// Constructs a new `Counter` of kind `CounterValueReference`. - pub fn counter_value_reference(counter_id: CounterId) -> Self { + pub(crate) fn counter_value_reference(counter_id: CounterId) -> Self { Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } } @@ -59,7 +57,7 @@ impl Counter { /// Must match the layout of `LLVMRustCounterExprKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum ExprKind { +pub(crate) enum ExprKind { Subtract = 0, Add = 1, } @@ -69,48 +67,13 @@ pub enum ExprKind { /// Must match the layout of `LLVMRustCounterExpression`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct CounterExpression { - pub kind: ExprKind, - pub lhs: Counter, - pub rhs: Counter, +pub(crate) struct CounterExpression { + pub(crate) kind: ExprKind, + pub(crate) lhs: Counter, + pub(crate) rhs: Counter, } -/// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegionKind`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -enum RegionKind { - /// A CodeRegion associates some code with a counter - CodeRegion = 0, - - /// An ExpansionRegion represents a file expansion region that associates - /// a source range with the expansion of a virtual source file, such as - /// for a macro instantiation or #include file. - ExpansionRegion = 1, - - /// A SkippedRegion represents a source range with code that was skipped - /// by a preprocessor or similar means. - SkippedRegion = 2, - - /// A GapRegion is like a CodeRegion, but its count is only set as the - /// line execution count when its the only region in the line. - GapRegion = 3, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion = 4, - - /// A DecisionRegion represents a top-level boolean expression and is - /// associated with a variable length bitmap index and condition number. - MCDCDecisionRegion = 5, - - /// A Branch Region can be extended to include IDs to facilitate MC/DC. - MCDCBranchRegion = 6, -} - -mod mcdc { +pub(crate) mod mcdc { use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; /// Must match the layout of `LLVMRustMCDCDecisionParameters`. @@ -121,8 +84,6 @@ mod mcdc { num_conditions: u16, } - // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at - // [19](https://github.com/llvm/llvm-project/pull/81257). type LLVMConditionId = i16; /// Must match the layout of `LLVMRustMCDCBranchParameters`. @@ -133,38 +94,6 @@ mod mcdc { condition_ids: [LLVMConditionId; 2], } - #[repr(C)] - #[derive(Clone, Copy, Debug)] - enum ParameterTag { - None = 0, - Decision = 1, - Branch = 2, - } - /// Same layout with `LLVMRustMCDCParameters` - #[repr(C)] - #[derive(Clone, Copy, Debug)] - pub(crate) struct Parameters { - tag: ParameterTag, - decision_params: DecisionParameters, - branch_params: BranchParameters, - } - - impl Parameters { - pub(crate) fn none() -> Self { - Self { - tag: ParameterTag::None, - decision_params: Default::default(), - branch_params: Default::default(), - } - } - pub(crate) fn decision(decision_params: DecisionParameters) -> Self { - Self { tag: ParameterTag::Decision, decision_params, branch_params: Default::default() } - } - pub(crate) fn branch(branch_params: BranchParameters) -> Self { - Self { tag: ParameterTag::Branch, decision_params: Default::default(), branch_params } - } - } - impl From<ConditionInfo> for BranchParameters { fn from(value: ConditionInfo) -> Self { let to_llvm_cond_id = |cond_id: Option<ConditionId>| { @@ -186,267 +115,68 @@ mod mcdc { } } -/// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the -/// coverage map, in accordance with the -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). -/// The struct composes fields representing the `Counter` type and value(s) (injected counter -/// ID, or expression type and operands), the source file (an indirect index into a "filenames -/// array", encoded separately), and source location (start and end positions of the represented -/// code region). +/// A span of source code coordinates to be embedded in coverage metadata. /// -/// Corresponds to struct `llvm::coverage::CounterMappingRegion`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegion`. -#[derive(Copy, Clone, Debug)] +/// Must match the layout of `LLVMRustCoverageSpan`. +#[derive(Clone, Debug)] #[repr(C)] -pub struct CounterMappingRegion { - /// The counter type and type-dependent counter data, if any. - counter: Counter, - - /// If the `RegionKind` is a `BranchRegion`, this represents the counter - /// for the false branch of the region. - false_counter: Counter, - - mcdc_params: mcdc::Parameters, - /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the - /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes - /// that, in turn, are used to look up the filename for this region. +pub(crate) struct CoverageSpan { + /// Local index into the function's local-to-global file ID table. + /// The value at that index is itself an index into the coverage filename + /// table in the CGU's `__llvm_covmap` section. file_id: u32, - /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find - /// the mapping regions created as a result of macro expansion, by checking if their file id - /// matches the expanded file id. - expanded_file_id: u32, - - /// 1-based starting line of the mapping region. + /// 1-based starting line of the source code span. start_line: u32, - - /// 1-based starting column of the mapping region. + /// 1-based starting column of the source code span. start_col: u32, - - /// 1-based ending line of the mapping region. + /// 1-based ending line of the source code span. end_line: u32, - - /// 1-based ending column of the mapping region. If the high bit is set, the current - /// mapping region is a gap area. + /// 1-based ending column of the source code span. High bit must be unset. end_col: u32, - - kind: RegionKind, } -impl CounterMappingRegion { - pub(crate) fn from_mapping( - mapping_kind: &MappingKind, - local_file_id: u32, - source_region: &SourceRegion, - ) -> Self { - let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = - source_region; - match *mapping_kind { - MappingKind::Code(term) => Self::code_region( - Counter::from_term(term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::Branch { true_term, false_term } => Self::branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { - Self::mcdc_branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - mcdc_params, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ) - } - MappingKind::MCDCDecision(decision_info) => Self::decision_region( - decision_info, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - } - } - - pub(crate) fn code_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::CodeRegion, - } +impl CoverageSpan { + pub(crate) fn from_source_region(file_id: u32, code_region: &SourceRegion) -> Self { + let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + // Internally, LLVM uses the high bit of `end_col` to distinguish between + // code regions and gap regions, so it can't be used by the column number. + assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}"); + Self { file_id, start_line, start_col, end_line, end_col } } +} - pub(crate) fn branch_region( - counter: Counter, - false_counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::BranchRegion, - } - } - - pub(crate) fn mcdc_branch_region( - counter: Counter, - false_counter: Counter, - condition_info: ConditionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::branch(condition_info.into()), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCBranchRegion, - } - } - - pub(crate) fn decision_region( - decision_info: DecisionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - let mcdc_params = mcdc::Parameters::decision(decision_info.into()); - - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params, - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCDecisionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageCodeRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct CodeRegion { + pub(crate) span: CoverageSpan, + pub(crate) counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn expansion_region( - file_id: u32, - expanded_file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::ExpansionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct BranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn skipped_region( - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::SkippedRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCBranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, + pub(crate) mcdc_branch_params: mcdc::BranchParameters, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn gap_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col: (1_u32 << 31) | end_col, - kind: RegionKind::GapRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCDecisionRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCDecisionRegion { + pub(crate) span: CoverageSpan, + pub(crate) mcdc_decision_params: mcdc::DecisionParameters, } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 267a2244916..61e474031bb 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,8 +1,11 @@ +use std::ffi::CStr; + use itertools::Itertools as _; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; +use rustc_middle::mir::coverage::MappingKind; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, mir}; use rustc_span::Symbol; @@ -10,33 +13,24 @@ use rustc_span::def_id::DefIdSet; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::ffi::CounterMappingRegion; +use crate::coverageinfo::ffi; use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; use crate::{coverageinfo, llvm}; -/// Generates and exports the Coverage Map. -/// -/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions -/// 6 and 7 (encoded as 5 and 6 respectively), as described at -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/18.0-2024-02-13/llvm/docs/CoverageMappingFormat.rst). -/// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) -/// distributed in the `llvm-tools-preview` rustup component. +/// Generates and exports the coverage map, which is embedded in special +/// linker sections in the final binary. /// -/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with -/// the same version. Clang's implementation of Coverage Map generation was referenced when -/// implementing this Rust version, and though the format documentation is very explicit and -/// detailed, some undocumented details in Clang's implementation (that may or may not be important) -/// were also replicated for Rust's Coverage Map. +/// Those sections are then read and understood by LLVM's `llvm-cov` tool, +/// which is distributed in the `llvm-tools` rustup component. pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure that LLVM is using a version of the coverage mapping format that // agrees with our Rust-side code. Expected versions (encoded as n-1) are: - // - `CovMapVersion::Version6` (5) used by LLVM 13-17 - // - `CovMapVersion::Version7` (6) used by LLVM 18 + // - `CovMapVersion::Version7` (6) used by LLVM 18-19 let covmap_version = { let llvm_covmap_version = coverageinfo::mapping_version(); - let expected_versions = 5..=6; + let expected_versions = 6..=6; assert!( expected_versions.contains(&llvm_covmap_version), "Coverage mapping version exposed by `llvm-wrapper` is out of sync; \ @@ -141,7 +135,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { .collect::<Vec<_>>(); let initializer = cx.const_array(cx.type_ptr(), &name_globals); - let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), "__llvm_coverage_names"); + let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), c"__llvm_coverage_names"); llvm::set_global_constant(array, true); llvm::set_linkage(array, llvm::Linkage::InternalLinkage); llvm::set_initializer(array, initializer); @@ -244,7 +238,10 @@ fn encode_mappings_for_function( let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); let mut virtual_file_mapping = VirtualFileMapping::default(); - let mut mapping_regions = Vec::with_capacity(counter_regions.len()); + let mut code_regions = vec![]; + let mut branch_regions = vec![]; + let mut mcdc_branch_regions = vec![]; + let mut mcdc_decision_regions = vec![]; // Group mappings into runs with the same filename, preserving the order // yielded by `FunctionCoverage`. @@ -264,11 +261,36 @@ fn encode_mappings_for_function( // form suitable for FFI. for (mapping_kind, region) in counter_regions_for_file { debug!("Adding counter {mapping_kind:?} to map for {region:?}"); - mapping_regions.push(CounterMappingRegion::from_mapping( - &mapping_kind, - local_file_id.as_u32(), - region, - )); + let span = ffi::CoverageSpan::from_source_region(local_file_id.as_u32(), region); + match mapping_kind { + MappingKind::Code(term) => { + code_regions + .push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); + } + MappingKind::Branch { true_term, false_term } => { + branch_regions.push(ffi::BranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + }); + } + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + mcdc_branch_regions.push(ffi::MCDCBranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), + }); + } + MappingKind::MCDCDecision(mcdc_decision_params) => { + mcdc_decision_regions.push(ffi::MCDCDecisionRegion { + span, + mcdc_decision_params: ffi::mcdc::DecisionParameters::from( + mcdc_decision_params, + ), + }); + } + } } } @@ -277,7 +299,10 @@ fn encode_mappings_for_function( coverageinfo::write_mapping_to_buffer( virtual_file_mapping.into_vec(), expressions, - mapping_regions, + &code_regions, + &branch_regions, + &mcdc_branch_regions, + &mcdc_decision_regions, buffer, ); }) @@ -314,7 +339,7 @@ fn generate_coverage_map<'ll>( /// specific, well-known section and name. fn save_function_record( cx: &CodegenCx<'_, '_>, - covfun_section_name: &str, + covfun_section_name: &CStr, mangled_function_name: &str, source_hash: u64, filenames_ref: u64, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index d7d29eebf85..b2956945911 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::ffi::{CStr, CString}; use libc::c_uint; use rustc_codegen_ssa::traits::{ @@ -12,11 +13,11 @@ use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use rustc_middle::ty::layout::HasTyCtxt; use rustc_target::abi::{Align, Size}; +use rustc_target::spec::HasTargetSpec; use tracing::{debug, instrument}; use crate::builder::Builder; use crate::common::CodegenCx; -use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; use crate::coverageinfo::map_data::FunctionCoverageCollector; use crate::llvm; @@ -255,8 +256,11 @@ pub(crate) fn write_filenames_section_to_buffer<'a>( pub(crate) fn write_mapping_to_buffer( virtual_file_mapping: Vec<u32>, - expressions: Vec<CounterExpression>, - mapping_regions: Vec<CounterMappingRegion>, + expressions: Vec<ffi::CounterExpression>, + code_regions: &[ffi::CodeRegion], + branch_regions: &[ffi::BranchRegion], + mcdc_branch_regions: &[ffi::MCDCBranchRegion], + mcdc_decision_regions: &[ffi::MCDCDecisionRegion], buffer: &RustString, ) { unsafe { @@ -265,8 +269,14 @@ pub(crate) fn write_mapping_to_buffer( virtual_file_mapping.len() as c_uint, expressions.as_ptr(), expressions.len() as c_uint, - mapping_regions.as_ptr(), - mapping_regions.len() as c_uint, + code_regions.as_ptr(), + code_regions.len() as c_uint, + branch_regions.as_ptr(), + branch_regions.len() as c_uint, + mcdc_branch_regions.as_ptr(), + mcdc_branch_regions.len() as c_uint, + mcdc_decision_regions.as_ptr(), + mcdc_decision_regions.len() as c_uint, buffer, ); } @@ -284,16 +294,16 @@ pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, cov_data_val: &'ll llvm::Value, ) { - let covmap_var_name = llvm::build_string(|s| unsafe { + let covmap_var_name = CString::new(llvm::build_byte_buffer(|s| unsafe { llvm::LLVMRustCoverageWriteMappingVarNameToString(s); - }) - .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); + })) + .unwrap(); debug!("covmap var name: {:?}", covmap_var_name); - let covmap_section_name = llvm::build_string(|s| unsafe { + let covmap_section_name = CString::new(llvm::build_byte_buffer(|s| unsafe { llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage section name failed UTF-8 conversion"); + })) + .expect("covmap section name should not contain NUL"); debug!("covmap section name: {:?}", covmap_section_name); let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); @@ -308,7 +318,7 @@ pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - covfun_section_name: &str, + covfun_section_name: &CStr, func_name_hash: u64, func_record_val: &'ll llvm::Value, is_used: bool, @@ -322,7 +332,8 @@ pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. let func_record_var_name = - format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); + CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) + .unwrap(); debug!("function record var name: {:?}", func_record_var_name); debug!("function record section name: {:?}", covfun_section_name); @@ -334,7 +345,9 @@ pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( llvm::set_section(llglobal, covfun_section_name); // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. llvm::set_alignment(llglobal, Align::EIGHT); - llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + if cx.target_spec().supports_comdat() { + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + } cx.add_used_global(llglobal); } @@ -349,9 +362,9 @@ pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( /// - `__llvm_covfun` on Linux /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) -pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String { - llvm::build_string(|s| unsafe { +pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage function record section name failed UTF-8 conversion") + })) + .expect("covfun section name should not contain NUL") } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 7be44dd51b5..33258cb46fa 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -84,10 +84,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - // Declare C ABI functions with the visibility used by C by default. - let visibility = Visibility::from_generic(self.tcx.sess.default_visibility()); - - declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type) + // Visibility should always be default for declarations, otherwise the linker may report an + // error. + declare_raw_fn(self, name, llvm::CCallConv, unnamed, Visibility::Default, fn_type) } /// Declare an entry Function diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index bb481d2a308..0d436e1891e 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -80,30 +80,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { } } -pub(crate) struct TargetFeatureDisableOrEnable<'a> { - pub features: &'a [&'a str], - pub span: Option<Span>, - pub missing_features: Option<MissingFeatures>, -} - -#[derive(Subdiagnostic)] -#[help(codegen_llvm_missing_features)] -pub(crate) struct MissingFeatures; - -impl<G: EmissionGuarantee> Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { - fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> { - let mut diag = Diag::new(dcx, level, fluent::codegen_llvm_target_feature_disable_or_enable); - if let Some(span) = self.span { - diag.span(span); - }; - if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); - } - diag.arg("features", self.features.join(", ")); - diag - } -} - #[derive(Diagnostic)] #[diag(codegen_llvm_lto_disallowed)] pub(crate) struct LtoDisallowed; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 30c6f08e894..c9a17c9852d 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -86,6 +86,11 @@ fn get_simple_intrinsic<'ll>( sym::fmaf64 => "llvm.fma.f64", sym::fmaf128 => "llvm.fma.f128", + sym::fmuladdf16 => "llvm.fmuladd.f16", + sym::fmuladdf32 => "llvm.fmuladd.f32", + sym::fmuladdf64 => "llvm.fmuladd.f64", + sym::fmuladdf128 => "llvm.fmuladd.f128", + sym::fabsf16 => "llvm.fabs.f16", sym::fabsf32 => "llvm.fabs.f32", sym::fabsf64 => "llvm.fabs.f64", @@ -782,7 +787,9 @@ fn codegen_msvc_try<'ll>( let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); unsafe { llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); - llvm::SetUniqueComdat(bx.llmod, tydesc); + if bx.cx.tcx.sess.target.supports_comdat() { + llvm::SetUniqueComdat(bx.llmod, tydesc); + } llvm::LLVMSetInitializer(tydesc, type_info); } @@ -1172,8 +1179,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => + && len + .try_to_target_usize(bx.tcx) + .expect("expected monomorphic const in codegen") + == expected_bytes => { let place = PlaceRef::alloca(bx, args[0].layout); args[0].val.store(bx, place); @@ -1238,12 +1247,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle_generic { - let idx = fn_args[2] - .expect_const() - .eval(tcx, ty::ParamEnv::reveal_all(), span) - .unwrap() - .1 - .unwrap_branch(); + let idx = fn_args[2].expect_const().try_to_valtree().unwrap().0.unwrap_branch(); let n = idx.len() as u64; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); @@ -1462,8 +1466,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => + && len + .try_to_target_usize(bx.tcx) + .expect("expected monomorphic const in codegen") + == expected_bytes => { // Zero-extend iN to the array length: let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 661debbb9f1..50e6c1494a8 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -646,6 +646,7 @@ unsafe extern "C" { pub type Attribute; pub type Metadata; pub type BasicBlock; + pub type Comdat; } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); @@ -1490,6 +1491,9 @@ unsafe extern "C" { pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + + pub fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; + pub fn LLVMSetComdat(V: &Value, C: &Comdat); } #[link(name = "llvm-wrapper", kind = "static")] @@ -1740,7 +1744,7 @@ unsafe extern "C" { ) -> bool; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + pub(crate) fn LLVMRustCoverageWriteFilenamesSectionToBuffer( Filenames: *const *const c_char, FilenamesLen: size_t, Lengths: *const size_t, @@ -1749,33 +1753,39 @@ unsafe extern "C" { ); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingToBuffer( + pub(crate) fn LLVMRustCoverageWriteMappingToBuffer( VirtualFileMappingIDs: *const c_uint, NumVirtualFileMappingIDs: c_uint, Expressions: *const crate::coverageinfo::ffi::CounterExpression, NumExpressions: c_uint, - MappingRegions: *const crate::coverageinfo::ffi::CounterMappingRegion, - NumMappingRegions: c_uint, + CodeRegions: *const crate::coverageinfo::ffi::CodeRegion, + NumCodeRegions: c_uint, + BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, + NumBranchRegions: c_uint, + MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, + NumMCDCBranchRegions: c_uint, + MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, + NumMCDCDecisionRegions: c_uint, BufferOut: &RustString, ); - pub fn LLVMRustCoverageCreatePGOFuncNameVar( + pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, FuncName: *const c_char, FuncNameLen: size_t, ) -> &Value; - pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + pub(crate) fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); - pub fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustDebugMetadataVersion() -> u32; pub fn LLVMRustVersionMajor() -> u32; pub fn LLVMRustVersionMinor() -> u32; @@ -2320,7 +2330,6 @@ unsafe extern "C" { pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); - pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index d0db350a149..e837022044e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -178,10 +178,10 @@ pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { // function. // For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 pub fn SetUniqueComdat(llmod: &Module, val: &Value) { - unsafe { - let name = get_value_name(val); - LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len()); - } + let name_buf = get_value_name(val).to_vec(); + let name = + CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap(); + set_comdat(llmod, val, &name); } pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { @@ -210,15 +210,13 @@ impl MemoryEffects { } } -pub fn set_section(llglobal: &Value, section_name: &str) { - let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); +pub fn set_section(llglobal: &Value, section_name: &CStr) { unsafe { - LLVMSetSection(llglobal, section_name_cstr.as_ptr()); + LLVMSetSection(llglobal, section_name.as_ptr()); } } -pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { - let name_cstr = CString::new(name).expect("unexpected CString error"); +pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } } @@ -252,9 +250,14 @@ pub fn set_alignment(llglobal: &Value, align: Align) { } } -pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { +/// Get the `name`d comdat from `llmod` and assign it to `llglobal`. +/// +/// Inserts the comdat into `llmod` if it does not exist. +/// It is an error to call this if the target does not support comdat. +pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { unsafe { - LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); + let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr()); + LLVMSetComdat(llglobal, comdat); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index bd847cd0068..57936215ff1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -6,6 +6,7 @@ use std::{ptr, slice, str}; use libc::c_int; use rustc_codegen_ssa::base::wants_wasm_eh; +use rustc_codegen_ssa::codegen_attrs::check_tied_features; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_data_structures::unord::UnordSet; @@ -19,8 +20,8 @@ use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATU use crate::back::write::create_informational_target_machine; use crate::errors::{ - FixedX18InvalidArch, InvalidTargetFeaturePrefix, PossibleFeature, TargetFeatureDisableOrEnable, - UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, + FixedX18InvalidArch, InvalidTargetFeaturePrefix, PossibleFeature, UnknownCTargetFeature, + UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; @@ -247,7 +248,9 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")), + // Before LLVM 20 those two features were packaged together as b16b16 + ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), + ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => { @@ -276,25 +279,6 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea } } -/// Given a map from target_features to whether they are enabled or disabled, -/// ensure only valid combinations are allowed. -pub(crate) fn check_tied_features( - sess: &Session, - features: &FxHashMap<&str, bool>, -) -> Option<&'static [&'static str]> { - if !features.is_empty() { - for tied in sess.target.tied_target_features() { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); - } - } - } - None -} - /// Used to generate cfg variables and apply features /// Must express features in the way Rust understands them pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { @@ -685,7 +669,7 @@ pub(crate) fn global_llvm_features( features.extend(feats); if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.dcx().emit_err(TargetFeatureDisableOrEnable { + sess.dcx().emit_err(rustc_codegen_ssa::errors::TargetFeatureDisableOrEnable { features: f, span: None, missing_features: None, diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 02e1995620b..bf6ef219873 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -64,7 +64,9 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); - if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { + if (linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR) + && self.tcx.sess.target.supports_comdat() + { llvm::SetUniqueComdat(self.llmod, lldecl); } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 7071dd86ee0..6be4c3f034f 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,11 +1,12 @@ use std::fmt::Write; +use rustc_abi::Primitive::{Float, Int, Pointer}; +use rustc_abi::{Abi, Align, FieldsShape, Scalar, Size, Variants}; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; -use rustc_target::abi::{Abi, Align, FieldsShape, Float, Int, Pointer, Scalar, Size, Variants}; use tracing::debug; use crate::common::*; @@ -190,7 +191,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. /// If the type is an unsized struct, the regular layout is generated, - /// with the inner-most trailing unsized field using the "minimal unit" + /// with the innermost trailing unsized field using the "minimal unit" /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { |