diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
21 files changed, 642 insertions, 628 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 9aa01bd1b95..d1804cb49ad 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -656,9 +656,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg) => "r", PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", - PowerPC(PowerPCInlineAsmRegClass::cr) - | PowerPC(PowerPCInlineAsmRegClass::xer) - | PowerPC(PowerPCInlineAsmRegClass::vreg) => { + PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", + PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -825,9 +824,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), - PowerPC(PowerPCInlineAsmRegClass::cr) - | PowerPC(PowerPCInlineAsmRegClass::xer) - | PowerPC(PowerPCInlineAsmRegClass::vreg) => { + PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4), + PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), @@ -1042,6 +1040,26 @@ fn llvm_fixup_input<'ll, 'tcx>( let value = bx.or(value, bx.const_u32(0xFFFF_0000)); bx.bitcast(value, bx.type_f32()) } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F32) => + { + let value = bx.insert_element( + bx.const_undef(bx.type_vector(bx.type_f32(), 4)), + value, + bx.const_usize(0), + ); + bx.bitcast(value, bx.type_vector(bx.type_f32(), 4)) + } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F64) => + { + let value = bx.insert_element( + bx.const_undef(bx.type_vector(bx.type_f64(), 2)), + value, + bx.const_usize(0), + ); + bx.bitcast(value, bx.type_vector(bx.type_f64(), 2)) + } _ => value, } } @@ -1177,6 +1195,18 @@ fn llvm_fixup_output<'ll, 'tcx>( let value = bx.trunc(value, bx.type_i16()); bx.bitcast(value, bx.type_f16()) } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F32) => + { + let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4)); + bx.extract_element(value, bx.const_usize(0)) + } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F64) => + { + let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2)); + bx.extract_element(value, bx.const_usize(0)) + } _ => value, } } @@ -1301,6 +1331,16 @@ fn llvm_fixup_output_type<'ll, 'tcx>( { cx.type_f32() } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F32) => + { + cx.type_vector(cx.type_f32(), 4) + } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F64) => + { + cx.type_vector(cx.type_f64(), 2) + } _ => layout.llvm_type(cx), } } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 48beb9be2b2..f6d2ec24da6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -148,7 +148,7 @@ fn prepare_lto( // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to // __llvm_profile_runtime, therefore we won't know until link time if this symbol // should have default visibility. - symbols_below_threshold.push(CString::new("__llvm_profile_counter_bias").unwrap()); + symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned()); Ok((symbols_below_threshold, upstream_modules)) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 00f7b479fa7..bde6668929c 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -61,6 +61,7 @@ fn write_output_file<'ll>( dwo_output: Option<&Path>, file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, + verify_llvm_ir: bool, ) -> Result<(), FatalError> { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { @@ -79,6 +80,7 @@ fn write_output_file<'ll>( output_c.as_ptr(), dwo_output_ptr, file_type, + verify_llvm_ir, ); // Record artifact sizes for self-profiling @@ -507,7 +509,7 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { } fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { - config.instrument_coverage.then(|| CString::new("default_%m_%p.profraw").unwrap()) + config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned()) } pub(crate) unsafe fn llvm_optimize( @@ -840,6 +842,7 @@ pub(crate) unsafe fn codegen( None, llvm::FileType::AssemblyFile, &cgcx.prof, + config.verify_llvm_ir, ) })?; } @@ -877,6 +880,7 @@ pub(crate) unsafe fn codegen( dwo_out, llvm::FileType::ObjectFile, &cgcx.prof, + config.verify_llvm_ir, ) })?; } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index e0a2de3366c..ec77f32caf4 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -104,7 +104,10 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t let is_hidden = if is_generic { // This is a monomorphization of a generic function. - if !cx.tcx.sess.opts.share_generics() { + if !(cx.tcx.sess.opts.share_generics() + || tcx.codegen_fn_attrs(instance_def_id).inline + == rustc_attr::InlineAttr::Never) + { // When not sharing generics, all instances are in the same // crate and have hidden visibility. true diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8852dec7d9f..adfe8aeb5c5 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -302,10 +302,9 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { (value, AddressSpace::DATA) } } - GlobalAlloc::Function { instance, .. } => ( - self.get_fn_addr(instance.polymorphize(self.tcx)), - self.data_layout().instruction_address_space, - ), + GlobalAlloc::Function { instance, .. } => { + (self.get_fn_addr(instance), self.data_layout().instruction_address_space) + } GlobalAlloc::VTable(ty, dyn_ty) => { let alloc = self .tcx diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 841c110b3c8..e0506c0c5fd 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -82,8 +82,8 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { pub isize_ty: &'ll Type, - /// Extra codegen state needed when coverage instrumentation is enabled. - pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, + /// Extra per-CGU codegen state needed when coverage instrumentation is enabled. + pub coverage_cx: Option<coverageinfo::CguCoverageContext<'ll, 'tcx>>, pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>, eh_personality: Cell<Option<&'ll Value>>, @@ -159,6 +159,11 @@ pub(crate) unsafe fn create_module<'ll>( // See https://github.com/llvm/llvm-project/pull/112084 target_data_layout = target_data_layout.replace("-i128:128", ""); } + if sess.target.arch.starts_with("powerpc64") { + // LLVM 20 updates the powerpc64 layout to correctly align 128 bit integers to 128 bit. + // See https://github.com/llvm/llvm-project/pull/118004 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } } // Ensure the data-layout values hardcoded remain the defaults. @@ -525,7 +530,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); let coverage_cx = - tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new); + tcx.sess.instrument_coverage().then(coverageinfo::CguCoverageContext::new); let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); @@ -576,7 +581,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// Extra state that is only available when coverage instrumentation is enabled. #[inline] #[track_caller] - pub(crate) fn coverage_cx(&self) -> &coverageinfo::CrateCoverageContext<'ll, 'tcx> { + pub(crate) fn coverage_cx(&self) -> &coverageinfo::CguCoverageContext<'ll, 'tcx> { self.coverage_cx.as_ref().expect("only called when coverage instrumentation is enabled") } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 19d6726002c..a6e07ea2a60 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,4 +1,6 @@ -use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion}; + +use crate::coverageinfo::mapgen::LocalFileId; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] @@ -124,23 +126,37 @@ pub(crate) struct CoverageSpan { /// Local index into the function's local-to-global file ID table. /// The value at that index is itself an index into the coverage filename /// table in the CGU's `__llvm_covmap` section. - pub(crate) file_id: u32, + file_id: u32, /// 1-based starting line of the source code span. - pub(crate) start_line: u32, + start_line: u32, /// 1-based starting column of the source code span. - pub(crate) start_col: u32, + start_col: u32, /// 1-based ending line of the source code span. - pub(crate) end_line: u32, + end_line: u32, /// 1-based ending column of the source code span. High bit must be unset. - pub(crate) end_col: u32, + end_col: u32, +} + +impl CoverageSpan { + pub(crate) fn from_source_region( + local_file_id: LocalFileId, + code_region: &SourceRegion, + ) -> Self { + let file_id = local_file_id.as_u32(); + let &SourceRegion { start_line, start_col, end_line, end_col } = code_region; + // Internally, LLVM uses the high bit of `end_col` to distinguish between + // code regions and gap regions, so it can't be used by the column number. + assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}"); + Self { file_id, start_line, start_col, end_line, end_col } + } } /// Must match the layout of `LLVMRustCoverageCodeRegion`. #[derive(Clone, Debug)] #[repr(C)] pub(crate) struct CodeRegion { - pub(crate) cov_span: CoverageSpan, + pub(crate) span: CoverageSpan, pub(crate) counter: Counter, } @@ -148,7 +164,7 @@ pub(crate) struct CodeRegion { #[derive(Clone, Debug)] #[repr(C)] pub(crate) struct BranchRegion { - pub(crate) cov_span: CoverageSpan, + pub(crate) span: CoverageSpan, pub(crate) true_counter: Counter, pub(crate) false_counter: Counter, } @@ -157,7 +173,7 @@ pub(crate) struct BranchRegion { #[derive(Clone, Debug)] #[repr(C)] pub(crate) struct MCDCBranchRegion { - pub(crate) cov_span: CoverageSpan, + pub(crate) span: CoverageSpan, pub(crate) true_counter: Counter, pub(crate) false_counter: Counter, pub(crate) mcdc_branch_params: mcdc::BranchParameters, @@ -167,6 +183,6 @@ pub(crate) struct MCDCBranchRegion { #[derive(Clone, Debug)] #[repr(C)] pub(crate) struct MCDCDecisionRegion { - pub(crate) cov_span: CoverageSpan, + pub(crate) span: CoverageSpan, pub(crate) mcdc_decision_params: mcdc::DecisionParameters, } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index 95746b88ced..c5d1ebdfe7c 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -1,201 +1,38 @@ use rustc_data_structures::captures::Captures; -use rustc_data_structures::fx::FxIndexSet; -use rustc_index::bit_set::BitSet; use rustc_middle::mir::coverage::{ - CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, MappingKind, Op, + CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, MappingKind, Op, + SourceRegion, }; -use rustc_middle::ty::Instance; -use rustc_span::Span; -use tracing::{debug, instrument}; use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; -/// Holds all of the coverage mapping data associated with a function instance, -/// collected during traversal of `Coverage` statements in the function's MIR. -#[derive(Debug)] -pub(crate) struct FunctionCoverageCollector<'tcx> { - /// Coverage info that was attached to this function by the instrumentor. - function_coverage_info: &'tcx FunctionCoverageInfo, - is_used: bool, - - /// Tracks which counters have been seen, so that we can identify mappings - /// to counters that were optimized out, and set them to zero. - counters_seen: BitSet<CounterId>, - /// Contains all expression IDs that have been seen in an `ExpressionUsed` - /// coverage statement, plus all expression IDs that aren't directly used - /// by any mappings (and therefore do not have expression-used statements). - /// After MIR traversal is finished, we can conclude that any IDs missing - /// from this set must have had their statements deleted by MIR opts. - expressions_seen: BitSet<ExpressionId>, +pub(crate) struct FunctionCoverage<'tcx> { + pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, + /// If `None`, the corresponding function is unused. + ids_info: Option<&'tcx CoverageIdsInfo>, } -impl<'tcx> FunctionCoverageCollector<'tcx> { - /// Creates a new set of coverage data for a used (called) function. - pub(crate) fn new( - instance: Instance<'tcx>, +impl<'tcx> FunctionCoverage<'tcx> { + pub(crate) fn new_used( function_coverage_info: &'tcx FunctionCoverageInfo, + ids_info: &'tcx CoverageIdsInfo, ) -> Self { - Self::create(instance, function_coverage_info, true) + Self { function_coverage_info, ids_info: Some(ids_info) } } - /// Creates a new set of coverage data for an unused (never called) function. - pub(crate) fn unused( - instance: Instance<'tcx>, - function_coverage_info: &'tcx FunctionCoverageInfo, - ) -> Self { - Self::create(instance, function_coverage_info, false) + pub(crate) fn new_unused(function_coverage_info: &'tcx FunctionCoverageInfo) -> Self { + Self { function_coverage_info, ids_info: None } } - fn create( - instance: Instance<'tcx>, - function_coverage_info: &'tcx FunctionCoverageInfo, - is_used: bool, - ) -> Self { - let num_counters = function_coverage_info.num_counters; - let num_expressions = function_coverage_info.expressions.len(); - debug!( - "FunctionCoverage::create(instance={instance:?}) has \ - num_counters={num_counters}, num_expressions={num_expressions}, is_used={is_used}" - ); - - // Create a filled set of expression IDs, so that expressions not - // directly used by mappings will be treated as "seen". - // (If they end up being unused, LLVM will delete them for us.) - let mut expressions_seen = BitSet::new_filled(num_expressions); - // For each expression ID that is directly used by one or more mappings, - // mark it as not-yet-seen. This indicates that we expect to see a - // corresponding `ExpressionUsed` statement during MIR traversal. - for mapping in function_coverage_info.mappings.iter() { - // Currently we only worry about ordinary code mappings. - // For branch and MC/DC mappings, expressions might not correspond - // to any particular point in the control-flow graph. - // (Keep this in sync with the injection of `ExpressionUsed` - // statements in the `InstrumentCoverage` MIR pass.) - if let MappingKind::Code(term) = mapping.kind - && let CovTerm::Expression(id) = term - { - expressions_seen.remove(id); - } - } - - Self { - function_coverage_info, - is_used, - counters_seen: BitSet::new_empty(num_counters), - expressions_seen, - } - } - - /// Marks a counter ID as having been seen in a counter-increment statement. - #[instrument(level = "debug", skip(self))] - pub(crate) fn mark_counter_id_seen(&mut self, id: CounterId) { - self.counters_seen.insert(id); - } - - /// Marks an expression ID as having been seen in an expression-used statement. - #[instrument(level = "debug", skip(self))] - pub(crate) fn mark_expression_id_seen(&mut self, id: ExpressionId) { - self.expressions_seen.insert(id); - } - - /// Identify expressions that will always have a value of zero, and note - /// their IDs in [`ZeroExpressions`]. Mappings that refer to a zero expression - /// can instead become mappings to a constant zero value. - /// - /// This method mainly exists to preserve the simplifications that were - /// already being performed by the Rust-side expression renumbering, so that - /// the resulting coverage mappings don't get worse. - fn identify_zero_expressions(&self) -> ZeroExpressions { - // The set of expressions that either were optimized out entirely, or - // have zero as both of their operands, and will therefore always have - // a value of zero. Other expressions that refer to these as operands - // can have those operands replaced with `CovTerm::Zero`. - let mut zero_expressions = ZeroExpressions::default(); - - // Simplify a copy of each expression based on lower-numbered expressions, - // and then update the set of always-zero expressions if necessary. - // (By construction, expressions can only refer to other expressions - // that have lower IDs, so one pass is sufficient.) - for (id, expression) in self.function_coverage_info.expressions.iter_enumerated() { - if !self.expressions_seen.contains(id) { - // If an expression was not seen, it must have been optimized away, - // so any operand that refers to it can be replaced with zero. - zero_expressions.insert(id); - continue; - } - - // We don't need to simplify the actual expression data in the - // expressions list; we can just simplify a temporary copy and then - // use that to update the set of always-zero expressions. - let Expression { mut lhs, op, mut rhs } = *expression; - - // If an expression has an operand that is also an expression, the - // operand's ID must be strictly lower. This is what lets us find - // all zero expressions in one pass. - let assert_operand_expression_is_lower = |operand_id: ExpressionId| { - assert!( - operand_id < id, - "Operand {operand_id:?} should be less than {id:?} in {expression:?}", - ) - }; - - // If an operand refers to a counter or expression that is always - // zero, then that operand can be replaced with `CovTerm::Zero`. - let maybe_set_operand_to_zero = |operand: &mut CovTerm| { - if let CovTerm::Expression(id) = *operand { - assert_operand_expression_is_lower(id); - } - - if is_zero_term(&self.counters_seen, &zero_expressions, *operand) { - *operand = CovTerm::Zero; - } - }; - maybe_set_operand_to_zero(&mut lhs); - maybe_set_operand_to_zero(&mut rhs); - - // Coverage counter values cannot be negative, so if an expression - // involves subtraction from zero, assume that its RHS must also be zero. - // (Do this after simplifications that could set the LHS to zero.) - if lhs == CovTerm::Zero && op == Op::Subtract { - rhs = CovTerm::Zero; - } - - // After the above simplifications, if both operands are zero, then - // we know that this expression is always zero too. - if lhs == CovTerm::Zero && rhs == CovTerm::Zero { - zero_expressions.insert(id); - } - } - - zero_expressions - } - - pub(crate) fn into_finished(self) -> FunctionCoverage<'tcx> { - let zero_expressions = self.identify_zero_expressions(); - let FunctionCoverageCollector { function_coverage_info, is_used, counters_seen, .. } = self; - - FunctionCoverage { function_coverage_info, is_used, counters_seen, zero_expressions } - } -} - -pub(crate) struct FunctionCoverage<'tcx> { - pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, - is_used: bool, - - counters_seen: BitSet<CounterId>, - zero_expressions: ZeroExpressions, -} - -impl<'tcx> FunctionCoverage<'tcx> { /// Returns true for a used (called) function, and false for an unused function. pub(crate) fn is_used(&self) -> bool { - self.is_used + self.ids_info.is_some() } /// Return the source hash, generated from the HIR node structure, and used to indicate whether /// or not the source code structure changed between different compilations. pub(crate) fn source_hash(&self) -> u64 { - if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } + if self.is_used() { self.function_coverage_info.function_source_hash } else { 0 } } /// Convert this function's coverage expression data into a form that can be @@ -220,16 +57,16 @@ impl<'tcx> FunctionCoverage<'tcx> { }) } - /// Yields all this function's coverage mappings, after simplifying away - /// unused counters and counter expressions. - pub(crate) fn mapping_spans( + /// Converts this function's coverage mappings into an intermediate form + /// that will be used by `mapgen` when preparing for FFI. + pub(crate) fn counter_regions( &self, - ) -> impl Iterator<Item = (MappingKind, Span)> + ExactSizeIterator + Captures<'_> { + ) -> impl Iterator<Item = (MappingKind, &SourceRegion)> + ExactSizeIterator { self.function_coverage_info.mappings.iter().map(move |mapping| { - let &Mapping { ref kind, span } = mapping; + let Mapping { kind, source_region } = mapping; let kind = kind.map_terms(|term| if self.is_zero_term(term) { CovTerm::Zero } else { term }); - (kind, span) + (kind, source_region) }) } @@ -238,37 +75,10 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn is_zero_term(&self, term: CovTerm) -> bool { - is_zero_term(&self.counters_seen, &self.zero_expressions, term) - } -} - -/// Set of expression IDs that are known to always evaluate to zero. -/// Any mapping or expression operand that refers to these expressions can have -/// that reference replaced with a constant zero value. -#[derive(Default)] -struct ZeroExpressions(FxIndexSet<ExpressionId>); - -impl ZeroExpressions { - fn insert(&mut self, id: ExpressionId) { - self.0.insert(id); - } - - fn contains(&self, id: ExpressionId) -> bool { - self.0.contains(&id) - } -} - -/// Returns `true` if the given term is known to have a value of zero, taking -/// into account knowledge of which counters are unused and which expressions -/// are always zero. -fn is_zero_term( - counters_seen: &BitSet<CounterId>, - zero_expressions: &ZeroExpressions, - term: CovTerm, -) -> bool { - match term { - CovTerm::Zero => true, - CovTerm::Counter(id) => !counters_seen.contains(id), - CovTerm::Expression(id) => zero_expressions.contains(id), + match self.ids_info { + Some(ids_info) => ids_info.is_zero_term(term), + // This function is unused, so all coverage counters/expressions are zero. + None => true, + } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index ed881418cb0..a6c3caf9e2b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,14 +1,12 @@ -mod spans; - use std::ffi::CString; -use std::sync::Arc; +use std::iter; use itertools::Itertools as _; use rustc_abi::Align; use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, }; -use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; +use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; use rustc_middle::mir::coverage::MappingKind; @@ -17,12 +15,12 @@ use rustc_middle::{bug, mir}; use rustc_session::RemapFileNameExt; use rustc_session::config::RemapPathScopeComponents; use rustc_span::def_id::DefIdSet; -use rustc_span::{SourceFile, StableSourceFileId}; +use rustc_span::{Span, Symbol}; use rustc_target::spec::HasTargetSpec; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; +use crate::coverageinfo::map_data::FunctionCoverage; use crate::coverageinfo::{ffi, llvm_cov}; use crate::llvm; @@ -65,20 +63,15 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { None => return, }; if function_coverage_map.is_empty() { - // This module has no functions with coverage instrumentation + // This CGU has no functions with coverage instrumentation. return; } - let function_coverage_entries = function_coverage_map - .into_iter() - .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) - .collect::<Vec<_>>(); - - let all_files = function_coverage_entries + let all_file_names = function_coverage_map .iter() .map(|(_, fn_cov)| fn_cov.function_coverage_info.body_span) - .map(|span| tcx.sess.source_map().lookup_source_file(span.lo())); - let global_file_table = GlobalFileTable::new(all_files); + .map(|span| span_file_name(tcx, span)); + let global_file_table = GlobalFileTable::new(all_file_names); // Encode all filenames referenced by coverage mappings in this CGU. let filenames_buffer = global_file_table.make_filenames_buffer(tcx); @@ -94,7 +87,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let mut unused_function_names = Vec::new(); // Encode coverage mappings and generate function records - for (instance, function_coverage) in function_coverage_entries { + for (instance, function_coverage) in function_coverage_map { debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); let mangled_function_name = tcx.symbol_name(instance).name; @@ -105,8 +98,15 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { encode_mappings_for_function(tcx, &global_file_table, &function_coverage); if coverage_mapping_buffer.is_empty() { - debug!("function has no mappings to embed; skipping"); - continue; + if function_coverage.is_used() { + bug!( + "A used function should have had coverage mapping data but did not: {}", + mangled_function_name + ); + } else { + debug!("unused function had no coverage mapping data: {}", mangled_function_name); + continue; + } } if !is_used { @@ -143,34 +143,29 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { } } -/// Maps "global" (per-CGU) file ID numbers to their underlying source files. +/// Maps "global" (per-CGU) file ID numbers to their underlying filenames. struct GlobalFileTable { - /// This "raw" table doesn't include the working dir, so a file's + /// This "raw" table doesn't include the working dir, so a filename's /// global ID is its index in this set **plus one**. - raw_file_table: FxIndexMap<StableSourceFileId, Arc<SourceFile>>, + raw_file_table: FxIndexSet<Symbol>, } impl GlobalFileTable { - fn new(all_files: impl IntoIterator<Item = Arc<SourceFile>>) -> Self { - // Collect all of the files into a set. Files usually come in contiguous - // runs, so we can dedup adjacent ones to save work. - let mut raw_file_table = all_files - .into_iter() - .dedup_by(|a, b| a.stable_id == b.stable_id) - .map(|f| (f.stable_id, f)) - .collect::<FxIndexMap<StableSourceFileId, Arc<SourceFile>>>(); + fn new(all_file_names: impl IntoIterator<Item = Symbol>) -> Self { + // Collect all of the filenames into a set. Filenames usually come in + // contiguous runs, so we can dedup adjacent ones to save work. + let mut raw_file_table = all_file_names.into_iter().dedup().collect::<FxIndexSet<Symbol>>(); - // Sort the file table by its underlying filenames. - raw_file_table.sort_unstable_by(|_, a, _, b| { - Ord::cmp(&a.name, &b.name).then_with(|| Ord::cmp(&a.stable_id, &b.stable_id)) - }); + // Sort the file table by its actual string values, not the arbitrary + // ordering of its symbols. + raw_file_table.sort_unstable_by(|a, b| a.as_str().cmp(b.as_str())); Self { raw_file_table } } - fn global_file_id_for_file(&self, file: &SourceFile) -> GlobalFileId { - let raw_id = self.raw_file_table.get_index_of(&file.stable_id).unwrap_or_else(|| { - bug!("file not found in prepared global file table: {:?}", file.name); + fn global_file_id_for_file_name(&self, file_name: Symbol) -> GlobalFileId { + let raw_id = self.raw_file_table.get_index_of(&file_name).unwrap_or_else(|| { + bug!("file name not found in prepared global file table: {file_name}"); }); // The raw file table doesn't include an entry for the working dir // (which has ID 0), so add 1 to get the correct ID. @@ -178,27 +173,24 @@ impl GlobalFileTable { } fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> { - let mut table = Vec::with_capacity(self.raw_file_table.len() + 1); - // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) // requires setting the first filename to the compilation directory. // Since rustc generates coverage maps with relative paths, the // compilation directory can be combined with the relative paths // to get absolute paths, if needed. - table.push( - tcx.sess - .opts - .working_dir - .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) - .to_string_lossy(), - ); - - // Add the regular entries after the base directory. - table.extend(self.raw_file_table.values().map(|file| { - file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy() - })); - - llvm_cov::write_filenames_to_buffer(table.iter().map(|f| f.as_ref())) + use rustc_session::RemapFileNameExt; + use rustc_session::config::RemapPathScopeComponents; + let working_dir: &str = &tcx + .sess + .opts + .working_dir + .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) + .to_string_lossy(); + + // Insert the working dir at index 0, before the other filenames. + let filenames = + iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)); + llvm_cov::write_filenames_to_buffer(filenames) } } @@ -211,7 +203,7 @@ rustc_index::newtype_index! { /// An index into a function's list of global file IDs. That underlying list /// of local-to-global mappings will be embedded in the function's record in /// the `__llvm_covfun` linker section. - struct LocalFileId {} + pub(crate) struct LocalFileId {} } /// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) @@ -237,6 +229,13 @@ impl VirtualFileMapping { } } +fn span_file_name(tcx: TyCtxt<'_>, span: Span) -> Symbol { + let source_file = tcx.sess.source_map().lookup_source_file(span.lo()); + let name = + source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy(); + Symbol::intern(&name) +} + /// Using the expressions and counter regions collected for a single function, /// generate the variable-sized payload of its corresponding `__llvm_covfun` /// entry. The payload is returned as a vector of bytes. @@ -247,13 +246,11 @@ fn encode_mappings_for_function( global_file_table: &GlobalFileTable, function_coverage: &FunctionCoverage<'_>, ) -> Vec<u8> { - let mapping_spans = function_coverage.mapping_spans(); - if mapping_spans.is_empty() { + let counter_regions = function_coverage.counter_regions(); + if counter_regions.is_empty() { return Vec::new(); } - let fn_cov_info = function_coverage.function_coverage_info; - let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); let mut virtual_file_mapping = VirtualFileMapping::default(); @@ -263,39 +260,34 @@ fn encode_mappings_for_function( let mut mcdc_decision_regions = vec![]; // Currently a function's mappings must all be in the same file as its body span. - let source_map = tcx.sess.source_map(); - let source_file = source_map.lookup_source_file(fn_cov_info.body_span.lo()); + let file_name = span_file_name(tcx, function_coverage.function_coverage_info.body_span); - // Look up the global file ID for that file. - let global_file_id = global_file_table.global_file_id_for_file(&source_file); + // Look up the global file ID for that filename. + let global_file_id = global_file_table.global_file_id_for_file_name(file_name); // Associate that global file ID with a local file ID for this function. let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); + debug!(" file id: {local_file_id:?} => {global_file_id:?} = '{file_name:?}'"); - let make_cov_span = |span| { - spans::make_coverage_span(local_file_id, source_map, fn_cov_info, &source_file, span) - }; - - // For each coverage mapping span in this function+file, convert it to a + // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - for (mapping_kind, span) in mapping_spans { - debug!("Adding counter {mapping_kind:?} to map for {span:?}"); - let Some(cov_span) = make_cov_span(span) else { continue }; + for (mapping_kind, region) in counter_regions { + debug!("Adding counter {mapping_kind:?} to map for {region:?}"); + let span = ffi::CoverageSpan::from_source_region(local_file_id, region); match mapping_kind { MappingKind::Code(term) => { - code_regions - .push(ffi::CodeRegion { cov_span, counter: ffi::Counter::from_term(term) }); + code_regions.push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); } MappingKind::Branch { true_term, false_term } => { branch_regions.push(ffi::BranchRegion { - cov_span, + span, true_counter: ffi::Counter::from_term(true_term), false_counter: ffi::Counter::from_term(false_term), }); } MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { mcdc_branch_regions.push(ffi::MCDCBranchRegion { - cov_span, + span, true_counter: ffi::Counter::from_term(true_term), false_counter: ffi::Counter::from_term(false_term), mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), @@ -303,7 +295,7 @@ fn encode_mappings_for_function( } MappingKind::MCDCDecision(mcdc_decision_params) => { mcdc_decision_regions.push(ffi::MCDCDecisionRegion { - cov_span, + span, mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), }); } @@ -538,9 +530,7 @@ fn add_unused_function_coverage<'tcx>( }), ); - // An unused function's mappings will automatically be rewritten to map to - // zero, because none of its counters/expressions are marked as seen. - let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); - + // An unused function's mappings will all be rewritten to map to zero. + let function_coverage = FunctionCoverage::new_unused(function_coverage_info); cx.coverage_cx().function_coverage_map.borrow_mut().insert(instance, function_coverage); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs deleted file mode 100644 index 4a7721879fd..00000000000 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs +++ /dev/null @@ -1,124 +0,0 @@ -use rustc_middle::mir::coverage::FunctionCoverageInfo; -use rustc_span::source_map::SourceMap; -use rustc_span::{BytePos, Pos, SourceFile, Span}; -use tracing::debug; - -use crate::coverageinfo::ffi; -use crate::coverageinfo::mapgen::LocalFileId; - -/// Converts the span into its start line and column, and end line and column. -/// -/// Line numbers and column numbers are 1-based. Unlike most column numbers emitted by -/// the compiler, these column numbers are denoted in **bytes**, because that's what -/// LLVM's `llvm-cov` tool expects to see in coverage maps. -/// -/// Returns `None` if the conversion failed for some reason. This shouldn't happen, -/// but it's hard to rule out entirely (especially in the presence of complex macros -/// or other expansions), and if it does happen then skipping a span or function is -/// better than an ICE or `llvm-cov` failure that the user might have no way to avoid. -pub(crate) fn make_coverage_span( - file_id: LocalFileId, - source_map: &SourceMap, - fn_cov_info: &FunctionCoverageInfo, - file: &SourceFile, - span: Span, -) -> Option<ffi::CoverageSpan> { - let span = ensure_non_empty_span(source_map, fn_cov_info, span)?; - - let lo = span.lo(); - let hi = span.hi(); - - // Column numbers need to be in bytes, so we can't use the more convenient - // `SourceMap` methods for looking up file coordinates. - let line_and_byte_column = |pos: BytePos| -> Option<(usize, usize)> { - let rpos = file.relative_position(pos); - let line_index = file.lookup_line(rpos)?; - let line_start = file.lines()[line_index]; - // Line numbers and column numbers are 1-based, so add 1 to each. - Some((line_index + 1, (rpos - line_start).to_usize() + 1)) - }; - - let (mut start_line, start_col) = line_and_byte_column(lo)?; - let (mut end_line, end_col) = line_and_byte_column(hi)?; - - // Apply an offset so that code in doctests has correct line numbers. - // FIXME(#79417): Currently we have no way to offset doctest _columns_. - start_line = source_map.doctest_offset_line(&file.name, start_line); - end_line = source_map.doctest_offset_line(&file.name, end_line); - - check_coverage_span(ffi::CoverageSpan { - file_id: file_id.as_u32(), - start_line: start_line as u32, - start_col: start_col as u32, - end_line: end_line as u32, - end_col: end_col as u32, - }) -} - -fn ensure_non_empty_span( - source_map: &SourceMap, - fn_cov_info: &FunctionCoverageInfo, - span: Span, -) -> Option<Span> { - if !span.is_empty() { - return Some(span); - } - - let lo = span.lo(); - let hi = span.hi(); - - // The span is empty, so try to expand it to cover an adjacent '{' or '}', - // but only within the bounds of the body span. - let try_next = hi < fn_cov_info.body_span.hi(); - let try_prev = fn_cov_info.body_span.lo() < lo; - if !(try_next || try_prev) { - return None; - } - - source_map - .span_to_source(span, |src, start, end| try { - // We're only checking for specific ASCII characters, so we don't - // have to worry about multi-byte code points. - if try_next && src.as_bytes()[end] == b'{' { - Some(span.with_hi(hi + BytePos(1))) - } else if try_prev && src.as_bytes()[start - 1] == b'}' { - Some(span.with_lo(lo - BytePos(1))) - } else { - None - } - }) - .ok()? -} - -/// If `llvm-cov` sees a source region that is improperly ordered (end < start), -/// it will immediately exit with a fatal error. To prevent that from happening, -/// discard regions that are improperly ordered, or might be interpreted in a -/// way that makes them improperly ordered. -fn check_coverage_span(cov_span: ffi::CoverageSpan) -> Option<ffi::CoverageSpan> { - let ffi::CoverageSpan { file_id: _, start_line, start_col, end_line, end_col } = cov_span; - - // Line/column coordinates are supposed to be 1-based. If we ever emit - // coordinates of 0, `llvm-cov` might misinterpret them. - let all_nonzero = [start_line, start_col, end_line, end_col].into_iter().all(|x| x != 0); - // Coverage mappings use the high bit of `end_col` to indicate that a - // region is actually a "gap" region, so make sure it's unset. - let end_col_has_high_bit_unset = (end_col & (1 << 31)) == 0; - // If a region is improperly ordered (end < start), `llvm-cov` will exit - // with a fatal error, which is inconvenient for users and hard to debug. - let is_ordered = (start_line, start_col) <= (end_line, end_col); - - if all_nonzero && end_col_has_high_bit_unset && is_ordered { - Some(cov_span) - } else { - debug!( - ?cov_span, - ?all_nonzero, - ?end_col_has_high_bit_unset, - ?is_ordered, - "Skipping source region that would be misinterpreted or rejected by LLVM" - ); - // If this happens in a debug build, ICE to make it easier to notice. - debug_assert!(false, "Improper source region: {cov_span:?}"); - None - } -} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index bf773cd2667..82b6677e203 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -13,7 +13,7 @@ use tracing::{debug, instrument}; use crate::builder::Builder; use crate::common::CodegenCx; -use crate::coverageinfo::map_data::FunctionCoverageCollector; +use crate::coverageinfo::map_data::FunctionCoverage; use crate::llvm; pub(crate) mod ffi; @@ -21,18 +21,17 @@ mod llvm_cov; pub(crate) mod map_data; mod mapgen; -/// A context object for maintaining all state needed by the coverageinfo module. -pub(crate) struct CrateCoverageContext<'ll, 'tcx> { +/// Extra per-CGU context/state needed for coverage instrumentation. +pub(crate) struct CguCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. - pub(crate) function_coverage_map: - RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, + pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, covfun_section_name: OnceCell<CString>, } -impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { +impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> { pub(crate) fn new() -> Self { Self { function_coverage_map: Default::default(), @@ -42,9 +41,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } - fn take_function_coverage_map( - &self, - ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { + fn take_function_coverage_map(&self) -> FxIndexMap<Instance<'tcx>, FunctionCoverage<'tcx>> { self.function_coverage_map.replace(FxIndexMap::default()) } @@ -143,6 +140,13 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let bx = self; + // Due to LocalCopy instantiation or MIR inlining, coverage statements + // can end up in a crate that isn't doing coverage instrumentation. + // When that happens, we currently just discard those statements, so + // the corresponding code will be undercounted. + // FIXME(Zalathar): Find a better solution for mixed-coverage builds. + let Some(coverage_cx) = &bx.cx.coverage_cx else { return }; + let Some(function_coverage_info) = bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() else { @@ -150,32 +154,27 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - // FIXME(#132395): Unwrapping `coverage_cx` here has led to ICEs in the - // wild, so keep this early-return until we understand why. - let mut coverage_map = match bx.coverage_cx { - Some(ref cx) => cx.function_coverage_map.borrow_mut(), - None => return, - }; - let func_coverage = coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); + // Mark the instance as used in this CGU, for coverage purposes. + // This includes functions that were not partitioned into this CGU, + // but were MIR-inlined into one of this CGU's functions. + coverage_cx.function_coverage_map.borrow_mut().entry(instance).or_insert_with(|| { + FunctionCoverage::new_used( + function_coverage_info, + bx.tcx.coverage_ids_info(instance.def), + ) + }); match *kind { CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( "marker statement {kind:?} should have been removed by CleanupPostBorrowck" ), CoverageKind::CounterIncrement { id } => { - func_coverage.mark_counter_id_seen(id); - // We need to explicitly drop the `RefMut` before calling into - // `instrprof_increment`, as that needs an exclusive borrow. - drop(coverage_map); - // The number of counters passed to `llvm.instrprof.increment` might // be smaller than the number originally inserted by the instrumentor, // if some high-numbered counters were removed by MIR optimizations. // If so, LLVM's profiler runtime will use fewer physical counters. let num_counters = - bx.tcx().coverage_ids_info(instance.def).max_counter_id.as_u32() + 1; + bx.tcx().coverage_ids_info(instance.def).num_counters_after_mir_opts(); assert!( num_counters as usize <= function_coverage_info.num_counters, "num_counters disagreement: query says {num_counters} but function info only has {}", @@ -192,23 +191,23 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { ); bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::ExpressionUsed { id } => { - func_coverage.mark_expression_id_seen(id); + CoverageKind::ExpressionUsed { id: _ } => { + // Expression-used statements are markers that are handled by + // `coverage_ids_info`, so there's nothing to codegen here. } CoverageKind::CondBitmapUpdate { index, decision_depth } => { - drop(coverage_map); - let cond_bitmap = bx - .coverage_cx() + let cond_bitmap = coverage_cx .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); let cond_index = bx.const_i32(index as i32); bx.mcdc_condbitmap_update(cond_index, cond_bitmap); } CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { - drop(coverage_map); - let cond_bitmap = bx.coverage_cx() - .try_get_mcdc_condition_bitmap(&instance, decision_depth) - .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); + let cond_bitmap = + coverage_cx.try_get_mcdc_condition_bitmap(&instance, decision_depth).expect( + "mcdc cond bitmap should have been allocated for merging \ + into the global bitmap", + ); assert!( bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits, "bitmap index of the decision out of range" diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index ef16e5bb459..59275254022 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -203,6 +203,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &ptr_type_debuginfo_name, + None, cx.size_and_align_of(ptr_type), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -259,6 +260,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( layout.fields.offset(abi::WIDE_PTR_ADDR), DIFlags::FlagZero, data_ptr_type_di_node, + None, ), build_field_di_node( cx, @@ -268,6 +270,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( layout.fields.offset(abi::WIDE_PTR_EXTRA), DIFlags::FlagZero, type_di_node(cx, extra_field.ty), + None, ), ] }, @@ -369,6 +372,7 @@ fn build_dyn_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &type_name, + None, cx.size_and_align_of(dyn_type), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -467,8 +471,6 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), - // Type parameters from polymorphized functions. - ty::Param(_) => build_param_type_di_node(cx, t), _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), }; @@ -722,6 +724,14 @@ fn build_cpp_f16_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> DINodeCreation // `f16`'s value to be displayed using a Natvis visualiser in `intrinsic.natvis`. let float_ty = cx.tcx.types.f16; let bits_ty = cx.tcx.types.u16; + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + match float_ty.kind() { + ty::Adt(def, _) => Some(file_metadata_from_def_id(cx, Some(def.did()))), + _ => None, + } + } else { + None + }; type_map::build_type_with_children( cx, type_map::stub( @@ -729,12 +739,21 @@ fn build_cpp_f16_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> DINodeCreation Stub::Struct, UniqueTypeId::for_ty(cx.tcx, float_ty), "f16", + def_location, cx.size_and_align_of(float_ty), NO_SCOPE_METADATA, DIFlags::FlagZero, ), // Fields: |cx, float_di_node| { + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + match bits_ty.kind() { + ty::Adt(def, _) => Some(def.did()), + _ => None, + } + } else { + None + }; smallvec![build_field_di_node( cx, float_di_node, @@ -743,6 +762,7 @@ fn build_cpp_f16_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> DINodeCreation Size::ZERO, DIFlags::FlagZero, type_di_node(cx, bits_ty), + def_id, )] }, NO_GENERICS, @@ -839,6 +859,7 @@ fn build_foreign_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &compute_debuginfo_type_name(cx.tcx, t, false), + None, cx.size_and_align_of(t), Some(get_namespace_for_item(cx, def_id)), DIFlags::FlagZero, @@ -848,26 +869,6 @@ fn build_foreign_type_di_node<'ll, 'tcx>( ) } -fn build_param_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - t: Ty<'tcx>, -) -> DINodeCreationResult<'ll> { - debug!("build_param_type_di_node: {:?}", t); - let name = format!("{t:?}"); - DINodeCreationResult { - di_node: unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_c_char_ptr(), - name.len(), - Size::ZERO.bits(), - DW_ATE_unsigned, - ) - }, - already_stored_in_typemap: false, - } -} - pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( tcx: TyCtxt<'tcx>, codegen_unit_name: &str, @@ -989,15 +990,22 @@ fn build_field_di_node<'ll, 'tcx>( offset: Size, flags: DIFlags, type_di_node: &'ll DIType, + def_id: Option<DefId>, ) -> &'ll DIType { + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; unsafe { llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), owner, name.as_c_char_ptr(), name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size_and_align.0.bits(), size_and_align.1.bits() as u32, offset.bits(), @@ -1041,6 +1049,11 @@ fn build_struct_type_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, adt_def.did()); let struct_type_and_layout = cx.layout_of(struct_type); let variant_def = adt_def.non_enum_variant(); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) + } else { + None + }; type_map::build_type_with_children( cx, @@ -1049,6 +1062,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &compute_debuginfo_type_name(cx.tcx, struct_type, false), + def_location, size_and_align_of(struct_type_and_layout), Some(containing_scope), visibility_di_flags(cx, adt_def.did(), adt_def.did()), @@ -1068,6 +1082,11 @@ fn build_struct_type_di_node<'ll, 'tcx>( Cow::Borrowed(f.name.as_str()) }; let field_layout = struct_type_and_layout.field(cx, i); + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(f.did) + } else { + None + }; build_field_di_node( cx, owner, @@ -1076,6 +1095,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( struct_type_and_layout.fields.offset(i), visibility_di_flags(cx, f.did, adt_def.did()), type_di_node(cx, field_layout.ty), + def_id, ) }) .collect() @@ -1125,6 +1145,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, up_var_ty), + None, ) }) .collect() @@ -1150,6 +1171,7 @@ fn build_tuple_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &type_name, + None, size_and_align_of(tuple_type_and_layout), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -1168,6 +1190,7 @@ fn build_tuple_type_di_node<'ll, 'tcx>( tuple_type_and_layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, component_type), + None, ) }) .collect() @@ -1189,6 +1212,12 @@ fn build_closure_env_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, def_id); let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(def_id))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -1196,6 +1225,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &type_name, + def_location, cx.size_and_align_of(closure_env_type), Some(containing_scope), DIFlags::FlagZero, @@ -1219,6 +1249,11 @@ fn build_union_type_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, union_def_id); let union_ty_and_layout = cx.layout_of(union_type); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(union_def_id))) + } else { + None + }; type_map::build_type_with_children( cx, @@ -1227,6 +1262,7 @@ fn build_union_type_di_node<'ll, 'tcx>( Stub::Union, unique_type_id, &type_name, + def_location, size_and_align_of(union_ty_and_layout), Some(containing_scope), DIFlags::FlagZero, @@ -1239,6 +1275,11 @@ fn build_union_type_di_node<'ll, 'tcx>( .enumerate() .map(|(i, f)| { let field_layout = union_ty_and_layout.field(cx, i); + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(f.did) + } else { + None + }; build_field_di_node( cx, owner, @@ -1247,6 +1288,7 @@ fn build_union_type_di_node<'ll, 'tcx>( Size::ZERO, DIFlags::FlagZero, type_di_node(cx, field_layout.ty), + def_id, ) }) .collect() @@ -1321,14 +1363,7 @@ pub(crate) fn build_global_var_di_node<'ll>( // We may want to remove the namespace scope if we're in an extern block (see // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952). let var_scope = get_namespace_for_item(cx, def_id); - let span = hygiene::walk_chain_collapsed(tcx.def_span(def_id), DUMMY_SP); - - let (file_metadata, line_number) = if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - (file_metadata(cx, &loc.file), loc.line) - } else { - (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) - }; + let (file_metadata, line_number) = file_metadata_from_def_id(cx, Some(def_id)); let is_local_to_unit = is_node_local_to_unit(cx, def_id); @@ -1418,6 +1453,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( Stub::VTableTy { vtable_holder }, unique_type_id, &vtable_type_name, + None, (size, pointer_align), NO_SCOPE_METADATA, DIFlags::FlagArtificial, @@ -1455,6 +1491,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( field_offset, DIFlags::FlagZero, field_type_di_node, + None, )) }) .collect() @@ -1606,3 +1643,20 @@ fn tuple_field_name(field_index: usize) -> Cow<'static, str> { .map(|s| Cow::from(*s)) .unwrap_or_else(|| Cow::from(format!("__{field_index}"))) } + +pub(crate) type DefinitionLocation<'ll> = (&'ll DIFile, c_uint); + +pub(crate) fn file_metadata_from_def_id<'ll>( + cx: &CodegenCx<'ll, '_>, + def_id: Option<DefId>, +) -> DefinitionLocation<'ll> { + if let Some(def_id) = def_id + && let span = hygiene::walk_chain_collapsed(cx.tcx.def_span(def_id), DUMMY_SP) + && !span.is_dummy() + { + let loc = cx.lookup_debug_loc(span.lo()); + (file_metadata(cx, &loc.file), loc.line) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 100b046cee2..d374767f187 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -4,7 +4,7 @@ use libc::c_uint; use rustc_abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; -use rustc_codegen_ssa::traits::ConstCodegenMethods; +use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods}; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -16,8 +16,8 @@ use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, NO_SCOPE_METADATA, SmallVec, UNKNOWN_LINE_NUMBER, - build_field_di_node, file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, - visibility_di_flags, + build_field_di_node, file_metadata, file_metadata_from_def_id, size_and_align_of, type_di_node, + unknown_file_metadata, visibility_di_flags, }; use crate::debuginfo::utils::DIB; use crate::llvm::debuginfo::{DIFile, DIFlags, DIType}; @@ -192,6 +192,12 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(enum_adt_def.did()))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -199,6 +205,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( type_map::Stub::Union, unique_type_id, &enum_type_name, + def_location, cx.size_and_align_of(enum_type), NO_SCOPE_METADATA, visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()), @@ -262,6 +269,14 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let coroutine_type = unique_type_id.expect_ty(); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + let &ty::Coroutine(coroutine_def_id, _) = coroutine_type.kind() else { + bug!("build_coroutine_di_node() called with non-coroutine type: `{:?}`", coroutine_type) + }; + Some(file_metadata_from_def_id(cx, Some(coroutine_def_id))) + } else { + None + }; let coroutine_type_and_layout = cx.layout_of(coroutine_type); let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); @@ -274,6 +289,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( type_map::Stub::Union, unique_type_id, &coroutine_type_name, + def_location, size_and_align_of(coroutine_type_and_layout), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -321,6 +337,12 @@ fn build_single_variant_union_fields<'ll, 'tcx>( let tag_base_type_di_node = type_di_node(cx, tag_base_type); let tag_base_type_align = cx.align_of(tag_base_type); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; + let variant_names_type_di_node = build_variant_names_type_di_node( cx, enum_type_di_node, @@ -328,6 +350,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_index, Cow::from(enum_adt_def.variant(variant_index).name.as_str()), )), + enum_adt_def_id, ); let variant_struct_type_wrapper_di_node = build_variant_struct_wrapper_type_di_node( @@ -341,6 +364,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( tag_base_type_di_node, tag_base_type, DiscrResult::NoDiscriminant, + None, ); smallvec![ @@ -354,6 +378,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( Size::ZERO, visibility_flags, variant_struct_type_wrapper_di_node, + None, ), unsafe { llvm::LLVMRustDIBuilderCreateStaticMemberType( @@ -383,6 +408,12 @@ fn build_union_fields_for_enum<'ll, 'tcx>( ) -> SmallVec<&'ll DIType> { let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; + let variant_names_type_di_node = build_variant_names_type_di_node( cx, enum_type_di_node, @@ -390,6 +421,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); (variant_index, variant_name) }), + enum_adt_def_id, ); let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); @@ -447,6 +479,7 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, containing_scope: &'ll DIType, variants: impl Iterator<Item = (VariantIdx, Cow<'tcx, str>)>, + enum_def_id: Option<rustc_span::def_id::DefId>, ) -> &'ll DIType { // Create an enumerator for each variant. super::build_enumeration_type_di_node( @@ -454,6 +487,7 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( "VariantNames", variant_names_enum_base_type(cx), variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32().into())), + enum_def_id, containing_scope, ) } @@ -469,6 +503,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( tag_base_type_di_node: &'ll DIType, tag_base_type: Ty<'tcx>, discr: DiscrResult, + source_info: Option<(&'ll DIFile, c_uint)>, ) -> &'ll DIType { type_map::build_type_with_children( cx, @@ -481,6 +516,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( variant_index, ), &variant_struct_wrapper_type_name(variant_index), + source_info, // NOTE: We use size and align of enum_type, not from variant_layout: size_and_align_of(enum_or_coroutine_type_and_layout), Some(enum_or_coroutine_type_di_node), @@ -530,6 +566,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( Size::ZERO, DIFlags::FlagZero, variant_struct_type_di_node, + None, )); let build_assoc_const = @@ -684,6 +721,11 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( variant_range .clone() .map(|variant_index| (variant_index, CoroutineArgs::variant_name(variant_index))), + if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(coroutine_def_id) + } else { + None + }, ); let discriminants: IndexVec<VariantIdx, DiscrResult> = { @@ -776,6 +818,11 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( tag_base_type_di_node, tag_base_type, variant_member_info.discr, + if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + variant_member_info.source_info + } else { + None + }, ); // We use LLVMRustDIBuilderCreateMemberType() member type directly because @@ -831,6 +878,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( lo_offset, di_flags, type_di_node, + None, )); unions_fields.push(build_field_di_node( @@ -841,6 +889,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( hi_offset, DIFlags::FlagZero, type_di_node, + None, )); } else { unions_fields.push(build_field_di_node( @@ -851,6 +900,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( enum_type_and_layout.fields.offset(tag_field), di_flags, tag_base_type_di_node, + None, )); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index b3d4a6642a1..65ab22ad89e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -3,6 +3,7 @@ use std::borrow::Cow; use rustc_abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; +use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::bug; @@ -16,8 +17,8 @@ use super::{SmallVec, size_and_align_of}; use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub}; use crate::debuginfo::metadata::{ - UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, type_di_node, - unknown_file_metadata, + UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, + file_metadata_from_def_id, type_di_node, unknown_file_metadata, }; use crate::debuginfo::utils::{DIB, create_DIArray, get_namespace_for_item}; use crate::llvm::debuginfo::{DIFlags, DIType}; @@ -68,6 +69,11 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( enum_type_and_layout: TyAndLayout<'tcx>, ) -> DINodeCreationResult<'ll> { let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; DINodeCreationResult { di_node: build_enumeration_type_di_node( cx, @@ -77,6 +83,7 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); (name, discr.val) }), + enum_adt_def_id, containing_scope, ), already_stored_in_typemap: false, @@ -92,6 +99,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( type_name: &str, base_type: Ty<'tcx>, enumerators: impl Iterator<Item = (Cow<'tcx, str>, u128)>, + def_id: Option<rustc_span::def_id::DefId>, containing_scope: &'ll DIType, ) -> &'ll DIType { let is_unsigned = match base_type.kind() { @@ -115,14 +123,21 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( }) .collect(); + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + unsafe { llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), containing_scope, type_name.as_c_char_ptr(), type_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size.bits(), align.bits() as u32, create_DIArray(DIB(cx), &enumerator_di_nodes[..]), @@ -193,6 +208,12 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( ) -> &'ll DIType { assert_eq!(variant_layout.ty, enum_type_and_layout.ty); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(variant_def.def_id))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -204,6 +225,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( variant_index, ), variant_def.name.as_str(), + def_location, // NOTE: We use size and align of enum_type, not from variant_layout: size_and_align_of(enum_type_and_layout), Some(enum_type_di_node), @@ -231,6 +253,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( variant_layout.fields.offset(field_index), di_flags, type_di_node(cx, field_layout.ty), + None, ) }) .collect::<SmallVec<_>>() @@ -286,6 +309,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &variant_name, + None, size_and_align_of(coroutine_type_and_layout), Some(coroutine_type_di_node), DIFlags::FlagZero, @@ -312,6 +336,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( variant_layout.fields.offset(field_index), DIFlags::FlagZero, type_di_node(cx, field_type), + None, ) }) .collect(); @@ -331,6 +356,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, upvar_ty), + None, ) }) .collect(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index d4006691d37..241bf167a81 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -4,7 +4,7 @@ use libc::c_uint; use rustc_abi::{Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; -use rustc_codegen_ssa::traits::ConstCodegenMethods; +use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self}; @@ -14,7 +14,8 @@ use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, SmallVec, UNKNOWN_LINE_NUMBER, file_metadata, - size_and_align_of, type_di_node, unknown_file_metadata, visibility_di_flags, + file_metadata_from_def_id, size_and_align_of, type_di_node, unknown_file_metadata, + visibility_di_flags, }; use crate::debuginfo::utils::{DIB, create_DIArray, get_namespace_for_item}; use crate::llvm::debuginfo::{DIFile, DIFlags, DIType}; @@ -55,6 +56,12 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(enum_adt_def.did()))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -62,6 +69,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &enum_type_name, + def_location, size_and_align_of(enum_type_and_layout), Some(containing_scope), visibility_flags, @@ -84,14 +92,27 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( enum_type_and_layout.for_variant(cx, variant_index), visibility_flags, ), - source_info: None, + source_info: if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id( + cx, + Some(enum_adt_def.variant(variant_index).def_id), + )) + } else { + None + }, }) .collect(); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; smallvec![build_enum_variant_part_di_node( cx, enum_type_and_layout, enum_type_di_node, + enum_adt_def_id, &variant_member_infos[..], )] }, @@ -134,6 +155,12 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(coroutine_def_id))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -141,6 +168,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &coroutine_type_name, + def_location, size_and_align_of(coroutine_type_and_layout), Some(containing_scope), DIFlags::FlagZero, @@ -197,10 +225,16 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( }) .collect(); + let coroutine_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(coroutine_def_id) + } else { + None + }; smallvec![build_enum_variant_part_di_node( cx, coroutine_type_and_layout, coroutine_type_di_node, + coroutine_def_id, &variant_struct_type_di_nodes[..], )] }, @@ -228,6 +262,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, + enum_type_def_id: Option<rustc_span::def_id::DefId>, variant_member_infos: &[VariantMemberInfo<'_, 'll>], ) -> &'ll DIType { let tag_member_di_node = @@ -236,6 +271,13 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( let variant_part_unique_type_id = UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, enum_type_def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + let stub = StubInfo::new( cx, variant_part_unique_type_id, @@ -246,8 +288,8 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( enum_type_di_node, variant_part_name.as_c_char_ptr(), variant_part_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 4e461476040..a37e719d43f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -8,7 +8,7 @@ use rustc_macros::HashStable; use rustc_middle::bug; use rustc_middle::ty::{self, PolyExistentialTraitRef, Ty, TyCtxt}; -use super::{SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; +use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; @@ -186,6 +186,7 @@ pub(super) fn stub<'ll, 'tcx>( kind: Stub<'ll>, unique_type_id: UniqueTypeId<'tcx>, name: &str, + def_location: Option<DefinitionLocation<'ll>>, (size, align): (Size, Align), containing_scope: Option<&'ll DIScope>, flags: DIFlags, @@ -193,6 +194,12 @@ pub(super) fn stub<'ll, 'tcx>( let empty_array = create_DIArray(DIB(cx), &[]); let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + let (file_metadata, line_number) = if let Some(def_location) = def_location { + (def_location.0, def_location.1) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + let metadata = match kind { Stub::Struct | Stub::VTableTy { .. } => { let vtable_holder = match kind { @@ -205,8 +212,8 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope, name.as_c_char_ptr(), name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size.bits(), align.bits() as u32, flags, @@ -225,8 +232,8 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope, name.as_c_char_ptr(), name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size.bits(), align.bits() as u32, flags, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index da7f94e8cf7..c38c5d4c644 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -352,84 +352,84 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::saturating_add | sym::saturating_sub => { let ty = arg_tys[0]; - match int_type_width_signed(ty, self) { - Some((width, signed)) => match name { - sym::ctlz | sym::cttz => { - let y = self.const_bool(false); - let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[ - args[0].immediate(), - y, - ]); - - self.intcast(ret, llret_ty, false) - } - sym::ctlz_nonzero => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.ctlz.i{width}"); - let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); - self.intcast(ret, llret_ty, false) - } - sym::cttz_nonzero => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.cttz.i{width}"); - let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); - self.intcast(ret, llret_ty, false) - } - sym::ctpop => { - let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[args - [0] - .immediate()]); - self.intcast(ret, llret_ty, false) - } - sym::bswap => { - if width == 8 { - args[0].immediate() // byte swap a u8/i8 is just a no-op - } else { - self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[ - args[0].immediate() - ]) - } - } - sym::bitreverse => self - .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[ + if !ty.is_integral() { + tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { + span, + name, + ty, + }); + return Ok(()); + } + let (size, signed) = ty.int_size_and_signed(self.tcx); + let width = size.bits(); + match name { + sym::ctlz | sym::cttz => { + let y = self.const_bool(false); + let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[ + args[0].immediate(), + y, + ]); + + self.intcast(ret, llret_ty, false) + } + sym::ctlz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.ctlz.i{width}"); + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) + } + sym::cttz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.cttz.i{width}"); + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) + } + sym::ctpop => { + let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[ + args[0].immediate() + ]); + self.intcast(ret, llret_ty, false) + } + sym::bswap => { + if width == 8 { + args[0].immediate() // byte swap a u8/i8 is just a no-op + } else { + self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[ args[0].immediate() - ]), - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same - let llvm_name = - &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); - - // llvm expects shift to be the same type as the values, but rust - // always uses `u32`. - let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); - - self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + ]) } - sym::saturating_add | sym::saturating_sub => { - let is_add = name == sym::saturating_add; - let lhs = args[0].immediate(); - let rhs = args[1].immediate(); - let llvm_name = &format!( - "llvm.{}{}.sat.i{}", - if signed { 's' } else { 'u' }, - if is_add { "add" } else { "sub" }, - width - ); - self.call_intrinsic(llvm_name, &[lhs, rhs]) - } - _ => bug!(), - }, - None => { - tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { - span, - name, - ty, - }); - return Ok(()); } + sym::bitreverse => self + .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[ + args[0].immediate() + ]), + sym::rotate_left | sym::rotate_right => { + let is_left = name == sym::rotate_left; + let val = args[0].immediate(); + let raw_shift = args[1].immediate(); + // rotate = funnel shift with first two args the same + let llvm_name = + &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + + // llvm expects shift to be the same type as the values, but rust + // always uses `u32`. + let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + } + sym::saturating_add | sym::saturating_sub => { + let is_add = name == sym::saturating_add; + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let llvm_name = &format!( + "llvm.{}{}.sat.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + self.call_intrinsic(llvm_name, &[lhs, rhs]) + } + _ => bug!(), } } @@ -1534,6 +1534,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), + sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), @@ -1572,6 +1573,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( | sym::simd_fpowi | sym::simd_fsin | sym::simd_fsqrt + | sym::simd_relaxed_fma | sym::simd_round | sym::simd_trunc ) { @@ -2529,19 +2531,3 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span_bug!(span, "unknown SIMD intrinsic"); } - -// Returns the width of an int Ty, and if it's signed or not -// Returns None if the type is not an integer -// FIXME: there’s multiple of this functions, investigate using some of the already existing -// stuffs. -fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { - match ty.kind() { - ty::Int(t) => { - Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), true)) - } - ty::Uint(t) => { - Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), false)) - } - _ => None, - } -} diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 9f398107fc6..5235891a18d 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -17,7 +17,6 @@ #![feature(iter_intersperse)] #![feature(let_chains)] #![feature(rustdoc_internals)] -#![feature(try_blocks)] #![warn(unreachable_pub)] // tidy-alphabetical-end @@ -37,7 +36,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed, FatalError}; +use rustc_errors::{DiagCtxtHandle, FatalError}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -371,19 +370,14 @@ impl CodegenBackend for LlvmCodegenBackend { (codegen_results, work_products) } - fn link( - &self, - sess: &Session, - codegen_results: CodegenResults, - outputs: &OutputFilenames, - ) -> Result<(), ErrorGuaranteed> { + fn link(&self, sess: &Session, codegen_results: CodegenResults, outputs: &OutputFilenames) { use rustc_codegen_ssa::back::link::link_binary; use crate::back::archive::LlvmArchiveBuilderBuilder; // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 17b0ec4b936..b1ace0033ba 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2240,6 +2240,7 @@ unsafe extern "C" { Output: *const c_char, DwoOutput: *const c_char, FileType: FileType, + VerifyIR: bool, ) -> LLVMRustResult; pub fn LLVMRustOptimize<'a>( M: &'a Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index db2b03d9aed..07eb89e6041 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -230,6 +230,8 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea "aarch64" } else if sess.target.arch == "sparc64" { "sparc" + } else if sess.target.arch == "powerpc64" { + "powerpc" } else { &*sess.target.arch }; @@ -289,6 +291,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), ("sparc", "v8plus") if get_version().0 < 19 => None, + ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), (_, s) => Some(LLVMFeature::new(s)), } } diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index e4c3e748cb5..8baa69cefe1 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -10,6 +10,15 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; +fn round_up_to_alignment<'ll>( + bx: &mut Builder<'_, 'll, '_>, + mut value: &'ll Value, + align: Align, +) -> &'ll Value { + value = bx.add(value, bx.cx().const_i32(align.bytes() as i32 - 1)); + return bx.and(value, bx.cx().const_i32(-(align.bytes() as i32))); +} + fn round_pointer_up_to_alignment<'ll>( bx: &mut Builder<'_, 'll, '_>, addr: &'ll Value, @@ -17,8 +26,7 @@ fn round_pointer_up_to_alignment<'ll>( ptr_ty: &'ll Type, ) -> &'ll Value { let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); - ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1)); - ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32))); + ptr_as_int = round_up_to_alignment(bx, ptr_as_int, align); bx.inttoptr(ptr_as_int, ptr_ty) } @@ -270,6 +278,106 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.load(val_type, val_addr, layout.align.abi) } +fn emit_xtensa_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of va_arg for Xtensa. There doesn't seem to be an authoritative source for + // this, other than "what GCC does". + // + // The va_list type has three fields: + // struct __va_list_tag { + // int32_t *va_stk; // Arguments passed on the stack + // int32_t *va_reg; // Arguments passed in registers, saved to memory by the prologue. + // int32_t va_ndx; // Offset into the arguments, in bytes + // }; + // + // The first 24 bytes (equivalent to 6 registers) come from va_reg, the rest from va_stk. + // Thus if va_ndx is less than 24, the next va_arg *may* read from va_reg, + // otherwise it must come from va_stk. + // + // Primitive arguments are never split between registers and the stack. For example, if loading an 8 byte + // primitive value and va_ndx = 20, we instead bump the offset and read everything from va_stk. + let va_list_addr = list.immediate(); + // FIXME: handle multi-field structs that split across regsave/stack? + let layout = bx.cx.layout_of(target_ty); + let from_stack = bx.append_sibling_block("va_arg.from_stack"); + let from_regsave = bx.append_sibling_block("va_arg.from_regsave"); + let end = bx.append_sibling_block("va_arg.end"); + + // (*va).va_ndx + let va_reg_offset = 4; + let va_ndx_offset = va_reg_offset + 4; + let offset_ptr = + bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_ndx_offset)]); + + let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi); + let offset = round_up_to_alignment(bx, offset, layout.align.abi); + + let slot_size = layout.size.align_to(Align::from_bytes(4).unwrap()).bytes() as i32; + + // Update the offset in va_list, by adding the slot's size. + let offset_next = bx.add(offset, bx.const_i32(slot_size)); + + // Figure out where to look for our value. We do that by checking the end of our slot (offset_next). + // If that is within the regsave area, then load from there. Otherwise load from the stack area. + let regsave_size = bx.const_i32(24); + let use_regsave = bx.icmp(IntPredicate::IntULE, offset_next, regsave_size); + bx.cond_br(use_regsave, from_regsave, from_stack); + + bx.switch_to_block(from_regsave); + // update va_ndx + bx.store(offset_next, offset_ptr, bx.tcx().data_layout.pointer_align.abi); + + // (*va).va_reg + let regsave_area_ptr = + bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_reg_offset)]); + let regsave_area = + bx.load(bx.type_ptr(), regsave_area_ptr, bx.tcx().data_layout.pointer_align.abi); + let regsave_value_ptr = bx.inbounds_gep(bx.type_i8(), regsave_area, &[offset]); + bx.br(end); + + bx.switch_to_block(from_stack); + + // The first time we switch from regsave to stack we needs to adjust our offsets a bit. + // va_stk is set up such that the first stack argument is always at va_stk + 32. + // The corrected offset is written back into the va_list struct. + + // let offset_corrected = cmp::max(offset, 32); + let stack_offset_start = bx.const_i32(32); + let needs_correction = bx.icmp(IntPredicate::IntULE, offset, stack_offset_start); + let offset_corrected = bx.select(needs_correction, stack_offset_start, offset); + + // let offset_next_corrected = offset_corrected + slot_size; + // va_ndx = offset_next_corrected; + let offset_next_corrected = bx.add(offset_next, bx.const_i32(slot_size)); + // update va_ndx + bx.store(offset_next_corrected, offset_ptr, bx.tcx().data_layout.pointer_align.abi); + + // let stack_value_ptr = unsafe { (*va).va_stk.byte_add(offset_corrected) }; + let stack_area_ptr = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(0)]); + let stack_area = bx.load(bx.type_ptr(), stack_area_ptr, bx.tcx().data_layout.pointer_align.abi); + let stack_value_ptr = bx.inbounds_gep(bx.type_i8(), stack_area, &[offset_corrected]); + bx.br(end); + + bx.switch_to_block(end); + + // On big-endian, for values smaller than the slot size we'd have to align the read to the end + // of the slot rather than the start. While the ISA and GCC support big-endian, all the Xtensa + // targets supported by rustc are litte-endian so don't worry about it. + + // if from_regsave { + // unsafe { *regsave_value_ptr } + // } else { + // unsafe { *stack_value_ptr } + // } + assert!(bx.tcx().sess.target.endian == Endian::Little); + let value_ptr = + bx.phi(bx.type_ptr(), &[regsave_value_ptr, stack_value_ptr], &[from_regsave, from_stack]); + return bx.load(layout.llvm_type(bx), value_ptr, layout.align.abi); +} + pub(super) fn emit_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, addr: OperandRef<'tcx, &'ll Value>, @@ -302,6 +410,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) } + "xtensa" => emit_xtensa_va_arg(bx, addr, target_ty), // For all other architecture/OS combinations fall back to using // the LLVM va_arg instruction. // https://llvm.org/docs/LangRef.html#va-arg-instruction |