diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
18 files changed, 536 insertions, 254 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 6ee80c08d4a..d1804cb49ad 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -645,6 +645,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> | Arm(ArmInlineAsmRegClass::qreg_low4) => "x", Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w", Hexagon(HexagonInlineAsmRegClass::reg) => "r", + Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"), LoongArch(LoongArchInlineAsmRegClass::reg) => "r", LoongArch(LoongArchInlineAsmRegClass::freg) => "f", Mips(MipsInlineAsmRegClass::reg) => "r", @@ -655,9 +656,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg) => "r", PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", - PowerPC(PowerPCInlineAsmRegClass::cr) - | PowerPC(PowerPCInlineAsmRegClass::xer) - | PowerPC(PowerPCInlineAsmRegClass::vreg) => { + PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", + PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -813,6 +813,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' | Arm(ArmInlineAsmRegClass::qreg_low8) | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2), Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + Hexagon(HexagonInlineAsmRegClass::preg) => unreachable!("clobber-only"), LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), @@ -823,9 +824,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), - PowerPC(PowerPCInlineAsmRegClass::cr) - | PowerPC(PowerPCInlineAsmRegClass::xer) - | PowerPC(PowerPCInlineAsmRegClass::vreg) => { + PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4), + PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), @@ -1040,6 +1040,26 @@ fn llvm_fixup_input<'ll, 'tcx>( let value = bx.or(value, bx.const_u32(0xFFFF_0000)); bx.bitcast(value, bx.type_f32()) } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F32) => + { + let value = bx.insert_element( + bx.const_undef(bx.type_vector(bx.type_f32(), 4)), + value, + bx.const_usize(0), + ); + bx.bitcast(value, bx.type_vector(bx.type_f32(), 4)) + } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F64) => + { + let value = bx.insert_element( + bx.const_undef(bx.type_vector(bx.type_f64(), 2)), + value, + bx.const_usize(0), + ); + bx.bitcast(value, bx.type_vector(bx.type_f64(), 2)) + } _ => value, } } @@ -1175,6 +1195,18 @@ fn llvm_fixup_output<'ll, 'tcx>( let value = bx.trunc(value, bx.type_i16()); bx.bitcast(value, bx.type_f16()) } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F32) => + { + let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4)); + bx.extract_element(value, bx.const_usize(0)) + } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F64) => + { + let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2)); + bx.extract_element(value, bx.const_usize(0)) + } _ => value, } } @@ -1299,6 +1331,16 @@ fn llvm_fixup_output_type<'ll, 'tcx>( { cx.type_f32() } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F32) => + { + cx.type_vector(cx.type_f32(), 4) + } + (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) + if s.primitive() == Primitive::Float(Float::F64) => + { + cx.type_vector(cx.type_f64(), 2) + } _ => layout.llvm_type(cx), } } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 48beb9be2b2..f6d2ec24da6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -148,7 +148,7 @@ fn prepare_lto( // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to // __llvm_profile_runtime, therefore we won't know until link time if this symbol // should have default visibility. - symbols_below_threshold.push(CString::new("__llvm_profile_counter_bias").unwrap()); + symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned()); Ok((symbols_below_threshold, upstream_modules)) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 00f7b479fa7..bde6668929c 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -61,6 +61,7 @@ fn write_output_file<'ll>( dwo_output: Option<&Path>, file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, + verify_llvm_ir: bool, ) -> Result<(), FatalError> { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { @@ -79,6 +80,7 @@ fn write_output_file<'ll>( output_c.as_ptr(), dwo_output_ptr, file_type, + verify_llvm_ir, ); // Record artifact sizes for self-profiling @@ -507,7 +509,7 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { } fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { - config.instrument_coverage.then(|| CString::new("default_%m_%p.profraw").unwrap()) + config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned()) } pub(crate) unsafe fn llvm_optimize( @@ -840,6 +842,7 @@ pub(crate) unsafe fn codegen( None, llvm::FileType::AssemblyFile, &cgcx.prof, + config.verify_llvm_ir, ) })?; } @@ -877,6 +880,7 @@ pub(crate) unsafe fn codegen( dwo_out, llvm::FileType::ObjectFile, &cgcx.prof, + config.verify_llvm_ir, ) })?; } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index e0a2de3366c..ec77f32caf4 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -104,7 +104,10 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t let is_hidden = if is_generic { // This is a monomorphization of a generic function. - if !cx.tcx.sess.opts.share_generics() { + if !(cx.tcx.sess.opts.share_generics() + || tcx.codegen_fn_attrs(instance_def_id).inline + == rustc_attr::InlineAttr::Never) + { // When not sharing generics, all instances are in the same // crate and have hidden visibility. true diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8852dec7d9f..adfe8aeb5c5 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -302,10 +302,9 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { (value, AddressSpace::DATA) } } - GlobalAlloc::Function { instance, .. } => ( - self.get_fn_addr(instance.polymorphize(self.tcx)), - self.data_layout().instruction_address_space, - ), + GlobalAlloc::Function { instance, .. } => { + (self.get_fn_addr(instance), self.data_layout().instruction_address_space) + } GlobalAlloc::VTable(ty, dyn_ty) => { let alloc = self .tcx diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 841c110b3c8..8218126ea29 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -82,8 +82,8 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { pub isize_ty: &'ll Type, - /// Extra codegen state needed when coverage instrumentation is enabled. - pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, + /// Extra per-CGU codegen state needed when coverage instrumentation is enabled. + pub coverage_cx: Option<coverageinfo::CguCoverageContext<'ll, 'tcx>>, pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>, eh_personality: Cell<Option<&'ll Value>>, @@ -525,7 +525,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); let coverage_cx = - tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new); + tcx.sess.instrument_coverage().then(coverageinfo::CguCoverageContext::new); let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); @@ -576,7 +576,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// Extra state that is only available when coverage instrumentation is enabled. #[inline] #[track_caller] - pub(crate) fn coverage_cx(&self) -> &coverageinfo::CrateCoverageContext<'ll, 'tcx> { + pub(crate) fn coverage_cx(&self) -> &coverageinfo::CguCoverageContext<'ll, 'tcx> { self.coverage_cx.as_ref().expect("only called when coverage instrumentation is enabled") } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index e3c0df27883..0752c718c70 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -1,12 +1,13 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxIndexSet; use rustc_index::bit_set::BitSet; +use rustc_middle::mir::CoverageIdsInfo; use rustc_middle::mir::coverage::{ CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, MappingKind, Op, SourceRegion, }; use rustc_middle::ty::Instance; -use tracing::{debug, instrument}; +use tracing::debug; use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; @@ -16,17 +17,8 @@ use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; pub(crate) struct FunctionCoverageCollector<'tcx> { /// Coverage info that was attached to this function by the instrumentor. function_coverage_info: &'tcx FunctionCoverageInfo, + ids_info: &'tcx CoverageIdsInfo, is_used: bool, - - /// Tracks which counters have been seen, so that we can identify mappings - /// to counters that were optimized out, and set them to zero. - counters_seen: BitSet<CounterId>, - /// Contains all expression IDs that have been seen in an `ExpressionUsed` - /// coverage statement, plus all expression IDs that aren't directly used - /// by any mappings (and therefore do not have expression-used statements). - /// After MIR traversal is finished, we can conclude that any IDs missing - /// from this set must have had their statements deleted by MIR opts. - expressions_seen: BitSet<ExpressionId>, } impl<'tcx> FunctionCoverageCollector<'tcx> { @@ -34,21 +26,24 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { pub(crate) fn new( instance: Instance<'tcx>, function_coverage_info: &'tcx FunctionCoverageInfo, + ids_info: &'tcx CoverageIdsInfo, ) -> Self { - Self::create(instance, function_coverage_info, true) + Self::create(instance, function_coverage_info, ids_info, true) } /// Creates a new set of coverage data for an unused (never called) function. pub(crate) fn unused( instance: Instance<'tcx>, function_coverage_info: &'tcx FunctionCoverageInfo, + ids_info: &'tcx CoverageIdsInfo, ) -> Self { - Self::create(instance, function_coverage_info, false) + Self::create(instance, function_coverage_info, ids_info, false) } fn create( instance: Instance<'tcx>, function_coverage_info: &'tcx FunctionCoverageInfo, + ids_info: &'tcx CoverageIdsInfo, is_used: bool, ) -> Self { let num_counters = function_coverage_info.num_counters; @@ -58,44 +53,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { num_counters={num_counters}, num_expressions={num_expressions}, is_used={is_used}" ); - // Create a filled set of expression IDs, so that expressions not - // directly used by mappings will be treated as "seen". - // (If they end up being unused, LLVM will delete them for us.) - let mut expressions_seen = BitSet::new_filled(num_expressions); - // For each expression ID that is directly used by one or more mappings, - // mark it as not-yet-seen. This indicates that we expect to see a - // corresponding `ExpressionUsed` statement during MIR traversal. - for mapping in function_coverage_info.mappings.iter() { - // Currently we only worry about ordinary code mappings. - // For branch and MC/DC mappings, expressions might not correspond - // to any particular point in the control-flow graph. - // (Keep this in sync with the injection of `ExpressionUsed` - // statements in the `InstrumentCoverage` MIR pass.) - if let MappingKind::Code(term) = mapping.kind - && let CovTerm::Expression(id) = term - { - expressions_seen.remove(id); - } - } - - Self { - function_coverage_info, - is_used, - counters_seen: BitSet::new_empty(num_counters), - expressions_seen, - } - } - - /// Marks a counter ID as having been seen in a counter-increment statement. - #[instrument(level = "debug", skip(self))] - pub(crate) fn mark_counter_id_seen(&mut self, id: CounterId) { - self.counters_seen.insert(id); - } - - /// Marks an expression ID as having been seen in an expression-used statement. - #[instrument(level = "debug", skip(self))] - pub(crate) fn mark_expression_id_seen(&mut self, id: ExpressionId) { - self.expressions_seen.insert(id); + Self { function_coverage_info, ids_info, is_used } } /// Identify expressions that will always have a value of zero, and note @@ -117,7 +75,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { // (By construction, expressions can only refer to other expressions // that have lower IDs, so one pass is sufficient.) for (id, expression) in self.function_coverage_info.expressions.iter_enumerated() { - if !self.expressions_seen.contains(id) { + if !self.is_used || !self.ids_info.expressions_seen.contains(id) { // If an expression was not seen, it must have been optimized away, // so any operand that refers to it can be replaced with zero. zero_expressions.insert(id); @@ -146,7 +104,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { assert_operand_expression_is_lower(id); } - if is_zero_term(&self.counters_seen, &zero_expressions, *operand) { + if is_zero_term(&self.ids_info.counters_seen, &zero_expressions, *operand) { *operand = CovTerm::Zero; } }; @@ -172,17 +130,17 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { pub(crate) fn into_finished(self) -> FunctionCoverage<'tcx> { let zero_expressions = self.identify_zero_expressions(); - let FunctionCoverageCollector { function_coverage_info, is_used, counters_seen, .. } = self; + let FunctionCoverageCollector { function_coverage_info, ids_info, is_used, .. } = self; - FunctionCoverage { function_coverage_info, is_used, counters_seen, zero_expressions } + FunctionCoverage { function_coverage_info, ids_info, is_used, zero_expressions } } } pub(crate) struct FunctionCoverage<'tcx> { pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, + ids_info: &'tcx CoverageIdsInfo, is_used: bool, - counters_seen: BitSet<CounterId>, zero_expressions: ZeroExpressions, } @@ -238,7 +196,7 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn is_zero_term(&self, term: CovTerm) -> bool { - is_zero_term(&self.counters_seen, &self.zero_expressions, term) + !self.is_used || is_zero_term(&self.ids_info.counters_seen, &self.zero_expressions, term) } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index b582dd967a7..8c24579fa7c 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -535,9 +535,12 @@ fn add_unused_function_coverage<'tcx>( }), ); - // An unused function's mappings will automatically be rewritten to map to - // zero, because none of its counters/expressions are marked as seen. - let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); + // An unused function's mappings will all be rewritten to map to zero. + let function_coverage = FunctionCoverageCollector::unused( + instance, + function_coverage_info, + tcx.coverage_ids_info(instance.def), + ); cx.coverage_cx().function_coverage_map.borrow_mut().insert(instance, function_coverage); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index bf773cd2667..c2fcb33f98b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -21,8 +21,8 @@ mod llvm_cov; pub(crate) mod map_data; mod mapgen; -/// A context object for maintaining all state needed by the coverageinfo module. -pub(crate) struct CrateCoverageContext<'ll, 'tcx> { +/// Extra per-CGU context/state needed for coverage instrumentation. +pub(crate) struct CguCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, @@ -32,7 +32,7 @@ pub(crate) struct CrateCoverageContext<'ll, 'tcx> { covfun_section_name: OnceCell<CString>, } -impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { +impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> { pub(crate) fn new() -> Self { Self { function_coverage_map: Default::default(), @@ -143,6 +143,13 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let bx = self; + // Due to LocalCopy instantiation or MIR inlining, coverage statements + // can end up in a crate that isn't doing coverage instrumentation. + // When that happens, we currently just discard those statements, so + // the corresponding code will be undercounted. + // FIXME(Zalathar): Find a better solution for mixed-coverage builds. + let Some(coverage_cx) = &bx.cx.coverage_cx else { return }; + let Some(function_coverage_info) = bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() else { @@ -150,32 +157,28 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - // FIXME(#132395): Unwrapping `coverage_cx` here has led to ICEs in the - // wild, so keep this early-return until we understand why. - let mut coverage_map = match bx.coverage_cx { - Some(ref cx) => cx.function_coverage_map.borrow_mut(), - None => return, - }; - let func_coverage = coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); + // Mark the instance as used in this CGU, for coverage purposes. + // This includes functions that were not partitioned into this CGU, + // but were MIR-inlined into one of this CGU's functions. + coverage_cx.function_coverage_map.borrow_mut().entry(instance).or_insert_with(|| { + FunctionCoverageCollector::new( + instance, + function_coverage_info, + bx.tcx.coverage_ids_info(instance.def), + ) + }); match *kind { CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( "marker statement {kind:?} should have been removed by CleanupPostBorrowck" ), CoverageKind::CounterIncrement { id } => { - func_coverage.mark_counter_id_seen(id); - // We need to explicitly drop the `RefMut` before calling into - // `instrprof_increment`, as that needs an exclusive borrow. - drop(coverage_map); - // The number of counters passed to `llvm.instrprof.increment` might // be smaller than the number originally inserted by the instrumentor, // if some high-numbered counters were removed by MIR optimizations. // If so, LLVM's profiler runtime will use fewer physical counters. let num_counters = - bx.tcx().coverage_ids_info(instance.def).max_counter_id.as_u32() + 1; + bx.tcx().coverage_ids_info(instance.def).num_counters_after_mir_opts(); assert!( num_counters as usize <= function_coverage_info.num_counters, "num_counters disagreement: query says {num_counters} but function info only has {}", @@ -192,23 +195,23 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { ); bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::ExpressionUsed { id } => { - func_coverage.mark_expression_id_seen(id); + CoverageKind::ExpressionUsed { id: _ } => { + // Expression-used statements are markers that are handled by + // `coverage_ids_info`, so there's nothing to codegen here. } CoverageKind::CondBitmapUpdate { index, decision_depth } => { - drop(coverage_map); - let cond_bitmap = bx - .coverage_cx() + let cond_bitmap = coverage_cx .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); let cond_index = bx.const_i32(index as i32); bx.mcdc_condbitmap_update(cond_index, cond_bitmap); } CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { - drop(coverage_map); - let cond_bitmap = bx.coverage_cx() - .try_get_mcdc_condition_bitmap(&instance, decision_depth) - .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); + let cond_bitmap = + coverage_cx.try_get_mcdc_condition_bitmap(&instance, decision_depth).expect( + "mcdc cond bitmap should have been allocated for merging \ + into the global bitmap", + ); assert!( bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits, "bitmap index of the decision out of range" diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index ef16e5bb459..59275254022 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -203,6 +203,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &ptr_type_debuginfo_name, + None, cx.size_and_align_of(ptr_type), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -259,6 +260,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( layout.fields.offset(abi::WIDE_PTR_ADDR), DIFlags::FlagZero, data_ptr_type_di_node, + None, ), build_field_di_node( cx, @@ -268,6 +270,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( layout.fields.offset(abi::WIDE_PTR_EXTRA), DIFlags::FlagZero, type_di_node(cx, extra_field.ty), + None, ), ] }, @@ -369,6 +372,7 @@ fn build_dyn_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &type_name, + None, cx.size_and_align_of(dyn_type), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -467,8 +471,6 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), - // Type parameters from polymorphized functions. - ty::Param(_) => build_param_type_di_node(cx, t), _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), }; @@ -722,6 +724,14 @@ fn build_cpp_f16_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> DINodeCreation // `f16`'s value to be displayed using a Natvis visualiser in `intrinsic.natvis`. let float_ty = cx.tcx.types.f16; let bits_ty = cx.tcx.types.u16; + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + match float_ty.kind() { + ty::Adt(def, _) => Some(file_metadata_from_def_id(cx, Some(def.did()))), + _ => None, + } + } else { + None + }; type_map::build_type_with_children( cx, type_map::stub( @@ -729,12 +739,21 @@ fn build_cpp_f16_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> DINodeCreation Stub::Struct, UniqueTypeId::for_ty(cx.tcx, float_ty), "f16", + def_location, cx.size_and_align_of(float_ty), NO_SCOPE_METADATA, DIFlags::FlagZero, ), // Fields: |cx, float_di_node| { + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + match bits_ty.kind() { + ty::Adt(def, _) => Some(def.did()), + _ => None, + } + } else { + None + }; smallvec![build_field_di_node( cx, float_di_node, @@ -743,6 +762,7 @@ fn build_cpp_f16_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> DINodeCreation Size::ZERO, DIFlags::FlagZero, type_di_node(cx, bits_ty), + def_id, )] }, NO_GENERICS, @@ -839,6 +859,7 @@ fn build_foreign_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &compute_debuginfo_type_name(cx.tcx, t, false), + None, cx.size_and_align_of(t), Some(get_namespace_for_item(cx, def_id)), DIFlags::FlagZero, @@ -848,26 +869,6 @@ fn build_foreign_type_di_node<'ll, 'tcx>( ) } -fn build_param_type_di_node<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - t: Ty<'tcx>, -) -> DINodeCreationResult<'ll> { - debug!("build_param_type_di_node: {:?}", t); - let name = format!("{t:?}"); - DINodeCreationResult { - di_node: unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( - DIB(cx), - name.as_c_char_ptr(), - name.len(), - Size::ZERO.bits(), - DW_ATE_unsigned, - ) - }, - already_stored_in_typemap: false, - } -} - pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( tcx: TyCtxt<'tcx>, codegen_unit_name: &str, @@ -989,15 +990,22 @@ fn build_field_di_node<'ll, 'tcx>( offset: Size, flags: DIFlags, type_di_node: &'ll DIType, + def_id: Option<DefId>, ) -> &'ll DIType { + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; unsafe { llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), owner, name.as_c_char_ptr(), name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size_and_align.0.bits(), size_and_align.1.bits() as u32, offset.bits(), @@ -1041,6 +1049,11 @@ fn build_struct_type_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, adt_def.did()); let struct_type_and_layout = cx.layout_of(struct_type); let variant_def = adt_def.non_enum_variant(); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) + } else { + None + }; type_map::build_type_with_children( cx, @@ -1049,6 +1062,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &compute_debuginfo_type_name(cx.tcx, struct_type, false), + def_location, size_and_align_of(struct_type_and_layout), Some(containing_scope), visibility_di_flags(cx, adt_def.did(), adt_def.did()), @@ -1068,6 +1082,11 @@ fn build_struct_type_di_node<'ll, 'tcx>( Cow::Borrowed(f.name.as_str()) }; let field_layout = struct_type_and_layout.field(cx, i); + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(f.did) + } else { + None + }; build_field_di_node( cx, owner, @@ -1076,6 +1095,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( struct_type_and_layout.fields.offset(i), visibility_di_flags(cx, f.did, adt_def.did()), type_di_node(cx, field_layout.ty), + def_id, ) }) .collect() @@ -1125,6 +1145,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, up_var_ty), + None, ) }) .collect() @@ -1150,6 +1171,7 @@ fn build_tuple_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &type_name, + None, size_and_align_of(tuple_type_and_layout), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -1168,6 +1190,7 @@ fn build_tuple_type_di_node<'ll, 'tcx>( tuple_type_and_layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, component_type), + None, ) }) .collect() @@ -1189,6 +1212,12 @@ fn build_closure_env_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, def_id); let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(def_id))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -1196,6 +1225,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &type_name, + def_location, cx.size_and_align_of(closure_env_type), Some(containing_scope), DIFlags::FlagZero, @@ -1219,6 +1249,11 @@ fn build_union_type_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, union_def_id); let union_ty_and_layout = cx.layout_of(union_type); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(union_def_id))) + } else { + None + }; type_map::build_type_with_children( cx, @@ -1227,6 +1262,7 @@ fn build_union_type_di_node<'ll, 'tcx>( Stub::Union, unique_type_id, &type_name, + def_location, size_and_align_of(union_ty_and_layout), Some(containing_scope), DIFlags::FlagZero, @@ -1239,6 +1275,11 @@ fn build_union_type_di_node<'ll, 'tcx>( .enumerate() .map(|(i, f)| { let field_layout = union_ty_and_layout.field(cx, i); + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(f.did) + } else { + None + }; build_field_di_node( cx, owner, @@ -1247,6 +1288,7 @@ fn build_union_type_di_node<'ll, 'tcx>( Size::ZERO, DIFlags::FlagZero, type_di_node(cx, field_layout.ty), + def_id, ) }) .collect() @@ -1321,14 +1363,7 @@ pub(crate) fn build_global_var_di_node<'ll>( // We may want to remove the namespace scope if we're in an extern block (see // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952). let var_scope = get_namespace_for_item(cx, def_id); - let span = hygiene::walk_chain_collapsed(tcx.def_span(def_id), DUMMY_SP); - - let (file_metadata, line_number) = if !span.is_dummy() { - let loc = cx.lookup_debug_loc(span.lo()); - (file_metadata(cx, &loc.file), loc.line) - } else { - (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) - }; + let (file_metadata, line_number) = file_metadata_from_def_id(cx, Some(def_id)); let is_local_to_unit = is_node_local_to_unit(cx, def_id); @@ -1418,6 +1453,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( Stub::VTableTy { vtable_holder }, unique_type_id, &vtable_type_name, + None, (size, pointer_align), NO_SCOPE_METADATA, DIFlags::FlagArtificial, @@ -1455,6 +1491,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( field_offset, DIFlags::FlagZero, field_type_di_node, + None, )) }) .collect() @@ -1606,3 +1643,20 @@ fn tuple_field_name(field_index: usize) -> Cow<'static, str> { .map(|s| Cow::from(*s)) .unwrap_or_else(|| Cow::from(format!("__{field_index}"))) } + +pub(crate) type DefinitionLocation<'ll> = (&'ll DIFile, c_uint); + +pub(crate) fn file_metadata_from_def_id<'ll>( + cx: &CodegenCx<'ll, '_>, + def_id: Option<DefId>, +) -> DefinitionLocation<'ll> { + if let Some(def_id) = def_id + && let span = hygiene::walk_chain_collapsed(cx.tcx.def_span(def_id), DUMMY_SP) + && !span.is_dummy() + { + let loc = cx.lookup_debug_loc(span.lo()); + (file_metadata(cx, &loc.file), loc.line) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 100b046cee2..d374767f187 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -4,7 +4,7 @@ use libc::c_uint; use rustc_abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; -use rustc_codegen_ssa::traits::ConstCodegenMethods; +use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods}; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -16,8 +16,8 @@ use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, NO_SCOPE_METADATA, SmallVec, UNKNOWN_LINE_NUMBER, - build_field_di_node, file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, - visibility_di_flags, + build_field_di_node, file_metadata, file_metadata_from_def_id, size_and_align_of, type_di_node, + unknown_file_metadata, visibility_di_flags, }; use crate::debuginfo::utils::DIB; use crate::llvm::debuginfo::{DIFile, DIFlags, DIType}; @@ -192,6 +192,12 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(enum_adt_def.did()))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -199,6 +205,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( type_map::Stub::Union, unique_type_id, &enum_type_name, + def_location, cx.size_and_align_of(enum_type), NO_SCOPE_METADATA, visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()), @@ -262,6 +269,14 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let coroutine_type = unique_type_id.expect_ty(); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + let &ty::Coroutine(coroutine_def_id, _) = coroutine_type.kind() else { + bug!("build_coroutine_di_node() called with non-coroutine type: `{:?}`", coroutine_type) + }; + Some(file_metadata_from_def_id(cx, Some(coroutine_def_id))) + } else { + None + }; let coroutine_type_and_layout = cx.layout_of(coroutine_type); let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); @@ -274,6 +289,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( type_map::Stub::Union, unique_type_id, &coroutine_type_name, + def_location, size_and_align_of(coroutine_type_and_layout), NO_SCOPE_METADATA, DIFlags::FlagZero, @@ -321,6 +337,12 @@ fn build_single_variant_union_fields<'ll, 'tcx>( let tag_base_type_di_node = type_di_node(cx, tag_base_type); let tag_base_type_align = cx.align_of(tag_base_type); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; + let variant_names_type_di_node = build_variant_names_type_di_node( cx, enum_type_di_node, @@ -328,6 +350,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_index, Cow::from(enum_adt_def.variant(variant_index).name.as_str()), )), + enum_adt_def_id, ); let variant_struct_type_wrapper_di_node = build_variant_struct_wrapper_type_di_node( @@ -341,6 +364,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( tag_base_type_di_node, tag_base_type, DiscrResult::NoDiscriminant, + None, ); smallvec![ @@ -354,6 +378,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( Size::ZERO, visibility_flags, variant_struct_type_wrapper_di_node, + None, ), unsafe { llvm::LLVMRustDIBuilderCreateStaticMemberType( @@ -383,6 +408,12 @@ fn build_union_fields_for_enum<'ll, 'tcx>( ) -> SmallVec<&'ll DIType> { let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; + let variant_names_type_di_node = build_variant_names_type_di_node( cx, enum_type_di_node, @@ -390,6 +421,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); (variant_index, variant_name) }), + enum_adt_def_id, ); let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); @@ -447,6 +479,7 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, containing_scope: &'ll DIType, variants: impl Iterator<Item = (VariantIdx, Cow<'tcx, str>)>, + enum_def_id: Option<rustc_span::def_id::DefId>, ) -> &'ll DIType { // Create an enumerator for each variant. super::build_enumeration_type_di_node( @@ -454,6 +487,7 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( "VariantNames", variant_names_enum_base_type(cx), variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32().into())), + enum_def_id, containing_scope, ) } @@ -469,6 +503,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( tag_base_type_di_node: &'ll DIType, tag_base_type: Ty<'tcx>, discr: DiscrResult, + source_info: Option<(&'ll DIFile, c_uint)>, ) -> &'ll DIType { type_map::build_type_with_children( cx, @@ -481,6 +516,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( variant_index, ), &variant_struct_wrapper_type_name(variant_index), + source_info, // NOTE: We use size and align of enum_type, not from variant_layout: size_and_align_of(enum_or_coroutine_type_and_layout), Some(enum_or_coroutine_type_di_node), @@ -530,6 +566,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( Size::ZERO, DIFlags::FlagZero, variant_struct_type_di_node, + None, )); let build_assoc_const = @@ -684,6 +721,11 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( variant_range .clone() .map(|variant_index| (variant_index, CoroutineArgs::variant_name(variant_index))), + if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(coroutine_def_id) + } else { + None + }, ); let discriminants: IndexVec<VariantIdx, DiscrResult> = { @@ -776,6 +818,11 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( tag_base_type_di_node, tag_base_type, variant_member_info.discr, + if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + variant_member_info.source_info + } else { + None + }, ); // We use LLVMRustDIBuilderCreateMemberType() member type directly because @@ -831,6 +878,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( lo_offset, di_flags, type_di_node, + None, )); unions_fields.push(build_field_di_node( @@ -841,6 +889,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( hi_offset, DIFlags::FlagZero, type_di_node, + None, )); } else { unions_fields.push(build_field_di_node( @@ -851,6 +900,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( enum_type_and_layout.fields.offset(tag_field), di_flags, tag_base_type_di_node, + None, )); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index b3d4a6642a1..65ab22ad89e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -3,6 +3,7 @@ use std::borrow::Cow; use rustc_abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; +use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::bug; @@ -16,8 +17,8 @@ use super::{SmallVec, size_and_align_of}; use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub}; use crate::debuginfo::metadata::{ - UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, type_di_node, - unknown_file_metadata, + UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, + file_metadata_from_def_id, type_di_node, unknown_file_metadata, }; use crate::debuginfo::utils::{DIB, create_DIArray, get_namespace_for_item}; use crate::llvm::debuginfo::{DIFlags, DIType}; @@ -68,6 +69,11 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( enum_type_and_layout: TyAndLayout<'tcx>, ) -> DINodeCreationResult<'ll> { let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; DINodeCreationResult { di_node: build_enumeration_type_di_node( cx, @@ -77,6 +83,7 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); (name, discr.val) }), + enum_adt_def_id, containing_scope, ), already_stored_in_typemap: false, @@ -92,6 +99,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( type_name: &str, base_type: Ty<'tcx>, enumerators: impl Iterator<Item = (Cow<'tcx, str>, u128)>, + def_id: Option<rustc_span::def_id::DefId>, containing_scope: &'ll DIType, ) -> &'ll DIType { let is_unsigned = match base_type.kind() { @@ -115,14 +123,21 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( }) .collect(); + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + unsafe { llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), containing_scope, type_name.as_c_char_ptr(), type_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size.bits(), align.bits() as u32, create_DIArray(DIB(cx), &enumerator_di_nodes[..]), @@ -193,6 +208,12 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( ) -> &'ll DIType { assert_eq!(variant_layout.ty, enum_type_and_layout.ty); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(variant_def.def_id))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -204,6 +225,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( variant_index, ), variant_def.name.as_str(), + def_location, // NOTE: We use size and align of enum_type, not from variant_layout: size_and_align_of(enum_type_and_layout), Some(enum_type_di_node), @@ -231,6 +253,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( variant_layout.fields.offset(field_index), di_flags, type_di_node(cx, field_layout.ty), + None, ) }) .collect::<SmallVec<_>>() @@ -286,6 +309,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &variant_name, + None, size_and_align_of(coroutine_type_and_layout), Some(coroutine_type_di_node), DIFlags::FlagZero, @@ -312,6 +336,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( variant_layout.fields.offset(field_index), DIFlags::FlagZero, type_di_node(cx, field_type), + None, ) }) .collect(); @@ -331,6 +356,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, upvar_ty), + None, ) }) .collect(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index d4006691d37..241bf167a81 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -4,7 +4,7 @@ use libc::c_uint; use rustc_abi::{Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; -use rustc_codegen_ssa::traits::ConstCodegenMethods; +use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self}; @@ -14,7 +14,8 @@ use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, SmallVec, UNKNOWN_LINE_NUMBER, file_metadata, - size_and_align_of, type_di_node, unknown_file_metadata, visibility_di_flags, + file_metadata_from_def_id, size_and_align_of, type_di_node, unknown_file_metadata, + visibility_di_flags, }; use crate::debuginfo::utils::{DIB, create_DIArray, get_namespace_for_item}; use crate::llvm::debuginfo::{DIFile, DIFlags, DIType}; @@ -55,6 +56,12 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(enum_adt_def.did()))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -62,6 +69,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &enum_type_name, + def_location, size_and_align_of(enum_type_and_layout), Some(containing_scope), visibility_flags, @@ -84,14 +92,27 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( enum_type_and_layout.for_variant(cx, variant_index), visibility_flags, ), - source_info: None, + source_info: if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id( + cx, + Some(enum_adt_def.variant(variant_index).def_id), + )) + } else { + None + }, }) .collect(); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; smallvec![build_enum_variant_part_di_node( cx, enum_type_and_layout, enum_type_di_node, + enum_adt_def_id, &variant_member_infos[..], )] }, @@ -134,6 +155,12 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(coroutine_def_id))) + } else { + None + }; + type_map::build_type_with_children( cx, type_map::stub( @@ -141,6 +168,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &coroutine_type_name, + def_location, size_and_align_of(coroutine_type_and_layout), Some(containing_scope), DIFlags::FlagZero, @@ -197,10 +225,16 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( }) .collect(); + let coroutine_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(coroutine_def_id) + } else { + None + }; smallvec![build_enum_variant_part_di_node( cx, coroutine_type_and_layout, coroutine_type_di_node, + coroutine_def_id, &variant_struct_type_di_nodes[..], )] }, @@ -228,6 +262,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, + enum_type_def_id: Option<rustc_span::def_id::DefId>, variant_member_infos: &[VariantMemberInfo<'_, 'll>], ) -> &'ll DIType { let tag_member_di_node = @@ -236,6 +271,13 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( let variant_part_unique_type_id = UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, enum_type_def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + let stub = StubInfo::new( cx, variant_part_unique_type_id, @@ -246,8 +288,8 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( enum_type_di_node, variant_part_name.as_c_char_ptr(), variant_part_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 4e461476040..a37e719d43f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -8,7 +8,7 @@ use rustc_macros::HashStable; use rustc_middle::bug; use rustc_middle::ty::{self, PolyExistentialTraitRef, Ty, TyCtxt}; -use super::{SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; +use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; @@ -186,6 +186,7 @@ pub(super) fn stub<'ll, 'tcx>( kind: Stub<'ll>, unique_type_id: UniqueTypeId<'tcx>, name: &str, + def_location: Option<DefinitionLocation<'ll>>, (size, align): (Size, Align), containing_scope: Option<&'ll DIScope>, flags: DIFlags, @@ -193,6 +194,12 @@ pub(super) fn stub<'ll, 'tcx>( let empty_array = create_DIArray(DIB(cx), &[]); let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + let (file_metadata, line_number) = if let Some(def_location) = def_location { + (def_location.0, def_location.1) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + let metadata = match kind { Stub::Struct | Stub::VTableTy { .. } => { let vtable_holder = match kind { @@ -205,8 +212,8 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope, name.as_c_char_ptr(), name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size.bits(), align.bits() as u32, flags, @@ -225,8 +232,8 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope, name.as_c_char_ptr(), name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, + file_metadata, + line_number, size.bits(), align.bits() as u32, flags, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index da7f94e8cf7..c38c5d4c644 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -352,84 +352,84 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::saturating_add | sym::saturating_sub => { let ty = arg_tys[0]; - match int_type_width_signed(ty, self) { - Some((width, signed)) => match name { - sym::ctlz | sym::cttz => { - let y = self.const_bool(false); - let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[ - args[0].immediate(), - y, - ]); - - self.intcast(ret, llret_ty, false) - } - sym::ctlz_nonzero => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.ctlz.i{width}"); - let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); - self.intcast(ret, llret_ty, false) - } - sym::cttz_nonzero => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.cttz.i{width}"); - let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); - self.intcast(ret, llret_ty, false) - } - sym::ctpop => { - let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[args - [0] - .immediate()]); - self.intcast(ret, llret_ty, false) - } - sym::bswap => { - if width == 8 { - args[0].immediate() // byte swap a u8/i8 is just a no-op - } else { - self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[ - args[0].immediate() - ]) - } - } - sym::bitreverse => self - .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[ + if !ty.is_integral() { + tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { + span, + name, + ty, + }); + return Ok(()); + } + let (size, signed) = ty.int_size_and_signed(self.tcx); + let width = size.bits(); + match name { + sym::ctlz | sym::cttz => { + let y = self.const_bool(false); + let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[ + args[0].immediate(), + y, + ]); + + self.intcast(ret, llret_ty, false) + } + sym::ctlz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.ctlz.i{width}"); + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) + } + sym::cttz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.cttz.i{width}"); + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) + } + sym::ctpop => { + let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[ + args[0].immediate() + ]); + self.intcast(ret, llret_ty, false) + } + sym::bswap => { + if width == 8 { + args[0].immediate() // byte swap a u8/i8 is just a no-op + } else { + self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[ args[0].immediate() - ]), - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same - let llvm_name = - &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); - - // llvm expects shift to be the same type as the values, but rust - // always uses `u32`. - let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); - - self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + ]) } - sym::saturating_add | sym::saturating_sub => { - let is_add = name == sym::saturating_add; - let lhs = args[0].immediate(); - let rhs = args[1].immediate(); - let llvm_name = &format!( - "llvm.{}{}.sat.i{}", - if signed { 's' } else { 'u' }, - if is_add { "add" } else { "sub" }, - width - ); - self.call_intrinsic(llvm_name, &[lhs, rhs]) - } - _ => bug!(), - }, - None => { - tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { - span, - name, - ty, - }); - return Ok(()); } + sym::bitreverse => self + .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[ + args[0].immediate() + ]), + sym::rotate_left | sym::rotate_right => { + let is_left = name == sym::rotate_left; + let val = args[0].immediate(); + let raw_shift = args[1].immediate(); + // rotate = funnel shift with first two args the same + let llvm_name = + &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + + // llvm expects shift to be the same type as the values, but rust + // always uses `u32`. + let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + } + sym::saturating_add | sym::saturating_sub => { + let is_add = name == sym::saturating_add; + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let llvm_name = &format!( + "llvm.{}{}.sat.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + self.call_intrinsic(llvm_name, &[lhs, rhs]) + } + _ => bug!(), } } @@ -1534,6 +1534,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), + sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), @@ -1572,6 +1573,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( | sym::simd_fpowi | sym::simd_fsin | sym::simd_fsqrt + | sym::simd_relaxed_fma | sym::simd_round | sym::simd_trunc ) { @@ -2529,19 +2531,3 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span_bug!(span, "unknown SIMD intrinsic"); } - -// Returns the width of an int Ty, and if it's signed or not -// Returns None if the type is not an integer -// FIXME: there’s multiple of this functions, investigate using some of the already existing -// stuffs. -fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { - match ty.kind() { - ty::Int(t) => { - Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), true)) - } - ty::Uint(t) => { - Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), false)) - } - _ => None, - } -} diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 3dfb86d422d..5235891a18d 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -36,7 +36,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed, FatalError}; +use rustc_errors::{DiagCtxtHandle, FatalError}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -370,19 +370,14 @@ impl CodegenBackend for LlvmCodegenBackend { (codegen_results, work_products) } - fn link( - &self, - sess: &Session, - codegen_results: CodegenResults, - outputs: &OutputFilenames, - ) -> Result<(), ErrorGuaranteed> { + fn link(&self, sess: &Session, codegen_results: CodegenResults, outputs: &OutputFilenames) { use rustc_codegen_ssa::back::link::link_binary; use crate::back::archive::LlvmArchiveBuilderBuilder; // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 17b0ec4b936..b1ace0033ba 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2240,6 +2240,7 @@ unsafe extern "C" { Output: *const c_char, DwoOutput: *const c_char, FileType: FileType, + VerifyIR: bool, ) -> LLVMRustResult; pub fn LLVMRustOptimize<'a>( M: &'a Module, diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index e4c3e748cb5..8baa69cefe1 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -10,6 +10,15 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; +fn round_up_to_alignment<'ll>( + bx: &mut Builder<'_, 'll, '_>, + mut value: &'ll Value, + align: Align, +) -> &'ll Value { + value = bx.add(value, bx.cx().const_i32(align.bytes() as i32 - 1)); + return bx.and(value, bx.cx().const_i32(-(align.bytes() as i32))); +} + fn round_pointer_up_to_alignment<'ll>( bx: &mut Builder<'_, 'll, '_>, addr: &'ll Value, @@ -17,8 +26,7 @@ fn round_pointer_up_to_alignment<'ll>( ptr_ty: &'ll Type, ) -> &'ll Value { let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); - ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1)); - ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32))); + ptr_as_int = round_up_to_alignment(bx, ptr_as_int, align); bx.inttoptr(ptr_as_int, ptr_ty) } @@ -270,6 +278,106 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.load(val_type, val_addr, layout.align.abi) } +fn emit_xtensa_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of va_arg for Xtensa. There doesn't seem to be an authoritative source for + // this, other than "what GCC does". + // + // The va_list type has three fields: + // struct __va_list_tag { + // int32_t *va_stk; // Arguments passed on the stack + // int32_t *va_reg; // Arguments passed in registers, saved to memory by the prologue. + // int32_t va_ndx; // Offset into the arguments, in bytes + // }; + // + // The first 24 bytes (equivalent to 6 registers) come from va_reg, the rest from va_stk. + // Thus if va_ndx is less than 24, the next va_arg *may* read from va_reg, + // otherwise it must come from va_stk. + // + // Primitive arguments are never split between registers and the stack. For example, if loading an 8 byte + // primitive value and va_ndx = 20, we instead bump the offset and read everything from va_stk. + let va_list_addr = list.immediate(); + // FIXME: handle multi-field structs that split across regsave/stack? + let layout = bx.cx.layout_of(target_ty); + let from_stack = bx.append_sibling_block("va_arg.from_stack"); + let from_regsave = bx.append_sibling_block("va_arg.from_regsave"); + let end = bx.append_sibling_block("va_arg.end"); + + // (*va).va_ndx + let va_reg_offset = 4; + let va_ndx_offset = va_reg_offset + 4; + let offset_ptr = + bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_ndx_offset)]); + + let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi); + let offset = round_up_to_alignment(bx, offset, layout.align.abi); + + let slot_size = layout.size.align_to(Align::from_bytes(4).unwrap()).bytes() as i32; + + // Update the offset in va_list, by adding the slot's size. + let offset_next = bx.add(offset, bx.const_i32(slot_size)); + + // Figure out where to look for our value. We do that by checking the end of our slot (offset_next). + // If that is within the regsave area, then load from there. Otherwise load from the stack area. + let regsave_size = bx.const_i32(24); + let use_regsave = bx.icmp(IntPredicate::IntULE, offset_next, regsave_size); + bx.cond_br(use_regsave, from_regsave, from_stack); + + bx.switch_to_block(from_regsave); + // update va_ndx + bx.store(offset_next, offset_ptr, bx.tcx().data_layout.pointer_align.abi); + + // (*va).va_reg + let regsave_area_ptr = + bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_reg_offset)]); + let regsave_area = + bx.load(bx.type_ptr(), regsave_area_ptr, bx.tcx().data_layout.pointer_align.abi); + let regsave_value_ptr = bx.inbounds_gep(bx.type_i8(), regsave_area, &[offset]); + bx.br(end); + + bx.switch_to_block(from_stack); + + // The first time we switch from regsave to stack we needs to adjust our offsets a bit. + // va_stk is set up such that the first stack argument is always at va_stk + 32. + // The corrected offset is written back into the va_list struct. + + // let offset_corrected = cmp::max(offset, 32); + let stack_offset_start = bx.const_i32(32); + let needs_correction = bx.icmp(IntPredicate::IntULE, offset, stack_offset_start); + let offset_corrected = bx.select(needs_correction, stack_offset_start, offset); + + // let offset_next_corrected = offset_corrected + slot_size; + // va_ndx = offset_next_corrected; + let offset_next_corrected = bx.add(offset_next, bx.const_i32(slot_size)); + // update va_ndx + bx.store(offset_next_corrected, offset_ptr, bx.tcx().data_layout.pointer_align.abi); + + // let stack_value_ptr = unsafe { (*va).va_stk.byte_add(offset_corrected) }; + let stack_area_ptr = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(0)]); + let stack_area = bx.load(bx.type_ptr(), stack_area_ptr, bx.tcx().data_layout.pointer_align.abi); + let stack_value_ptr = bx.inbounds_gep(bx.type_i8(), stack_area, &[offset_corrected]); + bx.br(end); + + bx.switch_to_block(end); + + // On big-endian, for values smaller than the slot size we'd have to align the read to the end + // of the slot rather than the start. While the ISA and GCC support big-endian, all the Xtensa + // targets supported by rustc are litte-endian so don't worry about it. + + // if from_regsave { + // unsafe { *regsave_value_ptr } + // } else { + // unsafe { *stack_value_ptr } + // } + assert!(bx.tcx().sess.target.endian == Endian::Little); + let value_ptr = + bx.phi(bx.type_ptr(), &[regsave_value_ptr, stack_value_ptr], &[from_regsave, from_stack]); + return bx.load(layout.llvm_type(bx), value_ptr, layout.align.abi); +} + pub(super) fn emit_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, addr: OperandRef<'tcx, &'ll Value>, @@ -302,6 +410,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) } + "xtensa" => emit_xtensa_va_arg(bx, addr, target_ty), // For all other architecture/OS combinations fall back to using // the LLVM va_arg instruction. // https://llvm.org/docs/LangRef.html#va-arg-instruction |