diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
24 files changed, 251 insertions, 147 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index aec11e9905f..a6a3f0f9646 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -2,6 +2,7 @@ use crate::attributes; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -425,6 +426,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()), ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (18, 0, 0) + { + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[ + llvm::AttributeKind::Writable.create_attr(cx.llcx), + llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx), + ], + ); + } } PassMode::Cast { cast, pad_i32: _ } => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); @@ -570,7 +583,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; if element_type_index >= 0 { let arg_ty = self.args[element_type_index as usize].layout.ty; - let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; + let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument"); let element_type_attr = unsafe { llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) }; diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index e09869cf425..db28c6857b7 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -16,6 +16,7 @@ use rustc_middle::{bug, span_bug, ty::Instance}; use rustc_span::{Pos, Span}; use rustc_target::abi::*; use rustc_target::asm::*; +use tracing::debug; use libc::{c_char, c_uint}; use smallvec::SmallVec; @@ -904,8 +905,8 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty Primitive::Int(Integer::I16, _) => cx.type_i16(), Primitive::Int(Integer::I32, _) => cx.type_i32(), Primitive::Int(Integer::I64, _) => cx.type_i64(), - Primitive::F32 => cx.type_f32(), - Primitive::F64 => cx.type_f64(), + Primitive::Float(Float::F32) => cx.type_f32(), + Primitive::Float(Float::F64) => cx.type_f64(), // FIXME(erikdesjardins): handle non-default addrspace ptr sizes Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()), _ => unreachable!(), @@ -950,7 +951,7 @@ fn llvm_fixup_input<'ll, 'tcx>( bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.primitive() == Primitive::F64 => + if s.primitive() == Primitive::Float(Float::F64) => { bx.bitcast(value, bx.cx.type_i64()) } @@ -986,8 +987,8 @@ fn llvm_fixup_input<'ll, 'tcx>( match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), - Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), - Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), + Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()), + Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()), _ => value, } } @@ -1027,7 +1028,7 @@ fn llvm_fixup_output<'ll, 'tcx>( bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.primitive() == Primitive::F64 => + if s.primitive() == Primitive::Float(Float::F64) => { bx.bitcast(value, bx.cx.type_f64()) } @@ -1064,8 +1065,8 @@ fn llvm_fixup_output<'ll, 'tcx>( // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), - Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), - Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), + Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()), + Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()), _ => value, } } @@ -1100,7 +1101,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( cx.type_vector(elem_ty, count * 2) } (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) - if s.primitive() == Primitive::F64 => + if s.primitive() == Primitive::Float(Float::F64) => { cx.type_i64() } @@ -1136,8 +1137,8 @@ fn llvm_fixup_output_type<'ll, 'tcx>( match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), - Primitive::F32 => cx.type_i32(), - Primitive::F64 => cx.type_i64(), + Primitive::Float(Float::F32) => cx.type_i32(), + Primitive::Float(Float::F64) => cx.type_i64(), _ => layout.llvm_type(cx), } } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index d4a3e39cef7..fa7e7e5377a 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -18,6 +18,7 @@ use rustc_codegen_ssa::back::archive::{ get_native_object_symbols, try_extract_macho_fat_archive, ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, UnknownArchiveKind, }; +use tracing::trace; use rustc_session::cstore::DllImport; use rustc_session::Session; @@ -200,21 +201,20 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { _ => panic!("unsupported arch {}", sess.target.arch), }; let mut dlltool_cmd = std::process::Command::new(&dlltool); - dlltool_cmd.args([ - "-d", - def_file_path.to_str().unwrap(), - "-D", - lib_name, - "-l", - output_path.to_str().unwrap(), - "-m", - dlltool_target_arch, - "-f", - dlltool_target_bitness, - "--no-leading-underscore", - "--temp-prefix", - temp_prefix.to_str().unwrap(), - ]); + dlltool_cmd + .arg("-d") + .arg(def_file_path) + .arg("-D") + .arg(lib_name) + .arg("-l") + .arg(&output_path) + .arg("-m") + .arg(dlltool_target_arch) + .arg("-f") + .arg(dlltool_target_bitness) + .arg("--no-leading-underscore") + .arg("--temp-prefix") + .arg(temp_prefix); match dlltool_cmd.output() { Err(e) => { diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index e61af863dc0..bb7c8fe2ea7 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -20,6 +20,7 @@ use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::config::{self, CrateType, Lto}; +use tracing::{debug, info}; use std::collections::BTreeMap; use std::ffi::{CStr, CString}; @@ -229,9 +230,12 @@ pub(crate) fn run_thin( thin_lto(cgcx, &dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold) } -pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { +pub(crate) fn prepare_thin( + module: ModuleCodegen<ModuleLlvm>, + emit_summary: bool, +) -> (String, ThinBuffer) { let name = module.name; - let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true, emit_summary); (name, buffer) } @@ -671,9 +675,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { + pub fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); ThinBuffer(buffer) } } @@ -687,6 +691,14 @@ impl ThinBufferMethods for ThinBuffer { slice::from_raw_parts(ptr, len) } } + + fn thin_link_data(&self) -> &[u8] { + unsafe { + let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); + slice::from_raw_parts(ptr, len) + } + } } impl Drop for ThinBuffer { diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 49f9d7ddab6..02e3bb06dda 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -35,6 +35,7 @@ use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; +use tracing::debug; use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_void, size_t}; @@ -708,13 +709,15 @@ pub(crate) unsafe fn codegen( // asm from LLVM and use `gcc` to create the object file. let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name); + let bc_summary_out = + cgcx.output_filenames.temp_path(OutputType::ThinLinkBitcode, module_name); let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name); if config.bitcode_needed() { let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod, config.emit_thin_lto); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto, config.emit_thin_lto_summary); let data = thin.data(); if let Some(bitcode_filename) = bc_out.file_name() { @@ -725,6 +728,25 @@ pub(crate) unsafe fn codegen( ); } + if config.emit_thin_lto_summary + && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name() + { + let summary_data = thin.thin_link_data(); + cgcx.prof.artifact_size( + "llvm_bitcode_summary", + thin_link_bitcode_filename.to_string_lossy(), + summary_data.len() as u64, + ); + + let _timer = cgcx.prof.generic_activity_with_arg( + "LLVM_module_codegen_emit_bitcode_summary", + &*module.name, + ); + if let Err(err) = fs::write(&bc_summary_out, summary_data) { + dcx.emit_err(WriteBytecode { path: &bc_summary_out, err }); + } + } + if config.emit_bc || config.emit_obj == EmitObj::Bitcode { let _timer = cgcx .prof diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index f7546039540..72ff9ea118e 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -17,7 +17,7 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; @@ -30,6 +30,7 @@ use std::borrow::Cow; use std::iter; use std::ops::Deref; use std::ptr; +use tracing::{debug, instrument}; // All Builders must have an llfn associated with them #[must_use] @@ -576,7 +577,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } } - abi::F16 | abi::F32 | abi::F64 | abi::F128 => {} + abi::Float(_) => {} } } @@ -976,6 +977,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { align: Align, flags: MemFlags, ) { + assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported"); let is_volatile = flags.contains(MemFlags::VOLATILE); unsafe { llvm::LLVMRustBuildMemSet( @@ -1704,12 +1706,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi_bundle } + /// Emits a call to `llvm.instrprof.mcdc.parameters`. + /// + /// This doesn't produce any code directly, but is used as input by + /// the LLVM pass that handles coverage instrumentation. + /// + /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.) + /// + /// [`CodeGenPGO::emitMCDCParameters`]: + /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 pub(crate) fn mcdc_parameters( &mut self, fn_name: &'ll Value, hash: &'ll Value, bitmap_bytes: &'ll Value, - ) -> &'ll Value { + ) { debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); @@ -1732,17 +1743,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { [].as_ptr(), 0 as c_uint, ); - // Create condition bitmap named `mcdc.addr`. - let mut bx = Builder::with_cx(self.cx); - bx.position_at_start(llvm::LLVMGetFirstBasicBlock(self.llfn())); - let cond_bitmap = { - let alloca = - llvm::LLVMBuildAlloca(bx.llbuilder, bx.cx.type_i32(), c"mcdc.addr".as_ptr()); - llvm::LLVMSetAlignment(alloca, 4); - alloca - }; - bx.store(self.const_i32(0), cond_bitmap, self.tcx().data_layout.i32_align.abi); - cond_bitmap } } @@ -1785,8 +1785,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { 0 as c_uint, ); } - let i32_align = self.tcx().data_layout.i32_align.abi; - self.store(self.const_i32(0), mcdc_temp, i32_align); + self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } pub(crate) fn mcdc_condbitmap_update( diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index e675362ac33..659c6ae0d86 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -12,6 +12,7 @@ use crate::value::Value; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; +use tracing::debug; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index ec33ce6292a..ab8036a1410 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -19,6 +19,7 @@ use rustc_target::spec::Target; use libc::{c_char, c_uint}; use std::fmt::Write; +use tracing::debug; /* * A note on nomenclature of linking: "extern", "foreign", and "upcall". diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 9e85c2d88f9..a2314f4850c 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -25,6 +25,7 @@ use rustc_target::abi::{ Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, }; use std::ops::Range; +use tracing::{debug, instrument, trace}; pub fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d32baa6dc02..53a098d178e 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -11,7 +11,8 @@ use crate::value::Value; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::base_n; +use rustc_data_structures::base_n::ToBaseN; +use rustc_data_structures::base_n::ALPHANUMERIC_ONLY; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; @@ -132,6 +133,15 @@ pub unsafe fn create_module<'ll>( } } + if llvm_version < (19, 0, 0) { + if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { + // LLVM 19 sets -Fn32 in its data layout string for 64-bit ARM + // Earlier LLVMs leave this default, so remove it. + // See https://github.com/llvm/llvm-project/pull/90702 + target_data_layout = target_data_layout.replace("-Fn32", ""); + } + } + // Ensure the data-layout values hardcoded remain the defaults. { let tm = crate::back::write::create_informational_target_machine(tcx.sess); @@ -1015,7 +1025,7 @@ impl CodegenCx<'_, '_> { let mut name = String::with_capacity(prefix.len() + 6); name.push_str(prefix); name.push('.'); - base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name); + name.push_str(&(idx as u64).to_base(ALPHANUMERIC_ONLY)); name } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index d85d9411f03..b969fe27a99 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -9,6 +9,7 @@ use rustc_middle::mir::coverage::{ }; use rustc_middle::ty::Instance; use rustc_span::Symbol; +use tracing::{debug, instrument}; /// Holds all of the coverage mapping data associated with a function instance, /// collected during traversal of `Coverage` statements in the function's MIR. diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 3f3969bbca3..d2c0f20c285 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -14,6 +14,7 @@ use rustc_middle::mir; use rustc_middle::ty::{self, TyCtxt}; use rustc_span::def_id::DefIdSet; use rustc_span::Symbol; +use tracing::debug; /// Generates and exports the Coverage Map. /// diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 085ce15d81f..7b7f8c885bb 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -13,10 +13,11 @@ use rustc_codegen_ssa::traits::{ use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::{CoverageKind, FunctionCoverageInfo}; +use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::Instance; -use rustc_target::abi::Align; +use rustc_target::abi::{Align, Size}; +use tracing::{debug, instrument}; use std::cell::RefCell; @@ -30,7 +31,7 @@ pub struct CrateCoverageContext<'ll, 'tcx> { pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, - pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, + pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -49,9 +50,20 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is called condition bitmap. - /// This value is named `mcdc.addr` (same as clang) and is a 32-bit integer. - fn try_get_mcdc_condition_bitmap(&self, instance: &Instance<'tcx>) -> Option<&'ll llvm::Value> { - self.mcdc_condition_bitmap_map.borrow().get(instance).copied() + /// In order to handle nested decisions, several condition bitmaps can be + /// allocated for a function body. + /// These values are named `mcdc.addr.{i}` and are a 32-bit integers. + /// They respectively hold the condition bitmaps for decisions with a depth of `i`. + fn try_get_mcdc_condition_bitmap( + &self, + instance: &Instance<'tcx>, + decision_depth: u16, + ) -> Option<&'ll llvm::Value> { + self.mcdc_condition_bitmap_map + .borrow() + .get(instance) + .and_then(|bitmap_map| bitmap_map.get(decision_depth as usize)) + .copied() // Dereference Option<&&Value> to Option<&Value> } } @@ -80,6 +92,42 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn init_coverage(&mut self, instance: Instance<'tcx>) { + let Some(function_coverage_info) = + self.tcx.instance_mir(instance.def).function_coverage_info.as_deref() + else { + return; + }; + + // If there are no MC/DC bitmaps to set up, return immediately. + if function_coverage_info.mcdc_bitmap_bytes == 0 { + return; + } + + let fn_name = self.get_pgo_func_name_var(instance); + let hash = self.const_u64(function_coverage_info.function_source_hash); + let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes); + self.mcdc_parameters(fn_name, hash, bitmap_bytes); + + // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps. + let mut cond_bitmaps = vec![]; + for i in 0..function_coverage_info.mcdc_num_condition_bitmaps { + // MC/DC intrinsics will perform loads/stores that use the ABI default + // alignment for i32, so our variable declaration should match. + let align = self.tcx.data_layout.i32_align.abi; + let cond_bitmap = self.alloca(Size::from_bytes(4), align); + llvm::set_value_name(cond_bitmap, format!("mcdc.addr.{i}").as_bytes()); + self.store(self.const_i32(0), cond_bitmap, align); + cond_bitmaps.push(cond_bitmap); + } + + self.coverage_context() + .expect("always present when coverage is enabled") + .mcdc_condition_bitmap_map + .borrow_mut() + .insert(instance, cond_bitmaps); + } + #[instrument(level = "debug", skip(self))] fn add_coverage(&mut self, instance: Instance<'tcx>, kind: &CoverageKind) { // Our caller should have already taken care of inlining subtleties, @@ -98,10 +146,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - if function_coverage_info.mcdc_bitmap_bytes > 0 { - ensure_mcdc_parameters(bx, instance, function_coverage_info); - } - let Some(coverage_context) = bx.coverage_context() else { return }; let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); let func_coverage = coverage_map @@ -143,7 +187,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::ExpressionUsed { id } => { func_coverage.mark_expression_id_seen(id); } - CoverageKind::CondBitmapUpdate { id, value, .. } => { + CoverageKind::CondBitmapUpdate { id, value, decision_depth } => { drop(coverage_map); assert_ne!( id.as_u32(), @@ -151,7 +195,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { "ConditionId of evaluated conditions should never be zero" ); let cond_bitmap = coverage_context - .try_get_mcdc_condition_bitmap(&instance) + .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); let cond_loc = bx.const_i32(id.as_u32() as i32 - 1); let bool_value = bx.const_bool(value); @@ -159,18 +203,13 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let hash = bx.const_u64(function_coverage_info.function_source_hash); bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value); } - CoverageKind::TestVectorBitmapUpdate { bitmap_idx } => { + CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { drop(coverage_map); let cond_bitmap = coverage_context - .try_get_mcdc_condition_bitmap(&instance) + .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); - let bitmap_bytes = bx.tcx().coverage_ids_info(instance.def).mcdc_bitmap_bytes; + let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes; assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range"); - assert!( - bitmap_bytes <= function_coverage_info.mcdc_bitmap_bytes, - "bitmap length disagreement: query says {bitmap_bytes} but function info only has {}", - function_coverage_info.mcdc_bitmap_bytes - ); let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); @@ -182,27 +221,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } -fn ensure_mcdc_parameters<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - instance: Instance<'tcx>, - function_coverage_info: &FunctionCoverageInfo, -) { - let Some(cx) = bx.coverage_context() else { return }; - if cx.mcdc_condition_bitmap_map.borrow().contains_key(&instance) { - return; - } - - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_coverage_info.function_source_hash); - let bitmap_bytes = bx.const_u32(function_coverage_info.mcdc_bitmap_bytes); - let cond_bitmap = bx.mcdc_parameters(fn_name, hash, bitmap_bytes); - bx.coverage_context() - .expect("already checked above") - .mcdc_condition_bitmap_map - .borrow_mut() - .insert(instance, cond_bitmap); -} - /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index e5fecddec52..10d3c0d0e74 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -41,6 +41,7 @@ use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; +use tracing::{debug, instrument}; use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; @@ -1281,7 +1282,7 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( let mut names = generics .parent .map_or_else(Vec::new, |def_id| get_parameter_names(cx, cx.tcx.generics_of(def_id))); - names.extend(generics.params.iter().map(|param| param.name)); + names.extend(generics.own_params.iter().map(|param| param.name)); names } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 657e9ce998f..bacd74f430f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -122,10 +122,7 @@ fn tag_base_type<'ll, 'tcx>( // Niche tags are always normalized to unsized integers of the correct size. match tag.primitive() { Primitive::Int(t, _) => t, - Primitive::F16 => Integer::I16, - Primitive::F32 => Integer::I32, - Primitive::F64 => Integer::I64, - Primitive::F128 => Integer::I128, + Primitive::Float(f) => Integer::from_size(f.size()).unwrap(), // FIXME(erikdesjardins): handle non-default addrspace ptr sizes Primitive::Pointer(_) => { // If the niche is the NULL value of a reference, then `discr_enum_ty` will be diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 1aec65cf949..e521d5e259c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -5,6 +5,7 @@ use rustc_data_structures::{ fx::FxHashMap, stable_hasher::{HashStable, StableHasher}, }; +use rustc_macros::HashStable; use rustc_middle::{ bug, ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}, @@ -23,6 +24,8 @@ use crate::{ use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER}; mod private { + use rustc_macros::HashStable; + // This type cannot be constructed outside of this module because // it has a private field. We make use of this in order to prevent // `UniqueTypeId` from being constructed directly, without asserting diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 4fdaa59e0e5..3486ce4becb 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -42,6 +42,7 @@ use std::cell::OnceCell; use std::cell::RefCell; use std::iter; use std::ops::Range; +use tracing::debug; mod create_scope_map; pub mod gdb; @@ -512,7 +513,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let mut names = generics.parent.map_or_else(Vec::new, |def_id| { get_parameter_names(cx, cx.tcx.generics_of(def_id)) }); - names.extend(generics.params.iter().map(|param| param.name)); + names.extend(generics.own_params.iter().map(|param| param.name)); names } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index c758010c581..155e7a89fd8 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -6,7 +6,7 @@ use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; use rustc_middle::ty::{self, Ty}; -use trace; +use tracing::trace; use crate::common::CodegenCx; use crate::llvm; diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 7117c4a0ed9..bf86d0e0569 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -24,6 +24,7 @@ use rustc_data_structures::fx::FxIndexSet; use rustc_middle::ty::{Instance, Ty}; use rustc_sanitizers::{cfi, kcfi}; use smallvec::SmallVec; +use tracing::debug; /// Declare a function. /// diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 41347333ea6..3d2ce550b23 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -14,12 +14,14 @@ use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; +use rustc_middle::mir::BinOp; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, Span, Symbol}; -use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size}; +use rustc_target::abi::{self, Align, Float, HasDataLayout, Primitive, Size}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; +use tracing::debug; use std::cmp::Ordering; @@ -231,13 +233,17 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { emit_va_arg(self, args[0], ret_ty) } } - Primitive::F16 => bug!("the va_arg intrinsic does not work with `f16`"), - Primitive::F64 | Primitive::Pointer(_) => { + Primitive::Float(Float::F16) => { + bug!("the va_arg intrinsic does not work with `f16`") + } + Primitive::Float(Float::F64) | Primitive::Pointer(_) => { emit_va_arg(self, args[0], ret_ty) } // `va_arg` should never be used with the return type f32. - Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), - Primitive::F128 => { + Primitive::Float(Float::F32) => { + bug!("the va_arg intrinsic does not work with `f32`") + } + Primitive::Float(Float::F128) => { bug!("the va_arg intrinsic does not work with `f128`") } } @@ -984,9 +990,9 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx, ty::Binder::dummy(tcx.mk_fn_sig( [i8p], - Ty::new_unit(tcx), + tcx.types.unit, false, - hir::Unsafety::Unsafe, + hir::Safety::Unsafe, Abi::Rust, )), ); @@ -995,9 +1001,9 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx, ty::Binder::dummy(tcx.mk_fn_sig( [i8p, i8p], - Ty::new_unit(tcx), + tcx.types.unit, false, - hir::Unsafety::Unsafe, + hir::Safety::Unsafe, Abi::Rust, )), ); @@ -1006,7 +1012,7 @@ fn get_rust_try_fn<'ll, 'tcx>( [try_fn_ty, i8p, catch_fn_ty], tcx.types.i32, false, - hir::Unsafety::Unsafe, + hir::Safety::Unsafe, Abi::Rust, )); let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); @@ -1100,12 +1106,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let in_ty = arg_tys[0]; let comparison = match name { - sym::simd_eq => Some(hir::BinOpKind::Eq), - sym::simd_ne => Some(hir::BinOpKind::Ne), - sym::simd_lt => Some(hir::BinOpKind::Lt), - sym::simd_le => Some(hir::BinOpKind::Le), - sym::simd_gt => Some(hir::BinOpKind::Gt), - sym::simd_ge => Some(hir::BinOpKind::Ge), + sym::simd_eq => Some(BinOp::Eq), + sym::simd_ne => Some(BinOp::Ne), + sym::simd_lt => Some(BinOp::Lt), + sym::simd_le => Some(BinOp::Le), + sym::simd_gt => Some(BinOp::Gt), + sym::simd_ge => Some(BinOp::Ge), _ => None, }; @@ -2332,7 +2338,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // Unary integer intrinsics - if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) { + if matches!( + name, + sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz + ) { let vec_ty = bx.cx.type_vector( match *in_elem.kind() { ty::Int(i) => bx.cx.type_int_from_ty(i), @@ -2350,31 +2359,38 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_bswap => "bswap", sym::simd_bitreverse => "bitreverse", sym::simd_ctlz => "ctlz", + sym::simd_ctpop => "ctpop", sym::simd_cttz => "cttz", _ => unreachable!(), }; let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits(); let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,); - return if name == sym::simd_bswap && int_size == 8 { + return match name { // byte swap is no-op for i8/u8 - Ok(args[0].immediate()) - } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) { - let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call( - fn_ty, - None, - None, - f, - &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], - None, - None, - )) - } else { - let fn_ty = bx.type_func(&[vec_ty], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()), + sym::simd_ctlz | sym::simd_cttz => { + // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison` + let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); + let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call( + fn_ty, + None, + None, + f, + &[args[0].immediate(), dont_poison_on_zero], + None, + None, + )) + } + sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => { + // simple unary argument cases + let fn_ty = bx.type_func(&[vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + } + _ => unreachable!(), }; } @@ -2383,7 +2399,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { span_bug!(span, "must be called with a vector of pointer types as first argument") }); - let layout = bx.layout_of(pointee.ty); + let layout = bx.layout_of(pointee); let ptrs = args[0].immediate(); // The second argument must be a ptr-sized integer. // (We don't care about the signedness, this is wrapping anyway.) diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index c84461e53eb..0029ec9d09a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -15,11 +15,6 @@ #![feature(let_chains)] #![feature(impl_trait_in_assoc_type)] -#[macro_use] -extern crate rustc_macros; -#[macro_use] -extern crate tracing; - use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; @@ -242,8 +237,11 @@ impl WriteBackendMethods for LlvmCodegenBackend { ) -> Result<CompiledModule, FatalError> { back::write::codegen(cgcx, dcx, module, config) } - fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module) + fn prepare_thin( + module: ModuleCodegen<Self::Module>, + emit_summary: bool, + ) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module, emit_summary) } fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer) { (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index a10dc61967e..132e1f9e8fd 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -200,6 +200,8 @@ pub enum AttributeKind { AllocAlign = 39, SanitizeSafeStack = 40, FnRetThunkExtern = 41, + Writable = 42, + DeadOnUnwind = 43, } /// LLVMIntPredicate @@ -2348,10 +2350,16 @@ extern "C" { #[allow(improper_ctypes)] pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); - pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; + pub fn LLVMRustThinLTOBufferCreate( + M: &Module, + is_thin: bool, + emit_summary: bool, + ) -> &'static mut ThinLTOBuffer; pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; + pub fn LLVMRustThinLTOBufferThinLinkDataPtr(M: &ThinLTOBuffer) -> *const c_char; + pub fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; pub fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, NumModules: c_uint, diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 29100a64171..a7df08421a3 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -13,6 +13,7 @@ use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; +use tracing::debug; impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 40ed6baa610..011d8ab57c7 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -6,8 +6,9 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty, TypeVisitableExt}; use rustc_target::abi::{Abi, Align, FieldsShape}; -use rustc_target::abi::{Int, Pointer, F128, F16, F32, F64}; +use rustc_target::abi::{Float, Int, Pointer}; use rustc_target::abi::{Scalar, Size, Variants}; +use tracing::debug; use std::fmt::Write; @@ -272,10 +273,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type { match scalar.primitive() { Int(i, _) => cx.type_from_integer(i), - F16 => cx.type_f16(), - F32 => cx.type_f32(), - F64 => cx.type_f64(), - F128 => cx.type_f128(), + Float(f) => cx.type_from_float(f), Pointer(address_space) => cx.type_ptr_ext(address_space), } } |