diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 34 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 92 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 24 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 |
8 files changed, 107 insertions, 70 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index c0b43b77897..123aef0f310 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -2,6 +2,7 @@ use crate::attributes; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -227,7 +228,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { // when passed by value, making it larger. let copy_bytes = cmp::min(scratch_size.bytes(), self.layout.size.bytes()); // Allocate some scratch space... - let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); + let llscratch = bx.alloca(scratch_size, scratch_align); bx.lifetime_start(llscratch, scratch_size); // ...store the value... bx.store(val, llscratch, scratch_align); @@ -425,6 +426,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()), ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (18, 0, 0) + { + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[ + llvm::AttributeKind::Writable.create_attr(cx.llcx), + llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx), + ], + ); + } } PassMode::Cast { cast, pad_i32: _ } => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 4c2bdb2f5ec..1a1b4ae3831 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -17,7 +17,7 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; @@ -468,9 +468,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { val } - fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { + fn alloca(&mut self, size: Size, align: Align) -> &'ll Value { let mut bx = Builder::with_cx(self.cx); bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); + let ty = self.cx().type_array(self.cx().type_i8(), size.bytes()); unsafe { let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); @@ -478,10 +479,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn byte_array_alloca(&mut self, len: &'ll Value, align: Align) -> &'ll Value { + fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value { unsafe { let alloca = - llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), len, UNNAMED); + llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); alloca } @@ -1703,12 +1704,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi_bundle } + /// Emits a call to `llvm.instrprof.mcdc.parameters`. + /// + /// This doesn't produce any code directly, but is used as input by + /// the LLVM pass that handles coverage instrumentation. + /// + /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.) + /// + /// [`CodeGenPGO::emitMCDCParameters`]: + /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 pub(crate) fn mcdc_parameters( &mut self, fn_name: &'ll Value, hash: &'ll Value, bitmap_bytes: &'ll Value, - ) -> &'ll Value { + ) { debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); @@ -1731,17 +1741,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { [].as_ptr(), 0 as c_uint, ); - // Create condition bitmap named `mcdc.addr`. - let mut bx = Builder::with_cx(self.cx); - bx.position_at_start(llvm::LLVMGetFirstBasicBlock(self.llfn())); - let cond_bitmap = { - let alloca = - llvm::LLVMBuildAlloca(bx.llbuilder, bx.cx.type_i32(), c"mcdc.addr".as_ptr()); - llvm::LLVMSetAlignment(alloca, 4); - alloca - }; - bx.store(self.const_i32(0), cond_bitmap, self.tcx().data_layout.i32_align.abi); - cond_bitmap } } @@ -1784,8 +1783,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { 0 as c_uint, ); } - let i32_align = self.tcx().data_layout.i32_align.abi; - self.store(self.const_i32(0), mcdc_temp, i32_align); + self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } pub(crate) fn mcdc_condbitmap_update( diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index d32baa6dc02..653abb4e41b 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -11,7 +11,8 @@ use crate::value::Value; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::base_n; +use rustc_data_structures::base_n::ToBaseN; +use rustc_data_structures::base_n::ALPHANUMERIC_ONLY; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; @@ -1015,7 +1016,7 @@ impl CodegenCx<'_, '_> { let mut name = String::with_capacity(prefix.len() + 6); name.push_str(prefix); name.push('.'); - base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name); + name.push_str(&(idx as u64).to_base(ALPHANUMERIC_ONLY)); name } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 085ce15d81f..c51a7744a30 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -13,10 +13,10 @@ use rustc_codegen_ssa::traits::{ use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::{CoverageKind, FunctionCoverageInfo}; +use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::Instance; -use rustc_target::abi::Align; +use rustc_target::abi::{Align, Size}; use std::cell::RefCell; @@ -30,7 +30,7 @@ pub struct CrateCoverageContext<'ll, 'tcx> { pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, - pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, + pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -49,9 +49,20 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is called condition bitmap. - /// This value is named `mcdc.addr` (same as clang) and is a 32-bit integer. - fn try_get_mcdc_condition_bitmap(&self, instance: &Instance<'tcx>) -> Option<&'ll llvm::Value> { - self.mcdc_condition_bitmap_map.borrow().get(instance).copied() + /// In order to handle nested decisions, several condition bitmaps can be + /// allocated for a function body. + /// These values are named `mcdc.addr.{i}` and are a 32-bit integers. + /// They respectively hold the condition bitmaps for decisions with a depth of `i`. + fn try_get_mcdc_condition_bitmap( + &self, + instance: &Instance<'tcx>, + decision_depth: u16, + ) -> Option<&'ll llvm::Value> { + self.mcdc_condition_bitmap_map + .borrow() + .get(instance) + .and_then(|bitmap_map| bitmap_map.get(decision_depth as usize)) + .copied() // Dereference Option<&&Value> to Option<&Value> } } @@ -80,6 +91,42 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn init_coverage(&mut self, instance: Instance<'tcx>) { + let Some(function_coverage_info) = + self.tcx.instance_mir(instance.def).function_coverage_info.as_deref() + else { + return; + }; + + // If there are no MC/DC bitmaps to set up, return immediately. + if function_coverage_info.mcdc_bitmap_bytes == 0 { + return; + } + + let fn_name = self.get_pgo_func_name_var(instance); + let hash = self.const_u64(function_coverage_info.function_source_hash); + let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes); + self.mcdc_parameters(fn_name, hash, bitmap_bytes); + + // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps. + let mut cond_bitmaps = vec![]; + for i in 0..function_coverage_info.mcdc_num_condition_bitmaps { + // MC/DC intrinsics will perform loads/stores that use the ABI default + // alignment for i32, so our variable declaration should match. + let align = self.tcx.data_layout.i32_align.abi; + let cond_bitmap = self.alloca(Size::from_bytes(4), align); + llvm::set_value_name(cond_bitmap, format!("mcdc.addr.{i}").as_bytes()); + self.store(self.const_i32(0), cond_bitmap, align); + cond_bitmaps.push(cond_bitmap); + } + + self.coverage_context() + .expect("always present when coverage is enabled") + .mcdc_condition_bitmap_map + .borrow_mut() + .insert(instance, cond_bitmaps); + } + #[instrument(level = "debug", skip(self))] fn add_coverage(&mut self, instance: Instance<'tcx>, kind: &CoverageKind) { // Our caller should have already taken care of inlining subtleties, @@ -98,10 +145,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - if function_coverage_info.mcdc_bitmap_bytes > 0 { - ensure_mcdc_parameters(bx, instance, function_coverage_info); - } - let Some(coverage_context) = bx.coverage_context() else { return }; let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); let func_coverage = coverage_map @@ -143,7 +186,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::ExpressionUsed { id } => { func_coverage.mark_expression_id_seen(id); } - CoverageKind::CondBitmapUpdate { id, value, .. } => { + CoverageKind::CondBitmapUpdate { id, value, decision_depth } => { drop(coverage_map); assert_ne!( id.as_u32(), @@ -151,7 +194,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { "ConditionId of evaluated conditions should never be zero" ); let cond_bitmap = coverage_context - .try_get_mcdc_condition_bitmap(&instance) + .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); let cond_loc = bx.const_i32(id.as_u32() as i32 - 1); let bool_value = bx.const_bool(value); @@ -159,10 +202,10 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let hash = bx.const_u64(function_coverage_info.function_source_hash); bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value); } - CoverageKind::TestVectorBitmapUpdate { bitmap_idx } => { + CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { drop(coverage_map); let cond_bitmap = coverage_context - .try_get_mcdc_condition_bitmap(&instance) + .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); let bitmap_bytes = bx.tcx().coverage_ids_info(instance.def).mcdc_bitmap_bytes; assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range"); @@ -182,27 +225,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } -fn ensure_mcdc_parameters<'ll, 'tcx>( - bx: &mut Builder<'_, 'll, 'tcx>, - instance: Instance<'tcx>, - function_coverage_info: &FunctionCoverageInfo, -) { - let Some(cx) = bx.coverage_context() else { return }; - if cx.mcdc_condition_bitmap_map.borrow().contains_key(&instance) { - return; - } - - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_coverage_info.function_source_hash); - let bitmap_bytes = bx.const_u32(function_coverage_info.mcdc_bitmap_bytes); - let cond_bitmap = bx.mcdc_parameters(fn_name, hash, bitmap_bytes); - bx.coverage_context() - .expect("already checked above") - .mcdc_condition_bitmap_map - .borrow_mut() - .insert(instance, cond_bitmap); -} - /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 1aec65cf949..e521d5e259c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -5,6 +5,7 @@ use rustc_data_structures::{ fx::FxHashMap, stable_hasher::{HashStable, StableHasher}, }; +use rustc_macros::HashStable; use rustc_middle::{ bug, ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}, @@ -23,6 +24,8 @@ use crate::{ use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER}; mod private { + use rustc_macros::HashStable; + // This type cannot be constructed outside of this module because // it has a private field. We make use of this in order to prevent // `UniqueTypeId` from being constructed directly, without asserting diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index ab9f20fdf63..56550dbfa4b 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -18,7 +18,7 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, Span, Symbol}; -use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; +use rustc_target::abi::{self, Align, HasDataLayout, Primitive, Size}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; @@ -649,8 +649,9 @@ fn codegen_msvc_try<'ll>( // } // // More information can be found in libstd's seh.rs implementation. + let ptr_size = bx.tcx().data_layout.pointer_size; let ptr_align = bx.tcx().data_layout.pointer_align.abi; - let slot = bx.alloca(bx.type_ptr(), ptr_align); + let slot = bx.alloca(ptr_size, ptr_align); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); @@ -920,15 +921,14 @@ fn codegen_emcc_try<'ll>( // We need to pass two values to catch_func (ptr and is_rust_panic), so // create an alloca and pass a pointer to that. + let ptr_size = bx.tcx().data_layout.pointer_size; let ptr_align = bx.tcx().data_layout.pointer_align.abi; let i8_align = bx.tcx().data_layout.i8_align.abi; - let catch_data_type = bx.type_struct(&[bx.type_ptr(), bx.type_bool()], false); - let catch_data = bx.alloca(catch_data_type, ptr_align); - let catch_data_0 = - bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); - bx.store(ptr, catch_data_0, ptr_align); - let catch_data_1 = - bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); + // Required in order for there to be no padding between the fields. + assert!(i8_align <= ptr_align); + let catch_data = bx.alloca(2 * ptr_size, ptr_align); + bx.store(ptr, catch_data, ptr_align); + let catch_data_1 = bx.inbounds_ptradd(catch_data, bx.const_usize(ptr_size.bytes())); bx.store(is_rust_panic, catch_data_1, i8_align); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); @@ -984,7 +984,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx, ty::Binder::dummy(tcx.mk_fn_sig( [i8p], - Ty::new_unit(tcx), + tcx.types.unit, false, hir::Unsafety::Unsafe, Abi::Rust, @@ -995,7 +995,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx, ty::Binder::dummy(tcx.mk_fn_sig( [i8p, i8p], - Ty::new_unit(tcx), + tcx.types.unit, false, hir::Unsafety::Unsafe, Abi::Rust, @@ -1374,7 +1374,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); // Convert the integer to a byte array - let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); + let ptr = bx.alloca(Size::from_bytes(expected_bytes), Align::ONE); bx.store(ze, ptr, Align::ONE); let array_ty = bx.type_array(bx.type_i8(), expected_bytes); return Ok(bx.load(array_ty, ptr, Align::ONE)); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index c84461e53eb..1cecf682e5d 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -16,8 +16,6 @@ #![feature(impl_trait_in_assoc_type)] #[macro_use] -extern crate rustc_macros; -#[macro_use] extern crate tracing; use back::owned_target_machine::OwnedTargetMachine; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index a10dc61967e..7a34e21628d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -200,6 +200,8 @@ pub enum AttributeKind { AllocAlign = 39, SanitizeSafeStack = 40, FnRetThunkExtern = 41, + Writable = 42, + DeadOnUnwind = 43, } /// LLVMIntPredicate |