diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 169 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 38 |
8 files changed, 68 insertions, 159 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index c6a7dc95d77..863cb7068f8 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -571,7 +571,9 @@ impl From<Conv> for llvm::CallConv { Conv::C | Conv::Rust | Conv::CCmseNonSecureCall | Conv::RiscvInterrupt { .. } => { llvm::CCallConv } - Conv::RustCold => llvm::ColdCallConv, + Conv::Cold => llvm::ColdCallConv, + Conv::PreserveMost => llvm::PreserveMost, + Conv::PreserveAll => llvm::PreserveAll, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index a82d2c5771a..f33075a8879 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -367,7 +367,7 @@ impl<'a> LlvmArchiveBuilder<'a> { match addition { Addition::File { path, name_in_archive } => { let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.clone())?; + let name = CString::new(name_in_archive.as_bytes())?; members.push(llvm::LLVMRustArchiveMemberNew( path.as_ptr(), name.as_ptr(), diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index b2d28cef899..c1d3392386c 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -441,7 +441,7 @@ fn thin_lto( for (i, (name, buffer)) in modules.into_iter().enumerate() { info!("local module: {} - {}", i, name); - let cname = CString::new(name.clone()).unwrap(); + let cname = CString::new(name.as_bytes()).unwrap(); thin_modules.push(llvm::ThinLTOModule { identifier: cname.as_ptr(), data: buffer.data().as_ptr(), diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 24fd5bbf8c5..8e8290279ab 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -10,6 +10,7 @@ use crate::value::Value; use cstr::cstr; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; +use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; @@ -1000,7 +1001,7 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { - span_bug!(span, "failed to get layout for `{ty}`: {err:?}") + self.tcx.sess.emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 621fd36b2a3..c70cb670e96 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -16,7 +16,7 @@ use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::{CodeRegion, CounterId, CoverageKind, ExpressionId, Op, Operand}; +use rustc_middle::mir::coverage::{CounterId, CoverageKind}; use rustc_middle::mir::Coverage; use rustc_middle::ty; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; @@ -104,144 +104,67 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { let bx = self; + let Some(coverage_context) = bx.coverage_context() else { return }; + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + let func_coverage = coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance)); + let Coverage { kind, code_region } = coverage.clone(); match kind { CoverageKind::Counter { function_source_hash, id } => { - if bx.set_function_source_hash(instance, function_source_hash) { - // If `set_function_source_hash()` returned true, the coverage map is enabled, - // so continue adding the counter. - if let Some(code_region) = code_region { - // Note: Some counters do not have code regions, but may still be referenced - // from expressions. In that case, don't add the counter to the coverage map, - // but do inject the counter intrinsic. - bx.add_coverage_counter(instance, id, code_region); - } - - let coverageinfo = bx.tcx().coverageinfo(instance.def); - - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_source_hash); - let num_counters = bx.const_u32(coverageinfo.num_counters); - let index = bx.const_u32(id.as_u32()); + debug!( + "ensuring function source hash is set for instance={:?}; function_source_hash={}", + instance, function_source_hash, + ); + func_coverage.set_function_source_hash(function_source_hash); + + if let Some(code_region) = code_region { + // Note: Some counters do not have code regions, but may still be referenced + // from expressions. In that case, don't add the counter to the coverage map, + // but do inject the counter intrinsic. debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", - fn_name, hash, num_counters, index, + "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", + instance, id, code_region, ); - bx.instrprof_increment(fn_name, hash, num_counters, index); + func_coverage.add_counter(id, code_region); } + // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, + // as that needs an exclusive borrow. + drop(coverage_map); + + let coverageinfo = bx.tcx().coverageinfo(instance.def); + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_source_hash); + let num_counters = bx.const_u32(coverageinfo.num_counters); + let index = bx.const_u32(id.as_u32()); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, index, + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); } CoverageKind::Expression { id, lhs, op, rhs } => { - bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); + debug!( + "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; region: {:?}", + instance, id, lhs, op, rhs, code_region, + ); + func_coverage.add_counter_expression(id, lhs, op, rhs, code_region); } CoverageKind::Unreachable => { - bx.add_coverage_unreachable( - instance, - code_region.expect("unreachable regions always have code regions"), + let code_region = + code_region.expect("unreachable regions always have code regions"); + debug!( + "adding unreachable code to coverage_map: instance={:?}, at {:?}", + instance, code_region, ); + func_coverage.add_unreachable_region(code_region); } } } } -// These methods used to be part of trait `CoverageInfoBuilderMethods`, but -// after moving most coverage code out of SSA they are now just ordinary methods. -impl<'tcx> Builder<'_, '_, 'tcx> { - /// Returns true if the function source hash was added to the coverage map (even if it had - /// already been added, for this instance). Returns false *only* if `-C instrument-coverage` is - /// not enabled (a coverage map is not being generated). - fn set_function_source_hash( - &mut self, - instance: Instance<'tcx>, - function_source_hash: u64, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "ensuring function source hash is set for instance={:?}; function_source_hash={}", - instance, function_source_hash, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .set_function_source_hash(function_source_hash); - true - } else { - false - } - } - - /// Returns true if the counter was added to the coverage map; false if `-C instrument-coverage` - /// is not enabled (a coverage map is not being generated). - fn add_coverage_counter( - &mut self, - instance: Instance<'tcx>, - id: CounterId, - region: CodeRegion, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", - instance, id, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter(id, region); - true - } else { - false - } - } - - /// Returns true if the expression was added to the coverage map; false if - /// `-C instrument-coverage` is not enabled (a coverage map is not being generated). - fn add_coverage_counter_expression( - &mut self, - instance: Instance<'tcx>, - id: ExpressionId, - lhs: Operand, - op: Op, - rhs: Operand, - region: Option<CodeRegion>, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ - region: {:?}", - instance, id, lhs, op, rhs, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter_expression(id, lhs, op, rhs, region); - true - } else { - false - } - } - - /// Returns true if the region was added to the coverage map; false if `-C instrument-coverage` - /// is not enabled (a coverage map is not being generated). - fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding unreachable code to coverage_map: instance={:?}, at {:?}", - instance, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_unreachable_region(region); - true - } else { - false - } - } -} - fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { let tcx = cx.tcx; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index f8cbcbd5ec8..ed938761694 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -445,9 +445,9 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) } - // Box<T, A> may have a non-ZST allocator A. In that case, we + // Box<T, A> may have a non-1-ZST allocator A. In that case, we // cannot treat Box<T, A> as just an owned alias of `*mut T`. - ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_zst() => { + ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_1zst() => { build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 84157d1e25c..01cbf7d3b11 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -83,12 +83,17 @@ pub enum LLVMModFlagBehavior { // Consts for the LLVM CallConv type, pre-cast to usize. /// LLVM CallingConv::ID. Should we wrap this? +/// +/// See <https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/CallingConv.h> #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] pub enum CallConv { CCallConv = 0, FastCallConv = 8, ColdCallConv = 9, + PreserveMost = 14, + PreserveAll = 15, + Tail = 18, X86StdcallCallConv = 64, X86FastcallCallConv = 65, ArmAapcsCallConv = 67, diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 831645579b9..dcc62d314ff 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -3,7 +3,7 @@ use crate::context::TypeLowering; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty, TypeVisitableExt}; use rustc_target::abi::HasDataLayout; @@ -215,20 +215,16 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + // This must produce the same result for `repr(transparent)` wrappers as for the inner type! + // In other words, this should generally not look at the type at all, but only at the + // layout. if let Abi::Scalar(scalar) = self.abi { // Use a different cache for scalars because pointers to DSTs // can be either fat or thin (data pointers of fat pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { return llty; } - let llty = match *self.ty.kind() { - ty::Ref(..) | ty::RawPtr(_) => cx.type_ptr(), - ty::Adt(def, _) if def.is_box() => cx.type_ptr(), - ty::FnPtr(sig) => { - cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty())) - } - _ => self.scalar_llvm_type_at(cx, scalar), - }; + let llty = self.scalar_llvm_type_at(cx, scalar); cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); return llty; } @@ -303,27 +299,9 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { index: usize, immediate: bool, ) -> &'a Type { - // HACK(eddyb) special-case fat pointers until LLVM removes - // pointee types, to avoid bitcasting every `OperandRef::deref`. - match *self.ty.kind() { - ty::Ref(..) | ty::RawPtr(_) => { - return self.field(cx, index).llvm_type(cx); - } - // only wide pointer boxes are handled as pointers - // thin pointer boxes with scalar allocators are handled by the general logic below - ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_zst() => { - let ptr_ty = Ty::new_mut_ptr(cx.tcx, self.ty.boxed_ty()); - return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); - } - // `dyn* Trait` has the same ABI as `*mut dyn Trait` - ty::Dynamic(bounds, region, ty::DynStar) => { - let ptr_ty = - Ty::new_mut_ptr(cx.tcx, Ty::new_dynamic(cx.tcx, bounds, region, ty::Dyn)); - return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); - } - _ => {} - } - + // This must produce the same result for `repr(transparent)` wrappers as for the inner type! + // In other words, this should generally not look at the type at all, but only at the + // layout. let Abi::ScalarPair(a, b) = self.abi else { bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); }; |