diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
39 files changed, 1698 insertions, 1873 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 3d75393bf06..1d35138b013 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,19 +1,21 @@ use std::cmp; use libc::c_uint; +use rustc_abi as abi; +pub(crate) use rustc_abi::ExternAbi; +use rustc_abi::Primitive::Int; +use rustc_abi::{HasDataLayout, Size}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::LayoutOf; -pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; +pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA}; use rustc_middle::{bug, ty}; use rustc_session::config; -pub(crate) use rustc_target::abi::call::*; -use rustc_target::abi::{self, HasDataLayout, Int, Size}; +pub(crate) use rustc_target::callconv::*; use rustc_target::spec::SanitizerSet; -pub(crate) use rustc_target::spec::abi::Abi; use smallvec::SmallVec; use crate::attributes::llfn_attrs_from_instance; @@ -413,7 +415,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { instance: Option<ty::Instance<'tcx>>, ) { let mut func_attrs = SmallVec::<[_; 3]>::new(); - if self.ret.layout.abi.is_uninhabited() { + if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); } if !self.can_unwind { @@ -434,7 +436,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i - 1 }; - let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| { + let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| { if cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() >= (19, 0, 0) && matches!(scalar.primitive(), Int(..)) @@ -456,7 +458,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { match &self.ret.mode { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); - if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + if let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr { apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); } } @@ -493,7 +495,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(attrs) => { let i = apply(attrs); - if let abi::Abi::Scalar(scalar) = arg.layout.abi { + if let abi::BackendRepr::Scalar(scalar) = arg.layout.backend_repr { apply_range_attr(llvm::AttributePlace::Argument(i), scalar); } } @@ -508,7 +510,9 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Pair(a, b) => { let i = apply(a); let ii = apply(b); - if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi { + if let abi::BackendRepr::ScalarPair(scalar_a, scalar_b) = + arg.layout.backend_repr + { apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a); apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b); } @@ -530,7 +534,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { let mut func_attrs = SmallVec::<[_; 2]>::new(); - if self.ret.layout.abi.is_uninhabited() { + if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); } if !self.can_unwind { @@ -568,7 +572,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } if bx.cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() < (19, 0, 0) - && let abi::Abi::Scalar(scalar) = self.ret.layout.abi + && let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr && matches!(scalar.primitive(), Int(..)) // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 2adac278c62..149ded28356 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -7,6 +7,7 @@ use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; +use crate::common::AsCCharPtr; use crate::llvm::{self, Context, False, Module, True, Type}; use crate::{ModuleLlvm, attributes, debuginfo}; @@ -76,19 +77,15 @@ pub(crate) unsafe fn codegen( unsafe { // __rust_alloc_error_handler_should_panic let name = OomStrategy::SYMBOL; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.default_hidden_visibility() { - llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); - } + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); + llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); let name = NO_ALLOC_SHIM_IS_UNSTABLE; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.default_hidden_visibility() { - llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); - } + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); + llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let llval = llvm::LLVMConstInt(i8, 0, False); llvm::LLVMSetInitializer(ll_g, llval); } @@ -119,7 +116,7 @@ fn create_wrapper_function( ); let llfn = llvm::LLVMRustGetOrInsertFunction( llmod, - from_name.as_ptr().cast(), + from_name.as_c_char_ptr(), from_name.len(), ty, ); @@ -132,9 +129,8 @@ fn create_wrapper_function( None }; - if tcx.sess.default_hidden_visibility() { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } + llvm::set_visibility(llfn, llvm::Visibility::from_generic(tcx.sess.default_visibility())); + if tcx.sess.must_emit_unwind_tables() { let uwtable = attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); @@ -142,12 +138,12 @@ fn create_wrapper_function( } let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_ptr().cast(), to_name.len(), ty); + llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_c_char_ptr(), to_name.len(), ty); if let Some(no_return) = no_return { // -> ! DIFlagNoReturn attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); } - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + llvm::set_visibility(callee, llvm::Visibility::Hidden); let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr()); @@ -158,7 +154,7 @@ fn create_wrapper_function( .enumerate() .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::<Vec<_>>(); - let ret = llvm::LLVMRustBuildCall( + let ret = llvm::LLVMBuildCallWithOperandBundles( llbuilder, ty, callee, @@ -166,6 +162,7 @@ fn create_wrapper_function( args.len() as c_uint, [].as_ptr(), 0 as c_uint, + c"".as_ptr(), ); llvm::LLVMSetTailCall(ret, True); if output.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 7621f111fe2..bb74dfe1487 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -1,6 +1,7 @@ use std::assert_matches::assert_matches; use libc::{c_char, c_uint}; +use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar}; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::mir::operand::OperandValue; use rustc_codegen_ssa::traits::*; @@ -9,13 +10,12 @@ use rustc_middle::ty::Instance; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::{bug, span_bug}; use rustc_span::{Pos, Span, Symbol, sym}; -use rustc_target::abi::*; use rustc_target::asm::*; use smallvec::SmallVec; use tracing::debug; use crate::builder::Builder; -use crate::common::Funclet; +use crate::common::{AsCCharPtr, Funclet}; use crate::context::CodegenCx; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -154,7 +154,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // We prefer the latter because it matches the behavior of // Clang. if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) { - constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string()); + constraints.push(reg_to_llvm(reg, Some(&in_value.layout))); } else { constraints.push(format!("{}", op_idx[&idx])); } @@ -268,6 +268,15 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::S390x => { constraints.push("~{cc}".to_string()); } + InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => { + // In LLVM, ~{icc} represents icc and xcc in 64-bit code. + // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64 + constraints.push("~{icc}".to_string()); + constraints.push("~{fcc0}".to_string()); + constraints.push("~{fcc1}".to_string()); + constraints.push("~{fcc2}".to_string()); + constraints.push("~{fcc3}".to_string()); + } InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} @@ -420,7 +429,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { unsafe { llvm::LLVMAppendModuleInlineAsm( self.llmod, - template_str.as_ptr().cast(), + template_str.as_c_char_ptr(), template_str.len(), ); } @@ -458,14 +467,14 @@ pub(crate) fn inline_asm_call<'ll>( let fty = bx.cx.type_func(&argtys, output); unsafe { // Ask LLVM to verify that the constraints are well-formed. - let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); + let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_c_char_ptr(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { let v = llvm::LLVMRustInlineAsm( fty, - asm.as_ptr().cast(), + asm.as_c_char_ptr(), asm.len(), - cons.as_ptr().cast(), + cons.as_c_char_ptr(), cons.len(), volatile, alignstack, @@ -542,57 +551,16 @@ fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { /// If the register is an AArch64 integer register then return its index. fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> { - use AArch64InlineAsmReg::*; - // Unlike `a64_vreg_index`, we can't subtract `x0` to get the u32 because - // `x19` and `x29` are missing and the integer constants for the - // `x0`..`x30` enum variants don't all match the register number. E.g. the - // integer constant for `x18` is 18, but the constant for `x20` is 19. - Some(match reg { - InlineAsmReg::AArch64(r) => match r { - x0 => 0, - x1 => 1, - x2 => 2, - x3 => 3, - x4 => 4, - x5 => 5, - x6 => 6, - x7 => 7, - x8 => 8, - x9 => 9, - x10 => 10, - x11 => 11, - x12 => 12, - x13 => 13, - x14 => 14, - x15 => 15, - x16 => 16, - x17 => 17, - x18 => 18, - // x19 is reserved - x20 => 20, - x21 => 21, - x22 => 22, - x23 => 23, - x24 => 24, - x25 => 25, - x26 => 26, - x27 => 27, - x28 => 28, - // x29 is reserved - x30 => 30, - _ => return None, - }, - _ => return None, - }) + match reg { + InlineAsmReg::AArch64(r) => r.reg_index(), + _ => None, + } } /// If the register is an AArch64 vector register then return its index. fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { - use AArch64InlineAsmReg::*; match reg { - InlineAsmReg::AArch64(reg) if reg as u32 >= v0 as u32 && reg as u32 <= v31 as u32 => { - Some(reg as u32 - v0 as u32) - } + InlineAsmReg::AArch64(reg) => reg.vreg_index(), _ => None, } } @@ -679,7 +647,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg) => "r", PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC(PowerPCInlineAsmRegClass::cr) + | PowerPC(PowerPCInlineAsmRegClass::xer) + | PowerPC(PowerPCInlineAsmRegClass::vreg) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -708,6 +678,11 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> S390x(S390xInlineAsmRegClass::reg) => "r", S390x(S390xInlineAsmRegClass::reg_addr) => "a", S390x(S390xInlineAsmRegClass::freg) => "f", + S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => { + unreachable!("clobber-only") + } + Sparc(SparcInlineAsmRegClass::reg) => "r", + Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"), Msp430(Msp430InlineAsmRegClass::reg) => "r", M68k(M68kInlineAsmRegClass::reg) => "r", M68k(M68kInlineAsmRegClass::reg_addr) => "a", @@ -801,6 +776,7 @@ fn modifier_to_llvm( }, Avr(_) => None, S390x(_) => None, + Sparc(_) => None, Msp430(_) => None, SpirV(SpirVInlineAsmRegClass::reg) => bug!("LLVM backend does not support SPIR-V"), M68k(_) => None, @@ -838,7 +814,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC(PowerPCInlineAsmRegClass::cr) + | PowerPC(PowerPCInlineAsmRegClass::xer) + | PowerPC(PowerPCInlineAsmRegClass::vreg) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), @@ -866,6 +844,11 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(), S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), + S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => { + unreachable!("clobber-only") + } + Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(), + Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"), Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), @@ -915,8 +898,8 @@ fn llvm_fixup_input<'ll, 'tcx>( ) -> &'ll Value { use InlineAsmRegClass::*; let dl = &bx.tcx.data_layout; - match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + match (reg, layout.backend_repr) { + (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) @@ -924,7 +907,7 @@ fn llvm_fixup_input<'ll, 'tcx>( value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s)) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(bx.cx, s); @@ -937,7 +920,7 @@ fn llvm_fixup_input<'ll, 'tcx>( } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); @@ -945,14 +928,14 @@ fn llvm_fixup_input<'ll, 'tcx>( let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } - (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F64) => { bx.bitcast(value, bx.cx.type_i64()) } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, + BackendRepr::Vector { .. }, ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), ( X86( @@ -960,7 +943,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if bx.sess().asm_arch == Some(InlineAsmArch::X86) && s.primitive() == Primitive::Float(Float::F128) => { @@ -972,7 +955,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if s.primitive() == Primitive::Float(Float::F16) => { let value = bx.insert_element( bx.const_undef(bx.type_vector(bx.type_f16(), 8)), @@ -987,11 +970,14 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Vector { element, count: count @ (8 | 16) }, + BackendRepr::Vector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_i16(), count)) } - (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => { + ( + Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + BackendRepr::Scalar(s), + ) => { if let Primitive::Int(Integer::I32, _) = s.primitive() { bx.bitcast(value, bx.cx.type_f32()) } else { @@ -1004,7 +990,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | ArmInlineAsmRegClass::dreg_low8 | ArmInlineAsmRegClass::dreg_low16, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) => { if let Primitive::Int(Integer::I64, _) = s.primitive() { bx.bitcast(value, bx.cx.type_f64()) @@ -1021,11 +1007,11 @@ fn llvm_fixup_input<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - Abi::Vector { element, count: count @ (4 | 8) }, + BackendRepr::Vector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_i16(), count)) } - (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => { match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), @@ -1034,7 +1020,7 @@ fn llvm_fixup_input<'ll, 'tcx>( _ => value, } } - (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s)) + (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F16) && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) => { @@ -1057,15 +1043,15 @@ fn llvm_fixup_output<'ll, 'tcx>( instance: Instance<'_>, ) -> &'ll Value { use InlineAsmRegClass::*; - match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + match (reg, layout.backend_repr) { + (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { bx.extract_element(value, bx.const_i32(0)) } else { value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s)) if s.primitive() != Primitive::Float(Float::F128) => { value = bx.extract_element(value, bx.const_i32(0)); @@ -1074,7 +1060,7 @@ fn llvm_fixup_output<'ll, 'tcx>( } value } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); @@ -1082,14 +1068,14 @@ fn llvm_fixup_output<'ll, 'tcx>( let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } - (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F64) => { bx.bitcast(value, bx.cx.type_f64()) } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, + BackendRepr::Vector { .. }, ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), ( X86( @@ -1097,7 +1083,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if bx.sess().asm_arch == Some(InlineAsmArch::X86) && s.primitive() == Primitive::Float(Float::F128) => { @@ -1109,7 +1095,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if s.primitive() == Primitive::Float(Float::F16) => { let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8)); bx.extract_element(value, bx.const_usize(0)) @@ -1120,11 +1106,14 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Vector { element, count: count @ (8 | 16) }, + BackendRepr::Vector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_f16(), count)) } - (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => { + ( + Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + BackendRepr::Scalar(s), + ) => { if let Primitive::Int(Integer::I32, _) = s.primitive() { bx.bitcast(value, bx.cx.type_i32()) } else { @@ -1137,7 +1126,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | ArmInlineAsmRegClass::dreg_low8 | ArmInlineAsmRegClass::dreg_low16, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) => { if let Primitive::Int(Integer::I64, _) = s.primitive() { bx.bitcast(value, bx.cx.type_i64()) @@ -1154,11 +1143,11 @@ fn llvm_fixup_output<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - Abi::Vector { element, count: count @ (4 | 8) }, + BackendRepr::Vector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_f16(), count)) } - (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => { match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), @@ -1168,7 +1157,7 @@ fn llvm_fixup_output<'ll, 'tcx>( _ => value, } } - (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s)) + (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F16) && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) => { @@ -1188,35 +1177,35 @@ fn llvm_fixup_output_type<'ll, 'tcx>( instance: Instance<'_>, ) -> &'ll Type { use InlineAsmRegClass::*; - match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + match (reg, layout.backend_repr) { + (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { cx.type_vector(cx.type_i8(), 8) } else { layout.llvm_type(cx) } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s)) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(cx, s); let count = 16 / layout.size.bytes(); cx.type_vector(elem_ty, count) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(cx, element); cx.type_vector(elem_ty, count * 2) } - (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F64) => { cx.type_i64() } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, + BackendRepr::Vector { .. }, ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), ( X86( @@ -1224,7 +1213,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if cx.sess().asm_arch == Some(InlineAsmArch::X86) && s.primitive() == Primitive::Float(Float::F128) => { @@ -1236,7 +1225,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8), ( X86( @@ -1244,11 +1233,14 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Vector { element, count: count @ (8 | 16) }, + BackendRepr::Vector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { cx.type_vector(cx.type_i16(), count) } - (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => { + ( + Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + BackendRepr::Scalar(s), + ) => { if let Primitive::Int(Integer::I32, _) = s.primitive() { cx.type_f32() } else { @@ -1261,7 +1253,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | ArmInlineAsmRegClass::dreg_low8 | ArmInlineAsmRegClass::dreg_low16, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) => { if let Primitive::Int(Integer::I64, _) = s.primitive() { cx.type_f64() @@ -1278,11 +1270,11 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - Abi::Vector { element, count: count @ (4 | 8) }, + BackendRepr::Vector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { cx.type_vector(cx.type_i16(), count) } - (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => { match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), @@ -1291,7 +1283,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( _ => layout.llvm_type(cx), } } - (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s)) + (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F16) && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) => { diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 489259da856..cb958c1d4d7 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -6,12 +6,11 @@ use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; -use rustc_span::symbol::sym; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; use crate::context::CodegenCx; -use crate::errors::{MissingFeatures, SanitizerMemtagRequiresMte, TargetFeatureDisableOrEnable}; +use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::value::Value; @@ -233,11 +232,6 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { return None; } - // probestack doesn't play nice either with gcov profiling. - if cx.sess().opts.unstable_opts.profile { - return None; - } - let attr_value = match cx.sess().target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support @@ -425,7 +419,10 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } - if let Some(PacRet { leaf, key }) = pac_ret { + if let Some(PacRet { leaf, pc, key }) = pac_ret { + if pc { + to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); + } to_add.push(llvm::CreateAttrStringValue( cx.llcx, "sign-return-address", @@ -502,26 +499,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); - if let Some(f) = llvm_util::check_tied_features( - cx.tcx.sess, - &function_features.iter().map(|f| (*f, true)).collect(), - ) { - let span = cx - .tcx - .get_attrs(instance.def_id(), sym::target_feature) - .next() - .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); - cx.tcx - .dcx() - .create_err(TargetFeatureDisableOrEnable { - features: f, - span: Some(span), - missing_features: Some(MissingFeatures), - }) - .emit(); - return; - } - let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 1b4f6682af0..33a956e552f 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -123,7 +123,7 @@ fn get_llvm_object_symbols( llvm::LLVMRustGetSymbols( buf.as_ptr(), buf.len(), - std::ptr::addr_of_mut!(*state) as *mut c_void, + (&raw mut *state) as *mut c_void, callback, error_callback, ) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 1f7a923dd2c..48beb9be2b2 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -165,13 +165,14 @@ fn get_bitcode_slice_from_object_data<'a>( // We drop the "__LLVM," prefix here because on Apple platforms there's a notion of "segment // name" which in the public API for sections gets treated as part of the section name, but // internally in MachOObjectFile.cpp gets treated separately. - let section_name = bitcode_section_name(cgcx).trim_start_matches("__LLVM,"); + let section_name = bitcode_section_name(cgcx).to_str().unwrap().trim_start_matches("__LLVM,"); let mut len = 0; let data = unsafe { llvm::LLVMRustGetSliceFromObjectDataByName( obj.as_ptr(), obj.len(), section_name.as_ptr(), + section_name.len(), &mut len, ) }; @@ -502,9 +503,9 @@ fn thin_lto( // upstream... let data = llvm::LLVMRustCreateThinLTOData( thin_modules.as_ptr(), - thin_modules.len() as u32, + thin_modules.len(), symbols_below_threshold.as_ptr(), - symbols_below_threshold.len() as u32, + symbols_below_threshold.len(), ) .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; @@ -569,7 +570,7 @@ fn thin_lto( info!(" - {}: re-compiled", module_name); opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { - shared: shared.clone(), + shared: Arc::clone(&shared), idx: module_index, })); } @@ -601,23 +602,9 @@ pub(crate) fn run_pass_manager( // This code is based off the code found in llvm's LTO code generator: // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); - unsafe { - if !llvm::LLVMRustHasModuleFlag( - module.module_llvm.llmod(), - "LTOPostLink".as_ptr().cast(), - 11, - ) { - llvm::LLVMRustAddModuleFlagU32( - module.module_llvm.llmod(), - llvm::LLVMModFlagBehavior::Error, - c"LTOPostLink".as_ptr(), - 1, - ); - } - let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; - let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage)?; - } + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + unsafe { write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage) }?; debug!("lto done"); Ok(()) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index afdd2b581b8..a65ae4df1e3 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,4 +1,4 @@ -use std::ffi::CString; +use std::ffi::{CStr, CString}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -9,6 +9,7 @@ use llvm::{ LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols, }; use rustc_codegen_ssa::back::link::ensure_removed; +use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -34,6 +35,7 @@ use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback, }; +use crate::common::AsCCharPtr; use crate::errors::{ CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression, WithLlvmError, WriteBytecode, @@ -210,7 +212,7 @@ pub(crate) fn target_machine_factory( singlethread = false; } - let triple = SmallCStr::new(&sess.target.llvm_target); + let triple = SmallCStr::new(&versioned_llvm_target(sess)); let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); let features = CString::new(target_features.join(",")).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); @@ -590,15 +592,14 @@ pub(crate) unsafe fn llvm_optimize( pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), - config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, llvm_selfprofiler, selfprofile_before_pass_callback, selfprofile_after_pass_callback, - extra_passes.as_ptr().cast(), + extra_passes.as_c_char_ptr(), extra_passes.len(), - llvm_plugins.as_ptr().cast(), + llvm_plugins.as_c_char_ptr(), llvm_plugins.len(), ) }; @@ -944,27 +945,13 @@ fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: asm } -fn target_is_apple(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { - let triple = cgcx.opts.target_triple.triple(); - triple.contains("-ios") - || triple.contains("-darwin") - || triple.contains("-tvos") - || triple.contains("-watchos") - || triple.contains("-visionos") -} - -fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { - cgcx.opts.target_triple.triple().contains("-aix") -} - -//FIXME use c string literals here too -pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static str { - if target_is_apple(cgcx) { - "__LLVM,__bitcode\0" - } else if target_is_aix(cgcx) { - ".ipa\0" +pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr { + if cgcx.target_is_like_osx { + c"__LLVM,__bitcode" + } else if cgcx.target_is_like_aix { + c".ipa" } else { - ".llvmbc\0" + c".llvmbc" } } @@ -1028,10 +1015,12 @@ unsafe fn embed_bitcode( // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). - let is_aix = target_is_aix(cgcx); - let is_apple = target_is_apple(cgcx); unsafe { - if is_apple || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") { + if cgcx.target_is_like_osx + || cgcx.target_is_like_aix + || cgcx.target_arch == "wasm32" + || cgcx.target_arch == "wasm64" + { // We don't need custom section flags, create LLVM globals. let llconst = common::bytes_in_context(llcx, bitcode); let llglobal = llvm::LLVMAddGlobal( @@ -1041,9 +1030,8 @@ unsafe fn embed_bitcode( ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = bitcode_section_name(cgcx); - llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, bitcode_section_name(cgcx)); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); @@ -1053,22 +1041,22 @@ unsafe fn embed_bitcode( c"rustc.embedded.cmdline".as_ptr(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { + let section = if cgcx.target_is_like_osx { c"__LLVM,__cmdline" - } else if is_aix { + } else if cgcx.target_is_like_aix { c".info" } else { c".llvmcmd" }; - llvm::LLVMSetSection(llglobal, section.as_ptr()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, section); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); } } } @@ -1096,7 +1084,7 @@ fn create_msvc_imps( let ptr_ty = Type::ptr_llcx(llcx); let globals = base::iter_globals(llmod) .filter(|&val| { - llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage + llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && llvm::LLVMIsDeclaration(val) == 0 }) .filter_map(|val| { @@ -1115,7 +1103,7 @@ fn create_msvc_imps( for (imp_name, val) in globals { let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr()); llvm::LLVMSetInitializer(imp, val); - llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); + llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 0ba8d82406a..32793894794 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -145,10 +145,8 @@ pub(crate) fn compile_codegen_unit( pub(crate) fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { let Some(sect) = attrs.link_section else { return }; - unsafe { - let buf = SmallCStr::new(sect.as_str()); - llvm::LLVMSetSection(llval, buf.as_ptr()); - } + let buf = SmallCStr::new(sect.as_str()); + llvm::set_section(llval, &buf); } pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 4e0c62c8bf8..b5bb7630ca6 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,6 +3,8 @@ use std::ops::Deref; use std::{iter, ptr}; use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::{Align, Size, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -12,15 +14,14 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; use rustc_span::Span; -use rustc_target::abi::call::FnAbi; -use rustc_target::abi::{self, Align, Size, WrappingRange}; +use rustc_target::callconv::FnAbi; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -80,9 +81,9 @@ impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { } } -impl<'tcx> ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { - fn param_env(&self) -> ty::ParamEnv<'tcx> { - self.cx.param_env() +impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> { + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.cx.typing_env() } } @@ -238,7 +239,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("invoke", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); - let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { bundles.push(funclet_bundle); @@ -249,13 +249,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - if let Some(kcfi_bundle) = kcfi_bundle { + if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { bundles.push(kcfi_bundle); } let invoke = unsafe { - llvm::LLVMRustBuildInvoke( + llvm::LLVMBuildInvokeWithOperandBundles( self.llbuilder, llty, llfn, @@ -473,7 +472,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { #[instrument(level = "trace", skip(self))] fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { if place.layout.is_unsized() { - let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.param_env()); + let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env()); if matches!(tail.kind(), ty::Foreign(..)) { // Unsized locals and, at least conceptually, even unsized arguments must be copied // around, which requires dynamically determining their size. Therefore, we cannot @@ -505,12 +504,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } match scalar.primitive() { - abi::Int(..) => { + abi::Primitive::Int(..) => { if !scalar.is_always_valid(bx) { bx.range_metadata(load, scalar.valid_range(bx)); } } - abi::Pointer(_) => { + abi::Primitive::Pointer(_) => { if !scalar.valid_range(bx).contains(0) { bx.nonnull_metadata(load); } @@ -521,7 +520,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } } - abi::Float(_) => {} + abi::Primitive::Float(_) => {} } } @@ -544,13 +543,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } let llval = const_llval.unwrap_or_else(|| { let load = self.load(llty, place.val.llval, place.val.align); - if let abi::Abi::Scalar(scalar) = place.layout.abi { + if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr { scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); } load }); OperandValue::Immediate(self.to_immediate(llval, place.layout)) - } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { + } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr { let b_offset = a.size(self).align_to(b.align(self).abi); let mut load = |i, scalar: abi::Scalar, layout, align, offset| { @@ -1164,39 +1163,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); } - fn instrprof_increment( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - num_counters: &'ll Value, - index: &'ll Value, - ) { - debug!( - "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, num_counters, index - ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, num_counters, index]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } - } - fn call( &mut self, llty: &'ll Type, @@ -1211,7 +1177,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("call", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); - let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { bundles.push(funclet_bundle); @@ -1222,13 +1187,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - if let Some(kcfi_bundle) = kcfi_bundle { + if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { bundles.push(kcfi_bundle); } let call = unsafe { - llvm::LLVMRustBuildCall( + llvm::LLVMBuildCallWithOperandBundles( self.llbuilder, llty, llfn, @@ -1236,6 +1200,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { args.len() as c_uint, bundles.as_ptr(), bundles.len() as c_uint, + c"".as_ptr(), ) }; if let Some(fn_abi) = fn_abi { @@ -1541,7 +1506,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { let args = self.check_call("callbr", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); - let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { bundles.push(funclet_bundle); @@ -1552,13 +1516,12 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - if let Some(kcfi_bundle) = kcfi_bundle { + if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { bundles.push(kcfi_bundle); } let callbr = unsafe { - llvm::LLVMRustBuildCallBr( + llvm::LLVMBuildCallBr( self.llbuilder, llty, llfn, @@ -1611,6 +1574,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { cfi::typeid_for_fnabi(self.tcx, fn_abi, options) }; let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + let dbg_loc = self.get_dbg_loc(); // Test whether the function pointer is associated with the type identifier. let cond = self.type_test(llfn, typeid_metadata); @@ -1619,10 +1583,16 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { self.cond_br(cond, bb_pass, bb_fail); self.switch_to_block(bb_fail); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } self.abort(); self.unreachable(); self.switch_to_block(bb_pass); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } } } @@ -1633,7 +1603,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, instance: Option<Instance<'tcx>>, llfn: &'ll Value, - ) -> Option<llvm::OperandBundleDef<'ll>> { + ) -> Option<llvm::OperandBundleOwned<'ll>> { let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi @@ -1659,13 +1629,25 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi::typeid_for_fnabi(self.tcx, fn_abi, options) }; - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + Some(llvm::OperandBundleOwned::new("kcfi", &[self.const_u32(kcfi_typeid)])) } else { None }; kcfi_bundle } + /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation. + #[instrument(level = "debug", skip(self))] + pub(crate) fn instrprof_increment( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); + } + /// Emits a call to `llvm.instrprof.mcdc.parameters`. /// /// This doesn't produce any code directly, but is used as input by @@ -1675,111 +1657,50 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { /// /// [`CodeGenPGO::emitMCDCParameters`]: /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_parameters( &mut self, fn_name: &'ll Value, hash: &'ll Value, - bitmap_bytes: &'ll Value, + bitmap_bits: &'ll Value, ) { - debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], - self.cx.type_void(), + assert!( + crate::llvm_util::get_version() >= (19, 0, 0), + "MCDC intrinsics require LLVM 19 or later" ); - let args = &[fn_name, hash, bitmap_bytes]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_tvbitmap_update( &mut self, fn_name: &'ll Value, hash: &'ll Value, - bitmap_bytes: &'ll Value, bitmap_index: &'ll Value, mcdc_temp: &'ll Value, ) { - debug!( - "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", - fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp + assert!( + crate::llvm_util::get_version() >= (19, 0, 0), + "MCDC intrinsics require LLVM 19 or later" ); + let args = &[fn_name, hash, bitmap_index, mcdc_temp]; + self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args); + } - let llfn = - unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[ - self.cx.type_ptr(), - self.cx.type_i64(), - self.cx.type_i32(), - self.cx.type_i32(), - self.cx.type_ptr(), - ], - self.cx.type_void(), - ); - let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp]; - let args = self.check_call("call", llty, llfn, args); - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + #[instrument(level = "debug", skip(self))] + pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) { self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } - pub(crate) fn mcdc_condbitmap_update( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - cond_loc: &'ll Value, - mcdc_temp: &'ll Value, - bool_value: &'ll Value, - ) { - debug!( - "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", - fn_name, hash, cond_loc, mcdc_temp, bool_value - ); - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[ - self.cx.type_ptr(), - self.cx.type_i64(), - self.cx.type_i32(), - self.cx.type_ptr(), - self.cx.type_i1(), - ], - self.cx.type_void(), + #[instrument(level = "debug", skip(self))] + pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { + assert!( + crate::llvm_util::get_version() >= (19, 0, 0), + "MCDC intrinsics require LLVM 19 or later" ); - let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value]; - self.check_call("call", llty, llfn, args); - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + let align = self.tcx.data_layout.i32_align.abi; + let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align); + let new_tv_index = self.add(current_tv_index, cond_index); + self.store(new_tv_index, mcdc_temp, align); } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 206a7069792..e0a2de3366c 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -5,7 +5,7 @@ //! closure. use rustc_codegen_ssa::common; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use tracing::debug; @@ -28,12 +28,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t } let sym = tcx.symbol_name(instance).name; - debug!( - "get_fn({:?}: {:?}) => {}", - instance, - instance.ty(cx.tcx(), ty::ParamEnv::reveal_all()), - sym - ); + debug!("get_fn({:?}: {:?}) => {}", instance, instance.ty(cx.tcx(), cx.typing_env()), sym); let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); @@ -44,6 +39,22 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t let llfn = if tcx.sess.target.arch == "x86" && let Some(dllimport) = crate::common::get_dllimport(tcx, instance_def_id, sym) { + // When calling functions in generated import libraries, MSVC needs + // the fully decorated name (as would have been in the declaring + // object file), but MinGW wants the name as exported (as would be + // in the def file) which may be missing decorations. + let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(&tcx.sess.target); + let llfn = cx.declare_fn( + &common::i686_decorated_name( + dllimport, + mingw_gnu_toolchain, + true, + !mingw_gnu_toolchain, + ), + fn_abi, + Some(instance), + ); + // Fix for https://github.com/rust-lang/rust/issues/104453 // On x86 Windows, LLVM uses 'L' as the prefix for any private // global symbols, so when we create an undecorated function symbol @@ -55,15 +66,6 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t // LLVM will prefix the name with `__imp_`. Ideally, we'd like the // existing logic below to set the Storage Class, but it has an // exemption for MinGW for backwards compatibility. - let llfn = cx.declare_fn( - &common::i686_decorated_name( - dllimport, - common::is_mingw_gnu_toolchain(&tcx.sess.target), - true, - ), - fn_abi, - Some(instance), - ); unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } @@ -95,11 +97,10 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t // whether we are sharing generics or not. The important thing here is // that the visibility we apply to the declaration is the same one that // has been applied to the definition (wherever that definition may be). - unsafe { - llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); - let is_generic = - instance.args.non_erasable_generics(tcx, instance.def_id()).next().is_some(); + llvm::set_linkage(llfn, llvm::Linkage::ExternalLinkage); + unsafe { + let is_generic = instance.args.non_erasable_generics().next().is_some(); let is_hidden = if is_generic { // This is a monomorphization of a generic function. @@ -136,7 +137,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t || !cx.tcx.is_reachable_non_generic(instance_def_id)) }; if is_hidden { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + llvm::set_visibility(llfn, llvm::Visibility::Hidden); } // MinGW: For backward compatibility we rely on the linker to decide whether it diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 31d59905446..8852dec7d9f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -1,7 +1,11 @@ //! Code that is useful in various codegen modules. use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::Primitive::Pointer; +use rustc_abi::{AddressSpace, HasDataLayout}; use rustc_ast::Mutability; +use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; use rustc_hir::def_id::DefId; @@ -9,12 +13,11 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::DllImport; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use tracing::debug; use crate::consts::const_alloc_to_llvm; pub(crate) use crate::context::CodegenCx; -use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, OperandBundleDef, True}; +use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, True}; use crate::type_::Type; use crate::value::Value; @@ -60,19 +63,19 @@ use crate::value::Value; /// the `OperandBundleDef` value created for MSVC landing pads. pub(crate) struct Funclet<'ll> { cleanuppad: &'ll Value, - operand: OperandBundleDef<'ll>, + operand: llvm::OperandBundleOwned<'ll>, } impl<'ll> Funclet<'ll> { pub(crate) fn new(cleanuppad: &'ll Value) -> Self { - Funclet { cleanuppad, operand: OperandBundleDef::new("funclet", &[cleanuppad]) } + Funclet { cleanuppad, operand: llvm::OperandBundleOwned::new("funclet", &[cleanuppad]) } } pub(crate) fn cleanuppad(&self) -> &'ll Value { self.cleanuppad } - pub(crate) fn bundle(&self) -> &OperandBundleDef<'ll> { + pub(crate) fn bundle(&self) -> &llvm::OperandBundle<'ll> { &self.operand } } @@ -144,6 +147,10 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_int" + ); unsafe { llvm::LLVMConstInt(t, i as u64, True) } } @@ -174,10 +181,18 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_uint" + ); unsafe { llvm::LLVMConstInt(t, i, False) } } fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_uint_big" + ); unsafe { let words = [u as u64, (u >> 64) as u64]; llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr()) @@ -204,8 +219,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMSetInitializer(g, sc); llvm::LLVMSetGlobalConstant(g, True); llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); } + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); (s.to_owned(), g) }) .1; @@ -377,3 +392,21 @@ pub(crate) fn get_dllimport<'tcx>( tcx.native_library(id) .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) } + +/// Extension trait for explicit casts to `*const c_char`. +pub(crate) trait AsCCharPtr { + /// Equivalent to `self.as_ptr().cast()`, but only casts to `*const c_char`. + fn as_c_char_ptr(&self) -> *const c_char; +} + +impl AsCCharPtr for str { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} + +impl AsCCharPtr for [u8] { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 33a85adeb87..6f5ffbb4b34 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,5 +1,8 @@ use std::ops::Range; +use rustc_abi::{ + Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, +}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::def::DefKind; @@ -10,16 +13,13 @@ use rustc_middle::mir::interpret::{ read_target_uint, }; use rustc_middle::mir::mono::MonoItem; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, Instance}; +use rustc_middle::ty::Instance; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::{bug, span_bug}; use rustc_session::config::Lto; -use rustc_target::abi::{ - Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, -}; use tracing::{debug, instrument, trace}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::errors::{ InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, }; @@ -172,40 +172,32 @@ fn check_and_apply_linkage<'ll, 'tcx>( if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); - unsafe { - // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(sym, cx.type_i8()); - llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); - - // Declare an internal global `extern_with_linkage_foo` which - // is initialized with the address of `foo`. If `foo` is - // discarded during linking (for example, if `foo` has weak - // linkage and there are no definitions), then - // `extern_with_linkage_foo` will instead be initialized to - // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); - let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { - span: cx.tcx.def_span(def_id), - symbol_name: sym, - }) - }); - llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); - llvm::LLVMSetInitializer(g2, g1); - g2 - } + // Declare a symbol `foo` with the desired linkage. + let g1 = cx.declare_global(sym, cx.type_i8()); + llvm::set_linkage(g1, base::linkage_to_llvm(linkage)); + + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + let mut real_name = "_rust_extern_with_linkage_".to_string(); + real_name.push_str(sym); + let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { + cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { + span: cx.tcx.def_span(def_id), + symbol_name: sym, + }) + }); + llvm::set_linkage(g2, llvm::Linkage::InternalLinkage); + unsafe { llvm::LLVMSetInitializer(g2, g1) }; + g2 } else if cx.tcx.sess.target.arch == "x86" + && common::is_mingw_gnu_toolchain(&cx.tcx.sess.target) && let Some(dllimport) = crate::common::get_dllimport(cx.tcx, def_id, sym) { - cx.declare_global( - &common::i686_decorated_name( - dllimport, - common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), - true, - ), - llty, - ) + cx.declare_global(&common::i686_decorated_name(dllimport, true, true, false), llty) } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global @@ -224,23 +216,21 @@ impl<'ll> CodegenCx<'ll, '_> { align: Align, kind: Option<&str>, ) -> &'ll Value { - unsafe { - let gv = match kind { - Some(kind) if !self.tcx.sess.fewer_names() => { - let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", name); - }); - llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); - gv - } - _ => self.define_private_global(self.val_ty(cv)), - }; - llvm::LLVMSetInitializer(gv, cv); - set_global_alignment(self, gv, align); - llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); - gv - } + let gv = match kind { + Some(kind) if !self.tcx.sess.fewer_names() => { + let name = self.generate_local_symbol_name(kind); + let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", name); + }); + llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); + gv + } + _ => self.define_private_global(self.val_ty(cv)), + }; + unsafe { llvm::LLVMSetInitializer(gv, cv) }; + set_global_alignment(self, gv, align); + llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + gv } #[instrument(level = "debug", skip(self))] @@ -254,7 +244,7 @@ impl<'ll> CodegenCx<'ll, '_> { let llty = if nested { self.type_i8() } else { - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let ty = instance.ty(self.tcx, self.typing_env()); trace!(?ty); self.layout_of(ty).llvm_type(self) }; @@ -292,9 +282,7 @@ impl<'ll> CodegenCx<'ll, '_> { let g = self.declare_global(sym, llty); if !self.tcx.is_reachable_non_generic(def_id) { - unsafe { - llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); - } + llvm::set_visibility(g, llvm::Visibility::Hidden); } g @@ -312,7 +300,7 @@ impl<'ll> CodegenCx<'ll, '_> { llvm::set_thread_local_mode(g, self.tls_model); } - let dso_local = unsafe { self.should_assume_dso_local(g, true) }; + let dso_local = self.should_assume_dso_local(g, true); if dso_local { unsafe { llvm::LLVMRustSetDSOLocal(g, true); @@ -401,18 +389,18 @@ impl<'ll> CodegenCx<'ll, '_> { let name = llvm::get_value_name(g).to_vec(); llvm::set_value_name(g, b""); - let linkage = llvm::LLVMRustGetLinkage(g); - let visibility = llvm::LLVMRustGetVisibility(g); + let linkage = llvm::get_linkage(g); + let visibility = llvm::get_visibility(g); let new_g = llvm::LLVMRustGetOrInsertGlobal( self.llmod, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), val_llty, ); - llvm::LLVMRustSetLinkage(new_g, linkage); - llvm::LLVMRustSetVisibility(new_g, visibility); + llvm::set_linkage(new_g, linkage); + llvm::set_visibility(new_g, visibility); // The old global has had its name removed but is returned by // get_static since it is in the instance cache. Provide an @@ -457,7 +445,7 @@ impl<'ll> CodegenCx<'ll, '_> { if let Some(section) = attrs.link_section { let section = llvm::LLVMMDStringInContext2( self.llcx, - section.as_str().as_ptr().cast(), + section.as_str().as_c_char_ptr(), section.as_str().len(), ); assert!(alloc.provenance().ptrs().is_empty()); @@ -468,7 +456,7 @@ impl<'ll> CodegenCx<'ll, '_> { let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); let alloc = - llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_c_char_ptr(), bytes.len()); let data = [section, alloc]; let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); let val = llvm::LLVMMetadataAsValue(self.llcx, meta); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 2b8912d1db2..841c110b3c8 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -3,6 +3,8 @@ use std::cell::{Cell, RefCell}; use std::ffi::{CStr, c_uint}; use std::str; +use rustc_abi::{HasDataLayout, TargetDataLayout, VariantIdx}; +use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; @@ -13,22 +15,22 @@ use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::PatchableFunctionEntry; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::Session; use rustc_session::config::{ - BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, PAuthKey, PacRet, + BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet, }; use rustc_span::source_map::Spanned; use rustc_span::{DUMMY_SP, Span}; -use rustc_target::abi::{HasDataLayout, TargetDataLayout, VariantIdx}; use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; +use crate::common::AsCCharPtr; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::{Metadata, MetadataType}; use crate::type_::Type; @@ -80,6 +82,7 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { pub isize_ty: &'ll Type, + /// Extra codegen state needed when coverage instrumentation is enabled. pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>, @@ -138,6 +141,26 @@ pub(crate) unsafe fn create_module<'ll>( } } + if llvm_version < (20, 0, 0) { + if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { + // LLVM 20 defines three additional address spaces for alternate + // pointer kinds used in Windows. + // See https://github.com/llvm/llvm-project/pull/111879 + target_data_layout = + target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); + } + if sess.target.arch.starts_with("sparc") { + // LLVM 20 updates the sparc layout to correctly align 128 bit integers to 128 bit. + // See https://github.com/llvm/llvm-project/pull/106951 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } + if sess.target.arch.starts_with("mips64") { + // LLVM 20 updates the mips64 layout to correctly align 128 bit integers to 128 bit. + // See https://github.com/llvm/llvm-project/pull/112084 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } + } + // Ensure the data-layout values hardcoded remain the defaults. { let tm = crate::back::write::create_informational_target_machine(tcx.sess, false); @@ -165,7 +188,7 @@ pub(crate) unsafe fn create_module<'ll>( llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()); } - let llvm_target = SmallCStr::new(&sess.target.llvm_target); + let llvm_target = SmallCStr::new(&versioned_llvm_target(sess)); unsafe { llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr()); } @@ -198,133 +221,121 @@ pub(crate) unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = c"RtLibUseGOT".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "RtLibUseGOT", 1); } // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { - let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - canonical_jump_tables, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "CFI Canonical Jump Tables", + 1, + ); } // If we're normalizing integers with CFI, ensure LLVM generated functions do the same. // See https://github.com/llvm/llvm-project/pull/104826 if sess.is_sanitizer_cfi_normalize_integers_enabled() { - let cfi_normalize_integers = c"cfi-normalize-integers".as_ptr().cast(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - cfi_normalize_integers, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cfi-normalize-integers", + 1, + ); } // Enable LTO unit splitting if specified or if CFI is enabled. (See // https://reviews.llvm.org/D53891.) if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { - let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - enable_split_lto_unit, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "EnableSplitLTOUnit", + 1, + ); } // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - let kcfi = c"kcfi".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Override, "kcfi", 1); // Add "kcfi-offset" module flag with -Z patchable-function-entry (See // https://reviews.llvm.org/D141172). let pfe = PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry); if pfe.prefix() > 0 { - let kcfi_offset = c"kcfi-offset".as_ptr().cast(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - kcfi_offset, - pfe.prefix().into(), - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "kcfi-offset", + pfe.prefix().into(), + ); } } - // Control Flow Guard is currently only supported by the MSVC linker on Windows. - if sess.target.is_like_msvc { - unsafe { - match sess.opts.cg.control_flow_guard { - CFGuard::Disabled => {} - CFGuard::NoChecks => { - // Set `cfguard=1` module flag to emit metadata only. - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, - 1, - ) - } - CFGuard::Checks => { - // Set `cfguard=2` module flag to emit metadata and checks. - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, - 2, - ) - } + // Control Flow Guard is currently only supported by MSVC and LLVM on Windows. + if sess.target.is_like_msvc + || (sess.target.options.os == "windows" + && sess.target.options.env == "gnu" + && sess.target.options.abi == "llvm") + { + match sess.opts.cg.control_flow_guard { + CFGuard::Disabled => {} + CFGuard::NoChecks => { + // Set `cfguard=1` module flag to emit metadata only. + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "cfguard", + 1, + ); + } + CFGuard::Checks => { + // Set `cfguard=2` module flag to emit metadata and checks. + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "cfguard", + 2, + ); } } } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch == "aarch64" { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"branch-target-enforcement".as_ptr(), - bti.into(), - ); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address".as_ptr(), - pac_ret.is_some().into(), - ); - let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-all".as_ptr(), - pac_opts.leaf.into(), - ); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-with-bkey".as_ptr(), - u32::from(pac_opts.key == PAuthKey::B), - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "branch-target-enforcement", + bti.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address", + pac_ret.is_some().into(), + ); + let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, pc: false, key: PAuthKey::A }); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "branch-protection-pauth-lr", + pac_opts.pc.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address-all", + pac_opts.leaf.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address-with-bkey", + u32::from(pac_opts.key == PAuthKey::B), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -335,46 +346,45 @@ pub(crate) unsafe fn create_module<'ll>( // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"cf-protection-branch".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cf-protection-branch", + 1, + ); } if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"cf-protection-return".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cf-protection-return", + 1, + ); } if sess.opts.unstable_opts.virtual_function_elimination { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Error, - c"Virtual Function Elim".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Virtual Function Elim", + 1, + ); } // Set module flag to enable Windows EHCont Guard (/guard:ehcont). if sess.opts.unstable_opts.ehcont_guard { - unsafe { - llvm::LLVMRustAddModuleFlagU32( + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "ehcontguard", 1); + } + + match sess.opts.unstable_opts.function_return { + FunctionReturn::Keep => {} + FunctionReturn::ThunkExtern => { + llvm::add_module_flag_u32( llmod, - llvm::LLVMModFlagBehavior::Warning, - c"ehcontguard".as_ptr() as *const _, + llvm::ModuleFlagMergeBehavior::Override, + "function_return_thunk_extern", 1, - ) + ); } } @@ -383,15 +393,12 @@ pub(crate) unsafe fn create_module<'ll>( // Set up the small-data optimization limit for architectures that use // an LLVM module flag to control this. (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => { - let flag = SmallCStr::new(flag.as_ref()); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Error, - flag.as_c_str().as_ptr(), - threshold as u32, - ) - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + &flag, + threshold as u32, + ); } _ => (), }; @@ -406,7 +413,7 @@ pub(crate) unsafe fn create_module<'ll>( let name_metadata = unsafe { llvm::LLVMMDStringInContext2( llcx, - rustc_producer.as_ptr().cast(), + rustc_producer.as_c_char_ptr(), rustc_producer.as_bytes().len(), ) }; @@ -425,33 +432,29 @@ pub(crate) unsafe fn create_module<'ll>( // If llvm_abiname is empty, emit nothing. let llvm_abiname = &sess.target.options.llvm_abiname; if matches!(sess.target.arch.as_ref(), "riscv32" | "riscv64") && !llvm_abiname.is_empty() { - unsafe { - llvm::LLVMRustAddModuleFlagString( - llmod, - llvm::LLVMModFlagBehavior::Error, - c"target-abi".as_ptr(), - llvm_abiname.as_ptr().cast(), - llvm_abiname.len(), - ); - } + llvm::add_module_flag_str( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "target-abi", + llvm_abiname, + ); } // Add module flags specified via -Z llvm_module_flag - for (key, value, behavior) in &sess.opts.unstable_opts.llvm_module_flag { - let key = format!("{key}\0"); - let behavior = match behavior.as_str() { - "error" => llvm::LLVMModFlagBehavior::Error, - "warning" => llvm::LLVMModFlagBehavior::Warning, - "require" => llvm::LLVMModFlagBehavior::Require, - "override" => llvm::LLVMModFlagBehavior::Override, - "append" => llvm::LLVMModFlagBehavior::Append, - "appendunique" => llvm::LLVMModFlagBehavior::AppendUnique, - "max" => llvm::LLVMModFlagBehavior::Max, - "min" => llvm::LLVMModFlagBehavior::Min, + for (key, value, merge_behavior) in &sess.opts.unstable_opts.llvm_module_flag { + let merge_behavior = match merge_behavior.as_str() { + "error" => llvm::ModuleFlagMergeBehavior::Error, + "warning" => llvm::ModuleFlagMergeBehavior::Warning, + "require" => llvm::ModuleFlagMergeBehavior::Require, + "override" => llvm::ModuleFlagMergeBehavior::Override, + "append" => llvm::ModuleFlagMergeBehavior::Append, + "appendunique" => llvm::ModuleFlagMergeBehavior::AppendUnique, + "max" => llvm::ModuleFlagMergeBehavior::Max, + "min" => llvm::ModuleFlagMergeBehavior::Min, // We already checked this during option parsing _ => unreachable!(), }; - unsafe { llvm::LLVMRustAddModuleFlagU32(llmod, behavior, key.as_ptr().cast(), *value) } + llvm::add_module_flag_u32(llmod, merge_behavior, key, *value); } llmod @@ -570,11 +573,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { &self.statics_to_rauw } + /// Extra state that is only available when coverage instrumentation is enabled. #[inline] - pub(crate) fn coverage_context( - &self, - ) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { - self.coverage_cx.as_ref() + #[track_caller] + pub(crate) fn coverage_cx(&self) -> &coverageinfo::CrateCoverageContext<'ll, 'tcx> { + self.coverage_cx.as_ref().expect("only called when coverage instrumentation is enabled") } pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { @@ -583,8 +586,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); + llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); + llvm::set_section(g, c"llvm.metadata"); } } } @@ -655,7 +658,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let llfn = match tcx.lang_items().eh_personality() { Some(def_id) if name.is_none() => self.get_fn_addr(ty::Instance::expect_resolve( tcx, - ty::ParamEnv::reveal_all(), + self.typing_env(), def_id, ty::List::empty(), DUMMY_SP, @@ -872,6 +875,11 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16); + ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32); + ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16); ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); @@ -1072,6 +1080,10 @@ impl<'ll> CodegenCx<'ll, '_> { if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); + if crate::llvm_util::get_version() >= (19, 0, 0) { + ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); + ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); + } } ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); @@ -1150,9 +1162,9 @@ impl<'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'_, 'tcx> { } } -impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { - fn param_env(&self) -> ty::ParamEnv<'tcx> { - ty::ParamEnv::reveal_all() +impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> { + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + ty::TypingEnv::fully_monomorphized() } } @@ -1175,10 +1187,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { span: Span, fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { - if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.tcx.dcx().emit_fatal(Spanned { span, node: err }) - } else { - match fn_abi_request { + match err { + FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + self.tcx.dcx().emit_fatal(Spanned { span, node: err }); + } + _ => match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } @@ -1188,7 +1201,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } - } + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 77821ca89bc..a6e07ea2a60 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,11 +1,11 @@ -use rustc_middle::mir::coverage::{ - ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, SourceRegion, -}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion}; + +use crate::coverageinfo::mapgen::LocalFileId; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum CounterKind { +pub(crate) enum CounterKind { Zero = 0, CounterValueReference = 1, Expression = 2, @@ -25,9 +25,9 @@ pub enum CounterKind { /// Must match the layout of `LLVMRustCounter`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct Counter { +pub(crate) struct Counter { // Important: The layout (order and types of fields) must match its C++ counterpart. - pub kind: CounterKind, + pub(crate) kind: CounterKind, id: u32, } @@ -36,7 +36,7 @@ impl Counter { pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; /// Constructs a new `Counter` of kind `CounterValueReference`. - pub fn counter_value_reference(counter_id: CounterId) -> Self { + pub(crate) fn counter_value_reference(counter_id: CounterId) -> Self { Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } } @@ -59,7 +59,7 @@ impl Counter { /// Must match the layout of `LLVMRustCounterExprKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum ExprKind { +pub(crate) enum ExprKind { Subtract = 0, Add = 1, } @@ -69,49 +69,14 @@ pub enum ExprKind { /// Must match the layout of `LLVMRustCounterExpression`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct CounterExpression { - pub kind: ExprKind, - pub lhs: Counter, - pub rhs: Counter, +pub(crate) struct CounterExpression { + pub(crate) kind: ExprKind, + pub(crate) lhs: Counter, + pub(crate) rhs: Counter, } -/// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegionKind`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -enum RegionKind { - /// A CodeRegion associates some code with a counter - CodeRegion = 0, - - /// An ExpansionRegion represents a file expansion region that associates - /// a source range with the expansion of a virtual source file, such as - /// for a macro instantiation or #include file. - ExpansionRegion = 1, - - /// A SkippedRegion represents a source range with code that was skipped - /// by a preprocessor or similar means. - SkippedRegion = 2, - - /// A GapRegion is like a CodeRegion, but its count is only set as the - /// line execution count when its the only region in the line. - GapRegion = 3, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion = 4, - - /// A DecisionRegion represents a top-level boolean expression and is - /// associated with a variable length bitmap index and condition number. - MCDCDecisionRegion = 5, - - /// A Branch Region can be extended to include IDs to facilitate MC/DC. - MCDCBranchRegion = 6, -} - -mod mcdc { - use rustc_middle::mir::coverage::{ConditionInfo, DecisionInfo}; +pub(crate) mod mcdc { + use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; /// Must match the layout of `LLVMRustMCDCDecisionParameters`. #[repr(C)] @@ -121,8 +86,6 @@ mod mcdc { num_conditions: u16, } - // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at - // [19](https://github.com/llvm/llvm-project/pull/81257). type LLVMConditionId = i16; /// Must match the layout of `LLVMRustMCDCBranchParameters`. @@ -133,46 +96,15 @@ mod mcdc { condition_ids: [LLVMConditionId; 2], } - #[repr(C)] - #[derive(Clone, Copy, Debug)] - enum ParameterTag { - None = 0, - Decision = 1, - Branch = 2, - } - /// Same layout with `LLVMRustMCDCParameters` - #[repr(C)] - #[derive(Clone, Copy, Debug)] - pub(crate) struct Parameters { - tag: ParameterTag, - decision_params: DecisionParameters, - branch_params: BranchParameters, - } - - impl Parameters { - pub(crate) fn none() -> Self { - Self { - tag: ParameterTag::None, - decision_params: Default::default(), - branch_params: Default::default(), - } - } - pub(crate) fn decision(decision_params: DecisionParameters) -> Self { - Self { tag: ParameterTag::Decision, decision_params, branch_params: Default::default() } - } - pub(crate) fn branch(branch_params: BranchParameters) -> Self { - Self { tag: ParameterTag::Branch, decision_params: Default::default(), branch_params } - } - } - impl From<ConditionInfo> for BranchParameters { fn from(value: ConditionInfo) -> Self { + let to_llvm_cond_id = |cond_id: Option<ConditionId>| { + cond_id.and_then(|id| LLVMConditionId::try_from(id.as_usize()).ok()).unwrap_or(-1) + }; + let ConditionInfo { condition_id, true_next_id, false_next_id } = value; Self { - condition_id: value.condition_id.as_u32() as LLVMConditionId, - condition_ids: [ - value.false_next_id.as_u32() as LLVMConditionId, - value.true_next_id.as_u32() as LLVMConditionId, - ], + condition_id: to_llvm_cond_id(Some(condition_id)), + condition_ids: [to_llvm_cond_id(false_next_id), to_llvm_cond_id(true_next_id)], } } } @@ -185,267 +117,72 @@ mod mcdc { } } -/// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the -/// coverage map, in accordance with the -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). -/// The struct composes fields representing the `Counter` type and value(s) (injected counter -/// ID, or expression type and operands), the source file (an indirect index into a "filenames -/// array", encoded separately), and source location (start and end positions of the represented -/// code region). +/// A span of source code coordinates to be embedded in coverage metadata. /// -/// Corresponds to struct `llvm::coverage::CounterMappingRegion`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegion`. -#[derive(Copy, Clone, Debug)] +/// Must match the layout of `LLVMRustCoverageSpan`. +#[derive(Clone, Debug)] #[repr(C)] -pub struct CounterMappingRegion { - /// The counter type and type-dependent counter data, if any. - counter: Counter, - - /// If the `RegionKind` is a `BranchRegion`, this represents the counter - /// for the false branch of the region. - false_counter: Counter, - - mcdc_params: mcdc::Parameters, - /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the - /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes - /// that, in turn, are used to look up the filename for this region. +pub(crate) struct CoverageSpan { + /// Local index into the function's local-to-global file ID table. + /// The value at that index is itself an index into the coverage filename + /// table in the CGU's `__llvm_covmap` section. file_id: u32, - /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find - /// the mapping regions created as a result of macro expansion, by checking if their file id - /// matches the expanded file id. - expanded_file_id: u32, - - /// 1-based starting line of the mapping region. + /// 1-based starting line of the source code span. start_line: u32, - - /// 1-based starting column of the mapping region. + /// 1-based starting column of the source code span. start_col: u32, - - /// 1-based ending line of the mapping region. + /// 1-based ending line of the source code span. end_line: u32, - - /// 1-based ending column of the mapping region. If the high bit is set, the current - /// mapping region is a gap area. + /// 1-based ending column of the source code span. High bit must be unset. end_col: u32, - - kind: RegionKind, } -impl CounterMappingRegion { - pub(crate) fn from_mapping( - mapping_kind: &MappingKind, - local_file_id: u32, - source_region: &SourceRegion, - ) -> Self { - let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = - source_region; - match *mapping_kind { - MappingKind::Code(term) => Self::code_region( - Counter::from_term(term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::Branch { true_term, false_term } => Self::branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { - Self::mcdc_branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - mcdc_params, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ) - } - MappingKind::MCDCDecision(decision_info) => Self::decision_region( - decision_info, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - } - } - - pub(crate) fn code_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, +impl CoverageSpan { + pub(crate) fn from_source_region( + local_file_id: LocalFileId, + code_region: &SourceRegion, ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::CodeRegion, - } - } - - pub(crate) fn branch_region( - counter: Counter, - false_counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::BranchRegion, - } + let file_id = local_file_id.as_u32(); + let &SourceRegion { start_line, start_col, end_line, end_col } = code_region; + // Internally, LLVM uses the high bit of `end_col` to distinguish between + // code regions and gap regions, so it can't be used by the column number. + assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}"); + Self { file_id, start_line, start_col, end_line, end_col } } +} - pub(crate) fn mcdc_branch_region( - counter: Counter, - false_counter: Counter, - condition_info: ConditionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::branch(condition_info.into()), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCBranchRegion, - } - } - - pub(crate) fn decision_region( - decision_info: DecisionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - let mcdc_params = mcdc::Parameters::decision(decision_info.into()); - - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params, - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCDecisionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageCodeRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct CodeRegion { + pub(crate) span: CoverageSpan, + pub(crate) counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn expansion_region( - file_id: u32, - expanded_file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::ExpansionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct BranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn skipped_region( - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::SkippedRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCBranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, + pub(crate) mcdc_branch_params: mcdc::BranchParameters, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn gap_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col: (1_u32 << 31) | end_col, - kind: RegionKind::GapRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCDecisionRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCDecisionRegion { + pub(crate) span: CoverageSpan, + pub(crate) mcdc_decision_params: mcdc::DecisionParameters, } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs new file mode 100644 index 00000000000..99c2d12b261 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -0,0 +1,100 @@ +//! Safe wrappers for coverage-specific FFI functions. + +use std::ffi::CString; + +use crate::common::AsCCharPtr; +use crate::coverageinfo::ffi; +use crate::llvm; + +pub(crate) fn covmap_var_name() -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteCovmapVarNameToString(s); + })) + .expect("covmap variable name should not contain NUL") +} + +pub(crate) fn covmap_section_name(llmod: &llvm::Module) -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteCovmapSectionNameToString(llmod, s); + })) + .expect("covmap section name should not contain NUL") +} + +pub(crate) fn covfun_section_name(llmod: &llvm::Module) -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteCovfunSectionNameToString(llmod, s); + })) + .expect("covfun section name should not contain NUL") +} + +pub(crate) fn create_pgo_func_name_var<'ll>( + llfn: &'ll llvm::Value, + mangled_fn_name: &str, +) -> &'ll llvm::Value { + unsafe { + llvm::LLVMRustCoverageCreatePGOFuncNameVar( + llfn, + mangled_fn_name.as_c_char_ptr(), + mangled_fn_name.len(), + ) + } +} + +pub(crate) fn write_filenames_to_buffer<'a>( + filenames: impl IntoIterator<Item = &'a str>, +) -> Vec<u8> { + let (pointers, lengths) = filenames + .into_iter() + .map(|s: &str| (s.as_c_char_ptr(), s.len())) + .unzip::<_, _, Vec<_>, Vec<_>>(); + + llvm::build_byte_buffer(|buffer| unsafe { + llvm::LLVMRustCoverageWriteFilenamesToBuffer( + pointers.as_ptr(), + pointers.len(), + lengths.as_ptr(), + lengths.len(), + buffer, + ); + }) +} + +pub(crate) fn write_function_mappings_to_buffer( + virtual_file_mapping: &[u32], + expressions: &[ffi::CounterExpression], + code_regions: &[ffi::CodeRegion], + branch_regions: &[ffi::BranchRegion], + mcdc_branch_regions: &[ffi::MCDCBranchRegion], + mcdc_decision_regions: &[ffi::MCDCDecisionRegion], +) -> Vec<u8> { + llvm::build_byte_buffer(|buffer| unsafe { + llvm::LLVMRustCoverageWriteFunctionMappingsToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len(), + expressions.as_ptr(), + expressions.len(), + code_regions.as_ptr(), + code_regions.len(), + branch_regions.as_ptr(), + branch_regions.len(), + mcdc_branch_regions.as_ptr(), + mcdc_branch_regions.len(), + mcdc_decision_regions.as_ptr(), + mcdc_decision_regions.len(), + buffer, + ) + }) +} + +/// Hashes some bytes into a 64-bit hash, via LLVM's `IndexedInstrProf::ComputeHash`, +/// as required for parts of the LLVM coverage mapping format. +pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { + unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_c_char_ptr(), bytes.len()) } +} + +/// Returns LLVM's `coverage::CovMapVersion::CurrentVersion` (CoverageMapping.h) +/// as a raw numeric value. For historical reasons, the numeric value is 1 less +/// than the number in the version's name, so `Version7` is actually `6u32`. +pub(crate) fn mapping_version() -> u32 { + unsafe { llvm::LLVMRustCoverageMappingVersion() } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index 5ed640b840e..e3c0df27883 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -6,7 +6,6 @@ use rustc_middle::mir::coverage::{ SourceRegion, }; use rustc_middle::ty::Instance; -use rustc_span::Symbol; use tracing::{debug, instrument}; use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; @@ -180,7 +179,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { } pub(crate) struct FunctionCoverage<'tcx> { - function_coverage_info: &'tcx FunctionCoverageInfo, + pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, is_used: bool, counters_seen: BitSet<CounterId>, @@ -199,11 +198,6 @@ impl<'tcx> FunctionCoverage<'tcx> { if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } } - /// Returns an iterator over all filenames used by this function's mappings. - pub(crate) fn all_file_names(&self) -> impl Iterator<Item = Symbol> + Captures<'_> { - self.function_coverage_info.mappings.iter().map(|mapping| mapping.source_region.file_name) - } - /// Convert this function's coverage expression data into a form that can be /// passed through FFI to LLVM. pub(crate) fn counter_expressions( diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 267a2244916..b582dd967a7 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,42 +1,43 @@ +use std::ffi::CString; +use std::iter; + use itertools::Itertools as _; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; +use rustc_abi::Align; +use rustc_codegen_ssa::traits::{ + BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, +}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; +use rustc_middle::mir::coverage::MappingKind; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, mir}; -use rustc_span::Symbol; +use rustc_session::RemapFileNameExt; +use rustc_session::config::RemapPathScopeComponents; use rustc_span::def_id::DefIdSet; +use rustc_span::{Span, Symbol}; +use rustc_target::spec::HasTargetSpec; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::ffi::CounterMappingRegion; use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; -use crate::{coverageinfo, llvm}; +use crate::coverageinfo::{ffi, llvm_cov}; +use crate::llvm; -/// Generates and exports the Coverage Map. -/// -/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions -/// 6 and 7 (encoded as 5 and 6 respectively), as described at -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/18.0-2024-02-13/llvm/docs/CoverageMappingFormat.rst). -/// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) -/// distributed in the `llvm-tools-preview` rustup component. +/// Generates and exports the coverage map, which is embedded in special +/// linker sections in the final binary. /// -/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with -/// the same version. Clang's implementation of Coverage Map generation was referenced when -/// implementing this Rust version, and though the format documentation is very explicit and -/// detailed, some undocumented details in Clang's implementation (that may or may not be important) -/// were also replicated for Rust's Coverage Map. +/// Those sections are then read and understood by LLVM's `llvm-cov` tool, +/// which is distributed in the `llvm-tools` rustup component. pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure that LLVM is using a version of the coverage mapping format that // agrees with our Rust-side code. Expected versions (encoded as n-1) are: - // - `CovMapVersion::Version6` (5) used by LLVM 13-17 - // - `CovMapVersion::Version7` (6) used by LLVM 18 + // - `CovMapVersion::Version7` (6) used by LLVM 18-19 let covmap_version = { - let llvm_covmap_version = coverageinfo::mapping_version(); - let expected_versions = 5..=6; + let llvm_covmap_version = llvm_cov::mapping_version(); + let expected_versions = 6..=6; assert!( expected_versions.contains(&llvm_covmap_version), "Coverage mapping version exposed by `llvm-wrapper` is out of sync; \ @@ -56,11 +57,11 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { add_unused_functions(cx); } - let function_coverage_map = match cx.coverage_context() { - Some(ctx) => ctx.take_function_coverage_map(), + // FIXME(#132395): Can this be none even when coverage is enabled? + let function_coverage_map = match cx.coverage_cx { + Some(ref cx) => cx.take_function_coverage_map(), None => return, }; - if function_coverage_map.is_empty() { // This module has no functions with coverage instrumentation return; @@ -71,8 +72,10 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) .collect::<Vec<_>>(); - let all_file_names = - function_coverage_entries.iter().flat_map(|(_, fn_cov)| fn_cov.all_file_names()); + let all_file_names = function_coverage_entries + .iter() + .map(|(_, fn_cov)| fn_cov.function_coverage_info.body_span) + .map(|span| span_file_name(tcx, span)); let global_file_table = GlobalFileTable::new(all_file_names); // Encode all filenames referenced by coverage mappings in this CGU. @@ -80,15 +83,13 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let filenames_size = filenames_buffer.len(); let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); + let filenames_ref = llvm_cov::hash_bytes(&filenames_buffer); // Generate the coverage map header, which contains the filenames used by // this CGU's coverage mappings, and store it in a well-known global. - let cov_data_val = generate_coverage_map(cx, covmap_version, filenames_size, filenames_val); - coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + generate_covmap_record(cx, covmap_version, filenames_size, filenames_val); let mut unused_function_names = Vec::new(); - let covfun_section_name = coverageinfo::covfun_section_name(cx); // Encode coverage mappings and generate function records for (instance, function_coverage) in function_coverage_entries { @@ -99,7 +100,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let is_used = function_coverage.is_used(); let coverage_mapping_buffer = - encode_mappings_for_function(&global_file_table, &function_coverage); + encode_mappings_for_function(tcx, &global_file_table, &function_coverage); if coverage_mapping_buffer.is_empty() { if function_coverage.is_used() { @@ -117,9 +118,8 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { unused_function_names.push(mangled_function_name); } - save_function_record( + generate_covfun_record( cx, - &covfun_section_name, mangled_function_name, source_hash, filenames_ref, @@ -141,7 +141,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { .collect::<Vec<_>>(); let initializer = cx.const_array(cx.type_ptr(), &name_globals); - let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), "__llvm_coverage_names"); + let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), c"__llvm_coverage_names"); llvm::set_global_constant(array, true); llvm::set_linkage(array, llvm::Linkage::InternalLinkage); llvm::set_initializer(array, initializer); @@ -168,13 +168,13 @@ impl GlobalFileTable { Self { raw_file_table } } - fn global_file_id_for_file_name(&self, file_name: Symbol) -> u32 { + fn global_file_id_for_file_name(&self, file_name: Symbol) -> GlobalFileId { let raw_id = self.raw_file_table.get_index_of(&file_name).unwrap_or_else(|| { bug!("file name not found in prepared global file table: {file_name}"); }); // The raw file table doesn't include an entry for the working dir // (which has ID 0), so add 1 to get the correct ID. - (raw_id + 1) as u32 + GlobalFileId::from_usize(raw_id + 1) } fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> { @@ -192,30 +192,35 @@ impl GlobalFileTable { .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) .to_string_lossy(); - llvm::build_byte_buffer(|buffer| { - coverageinfo::write_filenames_section_to_buffer( - // Insert the working dir at index 0, before the other filenames. - std::iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)), - buffer, - ); - }) + // Insert the working dir at index 0, before the other filenames. + let filenames = + iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)); + llvm_cov::write_filenames_to_buffer(filenames) } } rustc_index::newtype_index! { - struct LocalFileId {} + /// An index into the CGU's overall list of file paths. The underlying paths + /// will be embedded in the `__llvm_covmap` linker section. + struct GlobalFileId {} +} +rustc_index::newtype_index! { + /// An index into a function's list of global file IDs. That underlying list + /// of local-to-global mappings will be embedded in the function's record in + /// the `__llvm_covfun` linker section. + pub(crate) struct LocalFileId {} } /// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) /// file IDs. #[derive(Default)] struct VirtualFileMapping { - local_to_global: IndexVec<LocalFileId, u32>, - global_to_local: FxIndexMap<u32, LocalFileId>, + local_to_global: IndexVec<LocalFileId, GlobalFileId>, + global_to_local: FxIndexMap<GlobalFileId, LocalFileId>, } impl VirtualFileMapping { - fn local_id_for_global(&mut self, global_file_id: u32) -> LocalFileId { + fn local_id_for_global(&mut self, global_file_id: GlobalFileId) -> LocalFileId { *self .global_to_local .entry(global_file_id) @@ -223,16 +228,26 @@ impl VirtualFileMapping { } fn into_vec(self) -> Vec<u32> { - self.local_to_global.raw + // This conversion should be optimized away to ~zero overhead. + // In any case, it's probably not hot enough to worry about. + self.local_to_global.into_iter().map(|global| global.as_u32()).collect() } } +fn span_file_name(tcx: TyCtxt<'_>, span: Span) -> Symbol { + let source_file = tcx.sess.source_map().lookup_source_file(span.lo()); + let name = + source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy(); + Symbol::intern(&name) +} + /// Using the expressions and counter regions collected for a single function, /// generate the variable-sized payload of its corresponding `__llvm_covfun` /// entry. The payload is returned as a vector of bytes. /// /// Newly-encountered filenames will be added to the global file table. fn encode_mappings_for_function( + tcx: TyCtxt<'_>, global_file_table: &GlobalFileTable, function_coverage: &FunctionCoverage<'_>, ) -> Vec<u8> { @@ -244,54 +259,74 @@ fn encode_mappings_for_function( let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); let mut virtual_file_mapping = VirtualFileMapping::default(); - let mut mapping_regions = Vec::with_capacity(counter_regions.len()); - - // Group mappings into runs with the same filename, preserving the order - // yielded by `FunctionCoverage`. - // Prepare file IDs for each filename, and prepare the mapping data so that - // we can pass it through FFI to LLVM. - for (file_name, counter_regions_for_file) in - &counter_regions.group_by(|(_, region)| region.file_name) - { - // Look up the global file ID for this filename. - let global_file_id = global_file_table.global_file_id_for_file_name(file_name); - - // Associate that global file ID with a local file ID for this function. - let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); - debug!(" file id: {local_file_id:?} => global {global_file_id} = '{file_name:?}'"); - - // For each counter/region pair in this function+file, convert it to a - // form suitable for FFI. - for (mapping_kind, region) in counter_regions_for_file { - debug!("Adding counter {mapping_kind:?} to map for {region:?}"); - mapping_regions.push(CounterMappingRegion::from_mapping( - &mapping_kind, - local_file_id.as_u32(), - region, - )); + let mut code_regions = vec![]; + let mut branch_regions = vec![]; + let mut mcdc_branch_regions = vec![]; + let mut mcdc_decision_regions = vec![]; + + // Currently a function's mappings must all be in the same file as its body span. + let file_name = span_file_name(tcx, function_coverage.function_coverage_info.body_span); + + // Look up the global file ID for that filename. + let global_file_id = global_file_table.global_file_id_for_file_name(file_name); + + // Associate that global file ID with a local file ID for this function. + let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); + debug!(" file id: {local_file_id:?} => {global_file_id:?} = '{file_name:?}'"); + + // For each counter/region pair in this function+file, convert it to a + // form suitable for FFI. + for (mapping_kind, region) in counter_regions { + debug!("Adding counter {mapping_kind:?} to map for {region:?}"); + let span = ffi::CoverageSpan::from_source_region(local_file_id, region); + match mapping_kind { + MappingKind::Code(term) => { + code_regions.push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); + } + MappingKind::Branch { true_term, false_term } => { + branch_regions.push(ffi::BranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + }); + } + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + mcdc_branch_regions.push(ffi::MCDCBranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), + }); + } + MappingKind::MCDCDecision(mcdc_decision_params) => { + mcdc_decision_regions.push(ffi::MCDCDecisionRegion { + span, + mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), + }); + } } } // Encode the function's coverage mappings into a buffer. - llvm::build_byte_buffer(|buffer| { - coverageinfo::write_mapping_to_buffer( - virtual_file_mapping.into_vec(), - expressions, - mapping_regions, - buffer, - ); - }) + llvm_cov::write_function_mappings_to_buffer( + &virtual_file_mapping.into_vec(), + &expressions, + &code_regions, + &branch_regions, + &mcdc_branch_regions, + &mcdc_decision_regions, + ) } -/// Construct coverage map header and the array of function records, and combine them into the -/// coverage map. Save the coverage map data into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn generate_coverage_map<'ll>( +/// Generates the contents of the covmap record for this CGU, which mostly +/// consists of a header and a list of filenames. The record is then stored +/// as a global variable in the `__llvm_covmap` section. +fn generate_covmap_record<'ll>( cx: &CodegenCx<'ll, '_>, version: u32, filenames_size: usize, filenames_val: &'ll llvm::Value, -) -> &'ll llvm::Value { +) { debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); // Create the coverage data header (Note, fields 0 and 2 are now always zero, @@ -306,15 +341,25 @@ fn generate_coverage_map<'ll>( ); // Create the complete LLVM coverage data value to add to the LLVM IR - cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) + let covmap_data = + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(covmap_data), &llvm_cov::covmap_var_name()); + llvm::set_initializer(llglobal, covmap_data); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, &llvm_cov::covmap_section_name(cx.llmod)); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + cx.add_used_global(llglobal); } -/// Construct a function record and combine it with the function's coverage mapping data. -/// Save the function record into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn save_function_record( +/// Generates the contents of the covfun record for this function, which +/// contains the function's coverage mapping data. The record is then stored +/// as a global variable in the `__llvm_covfun` section. +fn generate_covfun_record( cx: &CodegenCx<'_, '_>, - covfun_section_name: &str, mangled_function_name: &str, source_hash: u64, filenames_ref: u64, @@ -325,7 +370,7 @@ fn save_function_record( let coverage_mapping_size = coverage_mapping_buffer.len(); let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); - let func_name_hash = coverageinfo::hash_bytes(mangled_function_name.as_bytes()); + let func_name_hash = llvm_cov::hash_bytes(mangled_function_name.as_bytes()); let func_name_hash_val = cx.const_u64(func_name_hash); let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); let source_hash_val = cx.const_u64(source_hash); @@ -341,13 +386,28 @@ fn save_function_record( /*packed=*/ true, ); - coverageinfo::save_func_record_to_mod( - cx, - covfun_section_name, - func_name_hash, - func_record_val, - is_used, - ); + // Choose a variable name to hold this function's covfun data. + // Functions that are used have a suffix ("u") to distinguish them from + // unused copies of the same function (from different CGUs), so that if a + // linker sees both it won't discard the used copy's data. + let func_record_var_name = + CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) + .unwrap(); + debug!("function record var name: {:?}", func_record_var_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, cx.covfun_section_name()); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + if cx.target_spec().supports_comdat() { + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + } + cx.add_used_global(llglobal); } /// Each CGU will normally only emit coverage metadata for the functions that it actually generates. @@ -479,9 +539,5 @@ fn add_unused_function_coverage<'tcx>( // zero, because none of its counters/expressions are marked as seen. let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); - if let Some(coverage_context) = cx.coverage_context() { - coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); - } else { - bug!("Could not get the `coverage_context`"); - } + cx.coverage_cx().function_coverage_map.borrow_mut().insert(instance, function_coverage); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 3a80d216f47..bf773cd2667 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,26 +1,23 @@ -use std::cell::RefCell; +use std::cell::{OnceCell, RefCell}; +use std::ffi::{CStr, CString}; -use libc::c_uint; +use rustc_abi::Size; use rustc_codegen_ssa::traits::{ - BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, - MiscCodegenMethods, StaticCodegenMethods, + BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, }; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; -use rustc_llvm::RustString; -use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use rustc_middle::ty::layout::HasTyCtxt; -use rustc_target::abi::{Align, Size}; use tracing::{debug, instrument}; use crate::builder::Builder; use crate::common::CodegenCx; -use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; use crate::coverageinfo::map_data::FunctionCoverageCollector; use crate::llvm; pub(crate) mod ffi; +mod llvm_cov; pub(crate) mod map_data; mod mapgen; @@ -31,6 +28,8 @@ pub(crate) struct CrateCoverageContext<'ll, 'tcx> { RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, + + covfun_section_name: OnceCell<CString>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -39,6 +38,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { function_coverage_map: Default::default(), pgo_func_name_var_map: Default::default(), mcdc_condition_bitmap_map: Default::default(), + covfun_section_name: Default::default(), } } @@ -65,27 +65,37 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } -// These methods used to be part of trait `CoverageInfoMethods`, which no longer -// exists after most coverage code was moved out of SSA. impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn coverageinfo_finalize(&self) { mapgen::finalize(self) } + /// Returns the section name to use when embedding per-function coverage information + /// in the object file, according to the target's object file format. LLVM's coverage + /// tools use information from this section when producing coverage reports. + /// + /// Typical values are: + /// - `__llvm_covfun` on Linux + /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) + /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) + fn covfun_section_name(&self) -> &CStr { + self.coverage_cx() + .covfun_section_name + .get_or_init(|| llvm_cov::covfun_section_name(self.llmod)) + } + /// For LLVM codegen, returns a function-specific `Value` for a global /// string, to hold the function name passed to LLVM intrinsic /// `instrprof.increment()`. The `Value` is only created once per instance. /// Multiple invocations with the same instance return the same `Value`. fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { - if let Some(coverage_context) = self.coverage_context() { - debug!("getting pgo_func_name_var for instance={:?}", instance); - let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); - pgo_func_name_var_map - .entry(instance) - .or_insert_with(|| create_pgo_func_name_var(self, instance)) - } else { - bug!("Could not get the `coverage_context`"); - } + debug!("getting pgo_func_name_var for instance={:?}", instance); + let mut pgo_func_name_var_map = self.coverage_cx().pgo_func_name_var_map.borrow_mut(); + pgo_func_name_var_map.entry(instance).or_insert_with(|| { + let llfn = self.get_fn(instance); + let mangled_fn_name: &str = self.tcx.symbol_name(instance).name; + llvm_cov::create_pgo_func_name_var(llfn, mangled_fn_name) + }) } } @@ -98,14 +108,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { }; // If there are no MC/DC bitmaps to set up, return immediately. - if function_coverage_info.mcdc_bitmap_bytes == 0 { + if function_coverage_info.mcdc_bitmap_bits == 0 { return; } let fn_name = self.get_pgo_func_name_var(instance); let hash = self.const_u64(function_coverage_info.function_source_hash); - let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes); - self.mcdc_parameters(fn_name, hash, bitmap_bytes); + let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32); + self.mcdc_parameters(fn_name, hash, bitmap_bits); // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps. let mut cond_bitmaps = vec![]; @@ -119,11 +129,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { cond_bitmaps.push(cond_bitmap); } - self.coverage_context() - .expect("always present when coverage is enabled") - .mcdc_condition_bitmap_map - .borrow_mut() - .insert(instance, cond_bitmaps); + self.coverage_cx().mcdc_condition_bitmap_map.borrow_mut().insert(instance, cond_bitmaps); } #[instrument(level = "debug", skip(self))] @@ -144,8 +150,12 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - let Some(coverage_context) = bx.coverage_context() else { return }; - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + // FIXME(#132395): Unwrapping `coverage_cx` here has led to ICEs in the + // wild, so keep this early-return until we understand why. + let mut coverage_map = match bx.coverage_cx { + Some(ref cx) => cx.function_coverage_map.borrow_mut(), + None => return, + }; let func_coverage = coverage_map .entry(instance) .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); @@ -185,180 +195,31 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::ExpressionUsed { id } => { func_coverage.mark_expression_id_seen(id); } - CoverageKind::CondBitmapUpdate { id, value, decision_depth } => { + CoverageKind::CondBitmapUpdate { index, decision_depth } => { drop(coverage_map); - assert_ne!( - id.as_u32(), - 0, - "ConditionId of evaluated conditions should never be zero" - ); - let cond_bitmap = coverage_context + let cond_bitmap = bx + .coverage_cx() .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); - let cond_loc = bx.const_i32(id.as_u32() as i32 - 1); - let bool_value = bx.const_bool(value); - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_coverage_info.function_source_hash); - bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value); + let cond_index = bx.const_i32(index as i32); + bx.mcdc_condbitmap_update(cond_index, cond_bitmap); } CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { drop(coverage_map); - let cond_bitmap = coverage_context + let cond_bitmap = bx.coverage_cx() .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); - let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes; - assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range"); + assert!( + bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits, + "bitmap index of the decision out of range" + ); let fn_name = bx.get_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); - let bitmap_bytes = bx.const_u32(bitmap_bytes); let bitmap_index = bx.const_u32(bitmap_idx); - bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap); + bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); + bx.mcdc_condbitmap_reset(cond_bitmap); } } } } - -/// Calls llvm::createPGOFuncNameVar() with the given function instance's -/// mangled function name. The LLVM API returns an llvm::GlobalVariable -/// containing the function name, with the specific variable name and linkage -/// required by LLVM InstrProf source-based coverage instrumentation. Use -/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per -/// `Instance`. -fn create_pgo_func_name_var<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - instance: Instance<'tcx>, -) -> &'ll llvm::Value { - let mangled_fn_name: &str = cx.tcx.symbol_name(instance).name; - let llfn = cx.get_fn(instance); - unsafe { - llvm::LLVMRustCoverageCreatePGOFuncNameVar( - llfn, - mangled_fn_name.as_ptr().cast(), - mangled_fn_name.len(), - ) - } -} - -pub(crate) fn write_filenames_section_to_buffer<'a>( - filenames: impl IntoIterator<Item = &'a str>, - buffer: &RustString, -) { - let (pointers, lengths) = filenames - .into_iter() - .map(|s: &str| (s.as_ptr().cast(), s.len())) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - unsafe { - llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( - pointers.as_ptr(), - pointers.len(), - lengths.as_ptr(), - lengths.len(), - buffer, - ); - } -} - -pub(crate) fn write_mapping_to_buffer( - virtual_file_mapping: Vec<u32>, - expressions: Vec<CounterExpression>, - mapping_regions: Vec<CounterMappingRegion>, - buffer: &RustString, -) { - unsafe { - llvm::LLVMRustCoverageWriteMappingToBuffer( - virtual_file_mapping.as_ptr(), - virtual_file_mapping.len() as c_uint, - expressions.as_ptr(), - expressions.len() as c_uint, - mapping_regions.as_ptr(), - mapping_regions.len() as c_uint, - buffer, - ); - } -} - -pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } -} - -pub(crate) fn mapping_version() -> u32 { - unsafe { llvm::LLVMRustCoverageMappingVersion() } -} - -pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - cov_data_val: &'ll llvm::Value, -) { - let covmap_var_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteMappingVarNameToString(s); - }) - .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); - debug!("covmap var name: {:?}", covmap_var_name); - - let covmap_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage section name failed UTF-8 conversion"); - debug!("covmap section name: {:?}", covmap_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); - llvm::set_initializer(llglobal, cov_data_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::set_section(llglobal, &covmap_section_name); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - llvm::set_alignment(llglobal, Align::EIGHT); - cx.add_used_global(llglobal); -} - -pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - covfun_section_name: &str, - func_name_hash: u64, - func_record_val: &'ll llvm::Value, - is_used: bool, -) { - // Assign a name to the function record. This is used to merge duplicates. - // - // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that - // are included-but-not-used. If (or when) Rust generates functions that are - // included-but-not-used, note that a dummy description for a function included-but-not-used - // in a Crate can be replaced by full description provided by a different Crate. The two kinds - // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by - // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. - let func_record_var_name = - format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); - debug!("function record var name: {:?}", func_record_var_name); - debug!("function record section name: {:?}", covfun_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); - llvm::set_initializer(llglobal, func_record_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); - llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, covfun_section_name); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - llvm::set_alignment(llglobal, Align::EIGHT); - llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); - cx.add_used_global(llglobal); -} - -/// Returns the section name string to pass through to the linker when embedding -/// per-function coverage information in the object file, according to the target -/// platform's object file format. -/// -/// LLVM's coverage tools read coverage mapping details from this section when -/// producing coverage reports. -/// -/// Typical values are: -/// - `__llvm_covfun` on Linux -/// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) -/// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) -pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String { - llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage function record section name failed UTF-8 conversion") -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index ac6c2fb1b83..07bd0f4d1c1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -1,11 +1,15 @@ +use std::collections::hash_map::Entry; + use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; use rustc_index::Idx; use rustc_index::bit_set::BitSet; use rustc_middle::mir::{Body, SourceScope}; -use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv}; use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; +use rustc_span::{BytePos, hygiene}; use super::metadata::file_metadata; use super::utils::DIB; @@ -37,10 +41,20 @@ pub(crate) fn compute_mir_scopes<'ll, 'tcx>( None }; let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); + let mut discriminators = FxHashMap::default(); // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { let scope = SourceScope::new(idx); - make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); + make_mir_scope( + cx, + instance, + mir, + &variables, + debug_context, + &mut instantiated, + &mut discriminators, + scope, + ); } assert!(instantiated.count() == mir.source_scopes.len()); } @@ -52,6 +66,7 @@ fn make_mir_scope<'ll, 'tcx>( variables: &Option<BitSet<SourceScope>>, debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, instantiated: &mut BitSet<SourceScope>, + discriminators: &mut FxHashMap<BytePos, u32>, scope: SourceScope, ) { if instantiated.contains(scope) { @@ -60,16 +75,33 @@ fn make_mir_scope<'ll, 'tcx>( let scope_data = &mir.source_scopes[scope]; let parent_scope = if let Some(parent) = scope_data.parent_scope { - make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); - debug_context.scopes[parent] + make_mir_scope( + cx, + instance, + mir, + variables, + debug_context, + instantiated, + discriminators, + parent, + ); + if let Some(parent_scope) = debug_context.scopes[parent] { + parent_scope + } else { + // If the parent scope could not be represented then no children + // can be either. + debug_context.scopes[scope] = None; + instantiated.insert(scope); + return; + } } else { // The root is the function itself. let file = cx.sess().source_map().lookup_source_file(mir.span.lo()); - debug_context.scopes[scope] = DebugScope { + debug_context.scopes[scope] = Some(DebugScope { file_start_pos: file.start_pos, file_end_pos: file.end_position(), - ..debug_context.scopes[scope] - }; + ..debug_context.scopes[scope].unwrap() + }); instantiated.insert(scope); return; }; @@ -80,7 +112,7 @@ fn make_mir_scope<'ll, 'tcx>( { // Do not create a DIScope if there are no variables defined in this // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. - debug_context.scopes[scope] = parent_scope; + debug_context.scopes[scope] = Some(parent_scope); instantiated.insert(scope); return; } @@ -94,7 +126,7 @@ fn make_mir_scope<'ll, 'tcx>( // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. let callee = cx.tcx.instantiate_and_normalize_erasing_regions( instance.args, - ty::ParamEnv::reveal_all(), + cx.typing_env(), ty::EarlyBinder::bind(callee), ); debug_context.inlined_function_scopes.entry(callee).or_insert_with(|| { @@ -113,18 +145,59 @@ fn make_mir_scope<'ll, 'tcx>( }, }; - let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { - // FIXME(eddyb) this doesn't account for the macro-related - // `Span` fixups that `rustc_codegen_ssa::mir::debuginfo` does. - let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); - cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span) - }); - - debug_context.scopes[scope] = DebugScope { + let mut debug_scope = Some(DebugScope { dbg_scope, - inlined_at: inlined_at.or(parent_scope.inlined_at), + inlined_at: parent_scope.inlined_at, file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_position(), - }; + }); + + if let Some((_, callsite_span)) = scope_data.inlined { + let callsite_span = hygiene::walk_chain_collapsed(callsite_span, mir.span); + let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); + let loc = cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span); + + // NB: In order to produce proper debug info for variables (particularly + // arguments) in multiply-inlined functions, LLVM expects to see a single + // DILocalVariable with multiple different DILocations in the IR. While + // the source information for each DILocation would be identical, their + // inlinedAt attributes will be unique to the particular callsite. + // + // We generate DILocations here based on the callsite's location in the + // source code. A single location in the source code usually can't + // produce multiple distinct calls so this mostly works, until + // macros get involved. A macro can generate multiple calls + // at the same span, which breaks the assumption that we're going to + // produce a unique DILocation for every scope we process here. We + // have to explicitly add discriminators if we see inlines into the + // same source code location. + // + // Note further that we can't key this hashtable on the span itself, + // because these spans could have distinct SyntaxContexts. We have + // to key on exactly what we're giving to LLVM. + let inlined_at = match discriminators.entry(callsite_span.lo()) { + Entry::Occupied(mut o) => { + *o.get_mut() += 1; + unsafe { llvm::LLVMRustDILocationCloneWithBaseDiscriminator(loc, *o.get()) } + } + Entry::Vacant(v) => { + v.insert(0); + Some(loc) + } + }; + match inlined_at { + Some(inlined_at) => { + debug_scope.as_mut().unwrap().inlined_at = Some(inlined_at); + } + None => { + // LLVM has a maximum discriminator that it can encode (currently + // it uses 12 bits for 4096 possible values). If we exceed that + // there is little we can do but drop the debug info. + debug_scope = None; + } + } + } + + debug_context.scopes[scope] = debug_scope; instantiated.insert(scope); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index f93d3e40b20..aef8642f199 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -72,11 +72,11 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr()); + llvm::set_section(section_var, c".debug_gdb_scripts"); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(section_var, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than // the string it contains. Otherwise we get a warning from GDB. llvm::LLVMSetAlignment(section_var, 1); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 964b83c0fa0..ef16e5bb459 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -5,22 +5,20 @@ use std::path::{Path, PathBuf}; use std::{iter, ptr}; use libc::{c_char, c_longlong, c_uint}; +use rustc_abi::{Align, Size}; use rustc_codegen_ssa::debuginfo::type_names::{VTableNameKind, cpp_like_debuginfo}; use rustc_codegen_ssa::traits::*; -use rustc_fs_util::path_to_c_string; use rustc_hir::def::{CtorKind, DefKind}; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, - Visibility, + self, AdtKind, CoroutineArgsExt, Instance, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, hygiene}; use rustc_symbol_mangling::typeid_for_trait_ref; -use rustc_target::abi::{Align, Size}; use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; use tracing::{debug, instrument}; @@ -32,9 +30,9 @@ use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_na use super::utils::{ DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, }; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::build_type_with_children; -use crate::debuginfo::utils::{FatPtrKind, fat_pointer_kind}; +use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind}; use crate::llvm::debuginfo::{ DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, DebugNameTableKind, @@ -161,7 +159,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { // The debuginfo generated by this function is only valid if `ptr_type` is really just - // a (fat) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. + // a (wide) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. assert_eq!( cx.size_and_align_of(ptr_type), cx.size_and_align_of(Ty::new_mut_ptr(cx.tcx, pointee_type)) @@ -174,7 +172,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( let data_layout = &cx.tcx.data_layout; let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); - match fat_pointer_kind(cx, pointee_type) { + match wide_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. assert_eq!( @@ -190,14 +188,14 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( data_layout.pointer_size.bits(), data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_ptr().cast(), + ptr_type_debuginfo_name.as_c_char_ptr(), ptr_type_debuginfo_name.len(), ) }; DINodeCreationResult { di_node, already_stored_in_typemap: false } } - Some(fat_pointer_kind) => { + Some(wide_pointer_kind) => { type_map::build_type_with_children( cx, type_map::stub( @@ -210,7 +208,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( DIFlags::FlagZero, ), |cx, owner| { - // FIXME: If this fat pointer is a `Box` then we don't want to use its + // FIXME: If this wide pointer is a `Box` then we don't want to use its // type layout and instead use the layout of the raw pointer inside // of it. // The proper way to handle this is to not treat Box as a pointer @@ -227,16 +225,16 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( }; let layout = cx.layout_of(layout_type); - let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); - let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); + let addr_field = layout.field(cx, abi::WIDE_PTR_ADDR); + let extra_field = layout.field(cx, abi::WIDE_PTR_EXTRA); - let (addr_field_name, extra_field_name) = match fat_pointer_kind { - FatPtrKind::Dyn => ("pointer", "vtable"), - FatPtrKind::Slice => ("data_ptr", "length"), + let (addr_field_name, extra_field_name) = match wide_pointer_kind { + WidePtrKind::Dyn => ("pointer", "vtable"), + WidePtrKind::Slice => ("data_ptr", "length"), }; - assert_eq!(abi::FAT_PTR_ADDR, 0); - assert_eq!(abi::FAT_PTR_EXTRA, 1); + assert_eq!(abi::WIDE_PTR_ADDR, 0); + assert_eq!(abi::WIDE_PTR_EXTRA, 1); // The data pointer type is a regular, thin pointer, regardless of whether this // is a slice or a trait object. @@ -258,7 +256,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( owner, addr_field_name, (addr_field.size, addr_field.align.abi), - layout.fields.offset(abi::FAT_PTR_ADDR), + layout.fields.offset(abi::WIDE_PTR_ADDR), DIFlags::FlagZero, data_ptr_type_di_node, ), @@ -267,7 +265,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( owner, extra_field_name, (extra_field.size, extra_field.align.abi), - layout.fields.offset(abi::FAT_PTR_EXTRA), + layout.fields.offset(abi::WIDE_PTR_EXTRA), DIFlags::FlagZero, type_di_node(cx, extra_field.ty), ), @@ -302,9 +300,8 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( .insert(unique_type_id, recursion_marker_type_di_node(cx)); let fn_ty = unique_type_id.expect_ty(); - let signature = cx - .tcx - .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), fn_ty.fn_sig(cx.tcx)); + let signature = + cx.tcx.normalize_erasing_late_bound_regions(cx.typing_env(), fn_ty.fn_sig(cx.tcx)); let signature_di_nodes: SmallVec<_> = iter::once( // return type @@ -348,7 +345,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( size, align, 0, // Ignore DWARF address space. - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), ) }; @@ -391,7 +388,7 @@ fn build_dyn_type_di_node<'ll, 'tcx>( /// /// NOTE: We currently emit just emit the debuginfo for the element type here /// (i.e. `T` for slices and `u8` for `str`), so that we end up with -/// `*const T` for the `data_ptr` field of the corresponding fat-pointer +/// `*const T` for the `data_ptr` field of the corresponding wide-pointer /// debuginfo of `&[T]`. /// /// It would be preferable and more accurate if we emitted a DIArray of T @@ -518,7 +515,7 @@ fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll D let name = "<recur_type>"; llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), cx.tcx.data_layout.pointer_size.bits(), DW_ATE_unsigned, @@ -630,6 +627,7 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, + rustc_span::SourceFileHashAlgorithm::Blake3 => llvm::ChecksumKind::None, }; let hash_value = hex_encode(source_file.src_hash.hash_bytes()); @@ -639,14 +637,14 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi unsafe { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), - file_name.as_ptr().cast(), + file_name.as_c_char_ptr(), file_name.len(), - directory.as_ptr().cast(), + directory.as_c_char_ptr(), directory.len(), hash_kind, - hash_value.as_ptr().cast(), + hash_value.as_c_char_ptr(), hash_value.len(), - source.map_or(ptr::null(), |x| x.as_ptr().cast()), + source.map_or(ptr::null(), |x| x.as_c_char_ptr()), source.map_or(0, |x| x.len()), ) } @@ -661,12 +659,12 @@ fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), - file_name.as_ptr().cast(), + file_name.as_c_char_ptr(), file_name.len(), - directory.as_ptr().cast(), + directory.as_c_char_ptr(), directory.len(), llvm::ChecksumKind::None, - hash_value.as_ptr().cast(), + hash_value.as_c_char_ptr(), hash_value.len(), ptr::null(), 0, @@ -787,7 +785,7 @@ fn build_basic_type_di_node<'ll, 'tcx>( let ty_di_node = unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), cx.size_of(t).bits(), encoding, @@ -809,7 +807,7 @@ fn build_basic_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_ptr().cast(), + typedef_name.as_c_char_ptr(), typedef_name.len(), unknown_file_metadata(cx), 0, @@ -860,7 +858,7 @@ fn build_param_type_di_node<'ll, 'tcx>( di_node: unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), Size::ZERO.bits(), DW_ATE_unsigned, @@ -947,9 +945,9 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( debug_context.builder, - name_in_debuginfo.as_ptr().cast(), + name_in_debuginfo.as_c_char_ptr(), name_in_debuginfo.len(), - work_dir.as_ptr().cast(), + work_dir.as_c_char_ptr(), work_dir.len(), llvm::ChecksumKind::None, ptr::null(), @@ -962,7 +960,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( debug_context.builder, DW_LANG_RUST, compile_unit_file, - producer.as_ptr().cast(), + producer.as_c_char_ptr(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, c"".as_ptr(), @@ -970,7 +968,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final // output(s). - split_name.as_ptr().cast(), + split_name.as_c_char_ptr(), split_name.len(), kind, 0, @@ -978,33 +976,8 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( debug_name_table_kind, ); - if tcx.sess.opts.unstable_opts.profile { - let default_gcda_path = &output_filenames.with_extension("gcda"); - let gcda_path = - tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); - - let gcov_cu_info = [ - path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), - path_to_mdstring(debug_context.llcontext, gcda_path), - unit_metadata, - ]; - let gcov_metadata = llvm::LLVMMDNodeInContext2( - debug_context.llcontext, - gcov_cu_info.as_ptr(), - gcov_cu_info.len(), - ); - let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); - - llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val); - } - return unit_metadata; }; - - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata { - let path_str = path_to_c_string(path); - unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) } - } } /// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. @@ -1021,7 +994,7 @@ fn build_field_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), owner, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -1134,9 +1107,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( } }; - assert!( - up_var_tys.iter().all(|t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) - ); + assert!(up_var_tys.iter().all(|t| t == cx.tcx.normalize_erasing_regions(cx.typing_env(), t))); let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); let layout = cx.layout_of(closure_or_coroutine_ty); @@ -1297,15 +1268,14 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( let template_params: SmallVec<_> = iter::zip(args, names) .filter_map(|(kind, name)| { kind.as_type().map(|ty| { - let actual_type = - cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); + let actual_type = cx.tcx.normalize_erasing_regions(cx.typing_env(), ty); let actual_type_di_node = type_di_node(cx, actual_type); let name = name.as_str(); unsafe { llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), actual_type_di_node, ) @@ -1366,7 +1336,7 @@ pub(crate) fn build_global_var_di_node<'ll>( if nested { return; } - let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); + let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, cx.typing_env()); let type_di_node = type_di_node(cx, variable_type); let var_name = tcx.item_name(def_id); let var_name = var_name.as_str(); @@ -1381,9 +1351,9 @@ pub(crate) fn build_global_var_di_node<'ll>( llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), Some(var_scope), - var_name.as_ptr().cast(), + var_name.as_c_char_ptr(), var_name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, line_number, @@ -1601,9 +1571,9 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), NO_SCOPE_METADATA, - vtable_name.as_ptr().cast(), + vtable_name.as_c_char_ptr(), vtable_name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 966788cf32f..100b046cee2 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use libc::c_uint; +use rustc_abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstCodegenMethods; @@ -8,10 +9,9 @@ use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; -use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ @@ -359,7 +359,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticMemberType( DIB(cx), enum_type_di_node, - TAG_FIELD_NAME.as_ptr().cast(), + TAG_FIELD_NAME.as_c_char_ptr(), TAG_FIELD_NAME.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -537,7 +537,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticMemberType( DIB(cx), wrapper_struct_type_di_node, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -785,7 +785,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), enum_type_di_node, - field_name.as_ptr().cast(), + field_name.as_c_char_ptr(), field_name.len(), file_di_node, line_number, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index fe1634146ff..b3d4a6642a1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use rustc_abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_hir::def::CtorKind; @@ -9,11 +10,10 @@ use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; use rustc_span::Symbol; -use rustc_target::abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub}; use crate::debuginfo::metadata::{ UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, type_di_node, @@ -106,7 +106,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( let value = [value as u64, (value >> 64) as u64]; Some(llvm::LLVMRustDIBuilderCreateEnumerator( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), value.as_ptr(), size.bits() as libc::c_uint, @@ -119,7 +119,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), containing_scope, - type_name.as_ptr().cast(), + type_name.as_c_char_ptr(), type_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 5e7dbdd921a..d4006691d37 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -1,16 +1,16 @@ use std::borrow::Cow; use libc::c_uint; +use rustc_abi::{Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstCodegenMethods; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self}; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, SmallVec, UNKNOWN_LINE_NUMBER, file_metadata, @@ -244,7 +244,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateVariantPart( DIB(cx), enum_type_di_node, - variant_part_name.as_ptr().cast(), + variant_part_name.as_c_char_ptr(), variant_part_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -253,7 +253,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( DIFlags::FlagZero, tag_member_di_node, create_DIArray(DIB(cx), &[]), - variant_part_unique_type_id_str.as_ptr().cast(), + variant_part_unique_type_id_str.as_c_char_ptr(), variant_part_unique_type_id_str.len(), ) }, @@ -327,7 +327,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( Some(llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), containing_scope, - tag_name.as_ptr().cast(), + tag_name.as_c_char_ptr(), tag_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -399,7 +399,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateVariantMemberType( DIB(cx), variant_part_di_node, - variant_member_info.variant_name.as_ptr().cast(), + variant_member_info.variant_name.as_c_char_ptr(), variant_member_info.variant_name.len(), file_di_node, line_number, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 714e3c0b145..4e461476040 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -1,15 +1,15 @@ use std::cell::RefCell; +use rustc_abi::{Align, Size, VariantIdx}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_macros::HashStable; use rustc_middle::bug; -use rustc_middle::ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}; -use rustc_target::abi::{Align, Size, VariantIdx}; +use rustc_middle::ty::{self, PolyExistentialTraitRef, Ty, TyCtxt}; use super::{SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; use crate::llvm::{self}; @@ -49,12 +49,15 @@ pub(super) enum UniqueTypeId<'tcx> { impl<'tcx> UniqueTypeId<'tcx> { pub(crate) fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { - assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); + assert_eq!(t, tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), t)); UniqueTypeId::Ty(t, private::HiddenZst) } pub(crate) fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { - assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty) + ); UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) } @@ -63,7 +66,10 @@ impl<'tcx> UniqueTypeId<'tcx> { enum_ty: Ty<'tcx>, variant_idx: VariantIdx, ) -> Self { - assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty) + ); UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) } @@ -72,7 +78,10 @@ impl<'tcx> UniqueTypeId<'tcx> { enum_ty: Ty<'tcx>, variant_idx: VariantIdx, ) -> Self { - assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty) + ); UniqueTypeId::VariantStructTypeCppLikeWrapper(enum_ty, variant_idx, private::HiddenZst) } @@ -81,10 +90,13 @@ impl<'tcx> UniqueTypeId<'tcx> { self_type: Ty<'tcx>, implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, ) -> Self { - assert_eq!(self_type, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type)); + assert_eq!( + self_type, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), self_type) + ); assert_eq!( implemented_trait, - tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), implemented_trait) ); UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst) } @@ -191,7 +203,7 @@ pub(super) fn stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStructType( DIB(cx), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -202,7 +214,7 @@ pub(super) fn stub<'ll, 'tcx>( empty_array, 0, vtable_holder, - unique_type_id_str.as_ptr().cast(), + unique_type_id_str.as_c_char_ptr(), unique_type_id_str.len(), ) } @@ -211,7 +223,7 @@ pub(super) fn stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateUnionType( DIB(cx), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -220,7 +232,7 @@ pub(super) fn stub<'ll, 'tcx>( flags, Some(empty_array), 0, - unique_type_id_str.as_ptr().cast(), + unique_type_id_str.as_c_char_ptr(), unique_type_id_str.len(), ) }, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 1a8153a54e8..a8fdfbed592 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -5,6 +5,7 @@ use std::ops::Range; use std::{iter, ptr}; use libc::c_uint; +use rustc_abi::Size; use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; @@ -14,15 +15,14 @@ use rustc_data_structures::unord::UnordMap; use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::IndexVec; use rustc_middle::mir; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, GenericArgsRef, Instance, ParamEnv, Ty, TypeVisitableExt}; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty, TypeVisitableExt}; use rustc_session::Session; use rustc_session::config::{self, DebugInfo}; use rustc_span::symbol::Symbol; use rustc_span::{ BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, }; -use rustc_target::abi::Size; use smallvec::SmallVec; use tracing::debug; @@ -31,7 +31,7 @@ use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; use crate::abi::FnAbi; use crate::builder::Builder; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::{ DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, @@ -55,7 +55,6 @@ const DW_TAG_arg_variable: c_uint = 0x101; /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { - llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>, @@ -78,9 +77,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { debug!("CodegenUnitDebugContext::new"); let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; // DIBuilder inherits context from the module, so we'd better use the same one - let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; CodegenUnitDebugContext { - llcontext, llmod, builder, created_files: Default::default(), @@ -91,45 +88,39 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { } pub(crate) fn finalize(&self, sess: &Session) { - unsafe { - llvm::LLVMRustDIBuilderFinalize(self.builder); - - if !sess.target.is_like_msvc { - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - let dwarf_version = sess - .opts - .unstable_opts - .dwarf_version - .unwrap_or(sess.target.default_dwarf_version); - llvm::LLVMRustAddModuleFlagU32( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"Dwarf Version".as_ptr(), - dwarf_version, - ); - } else { - // Indicate that we want CodeView debug information on MSVC - llvm::LLVMRustAddModuleFlagU32( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"CodeView".as_ptr(), - 1, - ) - } - - // Prevent bitcode readers from deleting the debug info. - llvm::LLVMRustAddModuleFlagU32( + unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder) }; + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = + sess.opts.unstable_opts.dwarf_version.unwrap_or(sess.target.default_dwarf_version); + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "Dwarf Version", + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC + llvm::add_module_flag_u32( self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"Debug Info Version".as_ptr(), - llvm::LLVMRustDebugMetadataVersion(), + llvm::ModuleFlagMergeBehavior::Warning, + "CodeView", + 1, ); } + + // Prevent bitcode readers from deleting the debug info. + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "Debug Info Version", + unsafe { llvm::LLVMRustDebugMetadataVersion() }, + ); } } @@ -215,6 +206,10 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { } } + fn get_dbg_loc(&self) -> Option<&'ll DILocation> { + unsafe { llvm::LLVMGetCurrentDebugLocation2(self.llbuilder) } + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { gdb::insert_reference_to_gdb_debug_scripts_section_global(self) } @@ -299,12 +294,12 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } // Initialize fn debug context (including scopes). - let empty_scope = DebugScope { + let empty_scope = Some(DebugScope { dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), inlined_at: None, file_start_pos: BytePos(0), file_end_pos: BytePos(0), - }; + }); let mut fn_debug_context = FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes), inlined_function_scopes: Default::default(), @@ -349,8 +344,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { type_names::push_generic_params( tcx, - tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), args), - enclosing_fn_def_id, + tcx.normalize_erasing_regions(self.typing_env(), args), &mut name, ); @@ -365,7 +359,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let mut flags = DIFlags::FlagPrototyped; - if fn_abi.ret.layout.abi.is_uninhabited() { + if fn_abi.ret.layout.is_uninhabited() { flags |= DIFlags::FlagNoReturn; } @@ -390,9 +384,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateMethod( DIB(self), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, loc.line, @@ -407,9 +401,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, loc.line, @@ -487,15 +481,14 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { iter::zip(args, names) .filter_map(|(kind, name)| { kind.as_type().map(|ty| { - let actual_type = - cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); + let actual_type = cx.tcx.normalize_erasing_regions(cx.typing_env(), ty); let actual_type_metadata = type_di_node(cx, actual_type); let name = name.as_str(); unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), actual_type_metadata, )) @@ -532,7 +525,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( instance.args, - ty::ParamEnv::reveal_all(), + cx.typing_env(), cx.tcx.type_of(impl_def_id), ); @@ -636,7 +629,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { DIB(self), dwarf_tag, scope_metadata, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), file_metadata, loc.line, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index 3578755aae0..33d9bc23890 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -5,7 +5,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::ty::{self, Instance}; use super::utils::{DIB, debug_context}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::DIScope; @@ -36,7 +36,7 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l llvm::LLVMRustDIBuilderCreateNameSpace( DIB(cx), parent_scope, - namespace_name_string.as_ptr().cast(), + namespace_name_string.as_c_char_ptr(), namespace_name_string.len(), false, // ExportSymbols (only relevant for C++ anonymous namespaces) ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index acb15449ce3..6e841293477 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -1,7 +1,7 @@ // Utility Functions. use rustc_hir::def_id::DefId; -use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Ty}; use tracing::trace; @@ -49,23 +49,23 @@ pub(crate) fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId } #[derive(Debug, PartialEq, Eq)] -pub(crate) enum FatPtrKind { +pub(crate) enum WidePtrKind { Slice, Dyn, } /// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. -/// if the second field of the fat pointer is a length or a vtable-pointer. -/// If `pointee_ty` does not require a fat pointer (because it is Sized) then +/// if the second field of the wide pointer is a length or a vtable-pointer. +/// If `pointee_ty` does not require a wide pointer (because it is Sized) then /// the function returns `None`. -pub(crate) fn fat_pointer_kind<'ll, 'tcx>( +pub(crate) fn wide_pointer_kind<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, pointee_ty: Ty<'tcx>, -) -> Option<FatPtrKind> { - let pointee_tail_ty = cx.tcx.struct_tail_for_codegen(pointee_ty, cx.param_env()); +) -> Option<WidePtrKind> { + let pointee_tail_ty = cx.tcx.struct_tail_for_codegen(pointee_ty, cx.typing_env()); let layout = cx.layout_of(pointee_tail_ty); trace!( - "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", + "wide_pointer_kind: {:?} has layout {:?} (is_unsized? {})", pointee_tail_ty, layout, layout.is_unsized() @@ -76,8 +76,8 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( } match *pointee_tail_ty.kind() { - ty::Str | ty::Slice(_) => Some(FatPtrKind::Slice), - ty::Dynamic(..) => Some(FatPtrKind::Dyn), + ty::Str | ty::Slice(_) => Some(WidePtrKind::Slice), + ty::Dynamic(..) => Some(WidePtrKind::Dyn), ty::Foreign(_) => { // Assert that pointers to foreign types really are thin: assert_eq!( @@ -90,7 +90,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( // For all other pointee types we should already have returned None // at the beginning of the function. panic!( - "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}" + "wide_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}" ) } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index b0b29ca1280..d338c848754 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -20,8 +20,10 @@ use smallvec::SmallVec; use tracing::debug; use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::common::AsCCharPtr; use crate::context::CodegenCx; use crate::llvm::AttributePlace::Function; +use crate::llvm::Visibility; use crate::type_::Type; use crate::value::Value; use crate::{attributes, llvm}; @@ -40,7 +42,7 @@ fn declare_raw_fn<'ll>( ) -> &'ll Value { debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); let llfn = unsafe { - llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_ptr().cast(), name.len(), ty) + llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_c_char_ptr(), name.len(), ty) }; llvm::SetFunctionCallConv(llfn, callconv); @@ -67,7 +69,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// return its Value instead. pub(crate) fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { debug!("declare_global(name={:?})", name); - unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } + unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_c_char_ptr(), name.len(), ty) } } /// Declare a C ABI function. @@ -83,14 +85,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - // Declare C ABI functions with the visibility used by C by default. - let visibility = if self.tcx.sess.default_hidden_visibility() { - llvm::Visibility::Hidden - } else { - llvm::Visibility::Default - }; - - declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type) + // Visibility should always be default for declarations, otherwise the linker may report an + // error. + declare_raw_fn(self, name, llvm::CCallConv, unnamed, Visibility::Default, fn_type) } /// Declare an entry Function @@ -107,11 +104,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - let visibility = if self.tcx.sess.default_hidden_visibility() { - llvm::Visibility::Hidden - } else { - llvm::Visibility::Default - }; + let visibility = Visibility::from_generic(self.tcx.sess.default_visibility()); declare_raw_fn(self, name, callconv, unnamed, visibility, fn_type) } @@ -217,7 +210,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// Gets declared value by name. pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); - unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } + unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_c_char_ptr(), name.len()) } } /// Gets defined or externally defined (AvailableExternally linkage) value by diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index bb481d2a308..3cdb5b971d9 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -31,6 +31,15 @@ pub(crate) struct UnstableCTargetFeature<'a> { pub feature: &'a str, } +#[derive(Diagnostic)] +#[diag(codegen_llvm_forbidden_ctarget_feature)] +#[note] +#[note(codegen_llvm_forbidden_ctarget_feature_issue)] +pub(crate) struct ForbiddenCTargetFeature<'a> { + pub feature: &'a str, + pub reason: &'a str, +} + #[derive(Subdiagnostic)] pub(crate) enum PossibleFeature<'a> { #[help(codegen_llvm_possible_feature)] @@ -80,30 +89,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { } } -pub(crate) struct TargetFeatureDisableOrEnable<'a> { - pub features: &'a [&'a str], - pub span: Option<Span>, - pub missing_features: Option<MissingFeatures>, -} - -#[derive(Subdiagnostic)] -#[help(codegen_llvm_missing_features)] -pub(crate) struct MissingFeatures; - -impl<G: EmissionGuarantee> Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { - fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> { - let mut diag = Diag::new(dcx, level, fluent::codegen_llvm_target_feature_disable_or_enable); - if let Some(span) = self.span { - diag.span(span); - }; - if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); - } - diag.arg("features", self.features.join(", ")); - diag - } -} - #[derive(Diagnostic)] #[diag(codegen_llvm_lto_disallowed)] pub(crate) struct LtoDisallowed; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c66c80da9fc..da7f94e8cf7 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,6 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; +use rustc_abi::{self as abi, Align, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; @@ -9,15 +10,14 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; use rustc_middle::mir::BinOp; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; -use rustc_target::abi::{self, Align, Float, HasDataLayout, Primitive, Size}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use tracing::debug; -use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; +use crate::abi::{ExternAbi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Metadata}; @@ -86,6 +86,11 @@ fn get_simple_intrinsic<'ll>( sym::fmaf64 => "llvm.fma.f64", sym::fmaf128 => "llvm.fma.f128", + sym::fmuladdf16 => "llvm.fmuladd.f16", + sym::fmuladdf32 => "llvm.fmuladd.f32", + sym::fmuladdf64 => "llvm.fmuladd.f64", + sym::fmuladdf128 => "llvm.fmuladd.f128", + sym::fabsf16 => "llvm.fabs.f16", sym::fabsf32 => "llvm.fabs.f32", sym::fabsf64 => "llvm.fabs.f64", @@ -158,14 +163,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { span: Span, ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; - let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); + let callee_ty = instance.ty(tcx, self.typing_env()); let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else { bug!("expected fn item type, found {}", callee_ty); }; let sig = callee_ty.fn_sig(tcx); - let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); + let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig); let arg_tys = sig.inputs(); let ret_ty = sig.output(); let name = tcx.item_name(def_id); @@ -187,7 +192,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Some(instance), ) } - sym::likely => self.expect(args[0].immediate(), true), sym::is_val_statically_known => { let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); let kind = self.type_kind(intrinsic_type); @@ -208,7 +212,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.const_bool(false) } } - sym::unlikely => self.expect(args[0].immediate(), false), sym::select_unpredictable => { let cond = args[0].immediate(); assert_eq!(args[1].layout, args[2].layout); @@ -253,8 +256,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) } sym::va_arg => { - match fn_abi.ret.layout.abi { - abi::Abi::Scalar(scalar) => { + match fn_abi.ret.layout.backend_repr { + abi::BackendRepr::Scalar(scalar) => { match scalar.primitive() { Primitive::Int(..) => { if self.cx().size_of(ret_ty).bytes() < 4 { @@ -431,13 +434,13 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::raw_eq => { - use abi::Abi::*; + use abi::BackendRepr::*; let tp_ty = fn_args.type_at(0); let layout = self.layout_of(tp_ty).layout; - let use_integer_compare = match layout.abi() { + let use_integer_compare = match layout.backend_repr() { Scalar(_) | ScalarPair(_, _) => true, Uninhabited | Vector { .. } => false, - Aggregate { .. } => { + Memory { .. } => { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), // so we re-use that same threshold here. @@ -544,7 +547,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } let llret_ty = if ret_ty.is_simd() - && let abi::Abi::Aggregate { .. } = self.layout_of(ret_ty).layout.abi + && let abi::BackendRepr::Memory { .. } = + self.layout_of(ret_ty).layout.backend_repr { let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx()); let elem_ll_ty = match elem_ty.kind() { @@ -780,11 +784,12 @@ fn codegen_msvc_try<'ll>( let type_info = bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false); let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); - unsafe { - llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + + llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + if bx.cx.tcx.sess.target.supports_comdat() { llvm::SetUniqueComdat(bx.llmod, tydesc); - llvm::LLVMSetInitializer(tydesc, type_info); } + unsafe { llvm::LLVMSetInitializer(tydesc, type_info) }; // The flag value of 8 indicates that we are catching the exception by // reference instead of by value. We can't use catch by value because @@ -1057,7 +1062,7 @@ fn gen_fn<'ll, 'tcx>( cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. - unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; + llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage); let llbb = Builder::append_block(cx, llfn, "entry-block"); let bx = Builder::build(cx, llbb); codegen(bx); @@ -1087,7 +1092,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.types.unit, false, hir::Safety::Unsafe, - Abi::Rust, + ExternAbi::Rust, )), ); // `unsafe fn(*mut i8, *mut i8) -> ()` @@ -1098,7 +1103,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.types.unit, false, hir::Safety::Unsafe, - Abi::Rust, + ExternAbi::Rust, )), ); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` @@ -1107,7 +1112,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.types.i32, false, hir::Safety::Unsafe, - Abi::Rust, + ExternAbi::Rust, )); let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); cx.rust_try_fn.set(Some(rust_try)); @@ -1147,8 +1152,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } let tcx = bx.tcx(); - let sig = - tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); + let sig = tcx.normalize_erasing_late_bound_regions(bx.typing_env(), callee_ty.fn_sig(tcx)); let arg_tys = sig.inputs(); // Sanity-check: all vector arguments must be immediates. @@ -1172,8 +1176,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => + && len + .try_to_target_usize(bx.tcx) + .expect("expected monomorphic const in codegen") + == expected_bytes => { let place = PlaceRef::alloca(bx, args[0].layout); args[0].val.store(bx, place); @@ -1238,12 +1244,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle_generic { - let idx = fn_args[2] - .expect_const() - .eval(tcx, ty::ParamEnv::reveal_all(), span) - .unwrap() - .1 - .unwrap_branch(); + let idx = fn_args[2].expect_const().try_to_valtree().unwrap().0.unwrap_branch(); let n = idx.len() as u64; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); @@ -1462,8 +1463,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => + && len + .try_to_target_usize(bx.tcx) + .expect("expected monomorphic const in codegen") + == expected_bytes => { // Zero-extend iN to the array length: let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); @@ -2183,9 +2186,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match in_elem.kind() { ty::RawPtr(p_ty, _) => { let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { - bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) + bx.tcx.normalize_erasing_regions(bx.typing_env(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer { + require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem @@ -2198,9 +2201,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match out_elem.kind() { ty::RawPtr(p_ty, _) => { let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { - bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) + bx.tcx.normalize_erasing_regions(bx.typing_env(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastFatPointer { + require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index bdfc0f626f8..3dfb86d422d 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -22,7 +22,6 @@ use std::any::Any; use std::ffi::CStr; -use std::io::Write; use std::mem::ManuallyDrop; use back::owned_target_machine::OwnedTargetMachine; @@ -165,30 +164,12 @@ impl WriteBackendMethods for LlvmCodegenBackend { type ThinData = back::lto::ThinData; type ThinBuffer = back::lto::ThinBuffer; fn print_pass_timings(&self) { - unsafe { - let mut size = 0; - let cstr = llvm::LLVMRustPrintPassTimings(std::ptr::addr_of_mut!(size)); - if cstr.is_null() { - println!("failed to get pass timings"); - } else { - let timings = std::slice::from_raw_parts(cstr as *const u8, size); - std::io::stdout().write_all(timings).unwrap(); - libc::free(cstr as *mut _); - } - } + let timings = llvm::build_string(|s| unsafe { llvm::LLVMRustPrintPassTimings(s) }).unwrap(); + print!("{timings}"); } fn print_statistics(&self) { - unsafe { - let mut size = 0; - let cstr = llvm::LLVMRustPrintStatistics(std::ptr::addr_of_mut!(size)); - if cstr.is_null() { - println!("failed to get pass stats"); - } else { - let stats = std::slice::from_raw_parts(cstr as *const u8, size); - std::io::stdout().write_all(stats).unwrap(); - libc::free(cstr as *mut _); - } - } + let stats = llvm::build_string(|s| unsafe { llvm::LLVMRustPrintStatistics(s) }).unwrap(); + print!("{stats}"); } fn run_link( cgcx: &CodegenContext<Self>, @@ -401,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend { // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs) } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 9aabfd794ba..7f59264824e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,9 +1,13 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] +use std::fmt::Debug; use std::marker::PhantomData; +use std::ptr; use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; +use rustc_macros::TryFromU32; +use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ @@ -18,6 +22,30 @@ pub type Bool = c_uint; pub const True: Bool = 1 as Bool; pub const False: Bool = 0 as Bool; +/// Wrapper for a raw enum value returned from LLVM's C APIs. +/// +/// For C enums returned by LLVM, it's risky to use a Rust enum as the return +/// type, because it would be UB if a later version of LLVM adds a new enum +/// value and returns it. Instead, return this raw wrapper, then convert to the +/// Rust-side enum explicitly. +#[repr(transparent)] +pub struct RawEnum<T> { + value: u32, + /// We don't own or consume a `T`, but we can produce one. + _rust_side_type: PhantomData<fn() -> T>, +} + +impl<T: TryFrom<u32>> RawEnum<T> { + #[track_caller] + pub(crate) fn to_rust(self) -> T + where + T::Error: Debug, + { + // If this fails, the Rust-side enum is out of sync with LLVM's enum. + T::try_from(self.value).expect("enum value returned by LLVM should be known") + } +} + #[derive(Copy, Clone, PartialEq)] #[repr(C)] #[allow(dead_code)] // Variants constructed by C++. @@ -58,7 +86,7 @@ pub enum LLVMMachineType { ARM = 0x01c0, } -/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. +/// Must match the layout of `LLVMRustModuleFlagMergeBehavior`. /// /// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are /// resolved according to the merge behaviors specified here. Flags differing only in merge @@ -66,9 +94,13 @@ pub enum LLVMMachineType { /// /// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, /// 'Error' and 'Warning' cannot be mixed for a given flag. +/// +/// There is a stable LLVM-C version of this enum (`LLVMModuleFlagBehavior`), +/// but as of LLVM 19 it does not support all of the enum values in the unstable +/// C++ API. #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum LLVMModFlagBehavior { +pub enum ModuleFlagMergeBehavior { Error = 1, Warning = 2, Require = 3, @@ -107,32 +139,52 @@ pub enum CallConv { AvrInterrupt = 85, } -/// LLVMRustLinkage -#[derive(Copy, Clone, PartialEq)] +/// Must match the layout of `LLVMLinkage`. +#[derive(Copy, Clone, PartialEq, TryFromU32)] #[repr(C)] pub enum Linkage { ExternalLinkage = 0, AvailableExternallyLinkage = 1, LinkOnceAnyLinkage = 2, LinkOnceODRLinkage = 3, - WeakAnyLinkage = 4, - WeakODRLinkage = 5, - AppendingLinkage = 6, - InternalLinkage = 7, - PrivateLinkage = 8, - ExternalWeakLinkage = 9, - CommonLinkage = 10, + #[deprecated = "marked obsolete by LLVM"] + LinkOnceODRAutoHideLinkage = 4, + WeakAnyLinkage = 5, + WeakODRLinkage = 6, + AppendingLinkage = 7, + InternalLinkage = 8, + PrivateLinkage = 9, + #[deprecated = "marked obsolete by LLVM"] + DLLImportLinkage = 10, + #[deprecated = "marked obsolete by LLVM"] + DLLExportLinkage = 11, + ExternalWeakLinkage = 12, + #[deprecated = "marked obsolete by LLVM"] + GhostLinkage = 13, + CommonLinkage = 14, + LinkerPrivateLinkage = 15, + LinkerPrivateWeakLinkage = 16, } -// LLVMRustVisibility +/// Must match the layout of `LLVMVisibility`. #[repr(C)] -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, TryFromU32)] pub enum Visibility { Default = 0, Hidden = 1, Protected = 2, } +impl Visibility { + pub fn from_generic(visibility: SymbolVisibility) -> Self { + match visibility { + SymbolVisibility::Hidden => Visibility::Hidden, + SymbolVisibility::Protected => Visibility::Protected, + SymbolVisibility::Interposable => Visibility::Default, + } + } +} + /// LLVMUnnamedAddr #[repr(C)] pub enum UnnamedAddr { @@ -152,7 +204,7 @@ pub enum DLLStorageClass { DllExport = 2, // Function to be accessible from DLL. } -/// Matches LLVMRustAttribute in LLVMWrapper.h +/// Must match the layout of `LLVMRustAttributeKind`. /// Semantically a subset of the C++ enum llvm::Attribute::AttrKind, /// though it is not ABI compatible (since it's a C++ enum) #[repr(C)] @@ -635,6 +687,7 @@ unsafe extern "C" { pub type Attribute; pub type Metadata; pub type BasicBlock; + pub type Comdat; } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); @@ -656,8 +709,9 @@ unsafe extern "C" { } #[repr(C)] pub struct RustArchiveMember<'a>(InvariantOpaque<'a>); +/// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] -pub struct OperandBundleDef<'a>(InvariantOpaque<'a>); +pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct Linker<'a>(InvariantOpaque<'a>); @@ -933,7 +987,11 @@ unsafe extern "C" { // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub fn LLVMGetLinkage(Global: &Value) -> RawEnum<Linkage>; + pub fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); pub fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub fn LLVMGetVisibility(Global: &Value) -> RawEnum<Visibility>; + pub fn LLVMSetVisibility(Global: &Value, Viz: Visibility); pub fn LLVMGetAlignment(Global: &Value) -> c_uint; pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); @@ -1005,6 +1063,7 @@ unsafe extern "C" { // Metadata pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); + pub fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>; // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; @@ -1479,6 +1538,53 @@ unsafe extern "C" { pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + + pub fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; + pub fn LLVMSetComdat(V: &Value, C: &Comdat); + + pub(crate) fn LLVMCreateOperandBundle( + Tag: *const c_char, + TagLen: size_t, + Args: *const &'_ Value, + NumArgs: c_uint, + ) -> *mut OperandBundle<'_>; + pub(crate) fn LLVMDisposeOperandBundle(Bundle: ptr::NonNull<OperandBundle<'_>>); + + pub(crate) fn LLVMBuildCallWithOperandBundles<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Bundles: *const &OperandBundle<'a>, + NumBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildInvokeWithOperandBundles<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Then: &'a BasicBlock, + Catch: &'a BasicBlock, + Bundles: *const &OperandBundle<'a>, + NumBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildCallBr<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + DefaultDest: &'a BasicBlock, + IndirectDests: *const &'a BasicBlock, + NumIndirectDests: c_uint, + Args: *const &'a Value, + NumArgs: c_uint, + Bundles: *const &OperandBundle<'a>, + NumBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; } #[link(name = "llvm-wrapper", kind = "static")] @@ -1506,10 +1612,6 @@ unsafe extern "C" { ) -> bool; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage; - pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); - pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; - pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); // Operations on global variables @@ -1568,52 +1670,11 @@ unsafe extern "C" { AttrsLen: size_t, ); - pub fn LLVMRustBuildInvoke<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - Args: *const &'a Value, - NumArgs: c_uint, - Then: &'a BasicBlock, - Catch: &'a BasicBlock, - OpBundles: *const &OperandBundleDef<'a>, - NumOpBundles: c_uint, - Name: *const c_char, - ) -> &'a Value; - - pub fn LLVMRustBuildCallBr<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - DefaultDest: &'a BasicBlock, - IndirectDests: *const &'a BasicBlock, - NumIndirectDests: c_uint, - Args: *const &'a Value, - NumArgs: c_uint, - OpBundles: *const &OperandBundleDef<'a>, - NumOpBundles: c_uint, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMRustSetFastMath(Instr: &Value); pub fn LLVMRustSetAlgebraicMath(Instr: &Value); pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value; - - pub fn LLVMRustBuildCall<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - Args: *const &'a Value, - NumArgs: c_uint, - OpBundles: *const &OperandBundleDef<'a>, - NumOpBundles: c_uint, - ) -> &'a Value; pub fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, Dst: &'a Value, @@ -1705,11 +1766,11 @@ unsafe extern "C" { /// Returns a string describing the last error caused by an LLVMRust* call. pub fn LLVMRustGetLastError() -> *const c_char; - /// Print the pass timings since static dtors aren't picking them up. - pub fn LLVMRustPrintPassTimings(size: *const size_t) -> *const c_char; + /// Prints the timing information collected by `-Ztime-llvm-passes`. + pub(crate) fn LLVMRustPrintPassTimings(OutStr: &RustString); - /// Print the statistics since static dtors aren't picking them up. - pub fn LLVMRustPrintStatistics(size: *const size_t) -> *const c_char; + /// Prints the statistics collected by `-Zprint-codegen-stats`. + pub(crate) fn LLVMRustPrintStatistics(OutStr: &RustString); /// Prepares inline assembly. pub fn LLVMRustInlineAsm( @@ -1729,8 +1790,7 @@ unsafe extern "C" { ConstraintsLen: size_t, ) -> bool; - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + pub(crate) fn LLVMRustCoverageWriteFilenamesToBuffer( Filenames: *const *const c_char, FilenamesLen: size_t, Lengths: *const size_t, @@ -1738,34 +1798,36 @@ unsafe extern "C" { BufferOut: &RustString, ); - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingToBuffer( + pub(crate) fn LLVMRustCoverageWriteFunctionMappingsToBuffer( VirtualFileMappingIDs: *const c_uint, - NumVirtualFileMappingIDs: c_uint, + NumVirtualFileMappingIDs: size_t, Expressions: *const crate::coverageinfo::ffi::CounterExpression, - NumExpressions: c_uint, - MappingRegions: *const crate::coverageinfo::ffi::CounterMappingRegion, - NumMappingRegions: c_uint, + NumExpressions: size_t, + CodeRegions: *const crate::coverageinfo::ffi::CodeRegion, + NumCodeRegions: size_t, + BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, + NumBranchRegions: size_t, + MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, + NumMCDCBranchRegions: size_t, + MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, + NumMCDCDecisionRegions: size_t, BufferOut: &RustString, ); - pub fn LLVMRustCoverageCreatePGOFuncNameVar( + pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, FuncName: *const c_char, FuncNameLen: size_t, ) -> &Value; - pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + pub(crate) fn LLVMRustCoverageHashBytes(Bytes: *const c_char, NumBytes: size_t) -> u64; - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteCovmapSectionNameToString(M: &Module, OutStr: &RustString); - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteCovfunSectionNameToString(M: &Module, OutStr: &RustString); - #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); - pub fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustDebugMetadataVersion() -> u32; pub fn LLVMRustVersionMajor() -> u32; pub fn LLVMRustVersionMinor() -> u32; @@ -1777,21 +1839,21 @@ unsafe extern "C" { /// "compatible" means depends on the merge behaviors involved. pub fn LLVMRustAddModuleFlagU32( M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: u32, + MergeBehavior: ModuleFlagMergeBehavior, + Name: *const c_char, + NameLen: size_t, + Value: u32, ); pub fn LLVMRustAddModuleFlagString( M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: *const c_char, - value_len: size_t, + MergeBehavior: ModuleFlagMergeBehavior, + Name: *const c_char, + NameLen: size_t, + Value: *const c_char, + ValueLen: size_t, ); - pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); @@ -2113,23 +2175,20 @@ unsafe extern "C" { Scope: &'a DIScope, InlinedAt: Option<&'a DILocation>, ) -> &'a DILocation; + pub fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( + Location: &'a DILocation, + BD: c_uint, + ) -> Option<&'a DILocation>; pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; pub fn LLVMRustDIBuilderCreateOpLLVMFragment() -> u64; - #[allow(improper_ctypes)] pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); - #[allow(improper_ctypes)] pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs( - T: &TargetMachine, - cpu: *const c_char, - print: unsafe extern "C" fn(out: *mut c_void, string: *const c_char, len: usize), - out: *mut c_void, - ); + pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2138,7 +2197,7 @@ unsafe extern "C" { Desc: &mut *const c_char, ); - pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; + pub fn LLVMRustGetHostCPUName(LenOut: &mut size_t) -> *const u8; // This function makes copies of pointed to data, so the data's lifetime may end after this // function returns. @@ -2203,7 +2262,6 @@ unsafe extern "C" { PGOUsePath: *const c_char, InstrumentCoverage: bool, InstrProfileOutput: *const c_char, - InstrumentGCOV: bool, PGOSampleUsePath: *const c_char, DebugInfoForProfiling: bool, llvm_selfprofiler: *mut c_void, @@ -2234,10 +2292,8 @@ unsafe extern "C" { pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); - #[allow(improper_ctypes)] pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); - #[allow(improper_ctypes)] pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( DI: &'a DiagnosticInfo, pass_name_out: &RustString, @@ -2255,7 +2311,6 @@ unsafe extern "C" { message_out: &mut Option<&'a Twine>, ); - #[allow(improper_ctypes)] pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; @@ -2264,7 +2319,6 @@ unsafe extern "C" { cookie_out: &mut c_uint, ) -> &'a SMDiagnostic; - #[allow(improper_ctypes)] pub fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, message_out: &RustString, @@ -2301,16 +2355,8 @@ unsafe extern "C" { pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustBuildOperandBundleDef( - Name: *const c_char, - Inputs: *const &'_ Value, - NumInputs: c_uint, - ) -> &mut OperandBundleDef<'_>; - pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); - pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); - pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); @@ -2319,7 +2365,6 @@ unsafe extern "C" { pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub fn LLVMRustModuleCost(M: &Module) -> u64; - #[allow(improper_ctypes)] pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); pub fn LLVMRustThinLTOBufferCreate( @@ -2334,9 +2379,9 @@ unsafe extern "C" { pub fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; pub fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, - NumModules: c_uint, + NumModules: size_t, PreservedSymbols: *const *const c_char, - PreservedSymbolsLen: c_uint, + PreservedSymbolsLen: size_t, ) -> Option<&'static mut ThinLTOData>; pub fn LLVMRustPrepareThinLTORename( Data: &ThinLTOData, @@ -2361,6 +2406,7 @@ unsafe extern "C" { data: *const u8, len: usize, name: *const u8, + name_len: usize, out_len: &mut usize, ) -> *const u8; @@ -2371,7 +2417,6 @@ unsafe extern "C" { bytecode_len: usize, ) -> bool; pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); - #[allow(improper_ctypes)] pub fn LLVMRustComputeLTOCacheKey( key_out: &RustString, mod_id: *const c_char, @@ -2394,7 +2439,6 @@ unsafe extern "C" { pgo_available: bool, ); - #[allow(improper_ctypes)] pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index d0db350a149..909afe35a17 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -1,14 +1,14 @@ #![allow(non_snake_case)] -use std::cell::RefCell; use std::ffi::{CStr, CString}; +use std::ops::Deref; +use std::ptr; use std::str::FromStr; use std::string::FromUtf8Error; use libc::c_uint; -use rustc_data_structures::small_c_str::SmallCStr; +use rustc_abi::{Align, Size, WrappingRange}; use rustc_llvm::RustString; -use rustc_target::abi::{Align, Size, WrappingRange}; pub use self::AtomicRmwBinOp::*; pub use self::CallConv::*; @@ -17,13 +17,13 @@ pub use self::IntPredicate::*; pub use self::Linkage::*; pub use self::MetadataType::*; pub use self::RealPredicate::*; +pub use self::ffi::*; +use crate::common::AsCCharPtr; pub mod archive_ro; pub mod diagnostic; mod ffi; -pub use self::ffi::*; - impl LLVMRustResult { pub fn into_result(self) -> Result<(), ()> { match self { @@ -53,9 +53,9 @@ pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) - unsafe { LLVMCreateStringAttribute( llcx, - attr.as_ptr().cast(), + attr.as_c_char_ptr(), attr.len().try_into().unwrap(), - value.as_ptr().cast(), + value.as_c_char_ptr(), value.len().try_into().unwrap(), ) } @@ -65,7 +65,7 @@ pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, - attr.as_ptr().cast(), + attr.as_c_char_ptr(), attr.len().try_into().unwrap(), std::ptr::null(), 0, @@ -178,10 +178,10 @@ pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { // function. // For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 pub fn SetUniqueComdat(llmod: &Module, val: &Value) { - unsafe { - let name = get_value_name(val); - LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len()); - } + let name_buf = get_value_name(val).to_vec(); + let name = + CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap(); + set_comdat(llmod, val, &name); } pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { @@ -210,15 +210,13 @@ impl MemoryEffects { } } -pub fn set_section(llglobal: &Value, section_name: &str) { - let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); +pub fn set_section(llglobal: &Value, section_name: &CStr) { unsafe { - LLVMSetSection(llglobal, section_name_cstr.as_ptr()); + LLVMSetSection(llglobal, section_name.as_ptr()); } } -pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { - let name_cstr = CString::new(name).expect("unexpected CString error"); +pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } } @@ -234,15 +232,23 @@ pub fn set_global_constant(llglobal: &Value, is_constant: bool) { } } +pub fn get_linkage(llglobal: &Value) -> Linkage { + unsafe { LLVMGetLinkage(llglobal) }.to_rust() +} + pub fn set_linkage(llglobal: &Value, linkage: Linkage) { unsafe { - LLVMRustSetLinkage(llglobal, linkage); + LLVMSetLinkage(llglobal, linkage); } } +pub fn get_visibility(llglobal: &Value) -> Visibility { + unsafe { LLVMGetVisibility(llglobal) }.to_rust() +} + pub fn set_visibility(llglobal: &Value, visibility: Visibility) { unsafe { - LLVMRustSetVisibility(llglobal, visibility); + LLVMSetVisibility(llglobal, visibility); } } @@ -252,9 +258,14 @@ pub fn set_alignment(llglobal: &Value, align: Align) { } } -pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { +/// Get the `name`d comdat from `llmod` and assign it to `llglobal`. +/// +/// Inserts the comdat into `llmod` if it does not exist. +/// It is an error to call this if the target does not support comdat. +pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { unsafe { - LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); + let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr()); + LLVMSetComdat(llglobal, comdat); } } @@ -283,21 +294,17 @@ pub fn get_value_name(value: &Value) -> &[u8] { /// Safe wrapper for `LLVMSetValueName2` from a byte slice pub fn set_value_name(value: &Value, name: &[u8]) { unsafe { - let data = name.as_ptr().cast(); + let data = name.as_c_char_ptr(); LLVMSetValueName2(value, data, name.len()); } } pub fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> { - let sr = RustString { bytes: RefCell::new(Vec::new()) }; - f(&sr); - String::from_utf8(sr.bytes.into_inner()) + String::from_utf8(RustString::build_byte_buffer(f)) } pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> { - let sr = RustString { bytes: RefCell::new(Vec::new()) }; - f(&sr); - sr.bytes.into_inner() + RustString::build_byte_buffer(f) } pub fn twine_to_string(tr: &Twine) -> String { @@ -320,24 +327,68 @@ pub fn last_error() -> Option<String> { } } -pub struct OperandBundleDef<'a> { - pub raw: &'a mut ffi::OperandBundleDef<'a>, +/// Owns an [`OperandBundle`], and will dispose of it when dropped. +pub(crate) struct OperandBundleOwned<'a> { + raw: ptr::NonNull<OperandBundle<'a>>, } -impl<'a> OperandBundleDef<'a> { - pub fn new(name: &str, vals: &[&'a Value]) -> Self { - let name = SmallCStr::new(name); - let def = unsafe { - LLVMRustBuildOperandBundleDef(name.as_ptr(), vals.as_ptr(), vals.len() as c_uint) +impl<'a> OperandBundleOwned<'a> { + pub(crate) fn new(name: &str, vals: &[&'a Value]) -> Self { + let raw = unsafe { + LLVMCreateOperandBundle( + name.as_c_char_ptr(), + name.len(), + vals.as_ptr(), + vals.len() as c_uint, + ) }; - OperandBundleDef { raw: def } + OperandBundleOwned { raw: ptr::NonNull::new(raw).unwrap() } } } -impl Drop for OperandBundleDef<'_> { +impl Drop for OperandBundleOwned<'_> { fn drop(&mut self) { unsafe { - LLVMRustFreeOperandBundleDef(&mut *(self.raw as *mut _)); + LLVMDisposeOperandBundle(self.raw); } } } + +impl<'a> Deref for OperandBundleOwned<'a> { + type Target = OperandBundle<'a>; + + fn deref(&self) -> &Self::Target { + // SAFETY: The returned reference is opaque and can only used for FFI. + // It is valid for as long as `&self` is. + unsafe { self.raw.as_ref() } + } +} + +pub(crate) fn add_module_flag_u32( + module: &Module, + merge_behavior: ModuleFlagMergeBehavior, + key: &str, + value: u32, +) { + unsafe { + LLVMRustAddModuleFlagU32(module, merge_behavior, key.as_c_char_ptr(), key.len(), value); + } +} + +pub(crate) fn add_module_flag_str( + module: &Module, + merge_behavior: ModuleFlagMergeBehavior, + key: &str, + value: &str, +) { + unsafe { + LLVMRustAddModuleFlagString( + module, + merge_behavior, + key.as_c_char_ptr(), + key.len(), + value.as_c_char_ptr(), + value.len(), + ); + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 71fd7afb148..db2b03d9aed 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,4 +1,5 @@ -use std::ffi::{CStr, CString, c_char, c_void}; +use std::collections::VecDeque; +use std::ffi::{CStr, CString}; use std::fmt::Write; use std::path::Path; use std::sync::Once; @@ -6,6 +7,7 @@ use std::{ptr, slice, str}; use libc::c_int; use rustc_codegen_ssa::base::wants_wasm_eh; +use rustc_codegen_ssa::codegen_attrs::check_tied_features; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_data_structures::unord::UnordSet; @@ -15,11 +17,11 @@ use rustc_session::Session; use rustc_session::config::{PrintKind, PrintRequest}; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy, SmallDataThresholdSupport}; -use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES}; +use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES, Stability}; use crate::back::write::create_informational_target_machine; use crate::errors::{ - FixedX18InvalidArch, InvalidTargetFeaturePrefix, PossibleFeature, TargetFeatureDisableOrEnable, + FixedX18InvalidArch, ForbiddenCTargetFeature, InvalidTargetFeaturePrefix, PossibleFeature, UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; @@ -226,6 +228,8 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea "x86" } else if sess.target.arch == "arm64ec" { "aarch64" + } else if sess.target.arch == "sparc64" { + "sparc" } else { &*sess.target.arch }; @@ -247,7 +251,10 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")), + ("aarch64", "pauth-lr") if get_version().0 < 19 => None, + // Before LLVM 20 those two features were packaged together as b16b16 + ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), + ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => { @@ -264,46 +271,51 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { Some(LLVMFeature::new("fast-unaligned-access")) } + // Filter out features that are not supported by the current LLVM version + ("riscv32" | "riscv64", "zaamo") if get_version().0 < 19 => None, + ("riscv32" | "riscv64", "zabha") if get_version().0 < 19 => None, + ("riscv32" | "riscv64", "zalrsc") if get_version().0 < 19 => None, // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => { Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))) } + // Support for `wide-arithmetic` will first land in LLVM 20 as part of + // llvm/llvm-project#111598 + ("wasm32" | "wasm64", "wide-arithmetic") if get_version() < (20, 0, 0) => None, + ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), + // In LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available and SPARC-V8+ ABI used". + // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L27-L28 + // Before LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available". + // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 + ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), + ("sparc", "v8plus") if get_version().0 < 19 => None, (_, s) => Some(LLVMFeature::new(s)), } } -/// Given a map from target_features to whether they are enabled or disabled, -/// ensure only valid combinations are allowed. -pub(crate) fn check_tied_features( - sess: &Session, - features: &FxHashMap<&str, bool>, -) -> Option<&'static [&'static str]> { - if !features.is_empty() { - for tied in sess.target.tied_target_features() { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); - } - } - } - None -} - -/// Used to generate cfg variables and apply features -/// Must express features in the way Rust understands them +/// Used to generate cfg variables and apply features. +/// Must express features in the way Rust understands them. +/// +/// We do not have to worry about RUSTC_SPECIFIC_FEATURES here, those are handled outside codegen. pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { - let mut features = vec![]; - - // Add base features for the target + let mut features: FxHashSet<Symbol> = Default::default(); + + // Add base features for the target. + // We do *not* add the -Ctarget-features there, and instead duplicate the logic for that below. + // The reason is that if LLVM considers a feature implied but we do not, we don't want that to + // show up in `cfg`. That way, `cfg` is entirely under our control -- except for the handling of + // the target CPU, that is still expanded to target features (with all their implied features) by + // LLVM. let target_machine = create_informational_target_machine(sess, true); + // Compute which of the known target features are enabled in the 'base' target machine. + // We only consider "supported" features; "forbidden" features are not reflected in `cfg` as of now. features.extend( sess.target - .supported_target_features() + .rust_target_features() .iter() + .filter(|(_, gate, _)| gate.is_supported()) .filter(|(feature, _, _)| { - // skip checking special features, as LLVM may not understands them + // skip checking special features, as LLVM may not understand them if RUSTC_SPECIAL_FEATURES.contains(feature) { return true; } @@ -332,18 +344,32 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { }) { if enabled { + // Also add all transitively implied features. features.extend(sess.target.implied_target_features(std::iter::once(feature))); } else { + // Remove transitively reverse-implied features. + + // We don't care about the order in `features` since the only thing we use it for is the + // `features.contains` below. + #[allow(rustc::potential_query_instability)] features.retain(|f| { - !sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) + if sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) { + // If `f` if implies `feature`, then `!feature` implies `!f`, so we have to + // remove `f`. (This is the standard logical contraposition principle.) + false + } else { + // We can keep `f`. + true + } }); } } // Filter enabled features based on feature gates sess.target - .supported_target_features() + .rust_target_features() .iter() + .filter(|(_, gate, _)| gate.is_supported()) .filter_map(|&(feature, gate, _)| { if sess.is_nightly_build() || allow_unstable || gate.is_stable() { Some(feature) @@ -398,14 +424,76 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { ret } -fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMachine) { +pub(crate) fn print(req: &PrintRequest, out: &mut String, sess: &Session) { + require_inited(); + let tm = create_informational_target_machine(sess, false); + match req.kind { + PrintKind::TargetCPUs => print_target_cpus(sess, &tm, out), + PrintKind::TargetFeatures => print_target_features(sess, &tm, out), + _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), + } +} + +fn print_target_cpus(sess: &Session, tm: &llvm::TargetMachine, out: &mut String) { + let cpu_names = llvm::build_string(|s| unsafe { + llvm::LLVMRustPrintTargetCPUs(&tm, s); + }) + .unwrap(); + + struct Cpu<'a> { + cpu_name: &'a str, + remark: String, + } + // Compare CPU against current target to label the default. + let target_cpu = handle_native(&sess.target.cpu); + let make_remark = |cpu_name| { + if cpu_name == target_cpu { + // FIXME(#132514): This prints the LLVM target string, which can be + // different from the Rust target string. Is that intended? + let target = &sess.target.llvm_target; + format!( + " - This is the default target CPU for the current build target (currently {target})." + ) + } else { + "".to_owned() + } + }; + let mut cpus = cpu_names + .lines() + .map(|cpu_name| Cpu { cpu_name, remark: make_remark(cpu_name) }) + .collect::<VecDeque<_>>(); + + // Only print the "native" entry when host and target are the same arch, + // since otherwise it could be wrong or misleading. + if sess.host.arch == sess.target.arch { + let host = get_host_cpu_name(); + cpus.push_front(Cpu { + cpu_name: "native", + remark: format!(" - Select the CPU of the current host (currently {host})."), + }); + } + + let max_name_width = cpus.iter().map(|cpu| cpu.cpu_name.len()).max().unwrap_or(0); + writeln!(out, "Available CPUs for this target:").unwrap(); + for Cpu { cpu_name, remark } in cpus { + // Only pad the CPU name if there's a remark to print after it. + let width = if remark.is_empty() { 0 } else { max_name_width }; + writeln!(out, " {cpu_name:<width$}{remark}").unwrap(); + } +} + +fn print_target_features(sess: &Session, tm: &llvm::TargetMachine, out: &mut String) { let mut llvm_target_features = llvm_target_features(tm); let mut known_llvm_target_features = FxHashSet::<&'static str>::default(); let mut rustc_target_features = sess .target - .supported_target_features() + .rust_target_features() .iter() - .filter_map(|(feature, _gate, _implied)| { + .filter_map(|(feature, gate, _implied)| { + if !gate.is_supported() { + // Only list (experimentally) supported features. + return None; + } // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these // strings. let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name; @@ -458,52 +546,31 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach writeln!(out, "and may be renamed or removed in a future version of LLVM or rustc.\n").unwrap(); } -pub(crate) fn print(req: &PrintRequest, mut out: &mut String, sess: &Session) { - require_inited(); - let tm = create_informational_target_machine(sess, false); - match req.kind { - PrintKind::TargetCPUs => { - // SAFETY generate a C compatible string from a byte slice to pass - // the target CPU name into LLVM, the lifetime of the reference is - // at least as long as the C function - let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) - .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); - unsafe extern "C" fn callback(out: *mut c_void, string: *const c_char, len: usize) { - let out = unsafe { &mut *(out as *mut &mut String) }; - let bytes = unsafe { slice::from_raw_parts(string as *const u8, len) }; - write!(out, "{}", String::from_utf8_lossy(bytes)).unwrap(); - } - unsafe { - llvm::LLVMRustPrintTargetCPUs( - &tm, - cpu_cstring.as_ptr(), - callback, - std::ptr::addr_of_mut!(out) as *mut c_void, - ); - } - } - PrintKind::TargetFeatures => print_target_features(out, sess, &tm), - _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), - } +/// Returns the host CPU name, according to LLVM. +fn get_host_cpu_name() -> &'static str { + let mut len = 0; + // SAFETY: The underlying C++ global function returns a `StringRef` that + // isn't tied to any particular backing buffer, so it must be 'static. + let slice: &'static [u8] = unsafe { + let ptr = llvm::LLVMRustGetHostCPUName(&mut len); + assert!(!ptr.is_null()); + slice::from_raw_parts(ptr, len) + }; + str::from_utf8(slice).expect("host CPU name should be UTF-8") } -fn handle_native(name: &str) -> &str { - if name != "native" { - return name; - } - - unsafe { - let mut len = 0; - let ptr = llvm::LLVMRustGetHostCPUName(&mut len); - str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() +/// If the given string is `"native"`, returns the host CPU name according to +/// LLVM. Otherwise, the string is returned as-is. +fn handle_native(cpu_name: &str) -> &str { + match cpu_name { + "native" => get_host_cpu_name(), + _ => cpu_name, } } pub(crate) fn target_cpu(sess: &Session) -> &str { - match sess.opts.cg.target_cpu { - Some(ref name) => handle_native(name), - None => handle_native(sess.target.cpu.as_ref()), - } + let cpu_name = sess.opts.cg.target_cpu.as_deref().unwrap_or_else(|| &sess.target.cpu); + handle_native(cpu_name) } /// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, @@ -536,6 +603,11 @@ pub(crate) fn global_llvm_features( // -Ctarget-cpu=native match sess.opts.cg.target_cpu { Some(ref s) if s == "native" => { + // We have already figured out the actual CPU name with `LLVMRustGetHostCPUName` and set + // that for LLVM, so the features implied by that CPU name will be available everywhere. + // However, that is not sufficient: e.g. `skylake` alone is not sufficient to tell if + // some of the instructions are available or not. So we have to also explicitly ask for + // the exact set of features available on the host, and enable all of them. let features_string = unsafe { let ptr = llvm::LLVMGetHostCPUFeatures(); let features_string = if !ptr.is_null() { @@ -564,6 +636,8 @@ pub(crate) fn global_llvm_features( .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(sess, v).is_some()) + // Drop +v8plus feature introduced in LLVM 20. + .filter(|v| *v != "+v8plus" || get_version() >= (20, 0, 0)) .map(String::from), ); @@ -573,7 +647,7 @@ pub(crate) fn global_llvm_features( // -Ctarget-features if !only_base_features { - let supported_features = sess.target.supported_target_features(); + let known_features = sess.target.rust_target_features(); let mut featsmap = FxHashMap::default(); // insert implied features @@ -607,50 +681,53 @@ pub(crate) fn global_llvm_features( } }; + // Get the backend feature name, if any. + // This excludes rustc-specific features, which do not get passed to LLVM. let feature = backend_feature_name(sess, s)?; // Warn against use of LLVM specific feature names and unstable features on the CLI. if diagnostics { - let feature_state = supported_features.iter().find(|&&(v, _, _)| v == feature); - if feature_state.is_none() { - let rust_feature = - supported_features.iter().find_map(|&(rust_feature, _, _)| { - let llvm_features = to_llvm_features(sess, rust_feature)?; - if llvm_features.contains(feature) - && !llvm_features.contains(rust_feature) - { - Some(rust_feature) - } else { - None + let feature_state = known_features.iter().find(|&&(v, _, _)| v == feature); + match feature_state { + None => { + let rust_feature = + known_features.iter().find_map(|&(rust_feature, _, _)| { + let llvm_features = to_llvm_features(sess, rust_feature)?; + if llvm_features.contains(feature) + && !llvm_features.contains(rust_feature) + { + Some(rust_feature) + } else { + None + } + }); + let unknown_feature = if let Some(rust_feature) = rust_feature { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::Some { rust_feature }, } - }); - let unknown_feature = if let Some(rust_feature) = rust_feature { - UnknownCTargetFeature { - feature, - rust_feature: PossibleFeature::Some { rust_feature }, - } - } else { - UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } - }; - sess.dcx().emit_warn(unknown_feature); - } else if feature_state - .is_some_and(|(_name, feature_gate, _implied)| !feature_gate.is_stable()) - { - // An unstable feature. Warn about using it. - sess.dcx().emit_warn(UnstableCTargetFeature { feature }); + } else { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::None, + } + }; + sess.dcx().emit_warn(unknown_feature); + } + Some((_, Stability::Stable, _)) => {} + Some((_, Stability::Unstable(_), _)) => { + // An unstable feature. Warn about using it. + sess.dcx().emit_warn(UnstableCTargetFeature { feature }); + } + Some((_, Stability::Forbidden { reason }, _)) => { + sess.dcx().emit_warn(ForbiddenCTargetFeature { feature, reason }); + } } - } - if diagnostics { // FIXME(nagisa): figure out how to not allocate a full hashset here. featsmap.insert(feature, enable_disable == '+'); } - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return None; - } - - // ... otherwise though we run through `to_llvm_features` when + // We run through `to_llvm_features` when // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. @@ -676,7 +753,7 @@ pub(crate) fn global_llvm_features( features.extend(feats); if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.dcx().emit_err(TargetFeatureDisableOrEnable { + sess.dcx().emit_err(rustc_codegen_ssa::errors::TargetFeatureDisableOrEnable { features: f, span: None, missing_features: None, @@ -704,12 +781,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { let feature = s .strip_prefix(&['+', '-'][..]) .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); - if s.is_empty() { - return None; - } // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + if s.is_empty() || RUSTC_SPECIFIC_FEATURES.contains(&feature) { return None; } Some(feature) diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 02e1995620b..33789c6261f 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -3,7 +3,7 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::mir::mono::{Linkage, Visibility}; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; +use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; @@ -26,11 +26,8 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure // out the llvm type from the actual evaluated initializer. - let ty = if nested { - self.tcx.types.unit - } else { - instance.ty(self.tcx, ty::ParamEnv::reveal_all()) - }; + let ty = + if nested { self.tcx.types.unit } else { instance.ty(self.tcx, self.typing_env()) }; let llty = self.layout_of(ty).llvm_type(self); let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { @@ -39,9 +36,9 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { .emit_fatal(SymbolAlreadyDefined { span: self.tcx.def_span(def_id), symbol_name }) }); + llvm::set_linkage(g, base::linkage_to_llvm(linkage)); + llvm::set_visibility(g, base::visibility_to_llvm(visibility)); unsafe { - llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); - llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); if self.should_assume_dso_local(g, false) { llvm::LLVMRustSetDSOLocal(g, true); } @@ -61,10 +58,12 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); - unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; + llvm::set_linkage(lldecl, base::linkage_to_llvm(linkage)); let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); - if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { + if (linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR) + && self.tcx.sess.target.supports_comdat() + { llvm::SetUniqueComdat(self.llmod, lldecl); } @@ -76,21 +75,15 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { && linkage != Linkage::Private && self.tcx.is_compiler_builtins(LOCAL_CRATE) { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); - } + llvm::set_visibility(lldecl, llvm::Visibility::Hidden); } else { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, base::visibility_to_llvm(visibility)); - } + llvm::set_visibility(lldecl, base::visibility_to_llvm(visibility)); } debug!("predefine_fn: instance = {:?}", instance); - unsafe { - if self.should_assume_dso_local(lldecl, false) { - llvm::LLVMRustSetDSOLocal(lldecl, true); - } + if self.should_assume_dso_local(lldecl, false) { + unsafe { llvm::LLVMRustSetDSOLocal(lldecl, true) }; } self.instances.borrow_mut().insert(instance, lldecl); @@ -100,13 +93,13 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { impl CodegenCx<'_, '_> { /// Whether a definition or declaration can be assumed to be local to a group of /// libraries that form a single DSO or executable. - pub(crate) unsafe fn should_assume_dso_local( + pub(crate) fn should_assume_dso_local( &self, llval: &llvm::Value, is_declaration: bool, ) -> bool { - let linkage = unsafe { llvm::LLVMRustGetLinkage(llval) }; - let visibility = unsafe { llvm::LLVMRustGetVisibility(llval) }; + let linkage = llvm::get_linkage(llval); + let visibility = llvm::get_visibility(llval); if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { return true; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index f1efc7a3dac..6aec078e0de 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -1,14 +1,14 @@ use std::{fmt, ptr}; use libc::{c_char, c_uint}; +use rustc_abi::{AddressSpace, Align, Integer, Size}; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_middle::bug; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty}; -use rustc_target::abi::call::{CastTarget, FnAbi, Reg}; -use rustc_target::abi::{AddressSpace, Align, Integer, Size}; +use rustc_target::callconv::{CastTarget, FnAbi, Reg}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::CodegenCx; diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 6e429a1674a..2b05e24a7ba 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,11 +1,12 @@ use std::fmt::Write; +use rustc_abi::Primitive::{Float, Int, Pointer}; +use rustc_abi::{Align, BackendRepr, FieldsShape, Scalar, Size, Variants}; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; -use rustc_target::abi::{Abi, Align, FieldsShape, Float, Int, Pointer, Scalar, Size, Variants}; use tracing::debug; use crate::common::*; @@ -16,13 +17,13 @@ fn uncached_llvm_type<'a, 'tcx>( layout: TyAndLayout<'tcx>, defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, ) -> &'a Type { - match layout.abi { - Abi::Scalar(_) => bug!("handled elsewhere"), - Abi::Vector { element, count } => { + match layout.backend_repr { + BackendRepr::Scalar(_) => bug!("handled elsewhere"), + BackendRepr::Vector { element, count } => { let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } - Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalarPair(..) => {} + BackendRepr::Uninhabited | BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {} } let name = match layout.ty.kind() { @@ -169,16 +170,21 @@ pub(crate) trait LayoutLlvmExt<'tcx> { impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { - match self.abi { - Abi::Scalar(_) | Abi::Vector { .. } => true, - Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, + match self.backend_repr { + BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true, + BackendRepr::ScalarPair(..) | BackendRepr::Uninhabited | BackendRepr::Memory { .. } => { + false + } } } fn is_llvm_scalar_pair(&self) -> bool { - match self.abi { - Abi::ScalarPair(..) => true, - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, + match self.backend_repr { + BackendRepr::ScalarPair(..) => true, + BackendRepr::Uninhabited + | BackendRepr::Scalar(_) + | BackendRepr::Vector { .. } + | BackendRepr::Memory { .. } => false, } } @@ -190,16 +196,16 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. /// If the type is an unsized struct, the regular layout is generated, - /// with the inner-most trailing unsized field using the "minimal unit" + /// with the innermost trailing unsized field using the "minimal unit" /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { // This must produce the same result for `repr(transparent)` wrappers as for the inner type! // In other words, this should generally not look at the type at all, but only at the // layout. - if let Abi::Scalar(scalar) = self.abi { + if let BackendRepr::Scalar(scalar) = self.backend_repr { // Use a different cache for scalars because pointers to DSTs - // can be either fat or thin (data pointers of fat pointers). + // can be either wide or thin (data pointers of wide pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { return llty; } @@ -247,13 +253,13 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - match self.abi { - Abi::Scalar(scalar) => { + match self.backend_repr { + BackendRepr::Scalar(scalar) => { if scalar.is_bool() { return cx.type_i1(); } } - Abi::ScalarPair(..) => { + BackendRepr::ScalarPair(..) => { // An immediate pair always contains just the two elements, without any padding // filler, as it should never be stored to memory. return cx.type_struct( @@ -286,7 +292,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // This must produce the same result for `repr(transparent)` wrappers as for the inner type! // In other words, this should generally not look at the type at all, but only at the // layout. - let Abi::ScalarPair(a, b) = self.abi else { + let BackendRepr::ScalarPair(a, b) = self.backend_repr else { bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); }; let scalar = [a, b][index]; diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index f12b94d5887..e4c3e748cb5 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -1,9 +1,9 @@ +use rustc_abi::{Align, Endian, HasDataLayout, Size}; use rustc_codegen_ssa::common::IntPredicate; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; use crate::builder::Builder; use crate::type_::Type; |