diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
33 files changed, 1154 insertions, 1368 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 2fe5ed32daa..1d35138b013 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -2,6 +2,7 @@ use std::cmp; use libc::c_uint; use rustc_abi as abi; +pub(crate) use rustc_abi::ExternAbi; use rustc_abi::Primitive::Int; use rustc_abi::{HasDataLayout, Size}; use rustc_codegen_ssa::MemFlags; @@ -13,9 +14,8 @@ use rustc_middle::ty::layout::LayoutOf; pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA}; use rustc_middle::{bug, ty}; use rustc_session::config; -pub(crate) use rustc_target::abi::call::*; +pub(crate) use rustc_target::callconv::*; use rustc_target::spec::SanitizerSet; -pub(crate) use rustc_target::spec::abi::Abi; use smallvec::SmallVec; use crate::attributes::llfn_attrs_from_instance; @@ -415,7 +415,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { instance: Option<ty::Instance<'tcx>>, ) { let mut func_attrs = SmallVec::<[_; 3]>::new(); - if self.ret.layout.abi.is_uninhabited() { + if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); } if !self.can_unwind { @@ -436,7 +436,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i - 1 }; - let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| { + let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| { if cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() >= (19, 0, 0) && matches!(scalar.primitive(), Int(..)) @@ -458,7 +458,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { match &self.ret.mode { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); - if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + if let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr { apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); } } @@ -495,7 +495,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(attrs) => { let i = apply(attrs); - if let abi::Abi::Scalar(scalar) = arg.layout.abi { + if let abi::BackendRepr::Scalar(scalar) = arg.layout.backend_repr { apply_range_attr(llvm::AttributePlace::Argument(i), scalar); } } @@ -510,7 +510,9 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Pair(a, b) => { let i = apply(a); let ii = apply(b); - if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi { + if let abi::BackendRepr::ScalarPair(scalar_a, scalar_b) = + arg.layout.backend_repr + { apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a); apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b); } @@ -532,7 +534,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { let mut func_attrs = SmallVec::<[_; 2]>::new(); - if self.ret.layout.abi.is_uninhabited() { + if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); } if !self.can_unwind { @@ -570,7 +572,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } if bx.cx.sess().opts.optimize != config::OptLevel::No && llvm_util::get_version() < (19, 0, 0) - && let abi::Abi::Scalar(scalar) = self.ret.layout.abi + && let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr && matches!(scalar.primitive(), Int(..)) // If the value is a boolean, the range is 0..2 and that ultimately // become 0..0 when the type becomes i1, which would be rejected diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 89f5305840b..149ded28356 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -7,6 +7,7 @@ use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; +use crate::common::AsCCharPtr; use crate::llvm::{self, Context, False, Module, True, Type}; use crate::{ModuleLlvm, attributes, debuginfo}; @@ -76,21 +77,15 @@ pub(crate) unsafe fn codegen( unsafe { // __rust_alloc_error_handler_should_panic let name = OomStrategy::SYMBOL; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - llvm::LLVMRustSetVisibility( - ll_g, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ); + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); + llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); let name = NO_ALLOC_SHIM_IS_UNSTABLE; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - llvm::LLVMRustSetVisibility( - ll_g, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ); + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); + llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let llval = llvm::LLVMConstInt(i8, 0, False); llvm::LLVMSetInitializer(ll_g, llval); } @@ -121,7 +116,7 @@ fn create_wrapper_function( ); let llfn = llvm::LLVMRustGetOrInsertFunction( llmod, - from_name.as_ptr().cast(), + from_name.as_c_char_ptr(), from_name.len(), ty, ); @@ -134,10 +129,7 @@ fn create_wrapper_function( None }; - llvm::LLVMRustSetVisibility( - llfn, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ); + llvm::set_visibility(llfn, llvm::Visibility::from_generic(tcx.sess.default_visibility())); if tcx.sess.must_emit_unwind_tables() { let uwtable = @@ -146,12 +138,12 @@ fn create_wrapper_function( } let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_ptr().cast(), to_name.len(), ty); + llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_c_char_ptr(), to_name.len(), ty); if let Some(no_return) = no_return { // -> ! DIFlagNoReturn attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); } - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + llvm::set_visibility(callee, llvm::Visibility::Hidden); let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr()); @@ -162,7 +154,7 @@ fn create_wrapper_function( .enumerate() .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::<Vec<_>>(); - let ret = llvm::LLVMRustBuildCall( + let ret = llvm::LLVMBuildCallWithOperandBundles( llbuilder, ty, callee, @@ -170,6 +162,7 @@ fn create_wrapper_function( args.len() as c_uint, [].as_ptr(), 0 as c_uint, + c"".as_ptr(), ); llvm::LLVMSetTailCall(ret, True); if output.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index d1d7d0cf4ce..a1ccf0d1719 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -1,6 +1,7 @@ use std::assert_matches::assert_matches; use libc::{c_char, c_uint}; +use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar}; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::mir::operand::OperandValue; use rustc_codegen_ssa::traits::*; @@ -9,13 +10,12 @@ use rustc_middle::ty::Instance; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::{bug, span_bug}; use rustc_span::{Pos, Span, Symbol, sym}; -use rustc_target::abi::*; use rustc_target::asm::*; use smallvec::SmallVec; use tracing::debug; use crate::builder::Builder; -use crate::common::Funclet; +use crate::common::{AsCCharPtr, Funclet}; use crate::context::CodegenCx; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -420,7 +420,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { unsafe { llvm::LLVMAppendModuleInlineAsm( self.llmod, - template_str.as_ptr().cast(), + template_str.as_c_char_ptr(), template_str.len(), ); } @@ -458,14 +458,14 @@ pub(crate) fn inline_asm_call<'ll>( let fty = bx.cx.type_func(&argtys, output); unsafe { // Ask LLVM to verify that the constraints are well-formed. - let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); + let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_c_char_ptr(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { let v = llvm::LLVMRustInlineAsm( fty, - asm.as_ptr().cast(), + asm.as_c_char_ptr(), asm.len(), - cons.as_ptr().cast(), + cons.as_c_char_ptr(), cons.len(), volatile, alignstack, @@ -638,7 +638,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg) => "r", PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC(PowerPCInlineAsmRegClass::cr) + | PowerPC(PowerPCInlineAsmRegClass::xer) + | PowerPC(PowerPCInlineAsmRegClass::vreg) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -800,7 +802,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC(PowerPCInlineAsmRegClass::cr) + | PowerPC(PowerPCInlineAsmRegClass::xer) + | PowerPC(PowerPCInlineAsmRegClass::vreg) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), @@ -880,8 +884,8 @@ fn llvm_fixup_input<'ll, 'tcx>( ) -> &'ll Value { use InlineAsmRegClass::*; let dl = &bx.tcx.data_layout; - match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + match (reg, layout.backend_repr) { + (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) @@ -889,7 +893,7 @@ fn llvm_fixup_input<'ll, 'tcx>( value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s)) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(bx.cx, s); @@ -902,7 +906,7 @@ fn llvm_fixup_input<'ll, 'tcx>( } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); @@ -910,14 +914,14 @@ fn llvm_fixup_input<'ll, 'tcx>( let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } - (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F64) => { bx.bitcast(value, bx.cx.type_i64()) } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, + BackendRepr::Vector { .. }, ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), ( X86( @@ -925,7 +929,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if bx.sess().asm_arch == Some(InlineAsmArch::X86) && s.primitive() == Primitive::Float(Float::F128) => { @@ -937,7 +941,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if s.primitive() == Primitive::Float(Float::F16) => { let value = bx.insert_element( bx.const_undef(bx.type_vector(bx.type_f16(), 8)), @@ -952,11 +956,14 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Vector { element, count: count @ (8 | 16) }, + BackendRepr::Vector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_i16(), count)) } - (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => { + ( + Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + BackendRepr::Scalar(s), + ) => { if let Primitive::Int(Integer::I32, _) = s.primitive() { bx.bitcast(value, bx.cx.type_f32()) } else { @@ -969,7 +976,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | ArmInlineAsmRegClass::dreg_low8 | ArmInlineAsmRegClass::dreg_low16, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) => { if let Primitive::Int(Integer::I64, _) = s.primitive() { bx.bitcast(value, bx.cx.type_f64()) @@ -986,11 +993,11 @@ fn llvm_fixup_input<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - Abi::Vector { element, count: count @ (4 | 8) }, + BackendRepr::Vector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_i16(), count)) } - (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => { match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), @@ -999,7 +1006,7 @@ fn llvm_fixup_input<'ll, 'tcx>( _ => value, } } - (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s)) + (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F16) && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) => { @@ -1022,15 +1029,15 @@ fn llvm_fixup_output<'ll, 'tcx>( instance: Instance<'_>, ) -> &'ll Value { use InlineAsmRegClass::*; - match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + match (reg, layout.backend_repr) { + (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { bx.extract_element(value, bx.const_i32(0)) } else { value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s)) if s.primitive() != Primitive::Float(Float::F128) => { value = bx.extract_element(value, bx.const_i32(0)); @@ -1039,7 +1046,7 @@ fn llvm_fixup_output<'ll, 'tcx>( } value } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); @@ -1047,14 +1054,14 @@ fn llvm_fixup_output<'ll, 'tcx>( let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) } - (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F64) => { bx.bitcast(value, bx.cx.type_f64()) } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, + BackendRepr::Vector { .. }, ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), ( X86( @@ -1062,7 +1069,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if bx.sess().asm_arch == Some(InlineAsmArch::X86) && s.primitive() == Primitive::Float(Float::F128) => { @@ -1074,7 +1081,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if s.primitive() == Primitive::Float(Float::F16) => { let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8)); bx.extract_element(value, bx.const_usize(0)) @@ -1085,11 +1092,14 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Vector { element, count: count @ (8 | 16) }, + BackendRepr::Vector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_f16(), count)) } - (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => { + ( + Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + BackendRepr::Scalar(s), + ) => { if let Primitive::Int(Integer::I32, _) = s.primitive() { bx.bitcast(value, bx.cx.type_i32()) } else { @@ -1102,7 +1112,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | ArmInlineAsmRegClass::dreg_low8 | ArmInlineAsmRegClass::dreg_low16, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) => { if let Primitive::Int(Integer::I64, _) = s.primitive() { bx.bitcast(value, bx.cx.type_i64()) @@ -1119,11 +1129,11 @@ fn llvm_fixup_output<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - Abi::Vector { element, count: count @ (4 | 8) }, + BackendRepr::Vector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_f16(), count)) } - (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => { match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), @@ -1133,7 +1143,7 @@ fn llvm_fixup_output<'ll, 'tcx>( _ => value, } } - (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s)) + (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F16) && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) => { @@ -1153,35 +1163,35 @@ fn llvm_fixup_output_type<'ll, 'tcx>( instance: Instance<'_>, ) -> &'ll Type { use InlineAsmRegClass::*; - match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + match (reg, layout.backend_repr) { + (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { cx.type_vector(cx.type_i8(), 8) } else { layout.llvm_type(cx) } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s)) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(cx, s); let count = 16 / layout.size.bytes(); cx.type_vector(elem_ty, count) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) + (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(cx, element); cx.type_vector(elem_ty, count * 2) } - (X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F64) => { cx.type_i64() } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - Abi::Vector { .. }, + BackendRepr::Vector { .. }, ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), ( X86( @@ -1189,7 +1199,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if cx.sess().asm_arch == Some(InlineAsmArch::X86) && s.primitive() == Primitive::Float(Float::F128) => { @@ -1201,7 +1211,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8), ( X86( @@ -1209,11 +1219,14 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - Abi::Vector { element, count: count @ (8 | 16) }, + BackendRepr::Vector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { cx.type_vector(cx.type_i16(), count) } - (Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), Abi::Scalar(s)) => { + ( + Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + BackendRepr::Scalar(s), + ) => { if let Primitive::Int(Integer::I32, _) = s.primitive() { cx.type_f32() } else { @@ -1226,7 +1239,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | ArmInlineAsmRegClass::dreg_low8 | ArmInlineAsmRegClass::dreg_low16, ), - Abi::Scalar(s), + BackendRepr::Scalar(s), ) => { if let Primitive::Int(Integer::I64, _) = s.primitive() { cx.type_f64() @@ -1243,11 +1256,11 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - Abi::Vector { element, count: count @ (4 | 8) }, + BackendRepr::Vector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { cx.type_vector(cx.type_i16(), count) } - (Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => { match s.primitive() { // MIPS only supports register-length arithmetics. Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), @@ -1256,7 +1269,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( _ => layout.llvm_type(cx), } } - (RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s)) + (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s)) if s.primitive() == Primitive::Float(Float::F16) && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) => { diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 2c5ec9dad59..cb958c1d4d7 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -232,11 +232,6 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { return None; } - // probestack doesn't play nice either with gcov profiling. - if cx.sess().opts.unstable_opts.profile { - return None; - } - let attr_value = match cx.sess().target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support @@ -424,7 +419,10 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } - if let Some(PacRet { leaf, key }) = pac_ret { + if let Some(PacRet { leaf, pc, key }) = pac_ret { + if pc { + to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); + } to_add.push(llvm::CreateAttrStringValue( cx.llcx, "sign-return-address", diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 1f7a923dd2c..48beb9be2b2 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -165,13 +165,14 @@ fn get_bitcode_slice_from_object_data<'a>( // We drop the "__LLVM," prefix here because on Apple platforms there's a notion of "segment // name" which in the public API for sections gets treated as part of the section name, but // internally in MachOObjectFile.cpp gets treated separately. - let section_name = bitcode_section_name(cgcx).trim_start_matches("__LLVM,"); + let section_name = bitcode_section_name(cgcx).to_str().unwrap().trim_start_matches("__LLVM,"); let mut len = 0; let data = unsafe { llvm::LLVMRustGetSliceFromObjectDataByName( obj.as_ptr(), obj.len(), section_name.as_ptr(), + section_name.len(), &mut len, ) }; @@ -502,9 +503,9 @@ fn thin_lto( // upstream... let data = llvm::LLVMRustCreateThinLTOData( thin_modules.as_ptr(), - thin_modules.len() as u32, + thin_modules.len(), symbols_below_threshold.as_ptr(), - symbols_below_threshold.len() as u32, + symbols_below_threshold.len(), ) .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; @@ -569,7 +570,7 @@ fn thin_lto( info!(" - {}: re-compiled", module_name); opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { - shared: shared.clone(), + shared: Arc::clone(&shared), idx: module_index, })); } @@ -601,23 +602,9 @@ pub(crate) fn run_pass_manager( // This code is based off the code found in llvm's LTO code generator: // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); - unsafe { - if !llvm::LLVMRustHasModuleFlag( - module.module_llvm.llmod(), - "LTOPostLink".as_ptr().cast(), - 11, - ) { - llvm::LLVMRustAddModuleFlagU32( - module.module_llvm.llmod(), - llvm::LLVMModFlagBehavior::Error, - c"LTOPostLink".as_ptr(), - 1, - ); - } - let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; - let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage)?; - } + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + unsafe { write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage) }?; debug!("lto done"); Ok(()) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index afdd2b581b8..a65ae4df1e3 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,4 +1,4 @@ -use std::ffi::CString; +use std::ffi::{CStr, CString}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -9,6 +9,7 @@ use llvm::{ LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols, }; use rustc_codegen_ssa::back::link::ensure_removed; +use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -34,6 +35,7 @@ use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback, }; +use crate::common::AsCCharPtr; use crate::errors::{ CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression, WithLlvmError, WriteBytecode, @@ -210,7 +212,7 @@ pub(crate) fn target_machine_factory( singlethread = false; } - let triple = SmallCStr::new(&sess.target.llvm_target); + let triple = SmallCStr::new(&versioned_llvm_target(sess)); let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); let features = CString::new(target_features.join(",")).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); @@ -590,15 +592,14 @@ pub(crate) unsafe fn llvm_optimize( pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), - config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, llvm_selfprofiler, selfprofile_before_pass_callback, selfprofile_after_pass_callback, - extra_passes.as_ptr().cast(), + extra_passes.as_c_char_ptr(), extra_passes.len(), - llvm_plugins.as_ptr().cast(), + llvm_plugins.as_c_char_ptr(), llvm_plugins.len(), ) }; @@ -944,27 +945,13 @@ fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: asm } -fn target_is_apple(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { - let triple = cgcx.opts.target_triple.triple(); - triple.contains("-ios") - || triple.contains("-darwin") - || triple.contains("-tvos") - || triple.contains("-watchos") - || triple.contains("-visionos") -} - -fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { - cgcx.opts.target_triple.triple().contains("-aix") -} - -//FIXME use c string literals here too -pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static str { - if target_is_apple(cgcx) { - "__LLVM,__bitcode\0" - } else if target_is_aix(cgcx) { - ".ipa\0" +pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static CStr { + if cgcx.target_is_like_osx { + c"__LLVM,__bitcode" + } else if cgcx.target_is_like_aix { + c".ipa" } else { - ".llvmbc\0" + c".llvmbc" } } @@ -1028,10 +1015,12 @@ unsafe fn embed_bitcode( // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). - let is_aix = target_is_aix(cgcx); - let is_apple = target_is_apple(cgcx); unsafe { - if is_apple || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") { + if cgcx.target_is_like_osx + || cgcx.target_is_like_aix + || cgcx.target_arch == "wasm32" + || cgcx.target_arch == "wasm64" + { // We don't need custom section flags, create LLVM globals. let llconst = common::bytes_in_context(llcx, bitcode); let llglobal = llvm::LLVMAddGlobal( @@ -1041,9 +1030,8 @@ unsafe fn embed_bitcode( ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = bitcode_section_name(cgcx); - llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, bitcode_section_name(cgcx)); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); @@ -1053,22 +1041,22 @@ unsafe fn embed_bitcode( c"rustc.embedded.cmdline".as_ptr(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { + let section = if cgcx.target_is_like_osx { c"__LLVM,__cmdline" - } else if is_aix { + } else if cgcx.target_is_like_aix { c".info" } else { c".llvmcmd" }; - llvm::LLVMSetSection(llglobal, section.as_ptr()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, section); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); } } } @@ -1096,7 +1084,7 @@ fn create_msvc_imps( let ptr_ty = Type::ptr_llcx(llcx); let globals = base::iter_globals(llmod) .filter(|&val| { - llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage + llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && llvm::LLVMIsDeclaration(val) == 0 }) .filter_map(|val| { @@ -1115,7 +1103,7 @@ fn create_msvc_imps( for (imp_name, val) in globals { let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr()); llvm::LLVMSetInitializer(imp, val); - llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); + llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 0ba8d82406a..32793894794 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -145,10 +145,8 @@ pub(crate) fn compile_codegen_unit( pub(crate) fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { let Some(sect) = attrs.link_section else { return }; - unsafe { - let buf = SmallCStr::new(sect.as_str()); - llvm::LLVMSetSection(llval, buf.as_ptr()); - } + let buf = SmallCStr::new(sect.as_str()); + llvm::set_section(llval, &buf); } pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index dbf5298d64b..751b2235dc8 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -21,7 +21,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; use rustc_span::Span; -use rustc_target::abi::call::FnAbi; +use rustc_target::callconv::FnAbi; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -239,7 +239,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("invoke", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); - let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { bundles.push(funclet_bundle); @@ -250,13 +249,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - if let Some(kcfi_bundle) = kcfi_bundle { + if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { bundles.push(kcfi_bundle); } let invoke = unsafe { - llvm::LLVMRustBuildInvoke( + llvm::LLVMBuildInvokeWithOperandBundles( self.llbuilder, llty, llfn, @@ -545,13 +543,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } let llval = const_llval.unwrap_or_else(|| { let load = self.load(llty, place.val.llval, place.val.align); - if let abi::Abi::Scalar(scalar) = place.layout.abi { + if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr { scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); } load }); OperandValue::Immediate(self.to_immediate(llval, place.layout)) - } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { + } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr { let b_offset = a.size(self).align_to(b.align(self).abi); let mut load = |i, scalar: abi::Scalar, layout, align, offset| { @@ -1165,39 +1163,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); } - fn instrprof_increment( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - num_counters: &'ll Value, - index: &'ll Value, - ) { - debug!( - "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, num_counters, index - ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, num_counters, index]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } - } - fn call( &mut self, llty: &'ll Type, @@ -1212,7 +1177,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("call", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); - let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { bundles.push(funclet_bundle); @@ -1223,13 +1187,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - if let Some(kcfi_bundle) = kcfi_bundle { + if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { bundles.push(kcfi_bundle); } let call = unsafe { - llvm::LLVMRustBuildCall( + llvm::LLVMBuildCallWithOperandBundles( self.llbuilder, llty, llfn, @@ -1237,6 +1200,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { args.len() as c_uint, bundles.as_ptr(), bundles.len() as c_uint, + c"".as_ptr(), ) }; if let Some(fn_abi) = fn_abi { @@ -1542,7 +1506,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { let args = self.check_call("callbr", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); - let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { bundles.push(funclet_bundle); @@ -1553,13 +1516,12 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - if let Some(kcfi_bundle) = kcfi_bundle { + if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { bundles.push(kcfi_bundle); } let callbr = unsafe { - llvm::LLVMRustBuildCallBr( + llvm::LLVMBuildCallBr( self.llbuilder, llty, llfn, @@ -1634,7 +1596,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, instance: Option<Instance<'tcx>>, llfn: &'ll Value, - ) -> Option<llvm::OperandBundleDef<'ll>> { + ) -> Option<llvm::OperandBundleOwned<'ll>> { let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi @@ -1660,13 +1622,25 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi::typeid_for_fnabi(self.tcx, fn_abi, options) }; - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + Some(llvm::OperandBundleOwned::new("kcfi", &[self.const_u32(kcfi_typeid)])) } else { None }; kcfi_bundle } + /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation. + #[instrument(level = "debug", skip(self))] + pub(crate) fn instrprof_increment( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); + } + /// Emits a call to `llvm.instrprof.mcdc.parameters`. /// /// This doesn't produce any code directly, but is used as input by @@ -1676,40 +1650,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { /// /// [`CodeGenPGO::emitMCDCParameters`]: /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_parameters( &mut self, fn_name: &'ll Value, hash: &'ll Value, bitmap_bits: &'ll Value, ) { - debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits); - assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, bitmap_bits]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_tvbitmap_update( &mut self, fn_name: &'ll Value, @@ -1717,39 +1672,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { bitmap_index: &'ll Value, mcdc_temp: &'ll Value, ) { - debug!( - "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, bitmap_index, mcdc_temp - ); assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" ); - - let llfn = - unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()], - self.cx.type_void(), - ); let args = &[fn_name, hash, bitmap_index, mcdc_temp]; - let args = self.check_call("call", llty, llfn, args); - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args); + } + + #[instrument(level = "debug", skip(self))] + pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) { self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { - debug!("mcdc_condbitmap_update() with args ({:?}, {:?})", cond_index, mcdc_temp); assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 206a7069792..25037b97375 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -95,11 +95,10 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t // whether we are sharing generics or not. The important thing here is // that the visibility we apply to the declaration is the same one that // has been applied to the definition (wherever that definition may be). - unsafe { - llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); - let is_generic = - instance.args.non_erasable_generics(tcx, instance.def_id()).next().is_some(); + llvm::set_linkage(llfn, llvm::Linkage::ExternalLinkage); + unsafe { + let is_generic = instance.args.non_erasable_generics().next().is_some(); let is_hidden = if is_generic { // This is a monomorphization of a generic function. @@ -136,7 +135,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t || !cx.tcx.is_reachable_non_generic(instance_def_id)) }; if is_hidden { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + llvm::set_visibility(llfn, llvm::Visibility::Hidden); } // MinGW: For backward compatibility we rely on the linker to decide whether it diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 0ced37b53a8..8852dec7d9f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -17,7 +17,7 @@ use tracing::debug; use crate::consts::const_alloc_to_llvm; pub(crate) use crate::context::CodegenCx; -use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, OperandBundleDef, True}; +use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, True}; use crate::type_::Type; use crate::value::Value; @@ -63,19 +63,19 @@ use crate::value::Value; /// the `OperandBundleDef` value created for MSVC landing pads. pub(crate) struct Funclet<'ll> { cleanuppad: &'ll Value, - operand: OperandBundleDef<'ll>, + operand: llvm::OperandBundleOwned<'ll>, } impl<'ll> Funclet<'ll> { pub(crate) fn new(cleanuppad: &'ll Value) -> Self { - Funclet { cleanuppad, operand: OperandBundleDef::new("funclet", &[cleanuppad]) } + Funclet { cleanuppad, operand: llvm::OperandBundleOwned::new("funclet", &[cleanuppad]) } } pub(crate) fn cleanuppad(&self) -> &'ll Value { self.cleanuppad } - pub(crate) fn bundle(&self) -> &OperandBundleDef<'ll> { + pub(crate) fn bundle(&self) -> &llvm::OperandBundle<'ll> { &self.operand } } @@ -219,8 +219,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMSetInitializer(g, sc); llvm::LLVMSetGlobalConstant(g, True); llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); } + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); (s.to_owned(), g) }) .1; @@ -392,3 +392,21 @@ pub(crate) fn get_dllimport<'tcx>( tcx.native_library(id) .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) } + +/// Extension trait for explicit casts to `*const c_char`. +pub(crate) trait AsCCharPtr { + /// Equivalent to `self.as_ptr().cast()`, but only casts to `*const c_char`. + fn as_c_char_ptr(&self) -> *const c_char; +} + +impl AsCCharPtr for str { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} + +impl AsCCharPtr for [u8] { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 33a85adeb87..bed64686a23 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,5 +1,8 @@ use std::ops::Range; +use rustc_abi::{ + Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, +}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::def::DefKind; @@ -14,12 +17,9 @@ use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use rustc_session::config::Lto; -use rustc_target::abi::{ - Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, -}; use tracing::{debug, instrument, trace}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::errors::{ InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, }; @@ -172,29 +172,27 @@ fn check_and_apply_linkage<'ll, 'tcx>( if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); - unsafe { - // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(sym, cx.type_i8()); - llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); - - // Declare an internal global `extern_with_linkage_foo` which - // is initialized with the address of `foo`. If `foo` is - // discarded during linking (for example, if `foo` has weak - // linkage and there are no definitions), then - // `extern_with_linkage_foo` will instead be initialized to - // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); - let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { - span: cx.tcx.def_span(def_id), - symbol_name: sym, - }) - }); - llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); - llvm::LLVMSetInitializer(g2, g1); - g2 - } + // Declare a symbol `foo` with the desired linkage. + let g1 = cx.declare_global(sym, cx.type_i8()); + llvm::set_linkage(g1, base::linkage_to_llvm(linkage)); + + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + let mut real_name = "_rust_extern_with_linkage_".to_string(); + real_name.push_str(sym); + let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { + cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { + span: cx.tcx.def_span(def_id), + symbol_name: sym, + }) + }); + llvm::set_linkage(g2, llvm::Linkage::InternalLinkage); + unsafe { llvm::LLVMSetInitializer(g2, g1) }; + g2 } else if cx.tcx.sess.target.arch == "x86" && let Some(dllimport) = crate::common::get_dllimport(cx.tcx, def_id, sym) { @@ -224,23 +222,21 @@ impl<'ll> CodegenCx<'ll, '_> { align: Align, kind: Option<&str>, ) -> &'ll Value { - unsafe { - let gv = match kind { - Some(kind) if !self.tcx.sess.fewer_names() => { - let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", name); - }); - llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); - gv - } - _ => self.define_private_global(self.val_ty(cv)), - }; - llvm::LLVMSetInitializer(gv, cv); - set_global_alignment(self, gv, align); - llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); - gv - } + let gv = match kind { + Some(kind) if !self.tcx.sess.fewer_names() => { + let name = self.generate_local_symbol_name(kind); + let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", name); + }); + llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); + gv + } + _ => self.define_private_global(self.val_ty(cv)), + }; + unsafe { llvm::LLVMSetInitializer(gv, cv) }; + set_global_alignment(self, gv, align); + llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + gv } #[instrument(level = "debug", skip(self))] @@ -292,9 +288,7 @@ impl<'ll> CodegenCx<'ll, '_> { let g = self.declare_global(sym, llty); if !self.tcx.is_reachable_non_generic(def_id) { - unsafe { - llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); - } + llvm::set_visibility(g, llvm::Visibility::Hidden); } g @@ -312,7 +306,7 @@ impl<'ll> CodegenCx<'ll, '_> { llvm::set_thread_local_mode(g, self.tls_model); } - let dso_local = unsafe { self.should_assume_dso_local(g, true) }; + let dso_local = self.should_assume_dso_local(g, true); if dso_local { unsafe { llvm::LLVMRustSetDSOLocal(g, true); @@ -401,18 +395,18 @@ impl<'ll> CodegenCx<'ll, '_> { let name = llvm::get_value_name(g).to_vec(); llvm::set_value_name(g, b""); - let linkage = llvm::LLVMRustGetLinkage(g); - let visibility = llvm::LLVMRustGetVisibility(g); + let linkage = llvm::get_linkage(g); + let visibility = llvm::get_visibility(g); let new_g = llvm::LLVMRustGetOrInsertGlobal( self.llmod, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), val_llty, ); - llvm::LLVMRustSetLinkage(new_g, linkage); - llvm::LLVMRustSetVisibility(new_g, visibility); + llvm::set_linkage(new_g, linkage); + llvm::set_visibility(new_g, visibility); // The old global has had its name removed but is returned by // get_static since it is in the instance cache. Provide an @@ -457,7 +451,7 @@ impl<'ll> CodegenCx<'ll, '_> { if let Some(section) = attrs.link_section { let section = llvm::LLVMMDStringInContext2( self.llcx, - section.as_str().as_ptr().cast(), + section.as_str().as_c_char_ptr(), section.as_str().len(), ); assert!(alloc.provenance().ptrs().is_empty()); @@ -468,7 +462,7 @@ impl<'ll> CodegenCx<'ll, '_> { let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); let alloc = - llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_c_char_ptr(), bytes.len()); let data = [section, alloc]; let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); let val = llvm::LLVMMetadataAsValue(self.llcx, meta); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 2f830d6f941..b7ab5d6d5f0 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -3,6 +3,8 @@ use std::cell::{Cell, RefCell}; use std::ffi::{CStr, c_uint}; use std::str; +use rustc_abi::{HasDataLayout, TargetDataLayout, VariantIdx}; +use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; @@ -23,12 +25,12 @@ use rustc_session::config::{ }; use rustc_span::source_map::Spanned; use rustc_span::{DUMMY_SP, Span}; -use rustc_target::abi::{HasDataLayout, TargetDataLayout, VariantIdx}; use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; +use crate::common::AsCCharPtr; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::{Metadata, MetadataType}; use crate::type_::Type; @@ -80,6 +82,7 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { pub isize_ty: &'ll Type, + /// Extra codegen state needed when coverage instrumentation is enabled. pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>, @@ -146,6 +149,11 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); } + if sess.target.arch.starts_with("sparc") { + // LLVM 20 updates the sparc layout to correctly align 128 bit integers to 128 bit. + // See https://github.com/llvm/llvm-project/pull/106951 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } } // Ensure the data-layout values hardcoded remain the defaults. @@ -175,7 +183,7 @@ pub(crate) unsafe fn create_module<'ll>( llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()); } - let llvm_target = SmallCStr::new(&sess.target.llvm_target); + let llvm_target = SmallCStr::new(&versioned_llvm_target(sess)); unsafe { llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr()); } @@ -208,133 +216,117 @@ pub(crate) unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = c"RtLibUseGOT".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "RtLibUseGOT", 1); } // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { - let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - canonical_jump_tables, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "CFI Canonical Jump Tables", + 1, + ); } // If we're normalizing integers with CFI, ensure LLVM generated functions do the same. // See https://github.com/llvm/llvm-project/pull/104826 if sess.is_sanitizer_cfi_normalize_integers_enabled() { - let cfi_normalize_integers = c"cfi-normalize-integers".as_ptr().cast(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - cfi_normalize_integers, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cfi-normalize-integers", + 1, + ); } // Enable LTO unit splitting if specified or if CFI is enabled. (See // https://reviews.llvm.org/D53891.) if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { - let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - enable_split_lto_unit, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "EnableSplitLTOUnit", + 1, + ); } // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - let kcfi = c"kcfi".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Override, "kcfi", 1); // Add "kcfi-offset" module flag with -Z patchable-function-entry (See // https://reviews.llvm.org/D141172). let pfe = PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry); if pfe.prefix() > 0 { - let kcfi_offset = c"kcfi-offset".as_ptr().cast(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - kcfi_offset, - pfe.prefix().into(), - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "kcfi-offset", + pfe.prefix().into(), + ); } } // Control Flow Guard is currently only supported by the MSVC linker on Windows. if sess.target.is_like_msvc { - unsafe { - match sess.opts.cg.control_flow_guard { - CFGuard::Disabled => {} - CFGuard::NoChecks => { - // Set `cfguard=1` module flag to emit metadata only. - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, - 1, - ) - } - CFGuard::Checks => { - // Set `cfguard=2` module flag to emit metadata and checks. - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, - 2, - ) - } + match sess.opts.cg.control_flow_guard { + CFGuard::Disabled => {} + CFGuard::NoChecks => { + // Set `cfguard=1` module flag to emit metadata only. + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "cfguard", + 1, + ); + } + CFGuard::Checks => { + // Set `cfguard=2` module flag to emit metadata and checks. + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "cfguard", + 2, + ); } } } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch == "aarch64" { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"branch-target-enforcement".as_ptr(), - bti.into(), - ); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address".as_ptr(), - pac_ret.is_some().into(), - ); - let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-all".as_ptr(), - pac_opts.leaf.into(), - ); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-with-bkey".as_ptr(), - u32::from(pac_opts.key == PAuthKey::B), - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "branch-target-enforcement", + bti.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address", + pac_ret.is_some().into(), + ); + let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, pc: false, key: PAuthKey::A }); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "branch-protection-pauth-lr", + pac_opts.pc.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address-all", + pac_opts.leaf.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address-with-bkey", + u32::from(pac_opts.key == PAuthKey::B), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -345,59 +337,46 @@ pub(crate) unsafe fn create_module<'ll>( // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"cf-protection-branch".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cf-protection-branch", + 1, + ); } if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"cf-protection-return".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cf-protection-return", + 1, + ); } if sess.opts.unstable_opts.virtual_function_elimination { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Error, - c"Virtual Function Elim".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Virtual Function Elim", + 1, + ); } // Set module flag to enable Windows EHCont Guard (/guard:ehcont). if sess.opts.unstable_opts.ehcont_guard { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"ehcontguard".as_ptr() as *const _, - 1, - ) - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "ehcontguard", 1); } match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => {} - FunctionReturn::ThunkExtern => unsafe { - llvm::LLVMRustAddModuleFlagU32( + FunctionReturn::ThunkExtern => { + llvm::add_module_flag_u32( llmod, - llvm::LLVMModFlagBehavior::Override, - c"function_return_thunk_extern".as_ptr(), + llvm::ModuleFlagMergeBehavior::Override, + "function_return_thunk_extern", 1, - ) - }, + ); + } } match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) @@ -405,15 +384,12 @@ pub(crate) unsafe fn create_module<'ll>( // Set up the small-data optimization limit for architectures that use // an LLVM module flag to control this. (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => { - let flag = SmallCStr::new(flag.as_ref()); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Error, - flag.as_c_str().as_ptr(), - threshold as u32, - ) - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + &flag, + threshold as u32, + ); } _ => (), }; @@ -428,7 +404,7 @@ pub(crate) unsafe fn create_module<'ll>( let name_metadata = unsafe { llvm::LLVMMDStringInContext2( llcx, - rustc_producer.as_ptr().cast(), + rustc_producer.as_c_char_ptr(), rustc_producer.as_bytes().len(), ) }; @@ -447,33 +423,29 @@ pub(crate) unsafe fn create_module<'ll>( // If llvm_abiname is empty, emit nothing. let llvm_abiname = &sess.target.options.llvm_abiname; if matches!(sess.target.arch.as_ref(), "riscv32" | "riscv64") && !llvm_abiname.is_empty() { - unsafe { - llvm::LLVMRustAddModuleFlagString( - llmod, - llvm::LLVMModFlagBehavior::Error, - c"target-abi".as_ptr(), - llvm_abiname.as_ptr().cast(), - llvm_abiname.len(), - ); - } + llvm::add_module_flag_str( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "target-abi", + llvm_abiname, + ); } // Add module flags specified via -Z llvm_module_flag - for (key, value, behavior) in &sess.opts.unstable_opts.llvm_module_flag { - let key = format!("{key}\0"); - let behavior = match behavior.as_str() { - "error" => llvm::LLVMModFlagBehavior::Error, - "warning" => llvm::LLVMModFlagBehavior::Warning, - "require" => llvm::LLVMModFlagBehavior::Require, - "override" => llvm::LLVMModFlagBehavior::Override, - "append" => llvm::LLVMModFlagBehavior::Append, - "appendunique" => llvm::LLVMModFlagBehavior::AppendUnique, - "max" => llvm::LLVMModFlagBehavior::Max, - "min" => llvm::LLVMModFlagBehavior::Min, + for (key, value, merge_behavior) in &sess.opts.unstable_opts.llvm_module_flag { + let merge_behavior = match merge_behavior.as_str() { + "error" => llvm::ModuleFlagMergeBehavior::Error, + "warning" => llvm::ModuleFlagMergeBehavior::Warning, + "require" => llvm::ModuleFlagMergeBehavior::Require, + "override" => llvm::ModuleFlagMergeBehavior::Override, + "append" => llvm::ModuleFlagMergeBehavior::Append, + "appendunique" => llvm::ModuleFlagMergeBehavior::AppendUnique, + "max" => llvm::ModuleFlagMergeBehavior::Max, + "min" => llvm::ModuleFlagMergeBehavior::Min, // We already checked this during option parsing _ => unreachable!(), }; - unsafe { llvm::LLVMRustAddModuleFlagU32(llmod, behavior, key.as_ptr().cast(), *value) } + llvm::add_module_flag_u32(llmod, merge_behavior, key, *value); } llmod @@ -592,11 +564,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { &self.statics_to_rauw } + /// Extra state that is only available when coverage instrumentation is enabled. #[inline] - pub(crate) fn coverage_context( - &self, - ) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { - self.coverage_cx.as_ref() + #[track_caller] + pub(crate) fn coverage_cx(&self) -> &coverageinfo::CrateCoverageContext<'ll, 'tcx> { + self.coverage_cx.as_ref().expect("only called when coverage instrumentation is enabled") } pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { @@ -605,8 +577,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); + llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); + llvm::set_section(g, c"llvm.metadata"); } } } @@ -1099,6 +1071,10 @@ impl<'ll> CodegenCx<'ll, '_> { if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); + if crate::llvm_util::get_version() >= (19, 0, 0) { + ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); + ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); + } } ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 90f7dd733ca..feac97f3e2b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,11 +1,9 @@ -use rustc_middle::mir::coverage::{ - ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, SourceRegion, -}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum CounterKind { +pub(crate) enum CounterKind { Zero = 0, CounterValueReference = 1, Expression = 2, @@ -25,9 +23,9 @@ pub enum CounterKind { /// Must match the layout of `LLVMRustCounter`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct Counter { +pub(crate) struct Counter { // Important: The layout (order and types of fields) must match its C++ counterpart. - pub kind: CounterKind, + pub(crate) kind: CounterKind, id: u32, } @@ -36,7 +34,7 @@ impl Counter { pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; /// Constructs a new `Counter` of kind `CounterValueReference`. - pub fn counter_value_reference(counter_id: CounterId) -> Self { + pub(crate) fn counter_value_reference(counter_id: CounterId) -> Self { Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } } @@ -59,7 +57,7 @@ impl Counter { /// Must match the layout of `LLVMRustCounterExprKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum ExprKind { +pub(crate) enum ExprKind { Subtract = 0, Add = 1, } @@ -69,48 +67,13 @@ pub enum ExprKind { /// Must match the layout of `LLVMRustCounterExpression`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct CounterExpression { - pub kind: ExprKind, - pub lhs: Counter, - pub rhs: Counter, +pub(crate) struct CounterExpression { + pub(crate) kind: ExprKind, + pub(crate) lhs: Counter, + pub(crate) rhs: Counter, } -/// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegionKind`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -enum RegionKind { - /// A CodeRegion associates some code with a counter - CodeRegion = 0, - - /// An ExpansionRegion represents a file expansion region that associates - /// a source range with the expansion of a virtual source file, such as - /// for a macro instantiation or #include file. - ExpansionRegion = 1, - - /// A SkippedRegion represents a source range with code that was skipped - /// by a preprocessor or similar means. - SkippedRegion = 2, - - /// A GapRegion is like a CodeRegion, but its count is only set as the - /// line execution count when its the only region in the line. - GapRegion = 3, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion = 4, - - /// A DecisionRegion represents a top-level boolean expression and is - /// associated with a variable length bitmap index and condition number. - MCDCDecisionRegion = 5, - - /// A Branch Region can be extended to include IDs to facilitate MC/DC. - MCDCBranchRegion = 6, -} - -mod mcdc { +pub(crate) mod mcdc { use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; /// Must match the layout of `LLVMRustMCDCDecisionParameters`. @@ -121,8 +84,6 @@ mod mcdc { num_conditions: u16, } - // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at - // [19](https://github.com/llvm/llvm-project/pull/81257). type LLVMConditionId = i16; /// Must match the layout of `LLVMRustMCDCBranchParameters`. @@ -133,38 +94,6 @@ mod mcdc { condition_ids: [LLVMConditionId; 2], } - #[repr(C)] - #[derive(Clone, Copy, Debug)] - enum ParameterTag { - None = 0, - Decision = 1, - Branch = 2, - } - /// Same layout with `LLVMRustMCDCParameters` - #[repr(C)] - #[derive(Clone, Copy, Debug)] - pub(crate) struct Parameters { - tag: ParameterTag, - decision_params: DecisionParameters, - branch_params: BranchParameters, - } - - impl Parameters { - pub(crate) fn none() -> Self { - Self { - tag: ParameterTag::None, - decision_params: Default::default(), - branch_params: Default::default(), - } - } - pub(crate) fn decision(decision_params: DecisionParameters) -> Self { - Self { tag: ParameterTag::Decision, decision_params, branch_params: Default::default() } - } - pub(crate) fn branch(branch_params: BranchParameters) -> Self { - Self { tag: ParameterTag::Branch, decision_params: Default::default(), branch_params } - } - } - impl From<ConditionInfo> for BranchParameters { fn from(value: ConditionInfo) -> Self { let to_llvm_cond_id = |cond_id: Option<ConditionId>| { @@ -186,267 +115,68 @@ mod mcdc { } } -/// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the -/// coverage map, in accordance with the -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). -/// The struct composes fields representing the `Counter` type and value(s) (injected counter -/// ID, or expression type and operands), the source file (an indirect index into a "filenames -/// array", encoded separately), and source location (start and end positions of the represented -/// code region). +/// A span of source code coordinates to be embedded in coverage metadata. /// -/// Corresponds to struct `llvm::coverage::CounterMappingRegion`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegion`. -#[derive(Copy, Clone, Debug)] +/// Must match the layout of `LLVMRustCoverageSpan`. +#[derive(Clone, Debug)] #[repr(C)] -pub struct CounterMappingRegion { - /// The counter type and type-dependent counter data, if any. - counter: Counter, - - /// If the `RegionKind` is a `BranchRegion`, this represents the counter - /// for the false branch of the region. - false_counter: Counter, - - mcdc_params: mcdc::Parameters, - /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the - /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes - /// that, in turn, are used to look up the filename for this region. +pub(crate) struct CoverageSpan { + /// Local index into the function's local-to-global file ID table. + /// The value at that index is itself an index into the coverage filename + /// table in the CGU's `__llvm_covmap` section. file_id: u32, - /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find - /// the mapping regions created as a result of macro expansion, by checking if their file id - /// matches the expanded file id. - expanded_file_id: u32, - - /// 1-based starting line of the mapping region. + /// 1-based starting line of the source code span. start_line: u32, - - /// 1-based starting column of the mapping region. + /// 1-based starting column of the source code span. start_col: u32, - - /// 1-based ending line of the mapping region. + /// 1-based ending line of the source code span. end_line: u32, - - /// 1-based ending column of the mapping region. If the high bit is set, the current - /// mapping region is a gap area. + /// 1-based ending column of the source code span. High bit must be unset. end_col: u32, - - kind: RegionKind, } -impl CounterMappingRegion { - pub(crate) fn from_mapping( - mapping_kind: &MappingKind, - local_file_id: u32, - source_region: &SourceRegion, - ) -> Self { - let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = - source_region; - match *mapping_kind { - MappingKind::Code(term) => Self::code_region( - Counter::from_term(term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::Branch { true_term, false_term } => Self::branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { - Self::mcdc_branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - mcdc_params, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ) - } - MappingKind::MCDCDecision(decision_info) => Self::decision_region( - decision_info, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - } - } - - pub(crate) fn code_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::CodeRegion, - } +impl CoverageSpan { + pub(crate) fn from_source_region(file_id: u32, code_region: &SourceRegion) -> Self { + let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + // Internally, LLVM uses the high bit of `end_col` to distinguish between + // code regions and gap regions, so it can't be used by the column number. + assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}"); + Self { file_id, start_line, start_col, end_line, end_col } } +} - pub(crate) fn branch_region( - counter: Counter, - false_counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::BranchRegion, - } - } - - pub(crate) fn mcdc_branch_region( - counter: Counter, - false_counter: Counter, - condition_info: ConditionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::branch(condition_info.into()), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCBranchRegion, - } - } - - pub(crate) fn decision_region( - decision_info: DecisionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - let mcdc_params = mcdc::Parameters::decision(decision_info.into()); - - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params, - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCDecisionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageCodeRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct CodeRegion { + pub(crate) span: CoverageSpan, + pub(crate) counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn expansion_region( - file_id: u32, - expanded_file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::ExpansionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct BranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn skipped_region( - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::SkippedRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCBranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, + pub(crate) mcdc_branch_params: mcdc::BranchParameters, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn gap_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col: (1_u32 << 31) | end_col, - kind: RegionKind::GapRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCDecisionRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCDecisionRegion { + pub(crate) span: CoverageSpan, + pub(crate) mcdc_decision_params: mcdc::DecisionParameters, } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index c2c261da79b..f6378199fe2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,16 +1,23 @@ +use std::ffi::CString; + use itertools::Itertools as _; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; +use rustc_abi::Align; +use rustc_codegen_ssa::traits::{ + BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, +}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; +use rustc_middle::mir::coverage::MappingKind; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, mir}; use rustc_span::Symbol; use rustc_span::def_id::DefIdSet; +use rustc_target::spec::HasTargetSpec; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::ffi::CounterMappingRegion; +use crate::coverageinfo::ffi; use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; use crate::{coverageinfo, llvm}; @@ -47,11 +54,11 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { add_unused_functions(cx); } - let function_coverage_map = match cx.coverage_context() { - Some(ctx) => ctx.take_function_coverage_map(), + // FIXME(#132395): Can this be none even when coverage is enabled? + let function_coverage_map = match cx.coverage_cx { + Some(ref cx) => cx.take_function_coverage_map(), None => return, }; - if function_coverage_map.is_empty() { // This module has no functions with coverage instrumentation return; @@ -75,11 +82,9 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { // Generate the coverage map header, which contains the filenames used by // this CGU's coverage mappings, and store it in a well-known global. - let cov_data_val = generate_coverage_map(cx, covmap_version, filenames_size, filenames_val); - coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + generate_covmap_record(cx, covmap_version, filenames_size, filenames_val); let mut unused_function_names = Vec::new(); - let covfun_section_name = coverageinfo::covfun_section_name(cx); // Encode coverage mappings and generate function records for (instance, function_coverage) in function_coverage_entries { @@ -108,9 +113,8 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { unused_function_names.push(mangled_function_name); } - save_function_record( + generate_covfun_record( cx, - &covfun_section_name, mangled_function_name, source_hash, filenames_ref, @@ -132,7 +136,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { .collect::<Vec<_>>(); let initializer = cx.const_array(cx.type_ptr(), &name_globals); - let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), "__llvm_coverage_names"); + let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), c"__llvm_coverage_names"); llvm::set_global_constant(array, true); llvm::set_linkage(array, llvm::Linkage::InternalLinkage); llvm::set_initializer(array, initializer); @@ -235,7 +239,10 @@ fn encode_mappings_for_function( let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); let mut virtual_file_mapping = VirtualFileMapping::default(); - let mut mapping_regions = Vec::with_capacity(counter_regions.len()); + let mut code_regions = vec![]; + let mut branch_regions = vec![]; + let mut mcdc_branch_regions = vec![]; + let mut mcdc_decision_regions = vec![]; // Group mappings into runs with the same filename, preserving the order // yielded by `FunctionCoverage`. @@ -255,11 +262,36 @@ fn encode_mappings_for_function( // form suitable for FFI. for (mapping_kind, region) in counter_regions_for_file { debug!("Adding counter {mapping_kind:?} to map for {region:?}"); - mapping_regions.push(CounterMappingRegion::from_mapping( - &mapping_kind, - local_file_id.as_u32(), - region, - )); + let span = ffi::CoverageSpan::from_source_region(local_file_id.as_u32(), region); + match mapping_kind { + MappingKind::Code(term) => { + code_regions + .push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); + } + MappingKind::Branch { true_term, false_term } => { + branch_regions.push(ffi::BranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + }); + } + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + mcdc_branch_regions.push(ffi::MCDCBranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), + }); + } + MappingKind::MCDCDecision(mcdc_decision_params) => { + mcdc_decision_regions.push(ffi::MCDCDecisionRegion { + span, + mcdc_decision_params: ffi::mcdc::DecisionParameters::from( + mcdc_decision_params, + ), + }); + } + } } } @@ -268,21 +300,24 @@ fn encode_mappings_for_function( coverageinfo::write_mapping_to_buffer( virtual_file_mapping.into_vec(), expressions, - mapping_regions, + &code_regions, + &branch_regions, + &mcdc_branch_regions, + &mcdc_decision_regions, buffer, ); }) } -/// Construct coverage map header and the array of function records, and combine them into the -/// coverage map. Save the coverage map data into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn generate_coverage_map<'ll>( +/// Generates the contents of the covmap record for this CGU, which mostly +/// consists of a header and a list of filenames. The record is then stored +/// as a global variable in the `__llvm_covmap` section. +fn generate_covmap_record<'ll>( cx: &CodegenCx<'ll, '_>, version: u32, filenames_size: usize, filenames_val: &'ll llvm::Value, -) -> &'ll llvm::Value { +) { debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); // Create the coverage data header (Note, fields 0 and 2 are now always zero, @@ -297,15 +332,37 @@ fn generate_coverage_map<'ll>( ); // Create the complete LLVM coverage data value to add to the LLVM IR - cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) + let covmap_data = + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false); + + let covmap_var_name = CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteMappingVarNameToString(s); + })) + .unwrap(); + debug!("covmap var name: {:?}", covmap_var_name); + + let covmap_section_name = CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); + })) + .expect("covmap section name should not contain NUL"); + debug!("covmap section name: {:?}", covmap_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(covmap_data), &covmap_var_name); + llvm::set_initializer(llglobal, covmap_data); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, &covmap_section_name); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + cx.add_used_global(llglobal); } -/// Construct a function record and combine it with the function's coverage mapping data. -/// Save the function record into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn save_function_record( +/// Generates the contents of the covfun record for this function, which +/// contains the function's coverage mapping data. The record is then stored +/// as a global variable in the `__llvm_covfun` section. +fn generate_covfun_record( cx: &CodegenCx<'_, '_>, - covfun_section_name: &str, mangled_function_name: &str, source_hash: u64, filenames_ref: u64, @@ -332,13 +389,28 @@ fn save_function_record( /*packed=*/ true, ); - coverageinfo::save_func_record_to_mod( - cx, - covfun_section_name, - func_name_hash, - func_record_val, - is_used, - ); + // Choose a variable name to hold this function's covfun data. + // Functions that are used have a suffix ("u") to distinguish them from + // unused copies of the same function (from different CGUs), so that if a + // linker sees both it won't discard the used copy's data. + let func_record_var_name = + CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) + .unwrap(); + debug!("function record var name: {:?}", func_record_var_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, cx.covfun_section_name()); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + if cx.target_spec().supports_comdat() { + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + } + cx.add_used_global(llglobal); } /// Each CGU will normally only emit coverage metadata for the functions that it actually generates. @@ -470,9 +542,5 @@ fn add_unused_function_coverage<'tcx>( // zero, because none of its counters/expressions are marked as seen. let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); - if let Some(coverage_context) = cx.coverage_context() { - coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); - } else { - bug!("Could not get the `coverage_context`"); - } + cx.coverage_cx().function_coverage_map.borrow_mut().insert(instance, function_coverage); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index d7d29eebf85..aaba0684c12 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,22 +1,20 @@ -use std::cell::RefCell; +use std::cell::{OnceCell, RefCell}; +use std::ffi::{CStr, CString}; use libc::c_uint; +use rustc_abi::Size; use rustc_codegen_ssa::traits::{ - BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, - MiscCodegenMethods, StaticCodegenMethods, + BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, }; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; -use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use rustc_middle::ty::layout::HasTyCtxt; -use rustc_target::abi::{Align, Size}; use tracing::{debug, instrument}; use crate::builder::Builder; -use crate::common::CodegenCx; -use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::coverageinfo::map_data::FunctionCoverageCollector; use crate::llvm; @@ -31,6 +29,8 @@ pub(crate) struct CrateCoverageContext<'ll, 'tcx> { RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, + + covfun_section_name: OnceCell<CString>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -39,6 +39,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { function_coverage_map: Default::default(), pgo_func_name_var_map: Default::default(), mcdc_condition_bitmap_map: Default::default(), + covfun_section_name: Default::default(), } } @@ -65,27 +66,38 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } -// These methods used to be part of trait `CoverageInfoMethods`, which no longer -// exists after most coverage code was moved out of SSA. impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn coverageinfo_finalize(&self) { mapgen::finalize(self) } + /// Returns the section name to use when embedding per-function coverage information + /// in the object file, according to the target's object file format. LLVM's coverage + /// tools use information from this section when producing coverage reports. + /// + /// Typical values are: + /// - `__llvm_covfun` on Linux + /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) + /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) + fn covfun_section_name(&self) -> &CStr { + self.coverage_cx().covfun_section_name.get_or_init(|| { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(self.llmod, s); + })) + .expect("covfun section name should not contain NUL") + }) + } + /// For LLVM codegen, returns a function-specific `Value` for a global /// string, to hold the function name passed to LLVM intrinsic /// `instrprof.increment()`. The `Value` is only created once per instance. /// Multiple invocations with the same instance return the same `Value`. fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { - if let Some(coverage_context) = self.coverage_context() { - debug!("getting pgo_func_name_var for instance={:?}", instance); - let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); - pgo_func_name_var_map - .entry(instance) - .or_insert_with(|| create_pgo_func_name_var(self, instance)) - } else { - bug!("Could not get the `coverage_context`"); - } + debug!("getting pgo_func_name_var for instance={:?}", instance); + let mut pgo_func_name_var_map = self.coverage_cx().pgo_func_name_var_map.borrow_mut(); + pgo_func_name_var_map + .entry(instance) + .or_insert_with(|| create_pgo_func_name_var(self, instance)) } } @@ -119,11 +131,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { cond_bitmaps.push(cond_bitmap); } - self.coverage_context() - .expect("always present when coverage is enabled") - .mcdc_condition_bitmap_map - .borrow_mut() - .insert(instance, cond_bitmaps); + self.coverage_cx().mcdc_condition_bitmap_map.borrow_mut().insert(instance, cond_bitmaps); } #[instrument(level = "debug", skip(self))] @@ -144,8 +152,12 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - let Some(coverage_context) = bx.coverage_context() else { return }; - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + // FIXME(#132395): Unwrapping `coverage_cx` here has led to ICEs in the + // wild, so keep this early-return until we understand why. + let mut coverage_map = match bx.coverage_cx { + Some(ref cx) => cx.function_coverage_map.borrow_mut(), + None => return, + }; let func_coverage = coverage_map .entry(instance) .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); @@ -187,7 +199,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } CoverageKind::CondBitmapUpdate { index, decision_depth } => { drop(coverage_map); - let cond_bitmap = coverage_context + let cond_bitmap = bx + .coverage_cx() .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); let cond_index = bx.const_i32(index as i32); @@ -195,7 +208,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { drop(coverage_map); - let cond_bitmap = coverage_context + let cond_bitmap = bx.coverage_cx() .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); assert!( @@ -207,6 +220,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let hash = bx.const_u64(function_coverage_info.function_source_hash); let bitmap_index = bx.const_u32(bitmap_idx); bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); + bx.mcdc_condbitmap_reset(cond_bitmap); } } } @@ -227,7 +241,7 @@ fn create_pgo_func_name_var<'ll, 'tcx>( unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar( llfn, - mangled_fn_name.as_ptr().cast(), + mangled_fn_name.as_c_char_ptr(), mangled_fn_name.len(), ) } @@ -239,7 +253,7 @@ pub(crate) fn write_filenames_section_to_buffer<'a>( ) { let (pointers, lengths) = filenames .into_iter() - .map(|s: &str| (s.as_ptr().cast(), s.len())) + .map(|s: &str| (s.as_c_char_ptr(), s.len())) .unzip::<_, _, Vec<_>, Vec<_>>(); unsafe { @@ -255,8 +269,11 @@ pub(crate) fn write_filenames_section_to_buffer<'a>( pub(crate) fn write_mapping_to_buffer( virtual_file_mapping: Vec<u32>, - expressions: Vec<CounterExpression>, - mapping_regions: Vec<CounterMappingRegion>, + expressions: Vec<ffi::CounterExpression>, + code_regions: &[ffi::CodeRegion], + branch_regions: &[ffi::BranchRegion], + mcdc_branch_regions: &[ffi::MCDCBranchRegion], + mcdc_decision_regions: &[ffi::MCDCDecisionRegion], buffer: &RustString, ) { unsafe { @@ -265,93 +282,23 @@ pub(crate) fn write_mapping_to_buffer( virtual_file_mapping.len() as c_uint, expressions.as_ptr(), expressions.len() as c_uint, - mapping_regions.as_ptr(), - mapping_regions.len() as c_uint, + code_regions.as_ptr(), + code_regions.len() as c_uint, + branch_regions.as_ptr(), + branch_regions.len() as c_uint, + mcdc_branch_regions.as_ptr(), + mcdc_branch_regions.len() as c_uint, + mcdc_decision_regions.as_ptr(), + mcdc_decision_regions.len() as c_uint, buffer, ); } } pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } + unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_c_char_ptr(), bytes.len()) } } pub(crate) fn mapping_version() -> u32 { unsafe { llvm::LLVMRustCoverageMappingVersion() } } - -pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - cov_data_val: &'ll llvm::Value, -) { - let covmap_var_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteMappingVarNameToString(s); - }) - .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); - debug!("covmap var name: {:?}", covmap_var_name); - - let covmap_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage section name failed UTF-8 conversion"); - debug!("covmap section name: {:?}", covmap_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); - llvm::set_initializer(llglobal, cov_data_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::set_section(llglobal, &covmap_section_name); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - llvm::set_alignment(llglobal, Align::EIGHT); - cx.add_used_global(llglobal); -} - -pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - covfun_section_name: &str, - func_name_hash: u64, - func_record_val: &'ll llvm::Value, - is_used: bool, -) { - // Assign a name to the function record. This is used to merge duplicates. - // - // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that - // are included-but-not-used. If (or when) Rust generates functions that are - // included-but-not-used, note that a dummy description for a function included-but-not-used - // in a Crate can be replaced by full description provided by a different Crate. The two kinds - // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by - // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. - let func_record_var_name = - format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); - debug!("function record var name: {:?}", func_record_var_name); - debug!("function record section name: {:?}", covfun_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); - llvm::set_initializer(llglobal, func_record_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); - llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, covfun_section_name); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - llvm::set_alignment(llglobal, Align::EIGHT); - llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); - cx.add_used_global(llglobal); -} - -/// Returns the section name string to pass through to the linker when embedding -/// per-function coverage information in the object file, according to the target -/// platform's object file format. -/// -/// LLVM's coverage tools read coverage mapping details from this section when -/// producing coverage reports. -/// -/// Typical values are: -/// - `__llvm_covfun` on Linux -/// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) -/// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) -pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String { - llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage function record section name failed UTF-8 conversion") -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index f93d3e40b20..aef8642f199 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -72,11 +72,11 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr()); + llvm::set_section(section_var, c".debug_gdb_scripts"); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(section_var, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than // the string it contains. Otherwise we get a warning from GDB. llvm::LLVMSetAlignment(section_var, 1); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 15d441a986d..151923a3bd2 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -5,9 +5,9 @@ use std::path::{Path, PathBuf}; use std::{iter, ptr}; use libc::{c_char, c_longlong, c_uint}; +use rustc_abi::{Align, Size}; use rustc_codegen_ssa::debuginfo::type_names::{VTableNameKind, cpp_like_debuginfo}; use rustc_codegen_ssa::traits::*; -use rustc_fs_util::path_to_c_string; use rustc_hir::def::{CtorKind, DefKind}; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; @@ -20,7 +20,6 @@ use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, hygiene}; use rustc_symbol_mangling::typeid_for_trait_ref; -use rustc_target::abi::{Align, Size}; use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; use tracing::{debug, instrument}; @@ -32,7 +31,7 @@ use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_na use super::utils::{ DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, }; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::build_type_with_children; use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind}; use crate::llvm::debuginfo::{ @@ -190,7 +189,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( data_layout.pointer_size.bits(), data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_ptr().cast(), + ptr_type_debuginfo_name.as_c_char_ptr(), ptr_type_debuginfo_name.len(), ) }; @@ -348,7 +347,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( size, align, 0, // Ignore DWARF address space. - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), ) }; @@ -518,7 +517,7 @@ fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll D let name = "<recur_type>"; llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), cx.tcx.data_layout.pointer_size.bits(), DW_ATE_unsigned, @@ -640,14 +639,14 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi unsafe { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), - file_name.as_ptr().cast(), + file_name.as_c_char_ptr(), file_name.len(), - directory.as_ptr().cast(), + directory.as_c_char_ptr(), directory.len(), hash_kind, - hash_value.as_ptr().cast(), + hash_value.as_c_char_ptr(), hash_value.len(), - source.map_or(ptr::null(), |x| x.as_ptr().cast()), + source.map_or(ptr::null(), |x| x.as_c_char_ptr()), source.map_or(0, |x| x.len()), ) } @@ -662,12 +661,12 @@ fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), - file_name.as_ptr().cast(), + file_name.as_c_char_ptr(), file_name.len(), - directory.as_ptr().cast(), + directory.as_c_char_ptr(), directory.len(), llvm::ChecksumKind::None, - hash_value.as_ptr().cast(), + hash_value.as_c_char_ptr(), hash_value.len(), ptr::null(), 0, @@ -788,7 +787,7 @@ fn build_basic_type_di_node<'ll, 'tcx>( let ty_di_node = unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), cx.size_of(t).bits(), encoding, @@ -810,7 +809,7 @@ fn build_basic_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_ptr().cast(), + typedef_name.as_c_char_ptr(), typedef_name.len(), unknown_file_metadata(cx), 0, @@ -861,7 +860,7 @@ fn build_param_type_di_node<'ll, 'tcx>( di_node: unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), Size::ZERO.bits(), DW_ATE_unsigned, @@ -948,9 +947,9 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( debug_context.builder, - name_in_debuginfo.as_ptr().cast(), + name_in_debuginfo.as_c_char_ptr(), name_in_debuginfo.len(), - work_dir.as_ptr().cast(), + work_dir.as_c_char_ptr(), work_dir.len(), llvm::ChecksumKind::None, ptr::null(), @@ -963,7 +962,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( debug_context.builder, DW_LANG_RUST, compile_unit_file, - producer.as_ptr().cast(), + producer.as_c_char_ptr(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, c"".as_ptr(), @@ -971,7 +970,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final // output(s). - split_name.as_ptr().cast(), + split_name.as_c_char_ptr(), split_name.len(), kind, 0, @@ -979,33 +978,8 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( debug_name_table_kind, ); - if tcx.sess.opts.unstable_opts.profile { - let default_gcda_path = &output_filenames.with_extension("gcda"); - let gcda_path = - tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); - - let gcov_cu_info = [ - path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), - path_to_mdstring(debug_context.llcontext, gcda_path), - unit_metadata, - ]; - let gcov_metadata = llvm::LLVMMDNodeInContext2( - debug_context.llcontext, - gcov_cu_info.as_ptr(), - gcov_cu_info.len(), - ); - let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); - - llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val); - } - return unit_metadata; }; - - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata { - let path_str = path_to_c_string(path); - unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) } - } } /// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. @@ -1022,7 +996,7 @@ fn build_field_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), owner, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -1306,7 +1280,7 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), actual_type_di_node, ) @@ -1382,9 +1356,9 @@ pub(crate) fn build_global_var_di_node<'ll>( llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), Some(var_scope), - var_name.as_ptr().cast(), + var_name.as_c_char_ptr(), var_name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, line_number, @@ -1602,9 +1576,9 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), NO_SCOPE_METADATA, - vtable_name.as_ptr().cast(), + vtable_name.as_c_char_ptr(), vtable_name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 966788cf32f..100b046cee2 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use libc::c_uint; +use rustc_abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstCodegenMethods; @@ -8,10 +9,9 @@ use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; -use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ @@ -359,7 +359,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticMemberType( DIB(cx), enum_type_di_node, - TAG_FIELD_NAME.as_ptr().cast(), + TAG_FIELD_NAME.as_c_char_ptr(), TAG_FIELD_NAME.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -537,7 +537,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticMemberType( DIB(cx), wrapper_struct_type_di_node, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -785,7 +785,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), enum_type_di_node, - field_name.as_ptr().cast(), + field_name.as_c_char_ptr(), field_name.len(), file_di_node, line_number, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index fe1634146ff..b3d4a6642a1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use rustc_abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_hir::def::CtorKind; @@ -9,11 +10,10 @@ use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; use rustc_span::Symbol; -use rustc_target::abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub}; use crate::debuginfo::metadata::{ UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, type_di_node, @@ -106,7 +106,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( let value = [value as u64, (value >> 64) as u64]; Some(llvm::LLVMRustDIBuilderCreateEnumerator( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), value.as_ptr(), size.bits() as libc::c_uint, @@ -119,7 +119,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), containing_scope, - type_name.as_ptr().cast(), + type_name.as_c_char_ptr(), type_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 5e7dbdd921a..d4006691d37 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -1,16 +1,16 @@ use std::borrow::Cow; use libc::c_uint; +use rustc_abi::{Size, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstCodegenMethods; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self}; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, SmallVec, UNKNOWN_LINE_NUMBER, file_metadata, @@ -244,7 +244,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateVariantPart( DIB(cx), enum_type_di_node, - variant_part_name.as_ptr().cast(), + variant_part_name.as_c_char_ptr(), variant_part_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -253,7 +253,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( DIFlags::FlagZero, tag_member_di_node, create_DIArray(DIB(cx), &[]), - variant_part_unique_type_id_str.as_ptr().cast(), + variant_part_unique_type_id_str.as_c_char_ptr(), variant_part_unique_type_id_str.len(), ) }, @@ -327,7 +327,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( Some(llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), containing_scope, - tag_name.as_ptr().cast(), + tag_name.as_c_char_ptr(), tag_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -399,7 +399,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateVariantMemberType( DIB(cx), variant_part_di_node, - variant_member_info.variant_name.as_ptr().cast(), + variant_member_info.variant_name.as_c_char_ptr(), variant_member_info.variant_name.len(), file_di_node, line_number, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 714e3c0b145..5120b63d173 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -1,15 +1,15 @@ use std::cell::RefCell; +use rustc_abi::{Align, Size, VariantIdx}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_macros::HashStable; use rustc_middle::bug; use rustc_middle::ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}; -use rustc_target::abi::{Align, Size, VariantIdx}; use super::{SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; use crate::llvm::{self}; @@ -191,7 +191,7 @@ pub(super) fn stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStructType( DIB(cx), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -202,7 +202,7 @@ pub(super) fn stub<'ll, 'tcx>( empty_array, 0, vtable_holder, - unique_type_id_str.as_ptr().cast(), + unique_type_id_str.as_c_char_ptr(), unique_type_id_str.len(), ) } @@ -211,7 +211,7 @@ pub(super) fn stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateUnionType( DIB(cx), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -220,7 +220,7 @@ pub(super) fn stub<'ll, 'tcx>( flags, Some(empty_array), 0, - unique_type_id_str.as_ptr().cast(), + unique_type_id_str.as_c_char_ptr(), unique_type_id_str.len(), ) }, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 1a8153a54e8..9e1e5127e80 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -5,6 +5,7 @@ use std::ops::Range; use std::{iter, ptr}; use libc::c_uint; +use rustc_abi::Size; use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; @@ -22,7 +23,6 @@ use rustc_span::symbol::Symbol; use rustc_span::{ BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, }; -use rustc_target::abi::Size; use smallvec::SmallVec; use tracing::debug; @@ -31,7 +31,7 @@ use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; use crate::abi::FnAbi; use crate::builder::Builder; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::{ DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, @@ -55,7 +55,6 @@ const DW_TAG_arg_variable: c_uint = 0x101; /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { - llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>, @@ -78,9 +77,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { debug!("CodegenUnitDebugContext::new"); let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; // DIBuilder inherits context from the module, so we'd better use the same one - let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; CodegenUnitDebugContext { - llcontext, llmod, builder, created_files: Default::default(), @@ -91,45 +88,39 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { } pub(crate) fn finalize(&self, sess: &Session) { - unsafe { - llvm::LLVMRustDIBuilderFinalize(self.builder); - - if !sess.target.is_like_msvc { - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - let dwarf_version = sess - .opts - .unstable_opts - .dwarf_version - .unwrap_or(sess.target.default_dwarf_version); - llvm::LLVMRustAddModuleFlagU32( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"Dwarf Version".as_ptr(), - dwarf_version, - ); - } else { - // Indicate that we want CodeView debug information on MSVC - llvm::LLVMRustAddModuleFlagU32( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"CodeView".as_ptr(), - 1, - ) - } - - // Prevent bitcode readers from deleting the debug info. - llvm::LLVMRustAddModuleFlagU32( + unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder) }; + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = + sess.opts.unstable_opts.dwarf_version.unwrap_or(sess.target.default_dwarf_version); + llvm::add_module_flag_u32( self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"Debug Info Version".as_ptr(), - llvm::LLVMRustDebugMetadataVersion(), + llvm::ModuleFlagMergeBehavior::Warning, + "Dwarf Version", + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "CodeView", + 1, ); } + + // Prevent bitcode readers from deleting the debug info. + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "Debug Info Version", + unsafe { llvm::LLVMRustDebugMetadataVersion() }, + ); } } @@ -350,7 +341,6 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { type_names::push_generic_params( tcx, tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), args), - enclosing_fn_def_id, &mut name, ); @@ -365,7 +355,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let mut flags = DIFlags::FlagPrototyped; - if fn_abi.ret.layout.abi.is_uninhabited() { + if fn_abi.ret.layout.is_uninhabited() { flags |= DIFlags::FlagNoReturn; } @@ -390,9 +380,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateMethod( DIB(self), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, loc.line, @@ -407,9 +397,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, loc.line, @@ -495,7 +485,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), actual_type_metadata, )) @@ -636,7 +626,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { DIB(self), dwarf_tag, scope_metadata, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), file_metadata, loc.line, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index 3578755aae0..33d9bc23890 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -5,7 +5,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::ty::{self, Instance}; use super::utils::{DIB, debug_context}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::DIScope; @@ -36,7 +36,7 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l llvm::LLVMRustDIBuilderCreateNameSpace( DIB(cx), parent_scope, - namespace_name_string.as_ptr().cast(), + namespace_name_string.as_c_char_ptr(), namespace_name_string.len(), false, // ExportSymbols (only relevant for C++ anonymous namespaces) ) diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 33258cb46fa..d338c848754 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -20,6 +20,7 @@ use smallvec::SmallVec; use tracing::debug; use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::common::AsCCharPtr; use crate::context::CodegenCx; use crate::llvm::AttributePlace::Function; use crate::llvm::Visibility; @@ -41,7 +42,7 @@ fn declare_raw_fn<'ll>( ) -> &'ll Value { debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); let llfn = unsafe { - llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_ptr().cast(), name.len(), ty) + llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_c_char_ptr(), name.len(), ty) }; llvm::SetFunctionCallConv(llfn, callconv); @@ -68,7 +69,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// return its Value instead. pub(crate) fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { debug!("declare_global(name={:?})", name); - unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } + unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_c_char_ptr(), name.len(), ty) } } /// Declare a C ABI function. @@ -209,7 +210,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// Gets declared value by name. pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); - unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } + unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_c_char_ptr(), name.len()) } } /// Gets defined or externally defined (AvailableExternally linkage) value by diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 0d436e1891e..3cdb5b971d9 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -31,6 +31,15 @@ pub(crate) struct UnstableCTargetFeature<'a> { pub feature: &'a str, } +#[derive(Diagnostic)] +#[diag(codegen_llvm_forbidden_ctarget_feature)] +#[note] +#[note(codegen_llvm_forbidden_ctarget_feature_issue)] +pub(crate) struct ForbiddenCTargetFeature<'a> { + pub feature: &'a str, + pub reason: &'a str, +} + #[derive(Subdiagnostic)] pub(crate) enum PossibleFeature<'a> { #[help(codegen_llvm_possible_feature)] diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index bfe623e7fc3..e9c687d75e3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,6 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; +use rustc_abi::{self as abi, Align, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; @@ -13,11 +14,10 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; -use rustc_target::abi::{self, Align, Float, HasDataLayout, Primitive, Size}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use tracing::debug; -use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; +use crate::abi::{ExternAbi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Metadata}; @@ -258,8 +258,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) } sym::va_arg => { - match fn_abi.ret.layout.abi { - abi::Abi::Scalar(scalar) => { + match fn_abi.ret.layout.backend_repr { + abi::BackendRepr::Scalar(scalar) => { match scalar.primitive() { Primitive::Int(..) => { if self.cx().size_of(ret_ty).bytes() < 4 { @@ -436,13 +436,13 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::raw_eq => { - use abi::Abi::*; + use abi::BackendRepr::*; let tp_ty = fn_args.type_at(0); let layout = self.layout_of(tp_ty).layout; - let use_integer_compare = match layout.abi() { + let use_integer_compare = match layout.backend_repr() { Scalar(_) | ScalarPair(_, _) => true, Uninhabited | Vector { .. } => false, - Aggregate { .. } => { + Memory { .. } => { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), // so we re-use that same threshold here. @@ -549,7 +549,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } let llret_ty = if ret_ty.is_simd() - && let abi::Abi::Aggregate { .. } = self.layout_of(ret_ty).layout.abi + && let abi::BackendRepr::Memory { .. } = + self.layout_of(ret_ty).layout.backend_repr { let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx()); let elem_ll_ty = match elem_ty.kind() { @@ -785,11 +786,12 @@ fn codegen_msvc_try<'ll>( let type_info = bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false); let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); - unsafe { - llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + + llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + if bx.cx.tcx.sess.target.supports_comdat() { llvm::SetUniqueComdat(bx.llmod, tydesc); - llvm::LLVMSetInitializer(tydesc, type_info); } + unsafe { llvm::LLVMSetInitializer(tydesc, type_info) }; // The flag value of 8 indicates that we are catching the exception by // reference instead of by value. We can't use catch by value because @@ -1062,7 +1064,7 @@ fn gen_fn<'ll, 'tcx>( cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. - unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; + llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage); let llbb = Builder::append_block(cx, llfn, "entry-block"); let bx = Builder::build(cx, llbb); codegen(bx); @@ -1092,7 +1094,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.types.unit, false, hir::Safety::Unsafe, - Abi::Rust, + ExternAbi::Rust, )), ); // `unsafe fn(*mut i8, *mut i8) -> ()` @@ -1103,7 +1105,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.types.unit, false, hir::Safety::Unsafe, - Abi::Rust, + ExternAbi::Rust, )), ); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` @@ -1112,7 +1114,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.types.i32, false, hir::Safety::Unsafe, - Abi::Rust, + ExternAbi::Rust, )); let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); cx.rust_try_fn.set(Some(rust_try)); @@ -1177,8 +1179,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => + && len + .try_to_target_usize(bx.tcx) + .expect("expected monomorphic const in codegen") + == expected_bytes => { let place = PlaceRef::alloca(bx, args[0].layout); args[0].val.store(bx, place); @@ -1243,12 +1247,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle_generic { - let idx = fn_args[2] - .expect_const() - .eval(tcx, ty::ParamEnv::reveal_all(), span) - .unwrap() - .1 - .unwrap_branch(); + let idx = fn_args[2].expect_const().try_to_valtree().unwrap().0.unwrap_branch(); let n = idx.len() as u64; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); @@ -1467,8 +1466,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) - == Some(expected_bytes) => + && len + .try_to_target_usize(bx.tcx) + .expect("expected monomorphic const in codegen") + == expected_bytes => { // Zero-extend iN to the array length: let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 661debbb9f1..d84ae8d8836 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,9 +1,12 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] +use std::fmt::Debug; use std::marker::PhantomData; +use std::ptr; use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; +use rustc_macros::TryFromU32; use rustc_target::spec::SymbolVisibility; use super::RustString; @@ -19,6 +22,30 @@ pub type Bool = c_uint; pub const True: Bool = 1 as Bool; pub const False: Bool = 0 as Bool; +/// Wrapper for a raw enum value returned from LLVM's C APIs. +/// +/// For C enums returned by LLVM, it's risky to use a Rust enum as the return +/// type, because it would be UB if a later version of LLVM adds a new enum +/// value and returns it. Instead, return this raw wrapper, then convert to the +/// Rust-side enum explicitly. +#[repr(transparent)] +pub struct RawEnum<T> { + value: u32, + /// We don't own or consume a `T`, but we can produce one. + _rust_side_type: PhantomData<fn() -> T>, +} + +impl<T: TryFrom<u32>> RawEnum<T> { + #[track_caller] + pub(crate) fn to_rust(self) -> T + where + T::Error: Debug, + { + // If this fails, the Rust-side enum is out of sync with LLVM's enum. + T::try_from(self.value).expect("enum value returned by LLVM should be known") + } +} + #[derive(Copy, Clone, PartialEq)] #[repr(C)] #[allow(dead_code)] // Variants constructed by C++. @@ -59,7 +86,7 @@ pub enum LLVMMachineType { ARM = 0x01c0, } -/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. +/// Must match the layout of `LLVMRustModuleFlagMergeBehavior`. /// /// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are /// resolved according to the merge behaviors specified here. Flags differing only in merge @@ -67,9 +94,13 @@ pub enum LLVMMachineType { /// /// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, /// 'Error' and 'Warning' cannot be mixed for a given flag. +/// +/// There is a stable LLVM-C version of this enum (`LLVMModuleFlagBehavior`), +/// but as of LLVM 19 it does not support all of the enum values in the unstable +/// C++ API. #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum LLVMModFlagBehavior { +pub enum ModuleFlagMergeBehavior { Error = 1, Warning = 2, Require = 3, @@ -108,26 +139,36 @@ pub enum CallConv { AvrInterrupt = 85, } -/// LLVMRustLinkage -#[derive(Copy, Clone, PartialEq)] +/// Must match the layout of `LLVMLinkage`. +#[derive(Copy, Clone, PartialEq, TryFromU32)] #[repr(C)] pub enum Linkage { ExternalLinkage = 0, AvailableExternallyLinkage = 1, LinkOnceAnyLinkage = 2, LinkOnceODRLinkage = 3, - WeakAnyLinkage = 4, - WeakODRLinkage = 5, - AppendingLinkage = 6, - InternalLinkage = 7, - PrivateLinkage = 8, - ExternalWeakLinkage = 9, - CommonLinkage = 10, + #[deprecated = "marked obsolete by LLVM"] + LinkOnceODRAutoHideLinkage = 4, + WeakAnyLinkage = 5, + WeakODRLinkage = 6, + AppendingLinkage = 7, + InternalLinkage = 8, + PrivateLinkage = 9, + #[deprecated = "marked obsolete by LLVM"] + DLLImportLinkage = 10, + #[deprecated = "marked obsolete by LLVM"] + DLLExportLinkage = 11, + ExternalWeakLinkage = 12, + #[deprecated = "marked obsolete by LLVM"] + GhostLinkage = 13, + CommonLinkage = 14, + LinkerPrivateLinkage = 15, + LinkerPrivateWeakLinkage = 16, } -// LLVMRustVisibility +/// Must match the layout of `LLVMVisibility`. #[repr(C)] -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, TryFromU32)] pub enum Visibility { Default = 0, Hidden = 1, @@ -646,6 +687,7 @@ unsafe extern "C" { pub type Attribute; pub type Metadata; pub type BasicBlock; + pub type Comdat; } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); @@ -667,8 +709,9 @@ unsafe extern "C" { } #[repr(C)] pub struct RustArchiveMember<'a>(InvariantOpaque<'a>); +/// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] -pub struct OperandBundleDef<'a>(InvariantOpaque<'a>); +pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct Linker<'a>(InvariantOpaque<'a>); @@ -944,7 +987,11 @@ unsafe extern "C" { // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub fn LLVMGetLinkage(Global: &Value) -> RawEnum<Linkage>; + pub fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); pub fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub fn LLVMGetVisibility(Global: &Value) -> RawEnum<Visibility>; + pub fn LLVMSetVisibility(Global: &Value, Viz: Visibility); pub fn LLVMGetAlignment(Global: &Value) -> c_uint; pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); @@ -1490,6 +1537,53 @@ unsafe extern "C" { pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + + pub fn LLVMGetOrInsertComdat(M: &Module, Name: *const c_char) -> &Comdat; + pub fn LLVMSetComdat(V: &Value, C: &Comdat); + + pub(crate) fn LLVMCreateOperandBundle( + Tag: *const c_char, + TagLen: size_t, + Args: *const &'_ Value, + NumArgs: c_uint, + ) -> *mut OperandBundle<'_>; + pub(crate) fn LLVMDisposeOperandBundle(Bundle: ptr::NonNull<OperandBundle<'_>>); + + pub(crate) fn LLVMBuildCallWithOperandBundles<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Bundles: *const &OperandBundle<'a>, + NumBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildInvokeWithOperandBundles<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Then: &'a BasicBlock, + Catch: &'a BasicBlock, + Bundles: *const &OperandBundle<'a>, + NumBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildCallBr<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + DefaultDest: &'a BasicBlock, + IndirectDests: *const &'a BasicBlock, + NumIndirectDests: c_uint, + Args: *const &'a Value, + NumArgs: c_uint, + Bundles: *const &OperandBundle<'a>, + NumBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; } #[link(name = "llvm-wrapper", kind = "static")] @@ -1517,10 +1611,6 @@ unsafe extern "C" { ) -> bool; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage; - pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); - pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; - pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); // Operations on global variables @@ -1579,51 +1669,11 @@ unsafe extern "C" { AttrsLen: size_t, ); - pub fn LLVMRustBuildInvoke<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - Args: *const &'a Value, - NumArgs: c_uint, - Then: &'a BasicBlock, - Catch: &'a BasicBlock, - OpBundles: *const &OperandBundleDef<'a>, - NumOpBundles: c_uint, - Name: *const c_char, - ) -> &'a Value; - - pub fn LLVMRustBuildCallBr<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - DefaultDest: &'a BasicBlock, - IndirectDests: *const &'a BasicBlock, - NumIndirectDests: c_uint, - Args: *const &'a Value, - NumArgs: c_uint, - OpBundles: *const &OperandBundleDef<'a>, - NumOpBundles: c_uint, - Name: *const c_char, - ) -> &'a Value; - pub fn LLVMRustSetFastMath(Instr: &Value); pub fn LLVMRustSetAlgebraicMath(Instr: &Value); pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; - - pub fn LLVMRustBuildCall<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Fn: &'a Value, - Args: *const &'a Value, - NumArgs: c_uint, - OpBundles: *const &OperandBundleDef<'a>, - NumOpBundles: c_uint, - ) -> &'a Value; pub fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, Dst: &'a Value, @@ -1740,7 +1790,7 @@ unsafe extern "C" { ) -> bool; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + pub(crate) fn LLVMRustCoverageWriteFilenamesSectionToBuffer( Filenames: *const *const c_char, FilenamesLen: size_t, Lengths: *const size_t, @@ -1749,33 +1799,39 @@ unsafe extern "C" { ); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingToBuffer( + pub(crate) fn LLVMRustCoverageWriteMappingToBuffer( VirtualFileMappingIDs: *const c_uint, NumVirtualFileMappingIDs: c_uint, Expressions: *const crate::coverageinfo::ffi::CounterExpression, NumExpressions: c_uint, - MappingRegions: *const crate::coverageinfo::ffi::CounterMappingRegion, - NumMappingRegions: c_uint, + CodeRegions: *const crate::coverageinfo::ffi::CodeRegion, + NumCodeRegions: c_uint, + BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, + NumBranchRegions: c_uint, + MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, + NumMCDCBranchRegions: c_uint, + MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, + NumMCDCDecisionRegions: c_uint, BufferOut: &RustString, ); - pub fn LLVMRustCoverageCreatePGOFuncNameVar( + pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, FuncName: *const c_char, FuncNameLen: size_t, ) -> &Value; - pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + pub(crate) fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); - pub fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustDebugMetadataVersion() -> u32; pub fn LLVMRustVersionMajor() -> u32; pub fn LLVMRustVersionMinor() -> u32; @@ -1787,21 +1843,21 @@ unsafe extern "C" { /// "compatible" means depends on the merge behaviors involved. pub fn LLVMRustAddModuleFlagU32( M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: u32, + MergeBehavior: ModuleFlagMergeBehavior, + Name: *const c_char, + NameLen: size_t, + Value: u32, ); pub fn LLVMRustAddModuleFlagString( M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: *const c_char, - value_len: size_t, + MergeBehavior: ModuleFlagMergeBehavior, + Name: *const c_char, + NameLen: size_t, + Value: *const c_char, + ValueLen: size_t, ); - pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); @@ -2134,12 +2190,8 @@ unsafe extern "C" { pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs( - T: &TargetMachine, - cpu: *const c_char, - print: unsafe extern "C" fn(out: *mut c_void, string: *const c_char, len: usize), - out: *mut c_void, - ); + #[allow(improper_ctypes)] + pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2148,7 +2200,7 @@ unsafe extern "C" { Desc: &mut *const c_char, ); - pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; + pub fn LLVMRustGetHostCPUName(LenOut: &mut size_t) -> *const u8; // This function makes copies of pointed to data, so the data's lifetime may end after this // function returns. @@ -2213,7 +2265,6 @@ unsafe extern "C" { PGOUsePath: *const c_char, InstrumentCoverage: bool, InstrProfileOutput: *const c_char, - InstrumentGCOV: bool, PGOSampleUsePath: *const c_char, DebugInfoForProfiling: bool, llvm_selfprofiler: *mut c_void, @@ -2311,16 +2362,8 @@ unsafe extern "C" { pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustBuildOperandBundleDef( - Name: *const c_char, - Inputs: *const &'_ Value, - NumInputs: c_uint, - ) -> &mut OperandBundleDef<'_>; - pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); - pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); - pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); @@ -2344,9 +2387,9 @@ unsafe extern "C" { pub fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; pub fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, - NumModules: c_uint, + NumModules: size_t, PreservedSymbols: *const *const c_char, - PreservedSymbolsLen: c_uint, + PreservedSymbolsLen: size_t, ) -> Option<&'static mut ThinLTOData>; pub fn LLVMRustPrepareThinLTORename( Data: &ThinLTOData, @@ -2371,6 +2414,7 @@ unsafe extern "C" { data: *const u8, len: usize, name: *const u8, + name_len: usize, out_len: &mut usize, ) -> *const u8; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index d0db350a149..3b0bf47366e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -2,13 +2,14 @@ use std::cell::RefCell; use std::ffi::{CStr, CString}; +use std::ops::Deref; +use std::ptr; use std::str::FromStr; use std::string::FromUtf8Error; use libc::c_uint; -use rustc_data_structures::small_c_str::SmallCStr; +use rustc_abi::{Align, Size, WrappingRange}; use rustc_llvm::RustString; -use rustc_target::abi::{Align, Size, WrappingRange}; pub use self::AtomicRmwBinOp::*; pub use self::CallConv::*; @@ -17,13 +18,13 @@ pub use self::IntPredicate::*; pub use self::Linkage::*; pub use self::MetadataType::*; pub use self::RealPredicate::*; +pub use self::ffi::*; +use crate::common::AsCCharPtr; pub mod archive_ro; pub mod diagnostic; mod ffi; -pub use self::ffi::*; - impl LLVMRustResult { pub fn into_result(self) -> Result<(), ()> { match self { @@ -53,9 +54,9 @@ pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) - unsafe { LLVMCreateStringAttribute( llcx, - attr.as_ptr().cast(), + attr.as_c_char_ptr(), attr.len().try_into().unwrap(), - value.as_ptr().cast(), + value.as_c_char_ptr(), value.len().try_into().unwrap(), ) } @@ -65,7 +66,7 @@ pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, - attr.as_ptr().cast(), + attr.as_c_char_ptr(), attr.len().try_into().unwrap(), std::ptr::null(), 0, @@ -178,10 +179,10 @@ pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { // function. // For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 pub fn SetUniqueComdat(llmod: &Module, val: &Value) { - unsafe { - let name = get_value_name(val); - LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len()); - } + let name_buf = get_value_name(val).to_vec(); + let name = + CString::from_vec_with_nul(name_buf).or_else(|buf| CString::new(buf.into_bytes())).unwrap(); + set_comdat(llmod, val, &name); } pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { @@ -210,15 +211,13 @@ impl MemoryEffects { } } -pub fn set_section(llglobal: &Value, section_name: &str) { - let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); +pub fn set_section(llglobal: &Value, section_name: &CStr) { unsafe { - LLVMSetSection(llglobal, section_name_cstr.as_ptr()); + LLVMSetSection(llglobal, section_name.as_ptr()); } } -pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { - let name_cstr = CString::new(name).expect("unexpected CString error"); +pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name_cstr: &CStr) -> &'a Value { unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } } @@ -234,15 +233,23 @@ pub fn set_global_constant(llglobal: &Value, is_constant: bool) { } } +pub fn get_linkage(llglobal: &Value) -> Linkage { + unsafe { LLVMGetLinkage(llglobal) }.to_rust() +} + pub fn set_linkage(llglobal: &Value, linkage: Linkage) { unsafe { - LLVMRustSetLinkage(llglobal, linkage); + LLVMSetLinkage(llglobal, linkage); } } +pub fn get_visibility(llglobal: &Value) -> Visibility { + unsafe { LLVMGetVisibility(llglobal) }.to_rust() +} + pub fn set_visibility(llglobal: &Value, visibility: Visibility) { unsafe { - LLVMRustSetVisibility(llglobal, visibility); + LLVMSetVisibility(llglobal, visibility); } } @@ -252,9 +259,14 @@ pub fn set_alignment(llglobal: &Value, align: Align) { } } -pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { +/// Get the `name`d comdat from `llmod` and assign it to `llglobal`. +/// +/// Inserts the comdat into `llmod` if it does not exist. +/// It is an error to call this if the target does not support comdat. +pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &CStr) { unsafe { - LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); + let comdat = LLVMGetOrInsertComdat(llmod, name.as_ptr()); + LLVMSetComdat(llglobal, comdat); } } @@ -283,7 +295,7 @@ pub fn get_value_name(value: &Value) -> &[u8] { /// Safe wrapper for `LLVMSetValueName2` from a byte slice pub fn set_value_name(value: &Value, name: &[u8]) { unsafe { - let data = name.as_ptr().cast(); + let data = name.as_c_char_ptr(); LLVMSetValueName2(value, data, name.len()); } } @@ -320,24 +332,68 @@ pub fn last_error() -> Option<String> { } } -pub struct OperandBundleDef<'a> { - pub raw: &'a mut ffi::OperandBundleDef<'a>, +/// Owns an [`OperandBundle`], and will dispose of it when dropped. +pub(crate) struct OperandBundleOwned<'a> { + raw: ptr::NonNull<OperandBundle<'a>>, } -impl<'a> OperandBundleDef<'a> { - pub fn new(name: &str, vals: &[&'a Value]) -> Self { - let name = SmallCStr::new(name); - let def = unsafe { - LLVMRustBuildOperandBundleDef(name.as_ptr(), vals.as_ptr(), vals.len() as c_uint) +impl<'a> OperandBundleOwned<'a> { + pub(crate) fn new(name: &str, vals: &[&'a Value]) -> Self { + let raw = unsafe { + LLVMCreateOperandBundle( + name.as_c_char_ptr(), + name.len(), + vals.as_ptr(), + vals.len() as c_uint, + ) }; - OperandBundleDef { raw: def } + OperandBundleOwned { raw: ptr::NonNull::new(raw).unwrap() } } } -impl Drop for OperandBundleDef<'_> { +impl Drop for OperandBundleOwned<'_> { fn drop(&mut self) { unsafe { - LLVMRustFreeOperandBundleDef(&mut *(self.raw as *mut _)); + LLVMDisposeOperandBundle(self.raw); } } } + +impl<'a> Deref for OperandBundleOwned<'a> { + type Target = OperandBundle<'a>; + + fn deref(&self) -> &Self::Target { + // SAFETY: The returned reference is opaque and can only used for FFI. + // It is valid for as long as `&self` is. + unsafe { self.raw.as_ref() } + } +} + +pub(crate) fn add_module_flag_u32( + module: &Module, + merge_behavior: ModuleFlagMergeBehavior, + key: &str, + value: u32, +) { + unsafe { + LLVMRustAddModuleFlagU32(module, merge_behavior, key.as_c_char_ptr(), key.len(), value); + } +} + +pub(crate) fn add_module_flag_str( + module: &Module, + merge_behavior: ModuleFlagMergeBehavior, + key: &str, + value: &str, +) { + unsafe { + LLVMRustAddModuleFlagString( + module, + merge_behavior, + key.as_c_char_ptr(), + key.len(), + value.as_c_char_ptr(), + value.len(), + ); + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 57936215ff1..6f2d86cc601 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,4 +1,5 @@ -use std::ffi::{CStr, CString, c_char, c_void}; +use std::collections::VecDeque; +use std::ffi::{CStr, CString}; use std::fmt::Write; use std::path::Path; use std::sync::Once; @@ -16,12 +17,12 @@ use rustc_session::Session; use rustc_session::config::{PrintKind, PrintRequest}; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy, SmallDataThresholdSupport}; -use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES}; +use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES, Stability}; use crate::back::write::create_informational_target_machine; use crate::errors::{ - FixedX18InvalidArch, InvalidTargetFeaturePrefix, PossibleFeature, UnknownCTargetFeature, - UnknownCTargetFeaturePrefix, UnstableCTargetFeature, + FixedX18InvalidArch, ForbiddenCTargetFeature, InvalidTargetFeaturePrefix, PossibleFeature, + UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; @@ -248,6 +249,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), + ("aarch64", "pauth-lr") if get_version().0 < 19 => None, // Before LLVM 20 those two features were packaged together as b16b16 ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), @@ -275,23 +277,36 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("x86", s) if s.starts_with("avx512") => { Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))) } + // Support for `wide-arithmetic` will first land in LLVM 20 as part of + // llvm/llvm-project#111598 + ("wasm32" | "wasm64", "wide-arithmetic") if get_version() < (20, 0, 0) => None, (_, s) => Some(LLVMFeature::new(s)), } } -/// Used to generate cfg variables and apply features -/// Must express features in the way Rust understands them +/// Used to generate cfg variables and apply features. +/// Must express features in the way Rust understands them. +/// +/// We do not have to worry about RUSTC_SPECIFIC_FEATURES here, those are handled outside codegen. pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { - let mut features = vec![]; - - // Add base features for the target + let mut features: FxHashSet<Symbol> = Default::default(); + + // Add base features for the target. + // We do *not* add the -Ctarget-features there, and instead duplicate the logic for that below. + // The reason is that if LLVM considers a feature implied but we do not, we don't want that to + // show up in `cfg`. That way, `cfg` is entirely under our control -- except for the handling of + // the target CPU, that is still expanded to target features (with all their implied features) by + // LLVM. let target_machine = create_informational_target_machine(sess, true); + // Compute which of the known target features are enabled in the 'base' target machine. + // We only consider "supported" features; "forbidden" features are not reflected in `cfg` as of now. features.extend( sess.target - .supported_target_features() + .rust_target_features() .iter() + .filter(|(_, gate, _)| gate.is_supported()) .filter(|(feature, _, _)| { - // skip checking special features, as LLVM may not understands them + // skip checking special features, as LLVM may not understand them if RUSTC_SPECIAL_FEATURES.contains(feature) { return true; } @@ -322,7 +337,12 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { if enabled { features.extend(sess.target.implied_target_features(std::iter::once(feature))); } else { + // We don't care about the order in `features` since the only thing we use it for is the + // `features.contains` below. + #[allow(rustc::potential_query_instability)] features.retain(|f| { + // Keep a feature if it does not imply `feature`. Or, equivalently, + // remove the reverse-dependencies of `feature`. !sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) }); } @@ -330,8 +350,9 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { // Filter enabled features based on feature gates sess.target - .supported_target_features() + .rust_target_features() .iter() + .filter(|(_, gate, _)| gate.is_supported()) .filter_map(|&(feature, gate, _)| { if sess.is_nightly_build() || allow_unstable || gate.is_stable() { Some(feature) @@ -386,14 +407,76 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { ret } -fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMachine) { +pub(crate) fn print(req: &PrintRequest, out: &mut String, sess: &Session) { + require_inited(); + let tm = create_informational_target_machine(sess, false); + match req.kind { + PrintKind::TargetCPUs => print_target_cpus(sess, &tm, out), + PrintKind::TargetFeatures => print_target_features(sess, &tm, out), + _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), + } +} + +fn print_target_cpus(sess: &Session, tm: &llvm::TargetMachine, out: &mut String) { + let cpu_names = llvm::build_string(|s| unsafe { + llvm::LLVMRustPrintTargetCPUs(&tm, s); + }) + .unwrap(); + + struct Cpu<'a> { + cpu_name: &'a str, + remark: String, + } + // Compare CPU against current target to label the default. + let target_cpu = handle_native(&sess.target.cpu); + let make_remark = |cpu_name| { + if cpu_name == target_cpu { + // FIXME(#132514): This prints the LLVM target string, which can be + // different from the Rust target string. Is that intended? + let target = &sess.target.llvm_target; + format!( + " - This is the default target CPU for the current build target (currently {target})." + ) + } else { + "".to_owned() + } + }; + let mut cpus = cpu_names + .lines() + .map(|cpu_name| Cpu { cpu_name, remark: make_remark(cpu_name) }) + .collect::<VecDeque<_>>(); + + // Only print the "native" entry when host and target are the same arch, + // since otherwise it could be wrong or misleading. + if sess.host.arch == sess.target.arch { + let host = get_host_cpu_name(); + cpus.push_front(Cpu { + cpu_name: "native", + remark: format!(" - Select the CPU of the current host (currently {host})."), + }); + } + + let max_name_width = cpus.iter().map(|cpu| cpu.cpu_name.len()).max().unwrap_or(0); + writeln!(out, "Available CPUs for this target:").unwrap(); + for Cpu { cpu_name, remark } in cpus { + // Only pad the CPU name if there's a remark to print after it. + let width = if remark.is_empty() { 0 } else { max_name_width }; + writeln!(out, " {cpu_name:<width$}{remark}").unwrap(); + } +} + +fn print_target_features(sess: &Session, tm: &llvm::TargetMachine, out: &mut String) { let mut llvm_target_features = llvm_target_features(tm); let mut known_llvm_target_features = FxHashSet::<&'static str>::default(); let mut rustc_target_features = sess .target - .supported_target_features() + .rust_target_features() .iter() - .filter_map(|(feature, _gate, _implied)| { + .filter_map(|(feature, gate, _implied)| { + if !gate.is_supported() { + // Only list (experimentally) supported features. + return None; + } // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these // strings. let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name; @@ -446,52 +529,31 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach writeln!(out, "and may be renamed or removed in a future version of LLVM or rustc.\n").unwrap(); } -pub(crate) fn print(req: &PrintRequest, mut out: &mut String, sess: &Session) { - require_inited(); - let tm = create_informational_target_machine(sess, false); - match req.kind { - PrintKind::TargetCPUs => { - // SAFETY generate a C compatible string from a byte slice to pass - // the target CPU name into LLVM, the lifetime of the reference is - // at least as long as the C function - let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) - .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); - unsafe extern "C" fn callback(out: *mut c_void, string: *const c_char, len: usize) { - let out = unsafe { &mut *(out as *mut &mut String) }; - let bytes = unsafe { slice::from_raw_parts(string as *const u8, len) }; - write!(out, "{}", String::from_utf8_lossy(bytes)).unwrap(); - } - unsafe { - llvm::LLVMRustPrintTargetCPUs( - &tm, - cpu_cstring.as_ptr(), - callback, - (&raw mut out) as *mut c_void, - ); - } - } - PrintKind::TargetFeatures => print_target_features(out, sess, &tm), - _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), - } +/// Returns the host CPU name, according to LLVM. +fn get_host_cpu_name() -> &'static str { + let mut len = 0; + // SAFETY: The underlying C++ global function returns a `StringRef` that + // isn't tied to any particular backing buffer, so it must be 'static. + let slice: &'static [u8] = unsafe { + let ptr = llvm::LLVMRustGetHostCPUName(&mut len); + assert!(!ptr.is_null()); + slice::from_raw_parts(ptr, len) + }; + str::from_utf8(slice).expect("host CPU name should be UTF-8") } -fn handle_native(name: &str) -> &str { - if name != "native" { - return name; - } - - unsafe { - let mut len = 0; - let ptr = llvm::LLVMRustGetHostCPUName(&mut len); - str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() +/// If the given string is `"native"`, returns the host CPU name according to +/// LLVM. Otherwise, the string is returned as-is. +fn handle_native(cpu_name: &str) -> &str { + match cpu_name { + "native" => get_host_cpu_name(), + _ => cpu_name, } } pub(crate) fn target_cpu(sess: &Session) -> &str { - match sess.opts.cg.target_cpu { - Some(ref name) => handle_native(name), - None => handle_native(sess.target.cpu.as_ref()), - } + let cpu_name = sess.opts.cg.target_cpu.as_deref().unwrap_or_else(|| &sess.target.cpu); + handle_native(cpu_name) } /// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, @@ -566,7 +628,7 @@ pub(crate) fn global_llvm_features( // -Ctarget-features if !only_base_features { - let supported_features = sess.target.supported_target_features(); + let known_features = sess.target.rust_target_features(); let mut featsmap = FxHashMap::default(); // insert implied features @@ -600,50 +662,53 @@ pub(crate) fn global_llvm_features( } }; + // Get the backend feature name, if any. + // This excludes rustc-specific features, which do not get passed to LLVM. let feature = backend_feature_name(sess, s)?; // Warn against use of LLVM specific feature names and unstable features on the CLI. if diagnostics { - let feature_state = supported_features.iter().find(|&&(v, _, _)| v == feature); - if feature_state.is_none() { - let rust_feature = - supported_features.iter().find_map(|&(rust_feature, _, _)| { - let llvm_features = to_llvm_features(sess, rust_feature)?; - if llvm_features.contains(feature) - && !llvm_features.contains(rust_feature) - { - Some(rust_feature) - } else { - None + let feature_state = known_features.iter().find(|&&(v, _, _)| v == feature); + match feature_state { + None => { + let rust_feature = + known_features.iter().find_map(|&(rust_feature, _, _)| { + let llvm_features = to_llvm_features(sess, rust_feature)?; + if llvm_features.contains(feature) + && !llvm_features.contains(rust_feature) + { + Some(rust_feature) + } else { + None + } + }); + let unknown_feature = if let Some(rust_feature) = rust_feature { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::Some { rust_feature }, } - }); - let unknown_feature = if let Some(rust_feature) = rust_feature { - UnknownCTargetFeature { - feature, - rust_feature: PossibleFeature::Some { rust_feature }, - } - } else { - UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } - }; - sess.dcx().emit_warn(unknown_feature); - } else if feature_state - .is_some_and(|(_name, feature_gate, _implied)| !feature_gate.is_stable()) - { - // An unstable feature. Warn about using it. - sess.dcx().emit_warn(UnstableCTargetFeature { feature }); + } else { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::None, + } + }; + sess.dcx().emit_warn(unknown_feature); + } + Some((_, Stability::Stable, _)) => {} + Some((_, Stability::Unstable(_), _)) => { + // An unstable feature. Warn about using it. + sess.dcx().emit_warn(UnstableCTargetFeature { feature }); + } + Some((_, Stability::Forbidden { reason }, _)) => { + sess.dcx().emit_warn(ForbiddenCTargetFeature { feature, reason }); + } } - } - if diagnostics { // FIXME(nagisa): figure out how to not allocate a full hashset here. featsmap.insert(feature, enable_disable == '+'); } - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return None; - } - - // ... otherwise though we run through `to_llvm_features` when + // We run through `to_llvm_features` when // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. @@ -697,12 +762,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { let feature = s .strip_prefix(&['+', '-'][..]) .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); - if s.is_empty() { - return None; - } // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + if s.is_empty() || RUSTC_SPECIFIC_FEATURES.contains(&feature) { return None; } Some(feature) diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 02e1995620b..ea8857b4739 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -39,9 +39,9 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { .emit_fatal(SymbolAlreadyDefined { span: self.tcx.def_span(def_id), symbol_name }) }); + llvm::set_linkage(g, base::linkage_to_llvm(linkage)); + llvm::set_visibility(g, base::visibility_to_llvm(visibility)); unsafe { - llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); - llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); if self.should_assume_dso_local(g, false) { llvm::LLVMRustSetDSOLocal(g, true); } @@ -61,10 +61,12 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); - unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; + llvm::set_linkage(lldecl, base::linkage_to_llvm(linkage)); let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); - if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { + if (linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR) + && self.tcx.sess.target.supports_comdat() + { llvm::SetUniqueComdat(self.llmod, lldecl); } @@ -76,21 +78,15 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { && linkage != Linkage::Private && self.tcx.is_compiler_builtins(LOCAL_CRATE) { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); - } + llvm::set_visibility(lldecl, llvm::Visibility::Hidden); } else { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, base::visibility_to_llvm(visibility)); - } + llvm::set_visibility(lldecl, base::visibility_to_llvm(visibility)); } debug!("predefine_fn: instance = {:?}", instance); - unsafe { - if self.should_assume_dso_local(lldecl, false) { - llvm::LLVMRustSetDSOLocal(lldecl, true); - } + if self.should_assume_dso_local(lldecl, false) { + unsafe { llvm::LLVMRustSetDSOLocal(lldecl, true) }; } self.instances.borrow_mut().insert(instance, lldecl); @@ -100,13 +96,13 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { impl CodegenCx<'_, '_> { /// Whether a definition or declaration can be assumed to be local to a group of /// libraries that form a single DSO or executable. - pub(crate) unsafe fn should_assume_dso_local( + pub(crate) fn should_assume_dso_local( &self, llval: &llvm::Value, is_declaration: bool, ) -> bool { - let linkage = unsafe { llvm::LLVMRustGetLinkage(llval) }; - let visibility = unsafe { llvm::LLVMRustGetVisibility(llval) }; + let linkage = llvm::get_linkage(llval); + let visibility = llvm::get_visibility(llval); if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { return true; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index f1efc7a3dac..6aec078e0de 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -1,14 +1,14 @@ use std::{fmt, ptr}; use libc::{c_char, c_uint}; +use rustc_abi::{AddressSpace, Align, Integer, Size}; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_middle::bug; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty}; -use rustc_target::abi::call::{CastTarget, FnAbi, Reg}; -use rustc_target::abi::{AddressSpace, Align, Integer, Size}; +use rustc_target::callconv::{CastTarget, FnAbi, Reg}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::CodegenCx; diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 1af666f818b..2b05e24a7ba 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,7 +1,7 @@ use std::fmt::Write; use rustc_abi::Primitive::{Float, Int, Pointer}; -use rustc_abi::{Abi, Align, FieldsShape, Scalar, Size, Variants}; +use rustc_abi::{Align, BackendRepr, FieldsShape, Scalar, Size, Variants}; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -17,13 +17,13 @@ fn uncached_llvm_type<'a, 'tcx>( layout: TyAndLayout<'tcx>, defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, ) -> &'a Type { - match layout.abi { - Abi::Scalar(_) => bug!("handled elsewhere"), - Abi::Vector { element, count } => { + match layout.backend_repr { + BackendRepr::Scalar(_) => bug!("handled elsewhere"), + BackendRepr::Vector { element, count } => { let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } - Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalarPair(..) => {} + BackendRepr::Uninhabited | BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {} } let name = match layout.ty.kind() { @@ -170,16 +170,21 @@ pub(crate) trait LayoutLlvmExt<'tcx> { impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { - match self.abi { - Abi::Scalar(_) | Abi::Vector { .. } => true, - Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, + match self.backend_repr { + BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true, + BackendRepr::ScalarPair(..) | BackendRepr::Uninhabited | BackendRepr::Memory { .. } => { + false + } } } fn is_llvm_scalar_pair(&self) -> bool { - match self.abi { - Abi::ScalarPair(..) => true, - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, + match self.backend_repr { + BackendRepr::ScalarPair(..) => true, + BackendRepr::Uninhabited + | BackendRepr::Scalar(_) + | BackendRepr::Vector { .. } + | BackendRepr::Memory { .. } => false, } } @@ -191,14 +196,14 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. /// If the type is an unsized struct, the regular layout is generated, - /// with the inner-most trailing unsized field using the "minimal unit" + /// with the innermost trailing unsized field using the "minimal unit" /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { // This must produce the same result for `repr(transparent)` wrappers as for the inner type! // In other words, this should generally not look at the type at all, but only at the // layout. - if let Abi::Scalar(scalar) = self.abi { + if let BackendRepr::Scalar(scalar) = self.backend_repr { // Use a different cache for scalars because pointers to DSTs // can be either wide or thin (data pointers of wide pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { @@ -248,13 +253,13 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - match self.abi { - Abi::Scalar(scalar) => { + match self.backend_repr { + BackendRepr::Scalar(scalar) => { if scalar.is_bool() { return cx.type_i1(); } } - Abi::ScalarPair(..) => { + BackendRepr::ScalarPair(..) => { // An immediate pair always contains just the two elements, without any padding // filler, as it should never be stored to memory. return cx.type_struct( @@ -287,7 +292,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // This must produce the same result for `repr(transparent)` wrappers as for the inner type! // In other words, this should generally not look at the type at all, but only at the // layout. - let Abi::ScalarPair(a, b) = self.abi else { + let BackendRepr::ScalarPair(a, b) = self.backend_repr else { bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); }; let scalar = [a, b][index]; diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index f12b94d5887..e4c3e748cb5 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -1,9 +1,9 @@ +use rustc_abi::{Align, Endian, HasDataLayout, Size}; use rustc_codegen_ssa::common::IntPredicate; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; use crate::builder::Builder; use crate::type_::Type; |