diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
39 files changed, 4077 insertions, 2525 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index d478efc863a..9e834b83df4 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -34,13 +34,6 @@ pub trait ArgAttributesExt { ); } -fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // LLVM prior to version 12 had known miscompiles in the presence of - // noalias attributes (see #54878), but we don't support earlier - // versions at all anymore. We now enable mutable noalias by default. - cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) -} - const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; @@ -88,9 +81,6 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' attrs.push(llattr.create_attr(cx.llcx)); } } - if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { - attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); - } } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects // memory sanitizer's behavior. @@ -221,17 +211,15 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) } else if self.is_unsized_indirect() { bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast(cast, _) = &self.mode { + } else if let PassMode::Cast { cast, pad_i32: _ } = &self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. let can_store_through_cast_ptr = false; if can_store_through_cast_ptr { - let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); - let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); - bx.store(val, cast_dst, self.layout.align.abi); + bx.store(val, dst.llval, self.layout.align.abi); } else { // The actual return type is a struct, but the ABI - // adaptation code has cast it into some scalar type. The + // adaptation code has cast it into some scalar type. The // code that follows is the only reliable way I have // found to do a transform like i64 -> {i32,i32}. // Basically we dump the data onto the stack then memcpy it. @@ -286,12 +274,12 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { PassMode::Pair(..) => { OperandValue::Pair(next(), next()).store(bx, dst); } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); } PassMode::Direct(_) - | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } - | PassMode::Cast(..) => { + | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } + | PassMode::Cast { .. } => { let next_arg = next(); self.store(bx, next_arg, dst); } @@ -344,39 +332,86 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { let llreturn_ty = match &self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast(cast, _) => cast.llvm_type(cx), + PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx), PassMode::Indirect { .. } => { - llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); + llargument_tys.push(cx.type_ptr()); cx.type_void() } }; for arg in args { + // Note that the exact number of arguments pushed here is carefully synchronized with + // code all over the place, both in the codegen_llvm and codegen_ssa crates. That's how + // other code then knows which LLVM argument(s) correspond to the n-th Rust argument. let llarg_ty = match &arg.mode { PassMode::Ignore => continue, - PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), + PassMode::Direct(_) => { + // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges), + // and for Scalar ABIs the LLVM type is fully determined by `layout.abi`, + // guarnateeing that we generate ABI-compatible LLVM IR. Things get tricky for + // aggregates... + if matches!(arg.layout.abi, abi::Abi::Aggregate { .. }) { + assert!( + arg.layout.is_sized(), + "`PassMode::Direct` for unsized type: {}", + arg.layout.ty + ); + // This really shouldn't happen, since `immediate_llvm_type` will use + // `layout.fields` to turn this Rust type into an LLVM type. This means all + // sorts of Rust type details leak into the ABI. However wasm sadly *does* + // currently use this mode so we have to allow it -- but we absolutely + // shouldn't let any more targets do that. + // (Also see <https://github.com/rust-lang/rust/issues/115666>.) + assert!( + matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64"), + "`PassMode::Direct` for aggregates only allowed on wasm targets\nProblematic type: {:#?}", + arg.layout, + ); + } + arg.layout.immediate_llvm_type(cx) + } PassMode::Pair(..) => { + // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges), + // so for ScalarPair we can easily be sure that we are generating ABI-compatible + // LLVM IR. + assert!( + matches!(arg.layout.abi, abi::Abi::ScalarPair(..)), + "PassMode::Pair for type {}", + arg.layout.ty + ); llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { - let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack } => { + // `Indirect` with metadata is only for unsized types, and doesn't work with + // on-stack passing. + assert!(arg.layout.is_unsized() && !on_stack); + // Construct the type of a (wide) pointer to `ty`, and pass its two fields. + // Any two ABI-compatible unsized types have the same metadata type and + // moreover the same metadata value leads to the same dynamic size and + // alignment, so this respects ABI compatibility. + let ptr_ty = Ty::new_mut_ptr(cx.tcx, arg.layout.ty); let ptr_layout = cx.layout_of(ptr_ty); llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Cast(cast, pad_i32) => { + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + assert!(arg.layout.is_sized()); + cx.type_ptr() + } + PassMode::Cast { cast, pad_i32 } => { + // `Cast` means "transmute to `CastType`"; that only makes sense for sized types. + assert!(arg.layout.is_sized()); // add padding if *pad_i32 { llargument_tys.push(Reg::i32().llvm_type(cx)); } + // Compute the LLVM type we use for this function from the cast type. + // We assume here that ABI-compatible Rust types have the same cast type. cast.llvm_type(cx) } - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { - cx.type_ptr_to(arg.memory_ty(cx)) - } }; llargument_tys.push(llarg_ty); } @@ -389,42 +424,24 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { - unsafe { - llvm::LLVMPointerType( - self.llvm_type(cx), - cx.data_layout().instruction_address_space.0 as c_uint, - ) - } + cx.type_ptr_ext(cx.data_layout().instruction_address_space) } fn llvm_cconv(&self) -> llvm::CallConv { - match self.conv { - Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, - Conv::RustCold => llvm::ColdCallConv, - Conv::AmdGpuKernel => llvm::AmdGpuKernel, - Conv::AvrInterrupt => llvm::AvrInterrupt, - Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, - Conv::ArmAapcs => llvm::ArmAapcsCallConv, - Conv::Msp430Intr => llvm::Msp430Intr, - Conv::PtxKernel => llvm::PtxKernel, - Conv::X86Fastcall => llvm::X86FastcallCallConv, - Conv::X86Intr => llvm::X86_Intr, - Conv::X86Stdcall => llvm::X86StdcallCallConv, - Conv::X86ThisCall => llvm::X86_ThisCall, - Conv::X86VectorCall => llvm::X86_VectorCall, - Conv::X86_64SysV => llvm::X86_64_SysV, - Conv::X86_64Win64 => llvm::X86_64_Win64, - } + self.conv.into() } fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { - let mut func_attrs = SmallVec::<[_; 2]>::new(); + let mut func_attrs = SmallVec::<[_; 3]>::new(); if self.ret.layout.abi.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); } if !self.can_unwind { func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); } + if let Conv::RiscvInterrupt { kind } = self.conv { + func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", kind.as_str())); + } attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); let mut i = 0; @@ -437,13 +454,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } - PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast, _) => { + PassMode::Cast { cast, pad_i32: _ } => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } _ => {} @@ -451,25 +468,25 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { for arg in self.args.iter() { match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { let i = apply(attrs); let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } PassMode::Direct(attrs) - | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { + | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { apply(attrs); } - PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { + PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); apply(attrs); - apply(extra_attrs); + apply(meta_attrs); } PassMode::Pair(a, b) => { apply(a); apply(b); } - PassMode::Cast(cast, pad_i32) => { + PassMode::Cast { cast, pad_i32 } => { if *pad_i32 { apply(&ArgAttributes::new()); } @@ -499,13 +516,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Direct(attrs) => { attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } - PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast, _) => { + PassMode::Cast { cast, pad_i32: _ } => { cast.attrs.apply_attrs_to_callsite( llvm::AttributePlace::ReturnValue, &bx.cx, @@ -527,7 +544,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { for arg in self.args.iter() { match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); attributes::apply_to_callsite( @@ -537,18 +554,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); } PassMode::Direct(attrs) - | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { + | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { apply(bx.cx, attrs); } - PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack: _ } => { apply(bx.cx, attrs); - apply(bx.cx, extra_attrs); + apply(bx.cx, meta_attrs); } PassMode::Pair(a, b) => { apply(bx.cx, a); apply(bx.cx, b); } - PassMode::Cast(cast, pad_i32) => { + PassMode::Cast { cast, pad_i32 } => { if *pad_i32 { apply(bx.cx, &ArgAttributes::new()); } @@ -596,3 +613,29 @@ impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { llvm::get_param(self.llfn(), index as c_uint) } } + +impl From<Conv> for llvm::CallConv { + fn from(conv: Conv) -> Self { + match conv { + Conv::C | Conv::Rust | Conv::CCmseNonSecureCall | Conv::RiscvInterrupt { .. } => { + llvm::CCallConv + } + Conv::Cold => llvm::ColdCallConv, + Conv::PreserveMost => llvm::PreserveMost, + Conv::PreserveAll => llvm::PreserveAll, + Conv::AmdGpuKernel => llvm::AmdGpuKernel, + Conv::AvrInterrupt => llvm::AvrInterrupt, + Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, + Conv::ArmAapcs => llvm::ArmAapcsCallConv, + Conv::Msp430Intr => llvm::Msp430Intr, + Conv::PtxKernel => llvm::PtxKernel, + Conv::X86Fastcall => llvm::X86FastcallCallConv, + Conv::X86Intr => llvm::X86_Intr, + Conv::X86Stdcall => llvm::X86StdcallCallConv, + Conv::X86ThisCall => llvm::X86_ThisCall, + Conv::X86VectorCall => llvm::X86_VectorCall, + Conv::X86_64SysV => llvm::X86_64_SysV, + Conv::X86_64Win64 => llvm::X86_64_Win64, + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index fed56cdd438..db5c1388ef8 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -1,13 +1,15 @@ use crate::attributes; use libc::c_uint; -use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_ast::expand::allocator::{ + alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy, + ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, +}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; -use rustc_span::symbol::sym; use crate::debuginfo; -use crate::llvm::{self, False, True}; +use crate::llvm::{self, Context, False, Module, True, Type}; use crate::ModuleLlvm; pub(crate) unsafe fn codegen( @@ -26,39 +28,107 @@ pub(crate) unsafe fn codegen( tws => bug!("Unsupported target word size for int: {}", tws), }; let i8 = llvm::LLVMInt8TypeInContext(llcx); - let i8p = llvm::LLVMPointerType(i8, 0); - let void = llvm::LLVMVoidTypeInContext(llcx); - - for method in ALLOCATOR_METHODS { - let mut args = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match *ty { - AllocatorTy::Layout => { - args.push(usize); // size - args.push(usize); // align + let i8p = llvm::LLVMPointerTypeInContext(llcx, 0); + + if kind == AllocatorKind::Default { + for method in ALLOCATOR_METHODS { + let mut args = Vec::with_capacity(method.inputs.len()); + for input in method.inputs.iter() { + match input.ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), } - AllocatorTy::Ptr => args.push(i8p), - AllocatorTy::Usize => args.push(usize), - - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), } + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + + let from_name = global_fn_name(method.name); + let to_name = default_fn_name(method.name); + + create_wrapper_function(tcx, llcx, llmod, &from_name, &to_name, &args, output, false); } - let output = match method.output { - AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, + } - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") - } - }; + // rust alloc error handler + create_wrapper_function( + tcx, + llcx, + llmod, + "__rust_alloc_error_handler", + &alloc_error_handler_name(alloc_error_handler_kind), + &[usize, usize], // size, align + None, + true, + ); + + // __rust_alloc_error_handler_should_panic + let name = OomStrategy::SYMBOL; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let val = tcx.sess.opts.unstable_opts.oom.should_panic(); + let llval = llvm::LLVMConstInt(i8, val as u64, False); + llvm::LLVMSetInitializer(ll_g, llval); + + let name = NO_ALLOC_SHIM_IS_UNSTABLE; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let llval = llvm::LLVMConstInt(i8, 0, False); + llvm::LLVMSetInitializer(ll_g, llval); + + if tcx.sess.opts.debuginfo != DebugInfo::None { + let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); + dbg_cx.finalize(tcx.sess); + } +} + +fn create_wrapper_function( + tcx: TyCtxt<'_>, + llcx: &Context, + llmod: &Module, + from_name: &str, + to_name: &str, + args: &[&Type], + output: Option<&Type>, + no_return: bool, +) { + unsafe { let ty = llvm::LLVMFunctionType( - output.unwrap_or(void), + output.unwrap_or_else(|| llvm::LLVMVoidTypeInContext(llcx)), args.as_ptr(), args.len() as c_uint, False, ); - let name = format!("__rust_{}", method.name); - let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + let llfn = llvm::LLVMRustGetOrInsertFunction( + llmod, + from_name.as_ptr().cast(), + from_name.len(), + ty, + ); + let no_return = if no_return { + // -> ! DIFlagNoReturn + let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); + Some(no_return) + } else { + None + }; if tcx.sess.target.default_hidden_visibility { llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); @@ -68,9 +138,12 @@ pub(crate) unsafe fn codegen( attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } - let callee = kind.fn_name(method.name); let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_ptr().cast(), to_name.len(), ty); + if let Some(no_return) = no_return { + // -> ! DIFlagNoReturn + attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); + } llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); @@ -88,7 +161,8 @@ pub(crate) unsafe fn codegen( callee, args.as_ptr(), args.len() as c_uint, - None, + [].as_ptr(), + 0 as c_uint, ); llvm::LLVMSetTailCall(ret, True); if output.is_some() { @@ -98,59 +172,4 @@ pub(crate) unsafe fn codegen( } llvm::LLVMDisposeBuilder(llbuilder); } - - // rust alloc error handler - let args = [usize, usize]; // size, align - - let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False); - let name = "__rust_alloc_error_handler"; - let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); - // -> ! DIFlagNoReturn - let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); - - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - - let callee = alloc_error_handler_kind.fn_name(sym::oom); - let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); - // -> ! DIFlagNoReturn - attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); - - let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); - llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); - let args = args - .iter() - .enumerate() - .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) - .collect::<Vec<_>>(); - let ret = - llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); - llvm::LLVMSetTailCall(ret, True); - llvm::LLVMBuildRetVoid(llbuilder); - llvm::LLVMDisposeBuilder(llbuilder); - - // __rust_alloc_error_handler_should_panic - let name = OomStrategy::SYMBOL; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); - } - let val = tcx.sess.opts.unstable_opts.oom.should_panic(); - let llval = llvm::LLVMConstInt(i8, val as u64, False); - llvm::LLVMSetInitializer(ll_g, llval); - - if tcx.sess.opts.debuginfo != DebugInfo::None { - let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); - debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); - dbg_cx.finalize(tcx.sess); - } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 219a4f8fa89..1323261ae92 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -44,9 +44,10 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let is_target_supported = |reg_class: InlineAsmRegClass| { for &(_, feature) in reg_class.supported_types(asm_arch) { if let Some(feature) = feature { - let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); - if self.tcx.sess.target_features.contains(&feature) - || codegen_fn_attrs.target_features.contains(&feature) + if self + .tcx + .asm_target_features(instance.def_id()) + .contains(&feature) { return true; } @@ -144,7 +145,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // We prefer the latter because it matches the behavior of // Clang. if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) { - constraints.push(format!("{}", reg_to_llvm(reg, Some(&in_value.layout)))); + constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string()); } else { constraints.push(format!("{}", op_idx[&idx])); } @@ -236,14 +237,32 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::Nvptx64 => {} InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} InlineAsmArch::Hexagon => {} + InlineAsmArch::LoongArch64 => { + constraints.extend_from_slice(&[ + "~{$fcc0}".to_string(), + "~{$fcc1}".to_string(), + "~{$fcc2}".to_string(), + "~{$fcc3}".to_string(), + "~{$fcc4}".to_string(), + "~{$fcc5}".to_string(), + "~{$fcc6}".to_string(), + "~{$fcc7}".to_string(), + ]); + } InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} - InlineAsmArch::S390x => {} + InlineAsmArch::S390x => { + constraints.push("~{cc}".to_string()); + } InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} InlineAsmArch::Msp430 => { constraints.push("~{sr}".to_string()); } + InlineAsmArch::M68k => { + constraints.push("~{ccr}".to_string()); + } + InlineAsmArch::CSKY => {} } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -381,7 +400,7 @@ impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { } unsafe { - llvm::LLVMRustAppendModuleInlineAsm( + llvm::LLVMAppendModuleInlineAsm( self.llmod, template_str.as_ptr().cast(), template_str.len(), @@ -439,13 +458,13 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, v, inputs, dest, catch, funclet) + bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) } else { - bx.call(fty, None, v, inputs, None) + bx.call(fty, None, None, v, inputs, None) }; // Store mark in a metadata node so we can map LLVM errors - // back to source locations. See #17552. + // back to source locations. See #17552. let key = "srcloc"; let kind = llvm::LLVMGetMDKindIDInContext( bx.llcx, @@ -630,6 +649,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", @@ -671,6 +692,11 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d", + InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::freg) => "f", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -713,6 +739,7 @@ fn modifier_to_llvm( } } InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::LoongArch(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, InlineAsmRegClass::PowerPC(_) => None, @@ -768,6 +795,8 @@ fn modifier_to_llvm( InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::M68k(_) => None, + InlineAsmRegClass::CSKY(_) => None, InlineAsmRegClass::Err => unreachable!(), } } @@ -796,6 +825,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' cx.type_vector(cx.type_i64(), 2) } InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), @@ -839,6 +870,11 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(), + InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -849,6 +885,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' /// Helper function to get the LLVM type for a Scalar. Pointers are returned as /// the equivalent integer type. fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { + let dl = &cx.tcx.data_layout; match scalar.primitive() { Primitive::Int(Integer::I8, _) => cx.type_i8(), Primitive::Int(Integer::I16, _) => cx.type_i16(), @@ -856,7 +893,8 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty Primitive::Int(Integer::I64, _) => cx.type_i64(), Primitive::F32 => cx.type_f32(), Primitive::F64 => cx.type_f64(), - Primitive::Pointer => cx.type_isize(), + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()), _ => unreachable!(), } } @@ -868,6 +906,7 @@ fn llvm_fixup_input<'ll, 'tcx>( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value { + let dl = &bx.tcx.data_layout; match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { @@ -881,8 +920,10 @@ fn llvm_fixup_input<'ll, 'tcx>( let elem_ty = llvm_asm_scalar_type(bx.cx, s); let count = 16 / layout.size.bytes(); let vec_ty = bx.cx.type_vector(elem_ty, count); - if let Primitive::Pointer = s.primitive() { - value = bx.ptrtoint(value, bx.cx.type_isize()); + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + if let Primitive::Pointer(_) = s.primitive() { + let t = bx.type_from_integer(dl.ptr_sized_integer()); + value = bx.ptrtoint(value, t); } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } @@ -958,7 +999,7 @@ fn llvm_fixup_output<'ll, 'tcx>( } (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { value = bx.extract_element(value, bx.const_i32(0)); - if let Primitive::Pointer = s.primitive() { + if let Primitive::Pointer(_) = s.primitive() { value = bx.inttoptr(value, layout.llvm_type(bx.cx)); } value diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index d96da5cc11d..b6c01545f30 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,7 +1,6 @@ //! Set and unset common attributes on LLVM values. use rustc_codegen_ssa::traits::*; -use rustc_data_structures::small_str::SmallStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::{self, TyCtxt}; @@ -12,6 +11,7 @@ use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtec use smallvec::SmallVec; use crate::attributes; +use crate::errors::{MissingFeatures, SanitizerMemtagRequiresMte, TargetFeatureDisableOrEnable}; use crate::llvm::AttributePlace::Function; use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::llvm_util; @@ -61,7 +61,7 @@ pub fn sanitize_attrs<'ll>( ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; - if enabled.contains(SanitizerSet::ADDRESS) { + if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } if enabled.contains(SanitizerSet::MEMORY) { @@ -82,11 +82,14 @@ pub fn sanitize_attrs<'ll>( let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.sess.err("`-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`"); + cx.tcx.sess.emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); } + if enabled.contains(SanitizerSet::SAFESTACK) { + attrs.push(llvm::AttributeKind::SanitizeSafeStack.create_attr(cx.llcx)); + } attrs } @@ -101,10 +104,10 @@ pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { let mut fp = cx.sess().target.frame_pointer; + let opts = &cx.sess().opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. - if cx.sess().instrument_mcount() || matches!(cx.sess().opts.cg.force_frame_pointers, Some(true)) - { + if opts.unstable_opts.instrument_mcount || matches!(opts.cg.force_frame_pointers, Some(true)) { fp = FramePointer::Always; } let attr_value = match fp { @@ -117,23 +120,62 @@ pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attr /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().instrument_mcount() { +fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { + let mut attrs = SmallVec::new(); + if cx.sess().opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = cx.sess().target.mcount.as_ref(); + let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), + None => cx.sess().target.mcount.as_ref(), + }; - Some(llvm::CreateAttrStringValue( + attrs.push(llvm::CreateAttrStringValue( cx.llcx, "instrument-function-entry-inlined", &mcount_name, - )) - } else { - None + )); + } + if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. + // Function prologue and epilogue are instrumented with NOP sleds, + // a runtime library later replaces them with detours into tracing code. + if options.always { + attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-always")); + } + if options.never { + attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-never")); + } + if options.ignore_loops { + attrs.push(llvm::CreateAttrString(cx.llcx, "xray-ignore-loops")); + } + // LLVM will not choose the default for us, but rather requires specific + // threshold in absence of "xray-always". Use the same default as Clang. + let threshold = options.instruction_threshold.unwrap_or(200); + attrs.push(llvm::CreateAttrStringValue( + cx.llcx, + "xray-instruction-threshold", + &threshold.to_string(), + )); + if options.skip_entry { + attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-entry")); + } + if options.skip_exit { + attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-exit")); + } + } + attrs +} + +fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + if !cx.sess().opts.unstable_opts.no_jump_tables { + return None; } + + Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { @@ -257,13 +299,12 @@ pub fn from_fn_attrs<'ll, 'tcx>( OptimizeAttr::Speed => {} } - let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - InlineAttr::Never - } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { - InlineAttr::Hint - } else { - codegen_fn_attrs.inline - }; + let inline = + if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { + InlineAttr::Hint + } else { + codegen_fn_attrs.inline + }; to_add.extend(inline_attr(cx, inline)); // The `uwtable` attribute according to LLVM is: @@ -293,9 +334,14 @@ pub fn from_fn_attrs<'ll, 'tcx>( // FIXME: none of these three functions interact with source level attributes. to_add.extend(frame_pointer_type_attr(cx)); to_add.extend(instrument_function_attr(cx)); + to_add.extend(nojumptables_attr(cx)); to_add.extend(probestack_attr(cx)); to_add.extend(stackprotector_attr(cx)); + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { + to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); } @@ -320,50 +366,44 @@ pub fn from_fn_attrs<'ll, 'tcx>( if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - // apply to argument place instead of function - let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); - to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); - let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { - flags |= AllocKindFlags::Uninitialized; - } else { - flags |= AllocKindFlags::Zeroed; - } - to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); // apply to return place instead of function (unlike all other attributes applied in this function) let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - to_add.push(llvm::CreateAllocKindAttr( - cx.llcx, - AllocKindFlags::Realloc | AllocKindFlags::Aligned, - )); - // applies to argument place instead of function place - let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); - // apply to argument place instead of function - let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); - to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); - } + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); - // applies to argument place instead of function place - let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); - } + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); @@ -393,20 +433,21 @@ pub fn from_fn_attrs<'ll, 'tcx>( .get_attrs(instance.def_id(), sym::target_feature) .next() .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); - let msg = format!( - "the target features {} must all be either enabled or disabled together", - f.join(", ") - ); - let mut err = cx.tcx.sess.struct_span_err(span, &msg); - err.help("add the missing features in a `target_feature` attribute"); - err.emit(); + cx.tcx + .sess + .create_err(TargetFeatureDisableOrEnable { + features: f, + span: Some(span), + missing_features: Some(MissingFeatures), + }) + .emit(); return; } let mut function_features = function_features .iter() .flat_map(|feat| { - llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{}", f)) + llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{f}")) }) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), @@ -431,7 +472,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // the WebAssembly specification, which has this feature. This won't be // needed when LLVM enables this `multivalue` feature by default. if !cx.tcx.is_closure(instance.def_id()) { - let abi = cx.tcx.fn_sig(instance.def_id()).abi(); + let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); if abi == Abi::Wasm { function_features.push("+multivalue".to_string()); } @@ -440,8 +481,8 @@ pub fn from_fn_attrs<'ll, 'tcx>( let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); let function_features = function_features.iter().map(|s| s.as_str()); - let target_features = - global_features.chain(function_features).intersperse(",").collect::<SmallStr<1024>>(); + let target_features: String = + global_features.chain(function_features).intersperse(",").collect(); if !target_features.is_empty() { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 082665bba38..f33075a8879 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,27 +1,30 @@ //! A helper class for dealing with static archives use std::env; -use std::ffi::{CStr, CString, OsString}; -use std::fs; -use std::io::{self, Write}; +use std::ffi::{c_char, c_void, CStr, CString, OsString}; +use std::io; use std::mem; use std::path::{Path, PathBuf}; use std::ptr; use std::str; -use object::read::macho::FatArch; - use crate::common; +use crate::errors::{ + DlltoolFailImportLibrary, ErrorCallingDllTool, ErrorCreatingImportLibrary, ErrorWritingDEFFile, +}; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; -use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; -use rustc_data_structures::memmap::Mmap; +use rustc_codegen_ssa::back::archive::{ + get_native_object_symbols, try_extract_macho_fat_archive, ArArchiveBuilder, + ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, UnknownArchiveKind, +}; + use rustc_session::cstore::DllImport; use rustc_session::Session; /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] -pub struct LlvmArchiveBuilder<'a> { +pub(crate) struct LlvmArchiveBuilder<'a> { sess: &'a Session, additions: Vec<Addition>, } @@ -53,58 +56,7 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { "x86" => LLVMMachineType::I386, "aarch64" => LLVMMachineType::ARM64, "arm" => LLVMMachineType::ARM, - _ => panic!("unsupported cpu type {}", cpu), - } -} - -fn try_filter_fat_archs( - archs: object::read::Result<&[impl FatArch]>, - target_arch: object::Architecture, - archive_path: &Path, - archive_map_data: &[u8], -) -> io::Result<Option<PathBuf>> { - let archs = archs.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; - - let desired = match archs.iter().filter(|a| a.architecture() == target_arch).next() { - Some(a) => a, - None => return Ok(None), - }; - - let (mut new_f, extracted_path) = tempfile::Builder::new() - .suffix(archive_path.file_name().unwrap()) - .tempfile()? - .keep() - .unwrap(); - - new_f.write_all( - desired.data(archive_map_data).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?, - )?; - - Ok(Some(extracted_path)) -} - -fn try_extract_macho_fat_archive( - sess: &Session, - archive_path: &Path, -) -> io::Result<Option<PathBuf>> { - let archive_map = unsafe { Mmap::map(fs::File::open(&archive_path)?)? }; - let target_arch = match sess.target.arch.as_ref() { - "aarch64" => object::Architecture::Aarch64, - "x86_64" => object::Architecture::X86_64, - _ => return Ok(None), - }; - - match object::macho::FatHeader::parse(&*archive_map) { - Ok(h) if h.magic.get(object::endian::BigEndian) == object::macho::FAT_MAGIC => { - let archs = object::macho::FatHeader::parse_arch32(&*archive_map); - try_filter_fat_archs(archs, target_arch, archive_path, &*archive_map) - } - Ok(h) if h.magic.get(object::endian::BigEndian) == object::macho::FAT_MAGIC_64 => { - let archs = object::macho::FatHeader::parse_arch64(&*archive_map); - try_filter_fat_archs(archs, target_arch, archive_path, &*archive_map) - } - // Not a FatHeader at all, just return None. - _ => Ok(None), + _ => panic!("unsupported cpu type {cpu}"), } } @@ -147,7 +99,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { fn build(mut self: Box<Self>, output: &Path) -> bool { match self.build_with_llvm(output) { Ok(any_members) => any_members, - Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), + Err(e) => self.sess.emit_fatal(ArchiveBuildFailure { error: e }), } } } @@ -156,7 +108,13 @@ pub struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { - Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + // FIXME use ArArchiveBuilder on most targets again once reading thin archives is + // implemented + if true { + Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + } else { + Box::new(ArArchiveBuilder::new(sess, get_llvm_object_symbols)) + } } fn create_dll_import_lib( @@ -170,7 +128,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { let name_suffix = if is_direct_dependency { "_imports" } else { "_imports_indirect" }; let output_path = { let mut output_path: PathBuf = tmpdir.to_path_buf(); - output_path.push(format!("{}{}", lib_name, name_suffix)); + output_path.push(format!("{lib_name}{name_suffix}")); output_path.with_extension("lib") }; @@ -195,10 +153,10 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { // The binutils linker used on -windows-gnu targets cannot read the import // libraries generated by LLVM: in our attempts, the linker produced an .EXE // that loaded but crashed with an AV upon calling one of the imported - // functions. Therefore, use binutils to create the import library instead, + // functions. Therefore, use binutils to create the import library instead, // by writing a .DEF file to the temp dir and calling binutils's dlltool. let def_file_path = - tmpdir.join(format!("{}{}", lib_name, name_suffix)).with_extension("def"); + tmpdir.join(format!("{lib_name}{name_suffix}")).with_extension("def"); let def_file_content = format!( "EXPORTS\n{}", @@ -206,7 +164,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { .into_iter() .map(|(name, ordinal)| { match ordinal { - Some(n) => format!("{} @{} NONAME", name, n), + Some(n) => format!("{name} @{n} NONAME"), None => name, } }) @@ -217,7 +175,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - sess.fatal(&format!("Error writing .DEF file: {}", e)); + sess.emit_fatal(ErrorWritingDEFFile { error: e }); } }; @@ -225,27 +183,58 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { // able to control the *exact* spelling of each of the symbols that are being imported: // hence we don't want `dlltool` adding leading underscores automatically. let dlltool = find_binutils_dlltool(sess); - let result = std::process::Command::new(dlltool) - .args([ - "-d", - def_file_path.to_str().unwrap(), - "-D", - lib_name, - "-l", - output_path.to_str().unwrap(), - "--no-leading-underscore", - ]) - .output(); - - match result { + let temp_prefix = { + let mut path = PathBuf::from(&output_path); + path.pop(); + path.push(lib_name); + path + }; + // dlltool target architecture args from: + // https://github.com/llvm/llvm-project-release-prs/blob/llvmorg-15.0.6/llvm/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp#L69 + let (dlltool_target_arch, dlltool_target_bitness) = match sess.target.arch.as_ref() { + "x86_64" => ("i386:x86-64", "--64"), + "x86" => ("i386", "--32"), + "aarch64" => ("arm64", "--64"), + "arm" => ("arm", "--32"), + _ => panic!("unsupported arch {}", sess.target.arch), + }; + let mut dlltool_cmd = std::process::Command::new(&dlltool); + dlltool_cmd.args([ + "-d", + def_file_path.to_str().unwrap(), + "-D", + lib_name, + "-l", + output_path.to_str().unwrap(), + "-m", + dlltool_target_arch, + "-f", + dlltool_target_bitness, + "--no-leading-underscore", + "--temp-prefix", + temp_prefix.to_str().unwrap(), + ]); + + match dlltool_cmd.output() { Err(e) => { - sess.fatal(&format!("Error calling dlltool: {}", e)); + sess.emit_fatal(ErrorCallingDllTool { + dlltool_path: dlltool.to_string_lossy(), + error: e, + }); + } + // dlltool returns '0' on failure, so check for error output instead. + Ok(output) if !output.stderr.is_empty() => { + sess.emit_fatal(DlltoolFailImportLibrary { + dlltool_path: dlltool.to_string_lossy(), + dlltool_args: dlltool_cmd + .get_args() + .map(|arg| arg.to_string_lossy()) + .collect::<Vec<_>>() + .join(" "), + stdout: String::from_utf8_lossy(&output.stdout), + stderr: String::from_utf8_lossy(&output.stderr), + }) } - Ok(output) if !output.status.success() => sess.fatal(&format!( - "Dlltool could not create import library: {}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )), _ => {} } } else { @@ -268,7 +257,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { // All import names are Rust identifiers and therefore cannot contain \0 characters. // FIXME: when support for #[link_name] is implemented, ensure that the import names - // still don't contain any \0 characters. Also need to check that the names don't + // still don't contain any \0 characters. Also need to check that the names don't // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved // in definition files. let cstring_import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = @@ -293,11 +282,10 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { }; if result == crate::llvm::LLVMRustResult::Failure { - sess.fatal(&format!( - "Error creating import library for {}: {}", + sess.emit_fatal(ErrorCreatingImportLibrary { lib_name, - llvm::last_error().unwrap_or("unknown LLVM error".to_string()) - )); + error: llvm::last_error().unwrap_or("unknown LLVM error".to_string()), + }); } }; @@ -305,12 +293,68 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { } } +// The object crate doesn't know how to get symbols for LLVM bitcode and COFF bigobj files. +// As such we need to use LLVM for them. +#[deny(unsafe_op_in_unsafe_fn)] +fn get_llvm_object_symbols( + buf: &[u8], + f: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> io::Result<bool> { + let is_bitcode = unsafe { llvm::LLVMRustIsBitcode(buf.as_ptr(), buf.len()) }; + + // COFF bigobj file, msvc LTO file or import library. See + // https://github.com/llvm/llvm-project/blob/453f27bc9/llvm/lib/BinaryFormat/Magic.cpp#L38-L51 + let is_unsupported_windows_obj_file = buf.get(0..4) == Some(b"\0\0\xFF\xFF"); + + if is_bitcode || is_unsupported_windows_obj_file { + let mut state = Box::new(f); + + let err = unsafe { + llvm::LLVMRustGetSymbols( + buf.as_ptr(), + buf.len(), + &mut *state as *mut &mut _ as *mut c_void, + callback, + error_callback, + ) + }; + + if err.is_null() { + return Ok(true); + } else { + return Err(unsafe { *Box::from_raw(err as *mut io::Error) }); + } + + unsafe extern "C" fn callback( + state: *mut c_void, + symbol_name: *const c_char, + ) -> *mut c_void { + let f = unsafe { &mut *(state as *mut &mut dyn FnMut(&[u8]) -> io::Result<()>) }; + match f(unsafe { CStr::from_ptr(symbol_name) }.to_bytes()) { + Ok(()) => std::ptr::null_mut(), + Err(err) => Box::into_raw(Box::new(err)) as *mut c_void, + } + } + + unsafe extern "C" fn error_callback(error: *const c_char) -> *mut c_void { + let error = unsafe { CStr::from_ptr(error) }; + Box::into_raw(Box::new(io::Error::new( + io::ErrorKind::Other, + format!("LLVM error: {}", error.to_string_lossy()), + ))) as *mut c_void + } + } else { + get_native_object_symbols(buf, f) + } +} + impl<'a> LlvmArchiveBuilder<'a> { fn build_with_llvm(&mut self, output: &Path) -> io::Result<bool> { let kind = &*self.sess.target.archive_format; - let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| { - self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) - }); + let kind = kind + .parse::<ArchiveKind>() + .map_err(|_| kind) + .unwrap_or_else(|kind| self.sess.emit_fatal(UnknownArchiveKind { kind })); let mut additions = mem::take(&mut self.additions); let mut strings = Vec::new(); @@ -323,7 +367,7 @@ impl<'a> LlvmArchiveBuilder<'a> { match addition { Addition::File { path, name_in_archive } => { let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.clone())?; + let name = CString::new(name_in_archive.as_bytes())?; members.push(llvm::LLVMRustArchiveMemberNew( path.as_ptr(), name.as_ptr(), @@ -391,33 +435,31 @@ impl<'a> LlvmArchiveBuilder<'a> { } fn string_to_io_error(s: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("bad archive: {}", s)) + io::Error::new(io::ErrorKind::Other, format!("bad archive: {s}")) } fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.cg.dlltool { return dlltool_path.clone().into_os_string(); } - let mut tool_name: OsString = if sess.host.arch != sess.target.arch { - // We are cross-compiling, so we need the tool with the prefix matching our target - if sess.target.arch == "x86" { - "i686-w64-mingw32-dlltool" - } else { - "x86_64-w64-mingw32-dlltool" - } + let tool_name: OsString = if sess.host.options.is_like_windows { + // If we're compiling on Windows, always use "dlltool.exe". + "dlltool.exe" } else { - // We are not cross-compiling, so we just want `dlltool` - "dlltool" + // On other platforms, use the architecture-specific name. + match sess.target.arch.as_ref() { + "x86_64" => "x86_64-w64-mingw32-dlltool", + "x86" => "i686-w64-mingw32-dlltool", + "aarch64" => "aarch64-w64-mingw32-dlltool", + + // For non-standard architectures (e.g., aarch32) fallback to "dlltool". + _ => "dlltool", + } } .into(); - if sess.host.options.is_like_windows { - // If we're compiling on Windows, add the .exe suffix - tool_name.push(".exe"); - } - // NOTE: it's not clear how useful it is to explicitly search PATH. for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { let full_path = dir.join(&tool_name); @@ -427,7 +469,7 @@ fn find_binutils_dlltool(sess: &Session) -> OsString { } // The user didn't specify the location of the dlltool binary, and we weren't able - // to find the appropriate one on the PATH. Just return the name of the tool + // to find the appropriate one on the PATH. Just return the name of the tool // and let the invocation fail with a hopefully useful error message. tool_name } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index a49cc7f8d66..cb5acf79135 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,10 +1,15 @@ -use crate::back::write::{self, save_temp_bitcode, DiagnosticHandlers}; +use crate::back::write::{ + self, bitcode_section_name, save_temp_bitcode, CodegenDiagnosticsStage, DiagnosticHandlers, +}; +use crate::errors::{ + DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro, +}; use crate::llvm::{self, build_string}; use crate::{LlvmCodegenBackend, ModuleLlvm}; use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; -use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; @@ -22,7 +27,6 @@ use std::fs::File; use std::io; use std::iter; use std::path::Path; -use std::ptr; use std::slice; use std::sync::Arc; @@ -32,8 +36,12 @@ pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { - CrateType::Executable | CrateType::Dylib | CrateType::Staticlib | CrateType::Cdylib => true, - CrateType::Rlib | CrateType::ProcMacro => false, + CrateType::Executable + | CrateType::Dylib + | CrateType::Staticlib + | CrateType::Cdylib + | CrateType::ProcMacro => true, + CrateType::Rlib => false, } } @@ -76,27 +84,23 @@ fn prepare_lto( // Make sure we actually can run LTO for crate_type in cgcx.crate_types.iter() { if !crate_type_allows_lto(*crate_type) { - let e = diag_handler.fatal( - "lto can only be run for executables, cdylibs and \ - static library outputs", - ); - return Err(e); + diag_handler.emit_err(LtoDisallowed); + return Err(FatalError); } else if *crate_type == CrateType::Dylib { if !cgcx.opts.unstable_opts.dylib_lto { - return Err(diag_handler - .fatal("lto cannot be used for `dylib` crate type without `-Zdylib-lto`")); + diag_handler.emit_err(LtoDylib); + return Err(FatalError); + } + } else if *crate_type == CrateType::ProcMacro { + if !cgcx.opts.unstable_opts.dylib_lto { + diag_handler.emit_err(LtoProcMacro); + return Err(FatalError); } } } if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto { - diag_handler - .struct_err("cannot prefer dynamic linking when performing LTO") - .note( - "only 'staticlib', 'bin', and 'cdylib' outputs are \ - supported with LTO", - ) - .emit(); + diag_handler.emit_err(DynamicLinkingWithLTO); return Err(FatalError); } @@ -127,24 +131,51 @@ fn prepare_lto( info!("adding bitcode from {}", name); match get_bitcode_slice_from_object_data( child.data(&*archive_data).expect("corrupt rlib"), + cgcx, ) { Ok(data) => { let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); } - Err(msg) => return Err(diag_handler.fatal(&msg)), + Err(e) => { + diag_handler.emit_err(e); + return Err(FatalError); + } } } } } + // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to + // __llvm_profile_runtime, therefore we won't know until link time if this symbol + // should have default visibility. + symbols_below_threshold.push(CString::new("__llvm_profile_counter_bias").unwrap()); Ok((symbols_below_threshold, upstream_modules)) } -fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { +fn get_bitcode_slice_from_object_data<'a>( + obj: &'a [u8], + cgcx: &CodegenContext<LlvmCodegenBackend>, +) -> Result<&'a [u8], LtoBitcodeFromRlib> { + // We're about to assume the data here is an object file with sections, but if it's raw LLVM IR that + // won't work. Fortunately, if that's what we have we can just return the object directly, so we sniff + // the relevant magic strings here and return. + if obj.starts_with(b"\xDE\xC0\x17\x0B") || obj.starts_with(b"BC\xC0\xDE") { + return Ok(obj); + } + // We drop the "__LLVM," prefix here because on Apple platforms there's a notion of "segment name" + // which in the public API for sections gets treated as part of the section name, but internally + // in MachOObjectFile.cpp gets treated separately. + let section_name = bitcode_section_name(cgcx).trim_start_matches("__LLVM,"); let mut len = 0; - let data = - unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; + let data = unsafe { + llvm::LLVMRustGetSliceFromObjectDataByName( + obj.as_ptr(), + obj.len(), + section_name.as_ptr(), + &mut len, + ) + }; if !data.is_null() { assert!(len != 0); let bc = unsafe { slice::from_raw_parts(data, len) }; @@ -156,8 +187,9 @@ fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { Ok(bc) } else { assert!(len == 0); - let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); - Err(format!("failed to get bitcode from object file for LTO ({})", msg)) + Err(LtoBitcodeFromRlib { + llvm_err: llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()), + }) } } @@ -165,7 +197,7 @@ fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { /// for further optimization. pub(crate) fn run_fat( cgcx: &CodegenContext<LlvmCodegenBackend>, - modules: Vec<FatLTOInput<LlvmCodegenBackend>>, + modules: Vec<FatLtoInput<LlvmCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> { let diag_handler = cgcx.create_diag_handler(); @@ -211,7 +243,7 @@ pub(crate) fn run_thin( } pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { - let name = module.name.clone(); + let name = module.name; let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -219,7 +251,7 @@ pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBu fn fat_lto( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, - modules: Vec<FatLTOInput<LlvmCodegenBackend>>, + modules: Vec<FatLtoInput<LlvmCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, symbols_below_threshold: &[*const libc::c_char], @@ -244,8 +276,8 @@ fn fat_lto( })); for module in modules { match module { - FatLTOInput::InMemory(m) => in_memory.push(m), - FatLTOInput::Serialized { name, buffer } => { + FatLtoInput::InMemory(m) => in_memory.push(m), + FatLtoInput::Serialized { name, buffer } => { info!("pushing serialized module {:?}", name); let buffer = SerializedModule::Local(buffer); serialized_modules.push((buffer, CString::new(name).unwrap())); @@ -301,7 +333,13 @@ fn fat_lto( // The linking steps below may produce errors and diagnostics within LLVM // which we'd like to handle and print, so set up our diagnostic handlers // (which get unregistered when they go out of scope below). - let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + let _handler = DiagnosticHandlers::new( + cgcx, + diag_handler, + llcx, + &module, + CodegenDiagnosticsStage::LTO, + ); // For all other modules we codegened we'll need to link them into our own // bitcode. All modules were codegened in their own LLVM context, however, @@ -325,14 +363,13 @@ fn fat_lto( let _timer = cgcx .prof .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| { - recorder.record_arg(format!("{:?}", name)) + recorder.record_arg(format!("{name:?}")) }); info!("linking {:?}", name); let data = bc_decoded.data(); - linker.add(data).map_err(|()| { - let msg = format!("failed to load bitcode of module {:?}", name); - write::llvm_err(diag_handler, &msg) - })?; + linker + .add(data) + .map_err(|()| write::llvm_err(diag_handler, LlvmError::LoadBitcode { name }))?; serialized_bitcode.push(bc_decoded); } drop(linker); @@ -426,7 +463,7 @@ fn thin_lto( info!("going for that thin, thin LTO"); let green_modules: FxHashMap<_, _> = - cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect(); + cached_modules.iter().map(|(_, wp)| (wp.cgu_name.clone(), wp.clone())).collect(); let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len(); let mut thin_buffers = Vec::with_capacity(modules.len()); @@ -435,7 +472,7 @@ fn thin_lto( for (i, (name, buffer)) in modules.into_iter().enumerate() { info!("local module: {} - {}", i, name); - let cname = CString::new(name.clone()).unwrap(); + let cname = CString::new(name.as_bytes()).unwrap(); thin_modules.push(llvm::ThinLTOModule { identifier: cname.as_ptr(), data: buffer.data().as_ptr(), @@ -490,7 +527,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len() as u32, ) - .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?; + .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::PrepareThinLtoContext))?; let data = ThinData(data); @@ -563,8 +600,7 @@ fn thin_lto( // session, overwriting the previous serialized data (if any). if let Some(path) = key_map_path { if let Err(err) = curr_key_map.save_to_file(&path) { - let msg = format!("Error while writing ThinLTO key data: {}", err); - return Err(write::llvm_err(diag_handler, &msg)); + return Err(write::llvm_err(diag_handler, LlvmError::WriteThinLtoKey { err })); } } @@ -578,7 +614,7 @@ pub(crate) fn run_pass_manager( module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, ) -> Result<(), FatalError> { - let _timer = cgcx.prof.verbose_generic_activity_with_arg("LLVM_lto_optimize", &*module.name); + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); let config = cgcx.config(module.kind); // Now we have one massive module inside of llmod. Time to run the @@ -690,8 +726,7 @@ pub unsafe fn optimize_thin_module( let module_name = &thin_module.shared.module_names[thin_module.idx]; let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); - let tm = - (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?; + let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, e))?; // Right now the implementation we've got only works over serialized // modules, so we create a fresh new LLVM context and parse the module @@ -710,18 +745,6 @@ pub unsafe fn optimize_thin_module( let llmod = module.module_llvm.llmod(); save_temp_bitcode(cgcx, &module, "thin-lto-input"); - // Before we do much else find the "main" `DICompileUnit` that we'll be - // using below. If we find more than one though then rustc has changed - // in a way we're not ready for, so generate an ICE by returning - // an error. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); - if !cu2.is_null() { - let msg = "multiple source DICompileUnits found"; - return Err(write::llvm_err(&diag_handler, msg)); - } - // Up next comes the per-module local analyses that we do for Thin LTO. // Each of these functions is basically copied from the LLVM // implementation and then tailored to suit this implementation. Ideally @@ -734,8 +757,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); } @@ -745,8 +767,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); } @@ -756,8 +777,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); } @@ -766,49 +786,11 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } - // Ok now this is a bit unfortunate. This is also something you won't - // find upstream in LLVM's ThinLTO passes! This is a hack for now to - // work around bugs in LLVM. - // - // First discovered in #45511 it was found that as part of ThinLTO - // importing passes LLVM will import `DICompileUnit` metadata - // information across modules. This means that we'll be working with one - // LLVM module that has multiple `DICompileUnit` instances in it (a - // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of - // bugs in LLVM's backend which generates invalid DWARF in a situation - // like this: - // - // https://bugs.llvm.org/show_bug.cgi?id=35212 - // https://bugs.llvm.org/show_bug.cgi?id=35562 - // - // While the first bug there is fixed the second ended up causing #46346 - // which was basically a resurgence of #45511 after LLVM's bug 35212 was - // fixed. - // - // This function below is a huge hack around this problem. The function - // below is defined in `PassWrapper.cpp` and will basically "merge" - // all `DICompileUnit` instances in a module. Basically it'll take all - // the objects, rewrite all pointers of `DISubprogram` to point to the - // first `DICompileUnit`, and then delete all the other units. - // - // This is probably mangling to the debug info slightly (but hopefully - // not too much) but for now at least gets LLVM to emit valid DWARF (or - // so it appears). Hopefully we can remove this once upstream bugs are - // fixed in LLVM. - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); - save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); - } - // Alright now that we've done everything related to the ThinLTO // analysis it's time to run some optimizations! Here we use the same // `run_pass_manager` as the "fat" LTO above except that we tell it to @@ -836,7 +818,7 @@ impl ThinLTOKeysMap { let file = File::create(path)?; let mut writer = io::BufWriter::new(file); for (module, key) in &self.keys { - writeln!(writer, "{} {}", module, key)?; + writeln!(writer, "{module} {key}")?; } Ok(()) } @@ -850,7 +832,7 @@ impl ThinLTOKeysMap { let mut split = line.split(' '); let module = split.next().unwrap(); let key = split.next().unwrap(); - assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line); + assert_eq!(split.next(), None, "Expected two space-separated values, found {line:?}"); keys.insert(module.to_string(), key.to_string()); } Ok(Self { keys }) @@ -887,11 +869,7 @@ pub fn parse_module<'a>( diag_handler: &Handler, ) -> Result<&'a llvm::Module, FatalError> { unsafe { - llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( - || { - let msg = "failed to parse bitcode for LTO module"; - write::llvm_err(diag_handler, msg) - }, - ) + llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()) + .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::ParseBitcode)) } } diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs new file mode 100644 index 00000000000..36484c3c3fc --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -0,0 +1,103 @@ +use std::{ + ffi::{c_char, CStr}, + marker::PhantomData, + ops::Deref, + ptr::NonNull, +}; + +use rustc_data_structures::small_c_str::SmallCStr; + +use crate::{errors::LlvmError, llvm}; + +/// Responsible for safely creating and disposing llvm::TargetMachine via ffi functions. +/// Not cloneable as there is no clone function for llvm::TargetMachine. +#[repr(transparent)] +pub struct OwnedTargetMachine { + tm_unique: NonNull<llvm::TargetMachine>, + phantom: PhantomData<llvm::TargetMachine>, +} + +impl OwnedTargetMachine { + pub fn new( + triple: &CStr, + cpu: &CStr, + features: &CStr, + abi: &CStr, + model: llvm::CodeModel, + reloc: llvm::RelocModel, + level: llvm::CodeGenOptLevel, + use_soft_fp: bool, + function_sections: bool, + data_sections: bool, + unique_section_names: bool, + trap_unreachable: bool, + singletree: bool, + asm_comments: bool, + emit_stack_size_section: bool, + relax_elf_relocations: bool, + use_init_array: bool, + split_dwarf_file: &CStr, + output_obj_file: &CStr, + debug_info_compression: &CStr, + force_emulated_tls: bool, + args_cstr_buff: &[u8], + ) -> Result<Self, LlvmError<'static>> { + assert!(args_cstr_buff.len() > 0); + assert!( + *args_cstr_buff.last().unwrap() == 0, + "The last character must be a null terminator." + ); + + // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data + let tm_ptr = unsafe { + llvm::LLVMRustCreateTargetMachine( + triple.as_ptr(), + cpu.as_ptr(), + features.as_ptr(), + abi.as_ptr(), + model, + reloc, + level, + use_soft_fp, + function_sections, + data_sections, + unique_section_names, + trap_unreachable, + singletree, + asm_comments, + emit_stack_size_section, + relax_elf_relocations, + use_init_array, + split_dwarf_file.as_ptr(), + output_obj_file.as_ptr(), + debug_info_compression.as_ptr(), + force_emulated_tls, + args_cstr_buff.as_ptr() as *const c_char, + args_cstr_buff.len(), + ) + }; + + NonNull::new(tm_ptr) + .map(|tm_unique| Self { tm_unique, phantom: PhantomData }) + .ok_or_else(|| LlvmError::CreateTargetMachine { triple: SmallCStr::from(triple) }) + } +} + +impl Deref for OwnedTargetMachine { + type Target = llvm::TargetMachine; + + fn deref(&self) -> &Self::Target { + // SAFETY: constructing ensures we have a valid pointer created by llvm::LLVMRustCreateTargetMachine + unsafe { self.tm_unique.as_ref() } + } +} + +impl Drop for OwnedTargetMachine { + fn drop(&mut self) { + // SAFETY: constructing ensures we have a valid pointer created by llvm::LLVMRustCreateTargetMachine + // OwnedTargetMachine is not copyable so there is no double free or use after free + unsafe { + llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut()); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 11053a8f6c4..c778a6e017f 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,15 +1,22 @@ use crate::back::lto::ThinBuffer; +use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, }; use crate::base; use crate::common; -use crate::consts; +use crate::errors::{ + CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression, + WithLlvmError, WriteBytecode, +}; use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; use crate::ModuleLlvm; +use llvm::{ + LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols, +}; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, @@ -28,6 +35,7 @@ use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; @@ -37,10 +45,10 @@ use std::slice; use std::str; use std::sync::Arc; -pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { +pub fn llvm_err<'a>(handler: &rustc_errors::Handler, err: LlvmError<'a>) -> FatalError { match llvm::last_error() { - Some(err) => handler.fatal(&format!("{}: {}", msg, err)), - None => handler.fatal(msg), + Some(llvm_err) => handler.emit_almost_fatal(WithLlvmError(err, llvm_err)), + None => handler.emit_almost_fatal(err), } } @@ -85,23 +93,22 @@ pub fn write_output_file<'ll>( } } - result.into_result().map_err(|()| { - let msg = format!("could not write output to {}", output.display()); - llvm_err(handler, &msg) - }) + result + .into_result() + .map_err(|()| llvm_err(handler, LlvmError::WriteOutput { path: output })) } } -pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { - let config = TargetMachineFactoryConfig { split_dwarf_file: None }; +pub fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine { + let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None }; // Can't use query system here quite yet because this function is invoked before the query // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) + .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise()) } -pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { +pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { tcx.output_filenames(()).split_dwarf_path( tcx.sess.split_debuginfo(), @@ -111,13 +118,17 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll } else { None }; - let config = TargetMachineFactoryConfig { split_dwarf_file }; + + let output_obj_file = + Some(tcx.output_filenames(()).temp_path(OutputType::Object, Some(mod_name))); + let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }; + target_machine_factory( &tcx.sess, tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), err).raise()) } pub fn to_llvm_opt_settings( @@ -203,7 +214,7 @@ pub fn target_machine_factory( sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; - let asm_comments = sess.asm_comments(); + let asm_comments = sess.opts.unstable_opts.asm_comments; let relax_elf_relocations = sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); @@ -212,37 +223,75 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); + let force_emulated_tls = sess.target.force_emulated_tls; + + // copy the exe path, followed by path all into one buffer + // null terminating them so we can use them as null terminated strings + let args_cstr_buff = { + let mut args_cstr_buff: Vec<u8> = Vec::new(); + let exe_path = std::env::current_exe().unwrap_or_default(); + let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default(); + + args_cstr_buff.extend_from_slice(exe_path_str.as_bytes()); + args_cstr_buff.push(0); + + for arg in sess.expanded_args.iter() { + args_cstr_buff.extend_from_slice(arg.as_bytes()); + args_cstr_buff.push(0); + } + + args_cstr_buff + }; + + let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); + match sess.opts.debuginfo_compression { + rustc_session::config::DebugInfoCompression::Zlib => { + if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { + sess.emit_warning(UnknownCompression { algorithm: "zlib" }); + } + } + rustc_session::config::DebugInfoCompression::Zstd => { + if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { + sess.emit_warning(UnknownCompression { algorithm: "zstd" }); + } + } + rustc_session::config::DebugInfoCompression::None => {} + }; + let debuginfo_compression = SmallCStr::new(&debuginfo_compression); + Arc::new(move |config: TargetMachineFactoryConfig| { - let split_dwarf_file = - path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; - let split_dwarf_file = CString::new(split_dwarf_file.to_str().unwrap()).unwrap(); - - let tm = unsafe { - llvm::LLVMRustCreateTargetMachine( - triple.as_ptr(), - cpu.as_ptr(), - features.as_ptr(), - abi.as_ptr(), - code_model, - reloc_model, - opt_level, - use_softfp, - ffunction_sections, - fdata_sections, - funique_section_names, - trap_unreachable, - singlethread, - asm_comments, - emit_stack_size_section, - relax_elf_relocations, - use_init_array, - split_dwarf_file.as_ptr(), - ) + let path_to_cstring_helper = |path: Option<PathBuf>| -> CString { + let path = path_mapping.map_prefix(path.unwrap_or_default()).0; + CString::new(path.to_str().unwrap()).unwrap() }; - tm.ok_or_else(|| { - format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) - }) + let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file); + let output_obj_file = path_to_cstring_helper(config.output_obj_file); + + OwnedTargetMachine::new( + &triple, + &cpu, + &features, + &abi, + code_model, + reloc_model, + opt_level, + use_softfp, + ffunction_sections, + fdata_sections, + funique_section_names, + trap_unreachable, + singlethread, + asm_comments, + emit_stack_size_section, + relax_elf_relocations, + use_init_array, + &split_dwarf_file, + &output_obj_file, + &debuginfo_compression, + force_emulated_tls, + &args_cstr_buff, + ) }) } @@ -255,7 +304,7 @@ pub(crate) fn save_temp_bitcode( return; } unsafe { - let ext = format!("{}.bc", name); + let ext = format!("{name}.bc"); let cgu = Some(&module.name[..]); let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); let cstr = path_to_c_string(&path); @@ -264,6 +313,16 @@ pub(crate) fn save_temp_bitcode( } } +/// In what context is a dignostic handler being attached to a codegen unit? +pub enum CodegenDiagnosticsStage { + /// Prelink optimization stage. + Opt, + /// LTO/ThinLTO postlink optimization stage. + LTO, + /// Code generation. + Codegen, +} + pub struct DiagnosticHandlers<'a> { data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a Handler), llcx: &'a llvm::Context, @@ -275,6 +334,8 @@ impl<'a> DiagnosticHandlers<'a> { cgcx: &'a CodegenContext<LlvmCodegenBackend>, handler: &'a Handler, llcx: &'a llvm::Context, + module: &ModuleCodegen<ModuleLlvm>, + stage: CodegenDiagnosticsStage, ) -> Self { let remark_passes_all: bool; let remark_passes: Vec<CString>; @@ -291,6 +352,21 @@ impl<'a> DiagnosticHandlers<'a> { }; let remark_passes: Vec<*const c_char> = remark_passes.iter().map(|name: &CString| name.as_ptr()).collect(); + let remark_file = cgcx + .remark_dir + .as_ref() + // Use the .opt.yaml file suffix, which is supported by LLVM's opt-viewer. + .map(|dir| { + let stage_suffix = match stage { + CodegenDiagnosticsStage::Codegen => "codegen", + CodegenDiagnosticsStage::Opt => "opt", + CodegenDiagnosticsStage::LTO => "lto", + }; + dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name)) + }) + .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok())); + + let pgo_available = cgcx.opts.cg.profile_use.is_some(); let data = Box::into_raw(Box::new((cgcx, handler))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); @@ -301,6 +377,10 @@ impl<'a> DiagnosticHandlers<'a> { remark_passes_all, remark_passes.as_ptr(), remark_passes.len(), + // The `as_ref()` is important here, otherwise the `CString` will be dropped + // too soon! + remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()), + pgo_available, ); DiagnosticHandlers { data, llcx, old_handler } } @@ -349,31 +429,36 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void } llvm::diagnostic::Optimization(opt) => { - let enabled = match cgcx.remark { - Passes::All => true, - Passes::Some(ref v) => v.iter().any(|s| *s == opt.pass_name), - }; - - if enabled { - diag_handler.note_without_error(&format!( - "{}:{}:{}: {}: {}", - opt.filename, opt.line, opt.column, opt.pass_name, opt.message, - )); - } + diag_handler.emit_note(FromLlvmOptimizationDiag { + filename: &opt.filename, + line: opt.line, + column: opt.column, + pass_name: &opt.pass_name, + kind: match opt.kind { + OptimizationDiagnosticKind::OptimizationRemark => "success", + OptimizationDiagnosticKind::OptimizationMissed + | OptimizationDiagnosticKind::OptimizationFailure => "missed", + OptimizationDiagnosticKind::OptimizationAnalysis + | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute + | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis", + OptimizationDiagnosticKind::OptimizationRemarkOther => "other", + }, + message: &opt.message, + }); } llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { - let msg = llvm::build_string(|s| { + let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.warn(&msg); + diag_handler.emit_warning(FromLlvmDiag { message }); } llvm::diagnostic::Unsupported(diagnostic_ref) => { - let msg = llvm::build_string(|s| { + let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.err(&msg); + diag_handler.emit_err(FromLlvmDiag { message }); } llvm::diagnostic::UnknownDiagnostic(..) => {} } @@ -409,11 +494,7 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { } fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { - if config.instrument_coverage { - Some(CString::new("default_%m_%p.profraw").unwrap()) - } else { - None - } + config.instrument_coverage.then(|| CString::new("default_%m_%p.profraw").unwrap()) } pub(crate) unsafe fn llvm_optimize( @@ -437,22 +518,27 @@ pub(crate) unsafe fn llvm_optimize( Some(llvm::SanitizerOptions { sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), + sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI), + sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI), sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), + sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS), + sanitize_kernel_address_recover: config + .sanitizer_recover + .contains(SanitizerSet::KERNELADDRESS), }) } else { None }; - let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { - Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) - } else { - None - }; + let mut llvm_profiler = cgcx + .prof + .llvm_recording_enabled() + .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())); let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); @@ -469,6 +555,7 @@ pub(crate) unsafe fn llvm_optimize( &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), opt_stage, + cgcx.opts.cg.linker_plugin_lto.enabled(), config.no_prepopulate_passes, config.verify_llvm_ir, using_thin_buffers, @@ -494,7 +581,7 @@ pub(crate) unsafe fn llvm_optimize( llvm_plugins.as_ptr().cast(), llvm_plugins.len(), ); - result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) + result.into_result().map_err(|()| llvm_err(diag_handler, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. @@ -508,7 +595,8 @@ pub(crate) unsafe fn optimize( let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; - let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + let _handlers = + DiagnosticHandlers::new(cgcx, diag_handler, llcx, module, CodegenDiagnosticsStage::Opt); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); @@ -547,8 +635,7 @@ pub(crate) fn link( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); linker.add(buffer.data()).map_err(|()| { - let msg = format!("failed to serialize module {:?}", module.name); - llvm_err(diag_handler, &msg) + llvm_err(diag_handler, LlvmError::SerializeModule { name: &module.name }) })?; } drop(linker); @@ -568,7 +655,13 @@ pub(crate) unsafe fn codegen( let tm = &*module.module_llvm.tm; let module_name = module.name.clone(); let module_name = Some(&module_name[..]); - let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + let _handlers = DiagnosticHandlers::new( + cgcx, + diag_handler, + llcx, + &module, + CodegenDiagnosticsStage::Codegen, + ); if cgcx.msvc_imps_needed { create_msvc_imps(cgcx, llcx, llmod); @@ -626,9 +719,8 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); - if let Err(e) = fs::write(&bc_out, data) { - let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); - diag_handler.err(&msg); + if let Err(err) = fs::write(&bc_out, data) { + diag_handler.emit_err(WriteBytecode { path: &bc_out, err }); } } @@ -664,7 +756,7 @@ pub(crate) unsafe fn codegen( let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 }; - if write!(cursor, "{:#}", demangled).is_err() { + if write!(cursor, "{demangled:#}").is_err() { // Possible only if provided buffer is not big enough return 0; } @@ -678,10 +770,9 @@ pub(crate) unsafe fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result.into_result().map_err(|()| { - let msg = format!("failed to write LLVM IR to {}", out.display()); - llvm_err(diag_handler, &msg) - })?; + result + .into_result() + .map_err(|()| llvm_err(diag_handler, LlvmError::WriteIr { path: &out }))?; } if config.emit_asm { @@ -749,8 +840,8 @@ pub(crate) unsafe fn codegen( EmitObj::Bitcode => { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); - if let Err(e) = link_or_copy(&bc_out, &obj_out) { - diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); + if let Err(err) = link_or_copy(&bc_out, &obj_out) { + diag_handler.emit_err(CopyBitcode { err }); } if !config.emit_bc { @@ -762,21 +853,31 @@ pub(crate) unsafe fn codegen( EmitObj::None => {} } - drop(handlers); + record_llvm_cgu_instructions_stats(&cgcx.prof, llmod); } + // `.dwo` files are only emitted if: + // + // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not + // produce dwarf objects, even if otherwise enabled) + // - Target supports Split DWARF + // - Split debuginfo is enabled + // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different + // sections in the `.o` files). + let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_)) + && cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split; Ok(module.into_compiled_module( config.emit_obj != EmitObj::None, - cgcx.target_can_use_split_dwarf - && cgcx.split_debuginfo != SplitDebuginfo::Off - && cgcx.split_dwarf_kind == SplitDwarfKind::Split, + dwarf_object_emitted, config.emit_bc, &cgcx.output_filenames, )) } fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec<u8> { - let mut asm = format!(".section {},\"{}\"\n", section_name, section_flags).into_bytes(); + let mut asm = format!(".section {section_name},\"{section_flags}\"\n").into_bytes(); asm.extend_from_slice(b".ascii \""); asm.reserve(data.len()); for &byte in data { @@ -798,6 +899,27 @@ fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: asm } +fn target_is_apple(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { + cgcx.opts.target_triple.triple().contains("-ios") + || cgcx.opts.target_triple.triple().contains("-darwin") + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos") +} + +fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { + cgcx.opts.target_triple.triple().contains("-aix") +} + +pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static str { + if target_is_apple(cgcx) { + "__LLVM,__bitcode\0" + } else if target_is_aix(cgcx) { + ".ipa\0" + } else { + ".llvmbc\0" + } +} + /// Embed the bitcode of an LLVM module in the LLVM module itself. /// /// This is done primarily for iOS where it appears to be standard to compile C @@ -852,14 +974,16 @@ unsafe fn embed_bitcode( // passed though then these sections will show up in the final output. // Additionally the flag that we need to set here is `SHF_EXCLUDE`. // + // * XCOFF - AIX linker ignores content in .ipa and .info if no auxiliary + // symbol associated with these sections. + // // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). - let is_apple = cgcx.opts.target_triple.triple().contains("-ios") - || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos") - || cgcx.opts.target_triple.triple().contains("-watchos"); + let is_aix = target_is_aix(cgcx); + let is_apple = target_is_apple(cgcx); if is_apple + || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") { @@ -872,7 +996,7 @@ unsafe fn embed_bitcode( ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; + let section = bitcode_section_name(cgcx); llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); @@ -884,22 +1008,28 @@ unsafe fn embed_bitcode( "rustc.embedded.cmdline\0".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; + let section = if is_apple { + "__LLVM,__cmdline\0" + } else if is_aix { + ".info\0" + } else { + ".llvmcmd\0" + }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } } // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. // This is required to satisfy `dllimport` references to static data in .rlibs -// when using MSVC linker. We do this only for data, as linker can fix up +// when using MSVC linker. We do this only for data, as linker can fix up // code references on its own. // See #26591, #27438 fn create_msvc_imps( @@ -917,7 +1047,7 @@ fn create_msvc_imps( let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; unsafe { - let i8p_ty = Type::i8p_llcx(llcx); + let ptr_ty = Type::ptr_llcx(llcx); let globals = base::iter_globals(llmod) .filter(|&val| { llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage @@ -937,8 +1067,8 @@ fn create_msvc_imps( .collect::<Vec<_>>(); for (imp_name, val) in globals { - let imp = llvm::LLVMAddGlobal(llmod, i8p_ty, imp_name.as_ptr().cast()); - llvm::LLVMSetInitializer(imp, consts::ptrcast(val, i8p_ty)); + let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr().cast()); + llvm::LLVMSetInitializer(imp, val); llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); } } @@ -965,3 +1095,23 @@ fn record_artifact_size( self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); } } + +fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) { + if !prof.enabled() { + return; + } + + let raw_stats = + llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(&llmod, s) }) + .expect("cannot get module instruction stats"); + + #[derive(serde::Deserialize)] + struct InstructionsStats { + module: String, + total: u64, + } + + let InstructionsStats { module, total } = + serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats"); + prof.artifact_size("cgu_instructions", module, total); +} diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5b2bbdb4bde..b659fd02eec 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -86,8 +86,8 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen { let cx = CodegenCx::new(tcx, cgu, &llvm_module); let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); - for &(mono_item, (linkage, visibility)) in &mono_items { - mono_item.predefine::<Builder<'_, '_, '_>>(&cx, linkage, visibility); + for &(mono_item, data) in &mono_items { + mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility); } // ... and now that we have everything pre-defined, fill out those definitions. @@ -123,8 +123,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen // happen after the llvm.used variables are created. for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { unsafe { - let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); - llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMReplaceAllUsesWith(old_g, new_g); llvm::LLVMDeleteGlobal(old_g); } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 9cb36ce7f18..4f9b86ec20a 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,7 +2,8 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; -use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock}; +use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -15,13 +16,16 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; -use rustc_target::spec::{HasTargetSpec, Target}; +use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; +use smallvec::SmallVec; use std::borrow::Cow; use std::ffi::CStr; use std::iter; @@ -215,6 +219,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -225,8 +230,22 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("invoke {:?} with args ({:?})", llfn, args); let args = self.check_call("invoke", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } + + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } let invoke = unsafe { llvm::LLVMRustBuildInvoke( @@ -237,7 +256,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { args.len() as c_uint, then, catch, - bundle, + bundles.as_ptr(), + bundles.len() as c_uint, UNNAMED, ) }; @@ -472,7 +492,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); if place.layout.is_zst() { - return OperandRef::new_zst(self, place.layout); + return OperandRef::zero_sized(place.layout); } #[instrument(level = "trace", skip(bx))] @@ -483,7 +503,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { layout: TyAndLayout<'tcx>, offset: Size, ) { - if !scalar.is_always_valid(bx) { + if !scalar.is_uninit_valid() { bx.noundef_metadata(load); } @@ -493,7 +513,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { bx.range_metadata(load, scalar.valid_range(bx)); } } - abi::Pointer => { + abi::Pointer(_) => { if !scalar.valid_range(bx).contains(0) { bx.nonnull_metadata(load); } @@ -556,15 +576,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn write_operand_repeatedly( - mut self, + &mut self, cg_elem: OperandRef<'tcx, &'ll Value>, count: u64, dest: PlaceRef<'tcx, &'ll Value>, - ) -> Self { + ) { let zero = self.const_usize(0); let count = self.const_usize(count); - let start = dest.project_index(&mut self, zero).llval; - let end = dest.project_index(&mut self, count).llval; let header_bb = self.append_sibling_block("repeat_loop_header"); let body_bb = self.append_sibling_block("repeat_loop_body"); @@ -573,26 +591,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.br(header_bb); let mut header_bx = Self::build(self.cx, header_bb); - let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); + let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]); - let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); + let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count); header_bx.cond_br(keep_going, body_bb, next_bb); let mut body_bx = Self::build(self.cx, body_bb); - let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); - cg_elem - .val - .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align)); - - let next = body_bx.inbounds_gep( - self.backend_type(cg_elem.layout), - current, - &[self.const_usize(1)], - ); + let dest_elem = dest.project_index(&mut body_bx, i); + cg_elem.val.store(&mut body_bx, dest_elem); + + let next = body_bx.unchecked_uadd(i, self.const_usize(1)); body_bx.br(header_bb); - header_bx.add_incoming_to_phi(current, next, body_bb); + header_bx.add_incoming_to_phi(i, next, body_bb); - Self::build(self.cx, next_bb) + *self = Self::build(self.cx, next_bb); } fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { @@ -641,7 +653,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { flags: MemFlags, ) -> &'ll Value { debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags); - let ptr = self.check_store(val, ptr); + assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer); unsafe { let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); let align = @@ -671,7 +683,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { size: Size, ) { debug!("Store {:?} -> {:?}", val, ptr); - let ptr = self.check_store(val, ptr); + assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer); unsafe { let store = llvm::LLVMRustBuildAtomicStore( self.llbuilder, @@ -823,7 +835,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + unsafe { + llvm::LLVMBuildIntCast2( + self.llbuilder, + val, + dest_ty, + if is_signed { True } else { False }, + UNNAMED, + ) + } } fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -854,8 +874,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - let dst = self.pointercast(dst, self.type_i8p()); - let src = self.pointercast(src, self.type_i8p()); unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, @@ -881,8 +899,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - let dst = self.pointercast(dst, self.type_i8p()); - let src = self.pointercast(src, self.type_i8p()); unsafe { llvm::LLVMRustBuildMemMove( self.llbuilder, @@ -905,7 +921,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { flags: MemFlags, ) { let is_volatile = flags.contains(MemFlags::VOLATILE); - let ptr = self.pointercast(ptr, self.type_i8p()); unsafe { llvm::LLVMRustBuildMemSet( self.llbuilder, @@ -961,15 +976,27 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn cleanup_landing_pad(&mut self, ty: &'ll Type, pers_fn: &'ll Value) -> &'ll Value { - let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { llvm::LLVMSetCleanup(landing_pad, llvm::True); } - landing_pad + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) + } + + fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 1); + self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[])); + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } - fn resume(&mut self, exn: &'ll Value) { + fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) { + let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false); + let mut exn = self.const_poison(ty); + exn = self.insert_value(exn, exn0, 0); + exn = self.insert_value(exn, exn1, 1); unsafe { llvm::LLVMBuildResume(self.llbuilder, exn); } @@ -978,11 +1005,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { let name = cstr!("cleanuppad"); let ret = unsafe { - llvm::LLVMRustBuildCleanupPad( + llvm::LLVMBuildCleanupPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), + args.len() as c_uint, name.as_ptr(), ) }; @@ -991,7 +1018,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { unsafe { - llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) .expect("LLVM does not have support for cleanupret"); } } @@ -999,11 +1026,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { let name = cstr!("catchpad"); let ret = unsafe { - llvm::LLVMRustBuildCatchPad( + llvm::LLVMBuildCatchPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), + args.len() as c_uint, name.as_ptr(), ) }; @@ -1018,7 +1045,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) -> &'ll Value { let name = cstr!("catchswitch"); let ret = unsafe { - llvm::LLVMRustBuildCatchSwitch( + llvm::LLVMBuildCatchSwitch( self.llbuilder, parent, unwind, @@ -1029,7 +1056,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let ret = ret.expect("LLVM does not have support for catchswitch"); for handler in handlers { unsafe { - llvm::LLVMRustAddHandler(ret, handler); + llvm::LLVMAddHandler(ret, handler); } } ret @@ -1130,7 +1157,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; let llty = self.cx.type_func( - &[self.cx.type_i8p(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], + &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], self.cx.type_void(), ); let args = &[fn_name, hash, num_counters, index]; @@ -1143,7 +1170,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, - None, + [].as_ptr(), + 0 as c_uint, ); } } @@ -1151,6 +1179,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1159,8 +1188,22 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("call {:?} with args ({:?})", llfn, args); let args = self.check_call("call", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } + + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } let call = unsafe { llvm::LLVMRustBuildCall( @@ -1169,7 +1212,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, - bundle, + bundles.as_ptr(), + bundles.len() as c_uint, ) }; if let Some(fn_abi) = fn_abi { @@ -1182,9 +1226,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn do_not_inline(&mut self, llret: &'ll Value) { - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { + if llvm_util::get_version() < (17, 0, 2) { + // Work around https://github.com/llvm/llvm-project/issues/66984. + let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + } else { + // Cleanup is always the cold path. + let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); + } } } @@ -1335,30 +1386,10 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { - let ret = - unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; + let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; ret.expect("LLVM does not have support for catchret") } - fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value { - let dest_ptr_ty = self.cx.val_ty(ptr); - let stored_ty = self.cx.val_ty(val); - let stored_ptr_ty = self.cx.type_ptr_to(stored_ty); - - assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer); - - if dest_ptr_ty == stored_ptr_ty { - ptr - } else { - debug!( - "type mismatch in store. \ - Expected {:?}, got {:?}; inserting bitcast", - dest_ptr_ty, stored_ptr_ty - ); - self.bitcast(ptr, stored_ptr_ty) - } - } - fn check_call<'b>( &mut self, typ: &str, @@ -1368,9 +1399,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) -> Cow<'b, [&'ll Value]> { assert!( self.cx.type_kind(fn_ty) == TypeKind::Function, - "builder::{} not passed a function, but {:?}", - typ, - fn_ty + "builder::{typ} not passed a function, but {fn_ty:?}" ); let param_tys = self.cx.func_params_types(fn_ty); @@ -1408,7 +1437,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, None, f, args, None) + self.call(ty, None, None, f, args, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { @@ -1421,7 +1450,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - let ptr = self.pointercast(ptr, self.cx.type_i8p()); self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); } @@ -1462,15 +1490,12 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { let instr = if signed { "fptosi" } else { "fptoui" }; let name = if let Some(vector_length) = vector_length { - format!( - "llvm.{}.sat.v{}i{}.v{}f{}", - instr, vector_length, int_width, vector_length, float_width - ) + format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}") } else { - format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + format!("llvm.{instr}.sat.i{int_width}.f{float_width}") }; let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None) + self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None) } pub(crate) fn landing_pad( @@ -1487,4 +1512,72 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) } } + + // Emits CFI pointer type membership tests. + fn cfi_type_test( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) { + let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; + if self.tcx.sess.is_sanitizer_cfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { + if let Some(fn_attrs) = fn_attrs && fn_attrs.no_sanitize.contains(SanitizerSet::CFI) { + return; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let typeid = typeid_for_fnabi(self.tcx, fn_abi, options); + let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + + // Test whether the function pointer is associated with the type identifier. + let cond = self.type_test(llfn, typeid_metadata); + let bb_pass = self.append_sibling_block("type_test.pass"); + let bb_fail = self.append_sibling_block("type_test.fail"); + self.cond_br(cond, bb_pass, bb_fail); + + self.switch_to_block(bb_fail); + self.abort(); + self.unreachable(); + + self.switch_to_block(bb_pass); + } + } + + // Emits KCFI operand bundles. + fn kcfi_operand_bundle( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) -> Option<llvm::OperandBundleDef<'ll>> { + let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; + let kcfi_bundle = + if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { + if let Some(fn_attrs) = fn_attrs && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI) { + return None; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + kcfi_bundle + } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 6f0d1b7ce84..5254c3f9c9a 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -4,16 +4,14 @@ //! and methods are represented as just a fn ptr and not a full //! closure. -use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common; use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; -use rustc_codegen_ssa::traits::*; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::{self, Instance, TypeVisitable}; +use rustc_middle::ty::{self, Instance, TypeVisitableExt}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -27,8 +25,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> debug!("get_fn(instance={:?})", instance); - assert!(!instance.substs.needs_infer()); - assert!(!instance.substs.has_escaping_bound_vars()); + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); if let Some(&llfn) = cx.instances.borrow().get(&instance) { return llfn; @@ -45,47 +43,28 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); let llfn = if let Some(llfn) = cx.get_declared_value(sym) { - // Create a fn pointer with the new signature. - let llptrty = fn_abi.ptr_to_llvm_type(cx); - - // This is subtle and surprising, but sometimes we have to bitcast - // the resulting fn pointer. The reason has to do with external - // functions. If you have two crates that both bind the same C - // library, they may not use precisely the same types: for - // example, they will probably each declare their own structs, - // which are distinct types from LLVM's point of view (nominal - // types). - // - // Now, if those two crates are linked into an application, and - // they contain inlined code, you can wind up with a situation - // where both of those functions wind up being loaded into this - // application simultaneously. In that case, the same function - // (from LLVM's point of view) requires two types. But of course - // LLVM won't allow one function to have two types. - // - // What we currently do, therefore, is declare the function with - // one of the two types (whichever happens to come first) and then - // bitcast as needed when the function is referenced to make sure - // it has the type we expect. - // - // This can occur on either a crate-local or crate-external - // reference. It also occurs when testing libcore and in some - // other weird situations. Annoying. - if cx.val_ty(llfn) != llptrty { - debug!("get_fn: casting {:?} to {:?}", llfn, llptrty); - cx.const_ptrcast(llfn, llptrty) - } else { - debug!("get_fn: not casting pointer!"); - llfn - } + llfn } else { let instance_def_id = instance.def_id(); let llfn = if tcx.sess.target.arch == "x86" && let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) { - cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi) + // Fix for https://github.com/rust-lang/rust/issues/104453 + // On x86 Windows, LLVM uses 'L' as the prefix for any private + // global symbols, so when we create an undecorated function symbol + // that begins with an 'L' LLVM misinterprets that as a private + // global symbol that it created and so fails the compilation at a + // later stage since such a symbol must have a definition. + // + // To avoid this, we set the Storage Class to "DllImport" so that + // LLVM will prefix the name with `__imp_`. Ideally, we'd like the + // existing logic below to set the Storage Class, but it has an + // exemption for MinGW for backwards compatability. + let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance)); + unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } + llfn } else { - cx.declare_fn(sym, fn_abi) + cx.declare_fn(sym, fn_abi, Some(instance)) }; debug!("get_fn: not casting pointer!"); @@ -116,7 +95,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); - let is_generic = instance.substs.non_erasable_generics().next().is_some(); + let is_generic = + instance.args.non_erasable_generics(tcx, instance.def_id()).next().is_some(); if is_generic { // This is a monomorphization. Its expected visibility depends diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index acee9134fb9..0b0816c27b6 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -1,22 +1,20 @@ //! Code that is useful in various codegen modules. -use crate::consts::{self, const_alloc_to_llvm}; +use crate::consts::const_alloc_to_llvm; pub use crate::context::CodegenCx; use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True}; use crate::type_::Type; -use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_ast::Mutability; -use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; +use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use rustc_target::spec::Target; use libc::{c_char, c_uint}; @@ -129,6 +127,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMGetUndef(t) } } + fn const_poison(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMGetPoison(t) } + } + fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { unsafe { llvm::LLVMConstInt(t, i as u64, True) } } @@ -164,6 +166,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.const_uint(self.type_i64(), i) } + fn const_u128(&self, i: u128) -> &'ll Value { + self.const_uint_big(self.type_i128(), i) + } + fn const_usize(&self, i: u64) -> &'ll Value { let bit_size = self.data_layout().pointer_size.bits(); if bit_size < 64 { @@ -203,11 +209,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) .1; let len = s.len(); - let cs = consts::ptrcast( - str_global, - self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)), - ); - (cs, self.const_usize(len as u64)) + (str_global, self.const_usize(len as u64)) } fn const_struct(&self, elts: &[&'ll Value], packed: bool) -> &'ll Value { @@ -236,7 +238,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { Scalar::Int(int) => { let data = int.assert_bits(layout.size(self)); let llval = self.const_uint_big(self.type_ix(bitsize), data); - if layout.primitive() == Pointer { + if matches!(layout.primitive(), Pointer(_)) { unsafe { llvm::LLVMConstIntToPtr(llval, llty) } } else { self.const_bitcast(llval, llty) @@ -252,8 +254,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), _ => self.static_addr_of(init, alloc.align, None), }; - if !self.sess().fewer_names() { - llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes()); + if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() { + let hash = self.tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + alloc.hash_stable(&mut hcx, &mut hasher); + hasher.finish::<Hash128>() + }); + llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes()); } (value, AddressSpace::DATA) } @@ -277,14 +284,14 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } }; let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( + llvm::LLVMConstInBoundsGEP2( self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)), + self.const_bitcast(base_addr, self.type_ptr_ext(base_addr_space)), &self.const_usize(offset.bytes()), 1, ) }; - if layout.primitive() != Pointer { + if !matches!(layout.primitive(), Pointer(_)) { unsafe { llvm::LLVMConstPtrToInt(llval, llty) } } else { self.const_bitcast(llval, llty) @@ -297,37 +304,19 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { const_alloc_to_llvm(self, alloc) } - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, &'ll Value> { - let alloc_align = alloc.inner().align; - assert_eq!(alloc_align, layout.align.abi); - let llty = self.type_ptr_to(layout.llvm_type(self)); - let llval = if layout.size == Size::ZERO { - let llval = self.const_usize(alloc_align.bytes()); - unsafe { llvm::LLVMConstIntToPtr(llval, llty) } - } else { - let init = const_alloc_to_llvm(self, alloc); - let base_addr = self.static_addr_of(init, alloc_align, None); - - let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( - self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p()), - &self.const_usize(offset.bytes()), - 1, - ) - }; - self.const_bitcast(llval, llty) - }; - PlaceRef::new_sized(llval, layout) - } - - fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { - consts::ptrcast(val, ty) + fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + self.const_bitcast(val, ty) + } + + fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { + unsafe { + llvm::LLVMConstInBoundsGEP2( + self.type_i8(), + base_addr, + &self.const_usize(offset.bytes()), + 1, + ) + } } } @@ -368,8 +357,7 @@ pub(crate) fn get_dllimport<'tcx>( name: &str, ) -> Option<&'tcx DllImport> { tcx.native_library(id) - .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) - .flatten() + .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) } pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { @@ -422,10 +410,10 @@ pub(crate) fn i686_decorated_name( DllCallingConvention::C => {} DllCallingConvention::Stdcall(arg_list_size) | DllCallingConvention::Fastcall(arg_list_size) => { - write!(&mut decorated_name, "@{}", arg_list_size).unwrap(); + write!(&mut decorated_name, "@{arg_list_size}").unwrap(); } DllCallingConvention::Vectorcall(arg_list_size) => { - write!(&mut decorated_name, "@@{}", arg_list_size).unwrap(); + write!(&mut decorated_name, "@@{arg_list_size}").unwrap(); } } } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index dd3c43ba5ca..95af2f8ef4a 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,32 +1,34 @@ use crate::base; use crate::common::{self, CodegenCx}; use crate::debuginfo; +use crate::errors::{ + InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, +}; use crate::llvm::{self, True}; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use cstr::cstr; -use libc::c_uint; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ - read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer, + read_target_uint, Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar, }; use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, span_bug}; +use rustc_session::config::Lto; use rustc_target::abi::{ - AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, + Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, }; use std::ops::Range; pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1); + let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); let dl = cx.data_layout(); let pointer_size = dl.pointer_size.bytes() as usize; @@ -38,9 +40,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< alloc: &'a Allocation, range: Range<usize>, ) { - let chunks = alloc - .init_mask() - .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end)); + let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); let chunk_to_llval = move |chunk| match chunk { InitChunk::Init(range) => { @@ -58,13 +58,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< // to avoid the cost of generating large complex const expressions. // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. - let max = if llvm_util::get_version() < (14, 0, 0) { - // Generating partially-uninit consts inhibits optimizations in LLVM < 14. - // See https://github.com/rust-lang/rust/issues/84565. - 1 - } else { - cx.sess().opts.unstable_opts.uninit_const_chunk_threshold - }; + let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold; let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; if allow_uninit_chunks { @@ -78,7 +72,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } let mut next_offset = 0; - for &(offset, alloc_id) in alloc.provenance().iter() { + for &(offset, alloc_id) in alloc.provenance().ptrs().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; @@ -98,12 +92,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< .expect("const_alloc_to_llvm: could not read relocation pointer") as u64; - let address_space = match cx.tcx.global_alloc(alloc_id) { - GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, - GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => { - AddressSpace::DATA - } - }; + let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); llvals.push(cx.scalar_to_backend( InterpScalar::from_pointer( @@ -111,10 +100,10 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< &cx.tcx, ), Scalar::Initialized { - value: Primitive::Pointer, + value: Primitive::Pointer(address_space), valid_range: WrappingRange::full(dl.pointer_size), }, - cx.type_i8p_ext(address_space), + cx.type_ptr_ext(address_space), )); next_offset = offset + pointer_size; } @@ -140,13 +129,18 @@ pub fn codegen_static_initializer<'ll, 'tcx>( fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { // The target may require greater alignment for globals than the type does. // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, - // which can force it to be smaller. Rust doesn't support this yet. + // which can force it to be smaller. Rust doesn't support this yet. if let Some(min) = cx.sess().target.min_global_align { match Align::from_bits(min) { Ok(min) => align = align.max(min), - Err(err) => { - cx.sess().err(&format!("invalid minimum global alignment: {}", err)); - } + Err(err) => match err { + AlignFromBytesError::NotPowerOfTwo(align) => { + cx.sess().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align }); + } + AlignFromBytesError::TooLarge(align) => { + cx.sess().emit_err(InvalidMinimumAlignmentTooLarge { align }); + } + }, } } unsafe { @@ -162,29 +156,16 @@ fn check_and_apply_linkage<'ll, 'tcx>( def_id: DefId, ) -> &'ll Value { let llty = cx.layout_of(ty).llvm_type(cx); - if let Some(linkage) = attrs.linkage { + if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); - // If this is a static with a linkage specified, then we need to handle - // it a little specially. The typesystem prevents things like &T and - // extern "C" fn() from being non-null, so we can't just declare a - // static and call it a day. Some linkages (like weak) will make it such - // that the static actually has a null value. - let llty2 = if let ty::RawPtr(ref mt) = ty.kind() { - cx.layout_of(mt.ty).llvm_type(cx) - } else { - cx.sess().span_fatal( - cx.tcx.def_span(def_id), - "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", - ) - }; unsafe { // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(sym, llty2); + let g1 = cx.declare_global(sym, cx.type_i8()); llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); // Declare an internal global `extern_with_linkage_foo` which - // is initialized with the address of `foo`. If `foo` is + // is initialized with the address of `foo`. If `foo` is // discarded during linking (for example, if `foo` has weak // linkage and there are no definitions), then // `extern_with_linkage_foo` will instead be initialized to @@ -192,10 +173,10 @@ fn check_and_apply_linkage<'ll, 'tcx>( let mut real_name = "_rust_extern_with_linkage_".to_string(); real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().span_fatal( - cx.tcx.def_span(def_id), - &format!("symbol `{}` is already defined", &sym), - ) + cx.sess().emit_fatal(SymbolAlreadyDefined { + span: cx.tcx.def_span(def_id), + symbol_name: sym, + }) }); llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); llvm::LLVMSetInitializer(g2, g1); @@ -212,10 +193,6 @@ fn check_and_apply_linkage<'ll, 'tcx>( } } -pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMConstPointerCast(val, ty) } -} - impl<'ll> CodegenCx<'ll, '_> { pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMConstBitCast(val, ty) } @@ -257,8 +234,7 @@ impl<'ll> CodegenCx<'ll, '_> { assert!( !defined_in_current_codegen_unit, "consts::get_static() should always hit the cache for \ - statics defined in the same CGU, but did not for `{:?}`", - def_id + statics defined in the same CGU, but did not for `{def_id:?}`" ); let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); @@ -270,7 +246,7 @@ impl<'ll> CodegenCx<'ll, '_> { let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { let llty = self.layout_of(ty).llvm_type(self); if let Some(g) = self.get_declared_value(sym) { - if self.val_ty(g) != self.type_ptr_to(llty) { + if self.val_ty(g) != self.type_ptr() { span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); } } @@ -298,12 +274,23 @@ impl<'ll> CodegenCx<'ll, '_> { llvm::set_thread_local_mode(g, self.tls_model); } + let dso_local = unsafe { self.should_assume_dso_local(g, true) }; + if dso_local { + unsafe { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + if !def_id.is_local() { let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && + // Local definitions can never be imported, so we must not apply + // the DLLImport annotation. + !dso_local && // ThinLTO can't handle this workaround in all cases, so we don't // emit the attrs. Instead we make them unnecessary by disallowing // dynamic linking when linker plugin based LTO is enabled. - !self.tcx.sess.opts.cg.linker_plugin_lto.enabled(); + !self.tcx.sess.opts.cg.linker_plugin_lto.enabled() && + self.tcx.sess.lto() != Lto::Thin; // If this assertion triggers, there's something wrong with commandline // argument validation. @@ -342,12 +329,6 @@ impl<'ll> CodegenCx<'ll, '_> { } } - unsafe { - if self.should_assume_dso_local(g, true) { - llvm::LLVMRustSetDSOLocal(g, true); - } - } - self.instances.borrow_mut().insert(instance, g); g } @@ -489,7 +470,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // happens to be zero. Instead, we should only check the value of defined bytes // and set all undefined bytes to zero if this allocation is headed for the // BSS. - let all_bytes_are_zero = alloc.provenance().is_empty() + let all_bytes_are_zero = alloc.provenance().ptrs().is_empty() && alloc .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) .iter() @@ -508,29 +489,27 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // go into custom sections of the wasm executable. if self.tcx.sess.target.is_like_wasm { if let Some(section) = attrs.link_section { - let section = llvm::LLVMMDStringInContext( + let section = llvm::LLVMMDStringInContext2( self.llcx, section.as_str().as_ptr().cast(), - section.as_str().len() as c_uint, + section.as_str().len(), ); - assert!(alloc.provenance().is_empty()); + assert!(alloc.provenance().ptrs().is_empty()); // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state (not // as part of the interpreter execution). let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); - let alloc = llvm::LLVMMDStringInContext( - self.llcx, - bytes.as_ptr().cast(), - bytes.len() as c_uint, - ); + let alloc = + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); let data = [section, alloc]; - let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); + let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); + let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, "wasm.custom_sections\0".as_ptr().cast(), - meta, + val, ); } } else { @@ -543,7 +522,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // The semantics of #[used] in Rust only require the symbol to make it into the // object file. It is explicitly allowed for the linker to strip the symbol if it - // is dead, which means we are allowed use `llvm.compiler.used` instead of + // is dead, which means we are allowed to use `llvm.compiler.used` instead of // `llvm.used` here. // // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique @@ -554,7 +533,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // That said, we only ever emit these when compiling for ELF targets, unless // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage // on other targets, in particular MachO targets have *their* static constructor - // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, + // lists broken if `llvm.compiler.used` is emitted rather than `llvm.used`. However, // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_hir_analysis`, // so we don't need to take care of it here. self.add_compiler_used_global(g); @@ -568,16 +547,14 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { } } - /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. + /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr. fn add_used_global(&self, global: &'ll Value) { - let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; - self.used_statics.borrow_mut().push(cast); + self.used_statics.borrow_mut().push(global); } /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, - /// an array of i8*. + /// an array of ptr. fn add_compiler_used_global(&self, global: &'ll Value) { - let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; - self.compiler_used_statics.borrow_mut().push(cast); + self.compiler_used_statics.borrow_mut().push(global); } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index c22ec128dac..b4b2ab1e1f8 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -9,7 +9,8 @@ use crate::type_::Type; use crate::value::Value; use cstr::cstr; -use rustc_codegen_ssa::base::wants_msvc_seh; +use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; +use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; @@ -26,12 +27,14 @@ use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; +use rustc_span::source_map::Spanned; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; +use libc::c_uint; use std::cell::{Cell, RefCell}; use std::ffi::CStr; use std::str; @@ -57,17 +60,6 @@ pub struct CodegenCx<'ll, 'tcx> { /// Cache of constant strings, pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>, - /// Reverse-direction for const ptrs cast from globals. - /// - /// Key is a Value holding a `*T`, - /// Val is a Value holding a `*[T]`. - /// - /// Needed because LLVM loses pointer->pointee association - /// when we ptrcast, and we have to ptrcast during codegen - /// of a `[T]` const because we form a slice, a `(*T,usize)` pair, not - /// a pointer to an LLVM array type. Similar for trait objects. - pub const_unsized: RefCell<FxHashMap<&'ll Value, &'ll Value>>, - /// Cache of emitted const globals (value -> global) pub const_globals: RefCell<FxHashMap<&'ll Value, &'ll Value>>, @@ -142,33 +134,35 @@ pub unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (14, 0, 0) { - if sess.target.llvm_target == "i686-pc-windows-msvc" - || sess.target.llvm_target == "i586-pc-windows-msvc" - { - target_data_layout = - "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" - .to_string(); - } - if sess.target.arch == "wasm32" { - target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); - } - } if llvm_version < (16, 0, 0) { if sess.target.arch == "s390x" { + // LLVM 16 data layout changed to always set 64-bit vector alignment, + // which is conditional in earlier LLVM versions. + // https://reviews.llvm.org/D131158 for the discussion. target_data_layout = target_data_layout.replace("-v128:64", ""); - } - - if sess.target.arch == "riscv64" { + } else if sess.target.arch == "riscv64" { + // LLVM 16 introduced this change so as to produce more efficient code. + // See https://reviews.llvm.org/D116735 for the discussion. target_data_layout = target_data_layout.replace("-n32:64-", "-n64-"); } } + if llvm_version < (17, 0, 0) { + if sess.target.arch.starts_with("powerpc") { + // LLVM 17 specifies function pointer alignment for ppc: + // https://reviews.llvm.org/D147016 + target_data_layout = target_data_layout + .replace("-Fn32", "") + .replace("-Fi32", "") + .replace("-Fn64", "") + .replace("-Fi64", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. if sess.target.is_builtin { + // tm is disposed by its drop impl let tm = crate::back::write::create_informational_target_machine(tcx.sess); - llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, tm); - llvm::LLVMRustDisposeTargetMachine(tm); + llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); let llvm_data_layout = llvm::LLVMGetDataLayoutStr(llmod); let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) @@ -190,7 +184,7 @@ pub unsafe fn create_module<'ll>( // // FIXME(#34960) let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = cfg_llvm_root.trim() != ""; + let custom_llvm_used = !cfg_llvm_root.trim().is_empty(); if !custom_llvm_used && target_data_layout != llvm_data_layout { bug!( @@ -216,7 +210,7 @@ pub unsafe fn create_module<'ll>( // PIE is potentially more effective than PIC, but can only be used in executables. // If all our outputs are executables, then we can relax PIC to PIE. if reloc_model == RelocModel::Pie - || sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) + || tcx.crate_types().iter().all(|ty| *ty == CrateType::Executable) { llvm::LLVMRustSetModulePIELevel(llmod); } @@ -236,18 +230,34 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. + // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) + if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Warning, + llvm::LLVMModFlagBehavior::Override, canonical_jump_tables, 1, ); } + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) + if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + enable_split_lto_unit, + 1, + ); + } + + // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) + if sess.is_sanitizer_kcfi_enabled() { + let kcfi = "kcfi\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); + } + // Control Flow Guard is currently only supported by the MSVC linker on Windows. if sess.target.is_like_msvc { match sess.opts.cg.control_flow_guard { @@ -274,34 +284,43 @@ pub unsafe fn create_module<'ll>( } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { - if sess.target.arch != "aarch64" { - sess.err("-Zbranch-protection is only supported on aarch64"); + let behavior = if llvm_version >= (15, 0, 0) { + llvm::LLVMModFlagBehavior::Min } else { + llvm::LLVMModFlagBehavior::Error + }; + + if sess.target.arch == "aarch64" { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "branch-target-enforcement\0".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "sign-return-address\0".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "sign-return-address-all\0".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, + behavior, "sign-return-address-with-bkey\0".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); + } else { + bug!( + "branch-protection used on non-AArch64 target; \ + this should be checked in rustc_session." + ); } } @@ -332,6 +351,23 @@ pub unsafe fn create_module<'ll>( ); } + // Insert `llvm.ident` metadata. + // + // On the wasm targets it will get hooked up to the "producer" sections + // `processed-by` information. + let rustc_producer = + format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION")); + let name_metadata = llvm::LLVMMDStringInContext( + llcx, + rustc_producer.as_ptr().cast(), + rustc_producer.as_bytes().len() as c_uint, + ); + llvm::LLVMAddNamedMetadataOperand( + llmod, + cstr!("llvm.ident").as_ptr(), + llvm::LLVMMDNodeInContext(llcx, &name_metadata, 1), + ); + llmod } @@ -401,12 +437,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); - let coverage_cx = if tcx.sess.instrument_coverage() { - let covctx = coverageinfo::CrateCoverageContext::new(); - Some(covctx) - } else { - None - }; + let coverage_cx = + tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new); let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); @@ -433,7 +465,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { instances: Default::default(), vtables: Default::default(), const_str_cache: Default::default(), - const_unsized: Default::default(), const_globals: Default::default(), statics_to_rauw: RefCell::new(Vec::new()), used_statics: RefCell::new(Vec::new()), @@ -464,7 +495,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let section = cstr!("llvm.metadata"); - let array = self.const_array(self.type_ptr_to(self.type_i8()), values); + let array = self.const_array(self.type_ptr(), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); @@ -515,24 +546,28 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { if let Some(llpersonality) = self.eh_personality.get() { return llpersonality; } + + let name = if wants_msvc_seh(self.sess()) { + Some("__CxxFrameHandler3") + } else if wants_wasm_eh(self.sess()) { + // LLVM specifically tests for the name of the personality function + // There is no need for this function to exist anywhere, it will + // not be called. However, its name has to be "__gxx_wasm_personality_v0" + // for native wasm exceptions. + Some("__gxx_wasm_personality_v0") + } else { + None + }; + let tcx = self.tcx; let llfn = match tcx.lang_items().eh_personality() { - Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( - ty::Instance::resolve( - tcx, - ty::ParamEnv::reveal_all(), - def_id, - tcx.intern_substs(&[]), - ) - .unwrap() - .unwrap(), + Some(def_id) if name.is_none() => self.get_fn_addr( + ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty()) + .unwrap() + .unwrap(), ), _ => { - let name = if wants_msvc_seh(self.sess()) { - "__CxxFrameHandler3" - } else { - "rust_eh_personality" - }; + let name = name.unwrap_or("rust_eh_personality"); if let Some(llfn) = self.get_declared_value(name) { llfn } else { @@ -574,8 +609,14 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { - if self.get_declared_value("main").is_none() { - Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) + let entry_name = self.sess().target.entry_name.as_ref(); + if self.get_declared_value(entry_name).is_none() { + Some(self.declare_entry_fn( + entry_name, + self.sess().target.entry_abi.into(), + llvm::UnnamedAddr::Global, + fn_type, + )) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} @@ -632,7 +673,7 @@ impl<'ll> CodegenCx<'ll, '_> { ($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false)) } - let i8p = self.type_i8p(); + let ptr = self.type_ptr(); let void = self.type_void(); let i1 = self.type_i1(); let t_i8 = self.type_i8(); @@ -644,6 +685,10 @@ impl<'ll> CodegenCx<'ll, '_> { let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); let t_metadata = self.type_metadata(); + let t_token = self.type_token(); + + ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr); + ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); @@ -678,7 +723,7 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.trap", fn() -> void); ifn!("llvm.debugtrap", fn() -> void); - ifn!("llvm.frameaddress", fn(t_i32) -> i8p); + ifn!("llvm.frameaddress", fn(t_i32) -> ptr); ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); @@ -732,9 +777,13 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.round.f32", fn(t_f32) -> t_f32); ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32); + ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64); + ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); @@ -841,43 +890,44 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128); - ifn!("llvm.lifetime.start.p0i8", fn(t_i64, i8p) -> void); - ifn!("llvm.lifetime.end.p0i8", fn(t_i64, i8p) -> void); + ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void); + ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void); ifn!("llvm.expect.i1", fn(i1, i1) -> i1); - ifn!("llvm.eh.typeid.for", fn(i8p) -> t_i32); + ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32); ifn!("llvm.localescape", fn(...) -> void); - ifn!("llvm.localrecover", fn(i8p, i8p, t_i32) -> i8p); - ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p); + ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr); + ifn!("llvm.x86.seh.recoverfp", fn(ptr, ptr) -> ptr); ifn!("llvm.assume", fn(i1) -> void); - ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + ifn!("llvm.prefetch", fn(ptr, t_i32, t_i32, t_i32) -> void); // This isn't an "LLVM intrinsic", but LLVM's optimization passes - // recognize it like one and we assume it exists in `core::slice::cmp` + // recognize it like one (including turning it into `bcmp` sometimes) + // and we use it to implement intrinsics like `raw_eq` and `compare_bytes` match self.sess().target.arch.as_ref() { - "avr" | "msp430" => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i16), - _ => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32), + "avr" | "msp430" => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i16), + _ => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i32), } // variadic intrinsics - ifn!("llvm.va_start", fn(i8p) -> void); - ifn!("llvm.va_end", fn(i8p) -> void); - ifn!("llvm.va_copy", fn(i8p, i8p) -> void); + ifn!("llvm.va_start", fn(ptr) -> void); + ifn!("llvm.va_end", fn(ptr) -> void); + ifn!("llvm.va_copy", fn(ptr, ptr) -> void); if self.sess().instrument_coverage() { - ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); + ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); } - ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); - ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); + ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1}); if self.sess().opts.debuginfo != DebugInfo::None { ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } - ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); + ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr); None } @@ -891,12 +941,10 @@ impl<'ll> CodegenCx<'ll, '_> { let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { Some(def_id) => self.get_static(def_id), _ => { - let ty = self - .type_struct(&[self.type_ptr_to(self.type_isize()), self.type_i8p()], false); + let ty = self.type_struct(&[self.type_ptr(), self.type_ptr()], false); self.declare_global("rust_eh_catch_typeinfo", ty) } }; - let eh_catch_typeinfo = self.const_bitcast(eh_catch_typeinfo, self.type_i8p()); self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo)); eh_catch_typeinfo } @@ -950,10 +998,10 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) = err { - self.sess().span_fatal(span, &err.to_string()) + if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { - span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + self.tcx.sess.emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) } } } @@ -969,25 +1017,16 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.sess().span_fatal(span, &err.to_string()) + self.sess().emit_fatal(Spanned { span, node: err }) } else { match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { - span_bug!( - span, - "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", - sig, - extra_args, - err - ); + span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } FnAbiRequest::OfInstance { instance, extra_args } => { span_bug!( span, - "`fn_abi_of_instance({}, {:?})` failed: {}", - instance, - extra_args, - err + "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs new file mode 100644 index 00000000000..763186a58bf --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -0,0 +1,279 @@ +use rustc_middle::mir::coverage::{CounterId, ExpressionId, Operand}; + +/// Must match the layout of `LLVMRustCounterKind`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum CounterKind { + Zero = 0, + CounterValueReference = 1, + Expression = 2, +} + +/// A reference to an instance of an abstract "counter" that will yield a value in a coverage +/// report. Note that `id` has different interpretations, depending on the `kind`: +/// * For `CounterKind::Zero`, `id` is assumed to be `0` +/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected +/// instrumentation counter (the `index` argument to the LLVM intrinsic +/// `instrprof.increment()`) +/// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of +/// counter expressions. +/// +/// Corresponds to struct `llvm::coverage::Counter`. +/// +/// Must match the layout of `LLVMRustCounter`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct Counter { + // Important: The layout (order and types of fields) must match its C++ counterpart. + pub kind: CounterKind, + id: u32, +} + +impl Counter { + /// A `Counter` of kind `Zero`. For this counter kind, the `id` is not used. + pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; + + /// Constructs a new `Counter` of kind `CounterValueReference`. + pub fn counter_value_reference(counter_id: CounterId) -> Self { + Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } + } + + /// Constructs a new `Counter` of kind `Expression`. + pub(crate) fn expression(expression_id: ExpressionId) -> Self { + Self { kind: CounterKind::Expression, id: expression_id.as_u32() } + } + + pub(crate) fn from_operand(operand: Operand) -> Self { + match operand { + Operand::Zero => Self::ZERO, + Operand::Counter(id) => Self::counter_value_reference(id), + Operand::Expression(id) => Self::expression(id), + } + } +} + +/// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`. +/// +/// Must match the layout of `LLVMRustCounterExprKind`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum ExprKind { + Subtract = 0, + Add = 1, +} + +/// Corresponds to struct `llvm::coverage::CounterExpression`. +/// +/// Must match the layout of `LLVMRustCounterExpression`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct CounterExpression { + pub kind: ExprKind, + pub lhs: Counter, + pub rhs: Counter, +} + +impl CounterExpression { + /// The dummy expression `(0 - 0)` has a representation of all zeroes, + /// making it marginally more efficient to initialize than `(0 + 0)`. + pub(crate) const DUMMY: Self = + Self { lhs: Counter::ZERO, kind: ExprKind::Subtract, rhs: Counter::ZERO }; + + pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { + Self { kind, lhs, rhs } + } +} + +/// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. +/// +/// Must match the layout of `LLVMRustCounterMappingRegionKind`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum RegionKind { + /// A CodeRegion associates some code with a counter + CodeRegion = 0, + + /// An ExpansionRegion represents a file expansion region that associates + /// a source range with the expansion of a virtual source file, such as + /// for a macro instantiation or #include file. + ExpansionRegion = 1, + + /// A SkippedRegion represents a source range with code that was skipped + /// by a preprocessor or similar means. + SkippedRegion = 2, + + /// A GapRegion is like a CodeRegion, but its count is only set as the + /// line execution count when its the only region in the line. + GapRegion = 3, + + /// A BranchRegion represents leaf-level boolean expressions and is + /// associated with two counters, each representing the number of times the + /// expression evaluates to true or false. + BranchRegion = 4, +} + +/// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the +/// coverage map, in accordance with the +/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). +/// The struct composes fields representing the `Counter` type and value(s) (injected counter +/// ID, or expression type and operands), the source file (an indirect index into a "filenames +/// array", encoded separately), and source location (start and end positions of the represented +/// code region). +/// +/// Corresponds to struct `llvm::coverage::CounterMappingRegion`. +/// +/// Must match the layout of `LLVMRustCounterMappingRegion`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct CounterMappingRegion { + /// The counter type and type-dependent counter data, if any. + counter: Counter, + + /// If the `RegionKind` is a `BranchRegion`, this represents the counter + /// for the false branch of the region. + false_counter: Counter, + + /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the + /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes + /// that, in turn, are used to look up the filename for this region. + file_id: u32, + + /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find + /// the mapping regions created as a result of macro expansion, by checking if their file id + /// matches the expanded file id. + expanded_file_id: u32, + + /// 1-based starting line of the mapping region. + start_line: u32, + + /// 1-based starting column of the mapping region. + start_col: u32, + + /// 1-based ending line of the mapping region. + end_line: u32, + + /// 1-based ending column of the mapping region. If the high bit is set, the current + /// mapping region is a gap area. + end_col: u32, + + kind: RegionKind, +} + +impl CounterMappingRegion { + pub(crate) fn code_region( + counter: Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: Counter::ZERO, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::CodeRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn branch_region( + counter: Counter, + false_counter: Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::BranchRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn expansion_region( + file_id: u32, + expanded_file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: Counter::ZERO, + false_counter: Counter::ZERO, + file_id, + expanded_file_id, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::ExpansionRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn skipped_region( + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: Counter::ZERO, + false_counter: Counter::ZERO, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::SkippedRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn gap_region( + counter: Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: Counter::ZERO, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col: (1_u32 << 31) | end_col, + kind: RegionKind::GapRegion, + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs new file mode 100644 index 00000000000..55f43aa5341 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -0,0 +1,294 @@ +use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; + +use rustc_data_structures::fx::FxIndexSet; +use rustc_index::IndexVec; +use rustc_middle::mir::coverage::{CodeRegion, CounterId, ExpressionId, Op, Operand}; +use rustc_middle::ty::Instance; +use rustc_middle::ty::TyCtxt; + +#[derive(Clone, Debug, PartialEq)] +pub struct Expression { + lhs: Operand, + op: Op, + rhs: Operand, + code_regions: Vec<CodeRegion>, +} + +/// Collects all of the coverage regions associated with (a) injected counters, (b) counter +/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), +/// for a given Function. This struct also stores the `function_source_hash`, +/// computed during instrumentation, and forwarded with counters. +/// +/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap +/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter +/// or expression), but the line or lines in the gap region are not executable (such as lines with +/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count +/// for a gap area is only used as the line execution count if there are no other regions on a +/// line." +#[derive(Debug)] +pub struct FunctionCoverage<'tcx> { + instance: Instance<'tcx>, + source_hash: u64, + is_used: bool, + counters: IndexVec<CounterId, Option<Vec<CodeRegion>>>, + expressions: IndexVec<ExpressionId, Option<Expression>>, + unreachable_regions: Vec<CodeRegion>, +} + +impl<'tcx> FunctionCoverage<'tcx> { + /// Creates a new set of coverage data for a used (called) function. + pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + Self::create(tcx, instance, true) + } + + /// Creates a new set of coverage data for an unused (never called) function. + pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + Self::create(tcx, instance, false) + } + + fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { + let coverageinfo = tcx.coverageinfo(instance.def); + debug!( + "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", + instance, coverageinfo, is_used + ); + Self { + instance, + source_hash: 0, // will be set with the first `add_counter()` + is_used, + counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), + expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), + unreachable_regions: Vec::new(), + } + } + + /// Returns true for a used (called) function, and false for an unused function. + pub fn is_used(&self) -> bool { + self.is_used + } + + /// Sets the function source hash value. If called multiple times for the same function, all + /// calls should have the same hash value. + pub fn set_function_source_hash(&mut self, source_hash: u64) { + if self.source_hash == 0 { + self.source_hash = source_hash; + } else { + debug_assert_eq!(source_hash, self.source_hash); + } + } + + /// Adds code regions to be counted by an injected counter intrinsic. + #[instrument(level = "debug", skip(self))] + pub(crate) fn add_counter(&mut self, id: CounterId, code_regions: &[CodeRegion]) { + if code_regions.is_empty() { + return; + } + + let slot = &mut self.counters[id]; + match slot { + None => *slot = Some(code_regions.to_owned()), + // If this counter ID slot has already been filled, it should + // contain identical information. + Some(ref previous_regions) => assert_eq!( + previous_regions, code_regions, + "add_counter: code regions for id changed" + ), + } + } + + /// Adds information about a coverage expression, along with zero or more + /// code regions mapped to that expression. + /// + /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other + /// expressions. These are tracked as separate variants of `Operand`, so there is no ambiguity + /// between operands that are counter IDs and operands that are expression IDs. + #[instrument(level = "debug", skip(self))] + pub(crate) fn add_counter_expression( + &mut self, + expression_id: ExpressionId, + lhs: Operand, + op: Op, + rhs: Operand, + code_regions: &[CodeRegion], + ) { + debug_assert!( + expression_id.as_usize() < self.expressions.len(), + "expression_id {} is out of range for expressions.len() = {} + for {:?}", + expression_id.as_usize(), + self.expressions.len(), + self, + ); + + let expression = Expression { lhs, op, rhs, code_regions: code_regions.to_owned() }; + let slot = &mut self.expressions[expression_id]; + match slot { + None => *slot = Some(expression), + // If this expression ID slot has already been filled, it should + // contain identical information. + Some(ref previous_expression) => assert_eq!( + previous_expression, &expression, + "add_counter_expression: expression for id changed" + ), + } + } + + /// Adds regions that will be marked as "unreachable", with a constant "zero counter". + #[instrument(level = "debug", skip(self))] + pub(crate) fn add_unreachable_regions(&mut self, code_regions: &[CodeRegion]) { + assert!(!code_regions.is_empty(), "unreachable regions always have code regions"); + self.unreachable_regions.extend_from_slice(code_regions); + } + + /// Perform some simplifications to make the final coverage mappings + /// slightly smaller. + /// + /// This method mainly exists to preserve the simplifications that were + /// already being performed by the Rust-side expression renumbering, so that + /// the resulting coverage mappings don't get worse. + pub(crate) fn simplify_expressions(&mut self) { + // The set of expressions that either were optimized out entirely, or + // have zero as both of their operands, and will therefore always have + // a value of zero. Other expressions that refer to these as operands + // can have those operands replaced with `Operand::Zero`. + let mut zero_expressions = FxIndexSet::default(); + + // For each expression, perform simplifications based on lower-numbered + // expressions, and then update the set of always-zero expressions if + // necessary. + // (By construction, expressions can only refer to other expressions + // that have lower IDs, so one simplification pass is sufficient.) + for (id, maybe_expression) in self.expressions.iter_enumerated_mut() { + let Some(expression) = maybe_expression else { + // If an expression is missing, it must have been optimized away, + // so any operand that refers to it can be replaced with zero. + zero_expressions.insert(id); + continue; + }; + + // If an operand refers to an expression that is always zero, then + // that operand can be replaced with `Operand::Zero`. + let maybe_set_operand_to_zero = |operand: &mut Operand| match &*operand { + Operand::Expression(id) if zero_expressions.contains(id) => { + *operand = Operand::Zero; + } + _ => (), + }; + maybe_set_operand_to_zero(&mut expression.lhs); + maybe_set_operand_to_zero(&mut expression.rhs); + + // Coverage counter values cannot be negative, so if an expression + // involves subtraction from zero, assume that its RHS must also be zero. + // (Do this after simplifications that could set the LHS to zero.) + if let Expression { lhs: Operand::Zero, op: Op::Subtract, .. } = expression { + expression.rhs = Operand::Zero; + } + + // After the above simplifications, if both operands are zero, then + // we know that this expression is always zero too. + if let Expression { lhs: Operand::Zero, rhs: Operand::Zero, .. } = expression { + zero_expressions.insert(id); + } + } + } + + /// Return the source hash, generated from the HIR node structure, and used to indicate whether + /// or not the source code structure changed between different compilations. + pub fn source_hash(&self) -> u64 { + self.source_hash + } + + /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their + /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create + /// `CounterMappingRegion`s. + pub fn get_expressions_and_counter_regions( + &self, + ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { + assert!( + self.source_hash != 0 || !self.is_used, + "No counters provided the source_hash for used function: {:?}", + self.instance + ); + + let counter_expressions = self.counter_expressions(); + // Expression IDs are indices into `self.expressions`, and on the LLVM + // side they will be treated as indices into `counter_expressions`, so + // the two vectors should correspond 1:1. + assert_eq!(self.expressions.len(), counter_expressions.len()); + + let counter_regions = self.counter_regions(); + let expression_regions = self.expression_regions(); + let unreachable_regions = self.unreachable_regions(); + + let counter_regions = + counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); + (counter_expressions, counter_regions) + } + + fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { + self.counters + .iter_enumerated() + // Filter out counter IDs that we never saw during MIR traversal. + // This can happen if a counter was optimized out by MIR transforms + // (and replaced with `CoverageKind::Unreachable` instead). + .filter_map(|(id, maybe_code_regions)| Some((id, maybe_code_regions.as_ref()?))) + .flat_map(|(id, code_regions)| { + let counter = Counter::counter_value_reference(id); + code_regions.iter().map(move |region| (counter, region)) + }) + } + + /// Convert this function's coverage expression data into a form that can be + /// passed through FFI to LLVM. + fn counter_expressions(&self) -> Vec<CounterExpression> { + // We know that LLVM will optimize out any unused expressions before + // producing the final coverage map, so there's no need to do the same + // thing on the Rust side unless we're confident we can do much better. + // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.) + + self.expressions + .iter() + .map(|expression| match expression { + None => { + // This expression ID was allocated, but we never saw the + // actual expression, so it must have been optimized out. + // Replace it with a dummy expression, and let LLVM take + // care of omitting it from the expression list. + CounterExpression::DUMMY + } + &Some(Expression { lhs, op, rhs, .. }) => { + // Convert the operands and operator as normal. + CounterExpression::new( + Counter::from_operand(lhs), + match op { + Op::Add => ExprKind::Add, + Op::Subtract => ExprKind::Subtract, + }, + Counter::from_operand(rhs), + ) + } + }) + .collect::<Vec<_>>() + } + + fn expression_regions(&self) -> Vec<(Counter, &CodeRegion)> { + // Find all of the expression IDs that weren't optimized out AND have + // one or more attached code regions, and return the corresponding + // mappings as counter/region pairs. + self.expressions + .iter_enumerated() + .filter_map(|(id, maybe_expression)| { + let code_regions = &maybe_expression.as_ref()?.code_regions; + Some((id, code_regions)) + }) + .flat_map(|(id, code_regions)| { + let counter = Counter::expression(id); + code_regions.iter().map(move |code_region| (counter, code_region)) + }) + .collect::<Vec<_>>() + } + + fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { + self.unreachable_regions.iter().map(|region| (Counter::ZERO, region)) + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 433f043209e..d4e77525698 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,25 +1,24 @@ use crate::common::CodegenCx; use crate::coverageinfo; +use crate::coverageinfo::ffi::CounterMappingRegion; +use crate::coverageinfo::map_data::FunctionCoverage; use crate::llvm; -use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; -use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; +use rustc_codegen_ssa::traits::ConstMethods; use rustc_data_structures::fx::FxIndexSet; use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefIdSet; -use rustc_llvm::RustString; +use rustc_hir::def_id::DefId; +use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::CodeRegion; use rustc_middle::ty::TyCtxt; - -use std::ffi::CString; +use rustc_span::Symbol; /// Generates and exports the Coverage Map. /// -/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions -/// 5 (LLVM 12, only) and 6 (zero-based encoded as 4 and 5, respectively), as defined at +/// Rust Coverage Map generation supports LLVM Coverage Mapping Format version +/// 6 (zero-based encoded as 5), as defined at /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). /// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) /// bundled with Rust's fork of LLVM. @@ -29,16 +28,13 @@ use std::ffi::CString; /// implementing this Rust version, and though the format documentation is very explicit and /// detailed, some undocumented details in Clang's implementation (that may or may not be important) /// were also replicated for Rust's Coverage Map. -pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { +pub fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; - // Ensure the installed version of LLVM supports at least Coverage Map - // Version 5 (encoded as a zero-based value: 4), which was introduced with - // LLVM 12. + // Ensure the installed version of LLVM supports Coverage Map Version 6 + // (encoded as a zero-based value: 5), which was introduced with LLVM 13. let version = coverageinfo::mapping_version(); - if version < 4 { - tcx.sess.fatal("rustc option `-C instrument-coverage` requires LLVM 12 or higher."); - } + assert_eq!(version, 5, "The `CoverageMappingVersion` exposed by `llvm-wrapper` is out of sync"); debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); @@ -60,21 +56,21 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { return; } - let mut mapgen = CoverageMapGenerator::new(tcx, version); + let mut global_file_table = GlobalFileTable::new(tcx); // Encode coverage mappings and generate function records let mut function_data = Vec::new(); - for (instance, function_coverage) in function_coverage_map { + for (instance, mut function_coverage) in function_coverage_map { debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); - let mangled_function_name = tcx.symbol_name(instance).to_string(); + function_coverage.simplify_expressions(); + let function_coverage = function_coverage; + + let mangled_function_name = tcx.symbol_name(instance).name; let source_hash = function_coverage.source_hash(); let is_used = function_coverage.is_used(); - let (expressions, counter_regions) = - function_coverage.get_expressions_and_counter_regions(); - let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| { - mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); - }); + let coverage_mapping_buffer = + encode_mappings_for_function(&mut global_file_table, &function_coverage); if coverage_mapping_buffer.is_empty() { if function_coverage.is_used() { @@ -92,20 +88,20 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { } // Encode all filenames referenced by counters/expressions in this module - let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| { - coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer); - }); + let filenames_buffer = global_file_table.into_filenames_buffer(); let filenames_size = filenames_buffer.len(); let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(filenames_buffer); + let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); // Generate the LLVM IR representation of the coverage map and store it in a well-known global - let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); + let cov_data_val = generate_coverage_map(cx, version, filenames_size, filenames_val); + let covfun_section_name = coverageinfo::covfun_section_name(cx); for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { save_function_record( cx, + &covfun_section_name, mangled_function_name, source_hash, filenames_ref, @@ -118,119 +114,129 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { coverageinfo::save_cov_data_to_mod(cx, cov_data_val); } -struct CoverageMapGenerator { - filenames: FxIndexSet<CString>, +struct GlobalFileTable { + global_file_table: FxIndexSet<Symbol>, } -impl CoverageMapGenerator { - fn new(tcx: TyCtxt<'_>, version: u32) -> Self { - let mut filenames = FxIndexSet::default(); - if version >= 5 { - // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) - // requires setting the first filename to the compilation directory. - // Since rustc generates coverage maps with relative paths, the - // compilation directory can be combined with the relative paths - // to get absolute paths, if needed. - let working_dir = tcx - .sess - .opts - .working_dir - .remapped_path_if_available() - .to_string_lossy() - .to_string(); - let c_filename = - CString::new(working_dir).expect("null error converting filename to C string"); - filenames.insert(c_filename); - } - Self { filenames } +impl GlobalFileTable { + fn new(tcx: TyCtxt<'_>) -> Self { + let mut global_file_table = FxIndexSet::default(); + // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) + // requires setting the first filename to the compilation directory. + // Since rustc generates coverage maps with relative paths, the + // compilation directory can be combined with the relative paths + // to get absolute paths, if needed. + let working_dir = Symbol::intern( + &tcx.sess.opts.working_dir.remapped_path_if_available().to_string_lossy(), + ); + global_file_table.insert(working_dir); + Self { global_file_table } } - /// Using the `expressions` and `counter_regions` collected for the current function, generate - /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use - /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into - /// the given `coverage_mapping` byte buffer, compliant with the LLVM Coverage Mapping format. - fn write_coverage_mapping<'a>( - &mut self, - expressions: Vec<CounterExpression>, - counter_regions: impl Iterator<Item = (Counter, &'a CodeRegion)>, - coverage_mapping_buffer: &RustString, - ) { - let mut counter_regions = counter_regions.collect::<Vec<_>>(); - if counter_regions.is_empty() { - return; - } + fn global_file_id_for_file_name(&mut self, file_name: Symbol) -> u32 { + let (global_file_id, _) = self.global_file_table.insert_full(file_name); + global_file_id as u32 + } - let mut virtual_file_mapping = Vec::new(); - let mut mapping_regions = Vec::new(); - let mut current_file_name = None; - let mut current_file_id = 0; - - // Convert the list of (Counter, CodeRegion) pairs to an array of `CounterMappingRegion`, sorted - // by filename and position. Capture any new files to compute the `CounterMappingRegion`s - // `file_id` (indexing files referenced by the current function), and construct the - // function-specific `virtual_file_mapping` from `file_id` to its index in the module's - // `filenames` array. - counter_regions.sort_unstable_by_key(|(_counter, region)| *region); - for (counter, region) in counter_regions { - let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; - let same_file = current_file_name.as_ref().map_or(false, |p| *p == file_name); - if !same_file { - if current_file_name.is_some() { - current_file_id += 1; - } - current_file_name = Some(file_name); - let c_filename = CString::new(file_name.to_string()) - .expect("null error converting filename to C string"); - debug!(" file_id: {} = '{:?}'", current_file_id, c_filename); - let (filenames_index, _) = self.filenames.insert_full(c_filename); - virtual_file_mapping.push(filenames_index as u32); - } - debug!("Adding counter {:?} to map for {:?}", counter, region); + fn into_filenames_buffer(self) -> Vec<u8> { + // This method takes `self` so that the caller can't accidentally + // modify the original file table after encoding it into a buffer. + + llvm::build_byte_buffer(|buffer| { + coverageinfo::write_filenames_section_to_buffer( + self.global_file_table.iter().map(Symbol::as_str), + buffer, + ); + }) + } +} + +/// Using the expressions and counter regions collected for a single function, +/// generate the variable-sized payload of its corresponding `__llvm_covfun` +/// entry. The payload is returned as a vector of bytes. +/// +/// Newly-encountered filenames will be added to the global file table. +fn encode_mappings_for_function( + global_file_table: &mut GlobalFileTable, + function_coverage: &FunctionCoverage<'_>, +) -> Vec<u8> { + let (expressions, counter_regions) = function_coverage.get_expressions_and_counter_regions(); + + let mut counter_regions = counter_regions.collect::<Vec<_>>(); + if counter_regions.is_empty() { + return Vec::new(); + } + + let mut virtual_file_mapping = IndexVec::<u32, u32>::new(); + let mut mapping_regions = Vec::with_capacity(counter_regions.len()); + + // Sort the list of (counter, region) mapping pairs by region, so that they + // can be grouped by filename. Prepare file IDs for each filename, and + // prepare the mapping data so that we can pass it through FFI to LLVM. + counter_regions.sort_by_key(|(_counter, region)| *region); + for counter_regions_for_file in + counter_regions.group_by(|(_, a), (_, b)| a.file_name == b.file_name) + { + // Look up (or allocate) the global file ID for this filename. + let file_name = counter_regions_for_file[0].1.file_name; + let global_file_id = global_file_table.global_file_id_for_file_name(file_name); + + // Associate that global file ID with a local file ID for this function. + let local_file_id: u32 = virtual_file_mapping.push(global_file_id); + debug!(" file id: local {local_file_id} => global {global_file_id} = '{file_name:?}'"); + + // For each counter/region pair in this function+file, convert it to a + // form suitable for FFI. + for &(counter, region) in counter_regions_for_file { + let CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = *region; + + debug!("Adding counter {counter:?} to map for {region:?}"); mapping_regions.push(CounterMappingRegion::code_region( counter, - current_file_id, + local_file_id, start_line, start_col, end_line, end_col, )); } + } - // Encode and append the current function's coverage mapping data + // Encode the function's coverage mappings into a buffer. + llvm::build_byte_buffer(|buffer| { coverageinfo::write_mapping_to_buffer( - virtual_file_mapping, + virtual_file_mapping.raw, expressions, mapping_regions, - coverage_mapping_buffer, + buffer, ); - } + }) +} - /// Construct coverage map header and the array of function records, and combine them into the - /// coverage map. Save the coverage map data into the LLVM IR as a static global using a - /// specific, well-known section and name. - fn generate_coverage_map<'ll>( - self, - cx: &CodegenCx<'ll, '_>, - version: u32, - filenames_size: usize, - filenames_val: &'ll llvm::Value, - ) -> &'ll llvm::Value { - debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); - - // Create the coverage data header (Note, fields 0 and 2 are now always zero, - // as of `llvm::coverage::CovMapVersion::Version4`.) - let zero_was_n_records_val = cx.const_u32(0); - let filenames_size_val = cx.const_u32(filenames_size as u32); - let zero_was_coverage_size_val = cx.const_u32(0); - let version_val = cx.const_u32(version); - let cov_data_header_val = cx.const_struct( - &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val], - /*packed=*/ false, - ); +/// Construct coverage map header and the array of function records, and combine them into the +/// coverage map. Save the coverage map data into the LLVM IR as a static global using a +/// specific, well-known section and name. +fn generate_coverage_map<'ll>( + cx: &CodegenCx<'ll, '_>, + version: u32, + filenames_size: usize, + filenames_val: &'ll llvm::Value, +) -> &'ll llvm::Value { + debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); + + // Create the coverage data header (Note, fields 0 and 2 are now always zero, + // as of `llvm::coverage::CovMapVersion::Version4`.) + let zero_was_n_records_val = cx.const_u32(0); + let filenames_size_val = cx.const_u32(filenames_size as u32); + let zero_was_coverage_size_val = cx.const_u32(0); + let version_val = cx.const_u32(version); + let cov_data_header_val = cx.const_struct( + &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val], + /*packed=*/ false, + ); - // Create the complete LLVM coverage data value to add to the LLVM IR - cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) - } + // Create the complete LLVM coverage data value to add to the LLVM IR + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) } /// Construct a function record and combine it with the function's coverage mapping data. @@ -238,7 +244,8 @@ impl CoverageMapGenerator { /// specific, well-known section and name. fn save_function_record( cx: &CodegenCx<'_, '_>, - mangled_function_name: String, + covfun_section_name: &str, + mangled_function_name: &str, source_hash: u64, filenames_ref: u64, coverage_mapping_buffer: Vec<u8>, @@ -248,7 +255,7 @@ fn save_function_record( let coverage_mapping_size = coverage_mapping_buffer.len(); let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); - let func_name_hash = coverageinfo::hash_str(&mangled_function_name); + let func_name_hash = coverageinfo::hash_bytes(mangled_function_name.as_bytes()); let func_name_hash_val = cx.const_u64(func_name_hash); let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); let source_hash_val = cx.const_u64(source_hash); @@ -264,7 +271,13 @@ fn save_function_record( /*packed=*/ true, ); - coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); + coverageinfo::save_func_record_to_mod( + cx, + covfun_section_name, + func_name_hash, + func_record_val, + is_used, + ); } /// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for @@ -283,14 +296,14 @@ fn save_function_record( /// "code coverage dead code cgu" during the partitioning process. This prevents us from generating /// code regions for the same function more than once which can lead to linker errors regarding /// duplicate symbols. -fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { +fn add_unused_functions(cx: &CodegenCx<'_, '_>) { assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); let tcx = cx.tcx; let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let eligible_def_ids: DefIdSet = tcx + let eligible_def_ids: Vec<DefId> = tcx .mir_keys(()) .iter() .filter_map(|local_def_id| { @@ -305,9 +318,8 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator ) { return None; - } else if ignore_unused_generics - && tcx.generics_of(def_id).requires_monomorphization(tcx) - { + } + if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { return None; } Some(local_def_id.to_def_id()) @@ -316,13 +328,15 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { let codegenned_def_ids = tcx.codegened_and_inlined_items(()); - for &non_codegenned_def_id in eligible_def_ids.difference(codegenned_def_ids) { + for non_codegenned_def_id in + eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) + { let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); - // If a function is marked `#[no_coverage]`, then skip generating a + // If a function is marked `#[coverage(off)]`, then skip generating a // dead code stub for it. if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { - debug!("skipping unused fn marked #[no_coverage]: {:?}", non_codegenned_def_id); + debug!("skipping unused fn marked #[coverage(off)]: {:?}", non_codegenned_def_id); continue; } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 964a632b6ee..dd2ce9b525b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -3,41 +3,40 @@ use crate::llvm; use crate::abi::Abi; use crate::builder::Builder; use crate::common::CodegenCx; +use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; +use crate::coverageinfo::map_data::FunctionCoverage; use libc::c_uint; -use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; use rustc_codegen_ssa::traits::{ - BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, - MiscMethods, StaticMethods, + BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, + StaticMethods, }; use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::{ - CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, -}; +use rustc_middle::mir::coverage::{CounterId, CoverageKind}; +use rustc_middle::mir::Coverage; use rustc_middle::ty; -use rustc_middle::ty::layout::FnAbiOf; -use rustc_middle::ty::subst::InternalSubsts; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; +use rustc_middle::ty::GenericArgs; use rustc_middle::ty::Instance; +use rustc_middle::ty::Ty; use std::cell::RefCell; -use std::ffi::CString; - -use std::iter; +pub(crate) mod ffi; +pub(crate) mod map_data; pub mod mapgen; -const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; +const UNUSED_FUNCTION_COUNTER_ID: CounterId = CounterId::START; const VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'ll, 'tcx> { - // Coverage data for each instrumented function identified by DefId. + /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -55,11 +54,17 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } -impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn coverageinfo_finalize(&self) { +// These methods used to be part of trait `CoverageInfoMethods`, which no longer +// exists after most coverage code was moved out of SSA. +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + pub(crate) fn coverageinfo_finalize(&self) { mapgen::finalize(self) } + /// For LLVM codegen, returns a function-specific `Value` for a global + /// string, to hold the function name passed to LLVM intrinsic + /// `instrprof.increment()`. The `Value` is only created once per instance. + /// Multiple invocations with the same instance return the same `Value`. fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { if let Some(coverage_context) = self.coverage_context() { debug!("getting pgo_func_name_var for instance={:?}", instance); @@ -84,9 +89,7 @@ impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { /// `function_coverage_map` (keyed by function `Instance`) during codegen. /// But in this case, since the unused function was _not_ previously /// codegenned, collect the coverage `CodeRegion`s from the MIR and add - /// them. The first `CodeRegion` is used to add a single counter, with the - /// same counter ID used in the injected `instrprof.increment` intrinsic - /// call. Since the function is never called, all other `CodeRegion`s can be + /// them. Since the function is never called, all of its `CodeRegion`s can be /// added as `unreachable_region`s. fn define_unused_fn(&self, def_id: DefId) { let instance = declare_unused_fn(self, def_id); @@ -96,89 +99,46 @@ impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn set_function_source_hash( - &mut self, - instance: Instance<'tcx>, - function_source_hash: u64, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "ensuring function source hash is set for instance={:?}; function_source_hash={}", - instance, function_source_hash, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .set_function_source_hash(function_source_hash); - true - } else { - false - } - } - - fn add_coverage_counter( - &mut self, - instance: Instance<'tcx>, - id: CounterValueReference, - region: CodeRegion, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", - instance, id, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter(id, region); - true - } else { - false - } - } - - fn add_coverage_counter_expression( - &mut self, - instance: Instance<'tcx>, - id: InjectedExpressionId, - lhs: ExpressionOperandId, - op: Op, - rhs: ExpressionOperandId, - region: Option<CodeRegion>, - ) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ - region: {:?}", - instance, id, lhs, op, rhs, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter_expression(id, lhs, op, rhs, region); - true - } else { - false - } - } - - fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { - if let Some(coverage_context) = self.coverage_context() { - debug!( - "adding unreachable code to coverage_map: instance={:?}, at {:?}", - instance, region, - ); - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); - coverage_map - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_unreachable_region(region); - true - } else { - false + fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { + let bx = self; + + let Some(coverage_context) = bx.coverage_context() else { return }; + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + let func_coverage = coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance)); + + let Coverage { kind, code_regions } = coverage; + match *kind { + CoverageKind::Counter { function_source_hash, id } => { + debug!( + "ensuring function source hash is set for instance={:?}; function_source_hash={}", + instance, function_source_hash, + ); + func_coverage.set_function_source_hash(function_source_hash); + func_coverage.add_counter(id, code_regions); + // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, + // as that needs an exclusive borrow. + drop(coverage_map); + + let coverageinfo = bx.tcx().coverageinfo(instance.def); + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_source_hash); + let num_counters = bx.const_u32(coverageinfo.num_counters); + let index = bx.const_u32(id.as_u32()); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, index, + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + } + CoverageKind::Expression { id, lhs, op, rhs } => { + func_coverage.add_counter_expression(id, lhs, op, rhs, code_regions); + } + CoverageKind::Unreachable => { + func_coverage.add_unreachable_regions(code_regions); + } } } } @@ -188,7 +148,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< let instance = Instance::new( def_id, - InternalSubsts::for_item(tcx, def_id, |param, _| { + GenericArgs::for_item(tcx, def_id, |param, _| { if let ty::GenericParamDefKind::Lifetime = param.kind { tcx.lifetimes.re_erased.into() } else { @@ -201,14 +161,15 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< tcx.symbol_name(instance).name, cx.fn_abi_of_fn_ptr( ty::Binder::dummy(tcx.mk_fn_sig( - iter::once(tcx.mk_unit()), - tcx.mk_unit(), + [Ty::new_unit(tcx)], + Ty::new_unit(tcx), false, hir::Unsafety::Unsafe, Abi::Rust, )), ty::List::empty(), ), + None, ); llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); @@ -244,14 +205,9 @@ fn add_unused_function_coverage<'tcx>( let tcx = cx.tcx; let mut function_coverage = FunctionCoverage::unused(tcx, instance); - for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { - if index == 0 { - // Insert at least one real counter so the LLVM CoverageMappingReader will find expected - // definitions. - function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); - } else { - function_coverage.add_unreachable_region(code_region.clone()); - } + for &code_region in tcx.covered_code_regions(def_id) { + let code_region = std::slice::from_ref(code_region); + function_coverage.add_unreachable_regions(code_region); } if let Some(coverage_context) = cx.coverage_context() { @@ -271,21 +227,32 @@ fn create_pgo_func_name_var<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, ) -> &'ll llvm::Value { - let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) - .expect("error converting function name to C string"); + let mangled_fn_name: &str = cx.tcx.symbol_name(instance).name; let llfn = cx.get_fn(instance); - unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } + unsafe { + llvm::LLVMRustCoverageCreatePGOFuncNameVar( + llfn, + mangled_fn_name.as_ptr().cast(), + mangled_fn_name.len(), + ) + } } pub(crate) fn write_filenames_section_to_buffer<'a>( - filenames: impl IntoIterator<Item = &'a CString>, + filenames: impl IntoIterator<Item = &'a str>, buffer: &RustString, ) { - let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>(); + let (pointers, lengths) = filenames + .into_iter() + .map(|s: &str| (s.as_ptr().cast(), s.len())) + .unzip::<_, _, Vec<_>, Vec<_>>(); + unsafe { llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( - c_str_vec.as_ptr(), - c_str_vec.len(), + pointers.as_ptr(), + pointers.len(), + lengths.as_ptr(), + lengths.len(), buffer, ); } @@ -310,12 +277,7 @@ pub(crate) fn write_mapping_to_buffer( } } -pub(crate) fn hash_str(strval: &str) -> u64 { - let strval = CString::new(strval).expect("null error converting hashable str to C string"); - unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } -} - -pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 { +pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } } @@ -350,6 +312,7 @@ pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, + covfun_section_name: &str, func_name_hash: u64, func_record_val: &'ll llvm::Value, is_used: bool, @@ -365,20 +328,33 @@ pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); debug!("function record var name: {:?}", func_record_var_name); - - let func_record_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage function record section name failed UTF-8 conversion"); - debug!("function record section name: {:?}", func_record_section_name); + debug!("function record section name: {:?}", covfun_section_name); let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); llvm::set_initializer(llglobal, func_record_val); llvm::set_global_constant(llglobal, true); llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, &func_record_section_name); + llvm::set_section(llglobal, covfun_section_name); llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); cx.add_used_global(llglobal); } + +/// Returns the section name string to pass through to the linker when embedding +/// per-function coverage information in the object file, according to the target +/// platform's object file format. +/// +/// LLVM's coverage tools read coverage mapping details from this section when +/// producing coverage reports. +/// +/// Typical values are: +/// - `__llvm_covfun` on Linux +/// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) +/// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) +pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String { + llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage function record section name failed UTF-8 conversion") +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 99e4ded62f1..aff764f0224 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; +use rustc_index::Idx; /// Produces DIScope DIEs for each MIR Scope which has variables defined in it. // FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. @@ -20,7 +20,7 @@ pub fn compute_mir_scopes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, ) { // Find all scopes with variables defined in them. let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { @@ -51,7 +51,7 @@ fn make_mir_scope<'ll, 'tcx>( instance: Instance<'tcx>, mir: &Body<'tcx>, variables: &Option<BitSet<SourceScope>>, - debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, instantiated: &mut BitSet<SourceScope>, scope: SourceScope, ) { @@ -65,10 +65,10 @@ fn make_mir_scope<'ll, 'tcx>( debug_context.scopes[parent] } else { // The root is the function itself. - let loc = cx.lookup_debug_loc(mir.span.lo()); + let file = cx.sess().source_map().lookup_source_file(mir.span.lo()); debug_context.scopes[scope] = DebugScope { - file_start_pos: loc.file.start_pos, - file_end_pos: loc.file.end_pos, + file_start_pos: file.start_pos, + file_end_pos: file.end_position(), ..debug_context.scopes[scope] }; instantiated.insert(scope); @@ -86,27 +86,31 @@ fn make_mir_scope<'ll, 'tcx>( let loc = cx.lookup_debug_loc(scope_data.span.lo()); let file_metadata = file_metadata(cx, &loc.file); - let dbg_scope = match scope_data.inlined { + let parent_dbg_scope = match scope_data.inlined { Some((callee, _)) => { // FIXME(eddyb) this would be `self.monomorphize(&callee)` // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. - let callee = cx.tcx.subst_and_normalize_erasing_regions( - instance.substs, + let callee = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, ty::ParamEnv::reveal_all(), - callee, + ty::EarlyBinder::bind(callee), ); - let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); - cx.dbg_scope_fn(callee, callee_fn_abi, None) + debug_context.inlined_function_scopes.entry(callee).or_insert_with(|| { + let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); + cx.dbg_scope_fn(callee, callee_fn_abi, None) + }) } - None => unsafe { - llvm::LLVMRustDIBuilderCreateLexicalBlock( - DIB(cx), - parent_scope.dbg_scope, - file_metadata, - loc.line, - loc.col, - ) - }, + None => parent_scope.dbg_scope, + }; + + let dbg_scope = unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_dbg_scope, + file_metadata, + loc.line, + loc.col, + ) }; let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { @@ -120,7 +124,7 @@ fn make_mir_scope<'ll, 'tcx>( dbg_scope, inlined_at: inlined_at.or(parent_scope.inlined_at), file_start_pos: loc.file.start_pos, - file_end_pos: loc.file.end_pos, + file_end_pos: loc.file.end_position(), }; instantiated.insert(scope); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 80fd9726fc7..425e935bc9f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -5,26 +5,24 @@ use crate::llvm; use crate::builder::Builder; use crate::common::CodegenCx; use crate::value::Value; +use rustc_ast::attr; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; +use rustc_middle::{bug, middle::debugger_visualizer::DebuggerVisualizerType}; use rustc_session::config::{CrateType, DebugInfo}; - use rustc_span::symbol::sym; -use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, '_, '_>) { if needs_gdb_debug_scripts_section(bx) { - let gdb_debug_scripts_section = - bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); + let gdb_debug_scripts_section = get_or_insert_gdb_debug_scripts_section_global(bx); // Load just the first byte as that's all that's necessary to force // LLVM to keep around the reference to the global. - let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); + let volatile_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); unsafe { - llvm::LLVMSetAlignment(volative_load_instruction, 1); + llvm::LLVMSetAlignment(volatile_load_instruction, 1); } } } @@ -55,7 +53,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' // The initial byte `4` instructs GDB that the following pretty printer // is defined inline as opposed to in a standalone file. section_contents.extend_from_slice(b"\x04"); - let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); + let vis_name = format!("pretty-printer-{crate_name}-{index}\n"); section_contents.extend_from_slice(vis_name.as_bytes()); section_contents.extend_from_slice(&visualizer.src); @@ -87,14 +85,14 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = - cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since // each rlib could produce a different set of visualizers that would be embedded // in the `.debug_gdb_scripts` section. For that reason, we make sure that the // section is only emitted for leaf crates. - let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { + let embed_visualizers = cx.tcx.crate_types().iter().any(|&crate_type| match crate_type { CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { // These are crate types for which we will embed pretty printers since they // are treated as leaf crates. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 163ccd9460c..ed938761694 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -27,11 +27,8 @@ use rustc_codegen_ssa::traits::*; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; -use rustc_middle::mir::{self, GeneratorLayout}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; @@ -113,7 +110,7 @@ macro_rules! return_if_di_node_created_in_meantime { /// Extract size and alignment from a TyAndLayout. #[inline] -fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { +fn size_and_align_of(ty_and_layout: TyAndLayout<'_>) -> (Size, Align) { (ty_and_layout.size, ty_and_layout.align.abi) } @@ -134,7 +131,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( let (size, align) = cx.size_and_align_of(array_type); - let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; + let upper_bound = len.eval_target_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; let subrange = unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; @@ -171,34 +168,31 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // a (fat) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. debug_assert_eq!( cx.size_and_align_of(ptr_type), - cx.size_and_align_of(cx.tcx.mk_mut_ptr(pointee_type)) + cx.size_and_align_of(Ty::new_mut_ptr(cx.tcx, pointee_type)) ); let pointee_type_di_node = type_di_node(cx, pointee_type); return_if_di_node_created_in_meantime!(cx, unique_type_id); - let (thin_pointer_size, thin_pointer_align) = - cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); + let data_layout = &cx.tcx.data_layout; let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); match fat_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. debug_assert_eq!( - (thin_pointer_size, thin_pointer_align), + (data_layout.pointer_size, data_layout.pointer_align.abi), cx.size_and_align_of(ptr_type), - "ptr_type={}, pointee_type={}", - ptr_type, - pointee_type, + "ptr_type={ptr_type}, pointee_type={pointee_type}", ); let di_node = unsafe { llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), pointee_type_di_node, - thin_pointer_size.bits(), - thin_pointer_align.bits() as u32, + data_layout.pointer_size.bits(), + data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. ptr_type_debuginfo_name.as_ptr().cast(), ptr_type_debuginfo_name.len(), @@ -227,8 +221,11 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // at all and instead emit regular struct debuginfo for it. We just // need to make sure that we don't break existing debuginfo consumers // by doing that (at least not without a warning period). - let layout_type = - if ptr_type.is_box() { cx.tcx.mk_mut_ptr(pointee_type) } else { ptr_type }; + let layout_type = if ptr_type.is_box() { + Ty::new_mut_ptr(cx.tcx, pointee_type) + } else { + ptr_type + }; let layout = cx.layout_of(layout_type); let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); @@ -434,7 +431,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D return existing_di_node; } - debug!("type_di_node: {:?}", t); + debug!("type_di_node: {:?} kind: {:?}", t, t.kind()); let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { @@ -448,9 +445,9 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) } - // Box<T, A> may have a non-ZST allocator A. In that case, we + // Box<T, A> may have a non-1-ZST allocator A. In that case, we // cannot treat Box<T, A> as just an owned alias of `*mut T`. - ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_1zst() => { build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), @@ -522,7 +519,7 @@ fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll D fn hex_encode(data: &[u8]) -> String { let mut hex_string = String::with_capacity(data.len() * 2); for byte in data.iter() { - write!(&mut hex_string, "{:02x}", byte).unwrap(); + write!(&mut hex_string, "{byte:02x}").unwrap(); } hex_string } @@ -740,7 +737,10 @@ fn build_foreign_type_di_node<'ll, 'tcx>( debug!("build_foreign_type_di_node: {:?}", t); let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { - bug!("build_foreign_type_di_node() called with unexpected type: {:?}", unique_type_id.expect_ty()); + bug!( + "build_foreign_type_di_node() called with unexpected type: {:?}", + unique_type_id.expect_ty() + ); }; build_type_with_children( @@ -764,7 +764,7 @@ fn build_param_type_di_node<'ll, 'tcx>( t: Ty<'tcx>, ) -> DINodeCreationResult<'ll> { debug!("build_param_type_di_node: {:?}", t); - let name = format!("{:?}", t); + let name = format!("{t:?}"); DINodeCreationResult { di_node: unsafe { llvm::LLVMRustDIBuilderCreateBasicType( @@ -784,10 +784,10 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( codegen_unit_name: &str, debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, ) -> &'ll DIDescriptor { - let mut name_in_debuginfo = match tcx.sess.local_crate_source_file { - Some(ref path) => path.clone(), - None => PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str()), - }; + let mut name_in_debuginfo = tcx + .sess + .local_crate_source_file() + .unwrap_or_else(|| PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str())); // To avoid breaking split DWARF, we need to ensure that each codegen unit // has a unique `DW_AT_name`. This is because there's a remote chance that @@ -810,10 +810,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( name_in_debuginfo.push(codegen_unit_name); debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); - let rustc_producer = - format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); + let rustc_producer = format!("rustc version {}", tcx.sess.cfg_version); // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. - let producer = format!("clang LLVM ({})", rustc_producer); + let producer = format!("clang LLVM ({rustc_producer})"); let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); @@ -833,24 +832,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( } .unwrap_or_default(); let split_name = split_name.to_str().unwrap(); - - // FIXME(#60020): - // - // This should actually be - // - // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); - // - // That is, we should set LLVM's emission kind to `LineTablesOnly` if - // we are compiling with "limited" debuginfo. However, some of the - // existing tools relied on slightly more debuginfo being generated than - // would be the case with `LineTablesOnly`, and we did not want to break - // these tools in a "drive-by fix", without a good idea or plan about - // what limited debuginfo should exactly look like. So for now we keep - // the emission kind as `FullDebug`. - // - // See https://github.com/rust-lang/rust/issues/60020 for details. - let kind = DebugEmissionKind::FullDebug; - assert!(tcx.sess.opts.debuginfo != DebugInfo::None); + let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( @@ -884,8 +866,6 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); if tcx.sess.opts.unstable_opts.profile { - let cu_desc_metadata = - llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); @@ -893,49 +873,25 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), path_to_mdstring(debug_context.llcontext, gcda_path), - cu_desc_metadata, + unit_metadata, ]; - let gcov_metadata = llvm::LLVMMDNodeInContext( + let gcov_metadata = llvm::LLVMMDNodeInContext2( debug_context.llcontext, gcov_cu_info.as_ptr(), - gcov_cu_info.len() as c_uint, + gcov_cu_info.len(), ); + let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); let llvm_gcov_ident = cstr!("llvm.gcov"); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - llvm_gcov_ident.as_ptr(), - gcov_metadata, - ); - } - - // Insert `llvm.ident` metadata on the wasm targets since that will - // get hooked up to the "producer" sections `processed-by` information. - if tcx.sess.target.is_like_wasm { - let name_metadata = llvm::LLVMMDStringInContext( - debug_context.llcontext, - rustc_producer.as_ptr().cast(), - rustc_producer.as_bytes().len() as c_uint, - ); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - cstr!("llvm.ident").as_ptr(), - llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), - ); + llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, llvm_gcov_ident.as_ptr(), val); } return unit_metadata; }; - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { + fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata { let path_str = path_to_c_string(path); - unsafe { - llvm::LLVMMDStringInContext( - llcx, - path_str.as_ptr(), - path_str.as_bytes().len() as c_uint, - ) - } + unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) } } } @@ -998,7 +954,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( .iter() .enumerate() .map(|(i, f)| { - let field_name = if variant_def.ctor_kind == CtorKind::Fn { + let field_name = if variant_def.ctor_kind() == Some(CtorKind::Fn) { // This is a tuple struct tuple_field_name(i) } else { @@ -1026,33 +982,6 @@ fn build_struct_type_di_node<'ll, 'tcx>( // Tuples //=----------------------------------------------------------------------------- -/// Returns names of captured upvars for closures and generators. -/// -/// Here are some examples: -/// - `name__field1__field2` when the upvar is captured by value. -/// - `_ref__name__field` when the upvar is captured by reference. -/// -/// For generators this only contains upvars that are shared by all states. -fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec<String> { - let body = tcx.optimized_mir(def_id); - - body.var_debug_info - .iter() - .filter_map(|var| { - let is_ref = match var.value { - mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => { - // The projection is either `[.., Field, Deref]` or `[.., Field]`. It - // implies whether the variable is captured by value or by reference. - matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) - } - _ => return None, - }; - let prefix = if is_ref { "_ref__" } else { "" }; - Some(prefix.to_owned() + var.name.as_str()) - }) - .collect() -} - /// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. /// For a generator, this will handle upvars shared by all states. fn build_upvar_field_di_nodes<'ll, 'tcx>( @@ -1061,14 +990,8 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_generator_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { - ty::Generator(def_id, substs, _) => { - let upvar_tys: SmallVec<_> = substs.as_generator().prefix_tys().collect(); - (def_id, upvar_tys) - } - ty::Closure(def_id, substs) => { - let upvar_tys: SmallVec<_> = substs.as_closure().upvar_tys().collect(); - (def_id, upvar_tys) - } + ty::Generator(def_id, args, _) => (def_id, args.as_generator().prefix_tys()), + ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), _ => { bug!( "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", @@ -1078,12 +1001,10 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( }; debug_assert!( - up_var_tys - .iter() - .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) + up_var_tys.iter().all(|t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) ); - let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); + let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); let layout = cx.layout_of(closure_or_generator_ty); up_var_tys @@ -1094,7 +1015,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( build_field_di_node( cx, closure_or_generator_di_node, - capture_name, + capture_name.as_str(), cx.size_and_align_of(up_var_ty), layout.fields.offset(index), DIFlags::FlagZero, @@ -1156,7 +1077,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let closure_env_type = unique_type_id.expect_ty(); - let &ty::Closure(def_id, _substs) = closure_env_type.kind() else { + let &ty::Closure(def_id, _args) = closure_env_type.kind() else { bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) }; let containing_scope = get_namespace_for_item(cx, def_id); @@ -1229,60 +1150,23 @@ fn build_union_type_di_node<'ll, 'tcx>( ) } -// FIXME(eddyb) maybe precompute this? Right now it's computed once -// per generator monomorphization, but it doesn't depend on substs. -fn generator_layout_and_saved_local_names<'tcx>( - tcx: TyCtxt<'tcx>, - def_id: DefId, -) -> (&'tcx GeneratorLayout<'tcx>, IndexVec<mir::GeneratorSavedLocal, Option<Symbol>>) { - let body = tcx.optimized_mir(def_id); - let generator_layout = body.generator_layout().unwrap(); - let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); - - let state_arg = mir::Local::new(1); - for var in &body.var_debug_info { - let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; - if place.local != state_arg { - continue; - } - match place.projection[..] { - [ - // Deref of the `Pin<&mut Self>` state argument. - mir::ProjectionElem::Field(..), - mir::ProjectionElem::Deref, - // Field of a variant of the state. - mir::ProjectionElem::Downcast(_, variant), - mir::ProjectionElem::Field(field, _), - ] => { - let name = &mut generator_saved_local_names - [generator_layout.variant_fields[variant][field]]; - if name.is_none() { - name.replace(var.name); - } - } - _ => {} - } - } - (generator_layout, generator_saved_local_names) -} - /// Computes the type parameters for a type, if any, for the given metadata. fn build_generic_type_param_di_nodes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, ) -> SmallVec<&'ll DIType> { - if let ty::Adt(def, substs) = *ty.kind() { - if substs.types().next().is_some() { + if let ty::Adt(def, args) = *ty.kind() { + if args.types().next().is_some() { let generics = cx.tcx.generics_of(def.did()); let names = get_parameter_names(cx, generics); - let template_params: SmallVec<_> = iter::zip(substs, names) + let template_params: SmallVec<_> = iter::zip(args, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_di_node = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -1290,10 +1174,8 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( name.len(), actual_type_di_node, ) - }) - } else { - None - } + } + }) }) .collect(); @@ -1397,7 +1279,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( // All function pointers are described as opaque pointers. This could be improved in the future // by describing them as actual function pointers. - let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); + let void_pointer_ty = Ty::new_imm_ptr(tcx, tcx.types.unit); let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); let usize_di_node = type_di_node(cx, tcx.types.usize); let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); @@ -1439,10 +1321,10 @@ fn build_vtable_type_di_node<'ll, 'tcx>( // Note: This code does not try to give a proper name to each method // because their might be multiple methods with the same name // (coming from different traits). - (format!("__method{}", index), void_pointer_type_di_node) + (format!("__method{index}"), void_pointer_type_di_node) } ty::VtblEntry::TraitVPtr(_) => { - (format!("__super_trait_ptr{}", index), void_pointer_type_di_node) + (format!("__super_trait_ptr{index}"), void_pointer_type_di_node) } ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node), ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node), @@ -1487,7 +1369,7 @@ fn vcall_visibility_metadata<'ll, 'tcx>( let trait_def_id = trait_ref_self.def_id(); let trait_vis = cx.tcx.visibility(trait_def_id); - let cgus = cx.sess().codegen_units(); + let cgus = cx.sess().codegen_units().as_usize(); let single_cgu = cgus == 1; let lto = cx.sess().lto(); @@ -1565,6 +1447,11 @@ pub fn create_vtable_di_node<'ll, 'tcx>( return; } + // When full debuginfo is enabled, we want to try and prevent vtables from being + // merged. Otherwise debuggers will have a hard time mapping from dyn pointer + // to concrete type. + llvm::SetUnnamedAddress(vtable, llvm::UnnamedAddr::No); + let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); @@ -1607,5 +1494,5 @@ pub fn tuple_field_name(field_index: usize) -> Cow<'static, str> { TUPLE_FIELD_NAMES .get(field_index) .map(|s| Cow::from(*s)) - .unwrap_or_else(|| Cow::from(format!("__{}", field_index))) + .unwrap_or_else(|| Cow::from(format!("__{field_index}"))) } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 129e336c7e4..88040557a9b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -6,13 +6,13 @@ use rustc_codegen_ssa::{ traits::ConstMethods, }; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::{ bug, ty::{ self, layout::{LayoutOf, TyAndLayout}, - AdtDef, GeneratorSubsts, Ty, + AdtDef, GeneratorArgs, Ty, }, }; use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; @@ -22,9 +22,9 @@ use crate::{ common::CodegenCx, debuginfo::{ metadata::{ - build_field_di_node, closure_saved_names_of_captured_variables, + build_field_di_node, enums::{tag_base_type, DiscrResult}, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER, @@ -62,7 +62,7 @@ const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; /// In CPP-like mode, we generate a union with a field for each variant and an /// explicit tag field. The field of each variant has a struct type -/// that encodes the discrimiant of the variant and it's data layout. +/// that encodes the discriminant of the variant and it's data layout. /// The union also has a nested enumeration type that is only used for encoding /// variant names in an efficient way. Its enumerator values do _not_ correspond /// to the enum's discriminant values. @@ -199,7 +199,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; + }; let enum_type_and_layout = cx.layout_of(enum_type); let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); @@ -462,7 +462,7 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( cx, "VariantNames", variant_names_enum_base_type(cx), - variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32() as u64)), + variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32().into())), containing_scope, ) } @@ -667,20 +667,21 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { - let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = generator_type_and_layout.variants else { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = + generator_type_and_layout.variants + else { bug!("This function only supports layouts with directly encoded tags.") }; - let (generator_def_id, generator_substs) = match generator_type_and_layout.ty.kind() { - &ty::Generator(def_id, substs, _) => (def_id, substs.as_generator()), + let (generator_def_id, generator_args) = match generator_type_and_layout.ty.kind() { + &ty::Generator(def_id, args, _) => (def_id, args.as_generator()), _ => unreachable!(), }; - let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + let generator_layout = cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); - let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); - let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); + let variant_range = generator_args.variant_range(generator_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); let tag_base_type = tag_base_type(cx, generator_type_and_layout); @@ -690,11 +691,11 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( generator_type_di_node, variant_range .clone() - .map(|variant_index| (variant_index, GeneratorSubsts::variant_name(variant_index))), + .map(|variant_index| (variant_index, GeneratorArgs::variant_name(variant_index))), ); let discriminants: IndexVec<VariantIdx, DiscrResult> = { - let discriminants_iter = generator_substs.discriminants(generator_def_id, cx.tcx); + let discriminants_iter = generator_args.discriminants(generator_def_id, cx.tcx); let mut discriminants: IndexVec<VariantIdx, DiscrResult> = IndexVec::with_capacity(variant_count); for (variant_index, discr) in discriminants_iter { @@ -714,7 +715,6 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( generator_type_and_layout, generator_type_di_node, generator_layout, - &state_specific_upvar_names, &common_upvar_names, ); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 14044d0f99b..d3239d5c358 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -3,18 +3,20 @@ use rustc_codegen_ssa::debuginfo::{ wants_c_like_enum_debuginfo, }; use rustc_hir::def::CtorKind; -use rustc_index::vec::IndexVec; +use rustc_index::IndexSlice; use rustc_middle::{ bug, - mir::{Field, GeneratorLayout, GeneratorSavedLocal}, + mir::GeneratorLayout, ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - AdtDef, GeneratorSubsts, Ty, VariantDef, + AdtDef, GeneratorArgs, Ty, VariantDef, }, }; use rustc_span::Symbol; -use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{ + FieldIdx, HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants, +}; use std::borrow::Cow; use crate::{ @@ -49,7 +51,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; + }; let enum_type_and_layout = cx.layout_of(enum_type); @@ -91,9 +93,7 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( tag_base_type(cx, enum_type_and_layout), enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); - // Is there anything we can do to support 128-bit C-Style enums? - let value = discr.val as u64; - (name, value) + (name, discr.val) }), containing_scope, ), @@ -124,7 +124,8 @@ fn tag_base_type<'ll, 'tcx>( Primitive::Int(t, _) => t, Primitive::F32 => Integer::I32, Primitive::F64 => Integer::I64, - Primitive::Pointer => { + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + Primitive::Pointer(_) => { // If the niche is the NULL value of a reference, then `discr_enum_ty` will be // a RawPtr. CodeView doesn't know what to do with enums whose base type is a // pointer so we fix this up to just be `usize`. @@ -147,14 +148,11 @@ fn tag_base_type<'ll, 'tcx>( /// This is a helper function and does not register anything in the type map by itself. /// /// `variants` is an iterator of (discr-value, variant-name). -/// -// NOTE: Handling of discriminant values is somewhat inconsistent. They can appear as u128, -// u64, and i64. Here everything gets mapped to i64 because that's what LLVM's API expects. fn build_enumeration_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, type_name: &str, base_type: Ty<'tcx>, - enumerators: impl Iterator<Item = (Cow<'tcx, str>, u64)>, + enumerators: impl Iterator<Item = (Cow<'tcx, str>, u128)>, containing_scope: &'ll DIType, ) -> &'ll DIType { let is_unsigned = match base_type.kind() { @@ -162,21 +160,22 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( ty::Uint(_) => true, _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), }; + let (size, align) = cx.size_and_align_of(base_type); let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = enumerators .map(|(name, value)| unsafe { + let value = [value as u64, (value >> 64) as u64]; Some(llvm::LLVMRustDIBuilderCreateEnumerator( DIB(cx), name.as_ptr().cast(), name.len(), - value as i64, + value.as_ptr(), + size.bits() as libc::c_uint, is_unsigned, )) }) .collect(); - let (size, align) = cx.size_and_align_of(base_type); - unsafe { llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), @@ -273,9 +272,10 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( |cx, struct_type_di_node| { (0..variant_layout.fields.count()) .map(|field_index| { - let field_name = if variant_def.ctor_kind != CtorKind::Fn { + let field_name = if variant_def.ctor_kind() != Some(CtorKind::Fn) { // Fields have names - Cow::from(variant_def.fields[field_index].name.as_str()) + let field = &variant_def.fields[FieldIdx::from_usize(field_index)]; + Cow::from(field.name.as_str()) } else { // Tuple-like super::tuple_field_name(field_index) @@ -323,10 +323,9 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, generator_layout: &GeneratorLayout<'tcx>, - state_specific_upvar_names: &IndexVec<GeneratorSavedLocal, Option<Symbol>>, - common_upvar_names: &[String], + common_upvar_names: &IndexSlice<FieldIdx, Symbol>, ) -> &'ll DIType { - let variant_name = GeneratorSubsts::variant_name(variant_index); + let variant_name = GeneratorArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( cx.tcx, generator_type_and_layout.ty, @@ -335,8 +334,8 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); - let generator_substs = match generator_type_and_layout.ty.kind() { - ty::Generator(_, substs, _) => substs.as_generator(), + let generator_args = match generator_type_and_layout.ty.kind() { + ty::Generator(_, args, _) => args.as_generator(), _ => unreachable!(), }; @@ -356,8 +355,8 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) .map(|field_index| { let generator_saved_local = generator_layout.variant_fields[variant_index] - [Field::from_usize(field_index)]; - let field_name_maybe = state_specific_upvar_names[generator_saved_local]; + [FieldIdx::from_usize(field_index)]; + let field_name_maybe = generator_layout.field_names[generator_saved_local]; let field_name = field_name_maybe .as_ref() .map(|s| Cow::from(s.as_str())) @@ -378,14 +377,16 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .collect(); // Fields that are common to all states - let common_fields: SmallVec<_> = generator_substs + let common_fields: SmallVec<_> = generator_args .prefix_tys() + .iter() + .zip(common_upvar_names) .enumerate() - .map(|(index, upvar_ty)| { + .map(|(index, (upvar_ty, upvar_name))| { build_field_di_node( cx, variant_struct_type_di_node, - &common_upvar_names[index], + upvar_name.as_str(), cx.size_and_align_of(upvar_ty), generator_type_and_layout.fields.offset(index), DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index becbccc434d..feac40d8c30 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -4,9 +4,8 @@ use crate::{ common::CodegenCx, debuginfo::{ metadata::{ - closure_saved_names_of_captured_variables, enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, StubInfo, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, UNKNOWN_LINE_NUMBER, @@ -58,7 +57,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; + }; let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); let enum_type_and_layout = cx.layout_of(enum_type); @@ -133,9 +132,9 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let generator_type = unique_type_id.expect_ty(); - let &ty::Generator(generator_def_id, _, _ ) = generator_type.kind() else { + let &ty::Generator(generator_def_id, _, _) = generator_type.kind() else { bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) - }; + }; let containing_scope = get_namespace_for_item(cx, generator_def_id); let generator_type_and_layout = cx.layout_of(generator_type); @@ -156,10 +155,12 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( DIFlags::FlagZero, ), |cx, generator_type_di_node| { - let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + let generator_layout = + cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); - let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = + generator_type_and_layout.variants + else { bug!( "Encountered generator with non-direct-tag layout: {:?}", generator_type_and_layout @@ -167,14 +168,14 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( }; let common_upvar_names = - closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() .map(|variant_index| { // FIXME: This is problematic because just a number is not a valid identifier. - // GeneratorSubsts::variant_name(variant_index), would be consistent + // GeneratorArgs::variant_name(variant_index), would be consistent // with enums? let variant_name = format!("{}", variant_index.as_usize()).into(); @@ -196,7 +197,6 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( generator_type_and_layout, generator_type_di_node, generator_layout, - &state_specific_upvar_names, &common_upvar_names, ), source_info, @@ -413,13 +413,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.opt_single_val().map(|value| { - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - cx.const_u64(value as u64) - }), + discr_value.opt_single_val().map(|value| cx.const_u128(value)), DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) @@ -439,6 +433,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( /// DW_TAG_structure_type (type of variant 1) /// DW_TAG_structure_type (type of variant 2) /// DW_TAG_structure_type (type of variant 3) +/// ``` struct VariantMemberInfo<'a, 'll> { variant_index: VariantIdx, variant_name: Cow<'a, str>, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index b23fe3fc9d5..30cc9ea9b82 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -21,13 +21,14 @@ use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::Hash128; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; +use rustc_middle::ty::GenericArgsRef; +use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -39,6 +40,7 @@ use smallvec::SmallVec; use std::cell::OnceCell; use std::cell::RefCell; use std::iter; +use std::ops::Range; mod create_scope_map; pub mod gdb; @@ -60,7 +62,7 @@ pub struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell<FxHashMap<Option<(u128, SourceFileHash)>, &'ll DIFile>>, + created_files: RefCell<FxHashMap<Option<(Hash128, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, @@ -163,12 +165,14 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { variable_alloca: Self::Value, direct_offset: Size, indirect_offsets: &[Size], + fragment: Option<Range<Size>>, ) { - // Convert the direct and indirect offsets to address ops. + // Convert the direct and indirect offsets and fragment byte range to address ops. // FIXME(eddyb) use `const`s instead of getting the values via FFI, // the values should match the ones in the DWARF standard anyway. let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; + let op_llvm_fragment = || unsafe { llvm::LLVMRustDIBuilderCreateOpLLVMFragment() }; let mut addr_ops = SmallVec::<[u64; 8]>::new(); if direct_offset.bytes() > 0 { @@ -182,6 +186,13 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { addr_ops.push(offset.bytes() as u64); } } + if let Some(fragment) = fragment { + // `DW_OP_LLVM_fragment` takes as arguments the fragment's + // offset and size, both of them in bits. + addr_ops.push(op_llvm_fragment()); + addr_ops.push(fragment.start.bits() as u64); + addr_ops.push((fragment.end - fragment.start).bits() as u64); + } unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. @@ -199,8 +210,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { unsafe { - let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc); } } @@ -253,11 +263,11 @@ impl CodegenCx<'_, '_> { pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { let (file, line, col) = match self.sess().source_map().lookup_line(pos) { Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.line_begin_pos(pos); + let line_pos = file.lines()[line]; // Use 1-based indexing. let line = (line + 1) as u32; - let col = (pos - line_pos).to_u32() + 1; + let col = (file.relative_position(pos) - line_pos).to_u32() + 1; (file, line, col) } @@ -282,7 +292,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: &'ll Value, mir: &mir::Body<'tcx>, - ) -> Option<FunctionDebugContext<&'ll DIScope, &'ll DILocation>> { + ) -> Option<FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>> { if self.sess().opts.debuginfo == DebugInfo::None { return None; } @@ -294,8 +304,10 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { file_start_pos: BytePos(0), file_end_pos: BytePos(0), }; - let mut fn_debug_context = - FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; + let mut fn_debug_context = FunctionDebugContext { + scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes), + inlined_function_scopes: Default::default(), + }; // Fill in all the scopes, with the information from the MIR body. compute_mir_scopes(self, instance, mir, &mut fn_debug_context); @@ -312,7 +324,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let def_id = instance.def_id(); - let containing_scope = get_containing_scope(self, instance); + let (containing_scope, is_method) = get_containing_scope(self, instance); let span = tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); @@ -322,25 +334,26 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), fn_signature) }; - let mut name = String::new(); + let mut name = String::with_capacity(64); type_names::push_item_name(tcx, def_id, false, &mut name); // Find the enclosing function, in case this is a closure. let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); - // We look up the generics of the enclosing function and truncate the substs + // We look up the generics of the enclosing function and truncate the args // to their length in order to cut off extra stuff that might be in there for // closures or generators. let generics = tcx.generics_of(enclosing_fn_def_id); - let substs = instance.substs.truncate_to(tcx, generics); + let args = instance.args.truncate_to(tcx, generics); type_names::push_generic_params( tcx, - tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), + tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), args), + enclosing_fn_def_id, &mut name, ); - let template_parameters = get_template_parameters(self, generics, substs); + let template_parameters = get_template_parameters(self, generics, args); let linkage_name = &mangled_name_of_instance(self, instance).name; // Omit the linkage_name if it is the same as subprogram name. @@ -368,8 +381,29 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - unsafe { - return llvm::LLVMRustDIBuilderCreateFunction( + // When we're adding a method to a type DIE, we only want a DW_AT_declaration there, because + // LLVM LTO can't unify type definitions when a child DIE is a full subprogram definition. + // When we use this `decl` below, the subprogram definition gets created at the CU level + // with a DW_AT_specification pointing back to the type's declaration. + let decl = is_method.then(|| unsafe { + llvm::LLVMRustDIBuilderCreateMethod( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + flags, + spflags & !DISPFlags::SPFlagDefinition, + template_parameters, + ) + }); + + return unsafe { + llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, name.as_ptr().cast(), @@ -384,15 +418,15 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { spflags, maybe_definition_llfn, template_parameters, - None, - ); - } + decl, + ) + }; fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> &'ll DIArray { - if cx.sess().opts.debuginfo == DebugInfo::Limited { + if cx.sess().opts.debuginfo != DebugInfo::Full { return create_DIArray(DIB(cx), &[]); } @@ -423,7 +457,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { ty::Array(ct, _) if (*ct == cx.tcx.types.u8) || cx.layout_of(*ct).is_zst() => { - cx.tcx.mk_imm_ptr(*ct) + Ty::new_imm_ptr(cx.tcx, *ct) } _ => t, }; @@ -440,23 +474,23 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn get_template_parameters<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, generics: &ty::Generics, - substs: SubstsRef<'tcx>, + args: GenericArgsRef<'tcx>, ) -> &'ll DIArray { - if substs.types().next().is_none() { + if args.types().next().is_none() { return create_DIArray(DIB(cx), &[]); } // Again, only create type information if full debuginfo is enabled let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { let names = get_parameter_names(cx, generics); - iter::zip(substs, names) + iter::zip(args, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_metadata = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -464,10 +498,8 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { name.len(), actual_type_metadata, )) - }) - } else { - None - } + } + }) }) .collect() } else { @@ -485,57 +517,53 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { names } + /// Returns a scope, plus `true` if that's a type scope for "class" methods, + /// otherwise `false` for plain namespace scopes. fn get_containing_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, - ) -> &'ll DIScope { + ) -> (&'ll DIScope, bool) { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( - instance.substs, + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, ty::ParamEnv::reveal_all(), cx.tcx.type_of(impl_def_id), ); // Only "class" methods are generally understood by LLVM, // so avoid methods on other types (e.g., `<*mut T>::null`). - match impl_self_ty.kind() { - ty::Adt(def, ..) if !def.is_box() => { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.needs_subst() - { - Some(type_di_node(cx, impl_self_ty)) - } else { - Some(namespace::item_namespace(cx, def.did())) - } + if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() + { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - _ => None, } } else { // For trait method impls we still use the "parallel namespace" // strategy - None } - }); + } - self_type.unwrap_or_else(|| { - namespace::item_namespace( - cx, - DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }, - ) - }) + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); + (scope, false) } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index d5ea48c311b..fa61c7dde18 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -28,7 +28,7 @@ pub fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DISco .map(|parent| item_namespace(cx, DefId { krate: def_id.krate, index: parent })); let namespace_name_string = { - let mut output = String::new(); + let mut output = String::with_capacity(64); type_names::push_item_name(cx.tcx, def_id, false, &mut output); output }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index a40cfc8b23f..c758010c581 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -5,7 +5,7 @@ use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; -use rustc_middle::ty::{self, DefIdTree, Ty}; +use rustc_middle::ty::{self, Ty}; use trace; use crate::common::CodegenCx; @@ -72,7 +72,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( layout.is_unsized() ); - if !layout.is_unsized() { + if layout.is_sized() { return None; } @@ -82,8 +82,8 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( ty::Foreign(_) => { // Assert that pointers to foreign types really are thin: debug_assert_eq!( - cx.size_of(cx.tcx.mk_imm_ptr(pointee_tail_ty)), - cx.size_of(cx.tcx.mk_imm_ptr(cx.tcx.types.u8)) + cx.size_of(Ty::new_imm_ptr(cx.tcx, pointee_tail_ty)), + cx.size_of(Ty::new_imm_ptr(cx.tcx, cx.tcx.types.u8)) ); None } @@ -91,8 +91,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( // For all other pointee types we should already have returned None // at the beginning of the function. panic!( - "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {:?}", - pointee_tail_ty + "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}" ) } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index f79ef11720d..164b12cf8d4 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -19,8 +19,11 @@ use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; use rustc_codegen_ssa::traits::TypeMembershipMethods; -use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::typeid_for_fnabi; +use rustc_middle::ty::{Instance, Ty}; +use rustc_symbol_mangling::typeid::{ + kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance, + TypeIdOptions, +}; use smallvec::SmallVec; /// Declare a function. @@ -90,11 +93,38 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type) } + /// Declare an entry Function + /// + /// The ABI of this function can change depending on the target (although for now the same as + /// `declare_cfn`) + /// + /// If there’s a value with the same name already declared, the function will + /// update the declaration and return existing Value instead. + pub fn declare_entry_fn( + &self, + name: &str, + callconv: llvm::CallConv, + unnamed: llvm::UnnamedAddr, + fn_type: &'ll Type, + ) -> &'ll Value { + let visibility = if self.tcx.sess.target.default_hidden_visibility { + llvm::Visibility::Hidden + } else { + llvm::Visibility::Default + }; + declare_raw_fn(self, name, callconv, unnamed, visibility, fn_type) + } + /// Declare a Rust function. /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + pub fn declare_fn( + &self, + name: &str, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + instance: Option<Instance<'tcx>>, + ) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); // Function addresses in Rust are never significant, allowing functions to @@ -110,8 +140,54 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.apply_attrs_llfn(self, llfn); if self.tcx.sess.is_sanitizer_cfi_enabled() { - let typeid = typeid_for_fnabi(self.tcx, fn_abi); - self.set_type_metadata(llfn, typeid); + if let Some(instance) = instance { + let typeid = typeid_for_instance(self.tcx, &instance, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_instance( + self.tcx, + &instance, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } else { + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi( + self.tcx, + fn_abi, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } + } + + if self.tcx.sess.is_sanitizer_kcfi_enabled() { + // LLVM KCFI does not support multiple !kcfi_type attachments + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + if let Some(instance) = instance { + let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, &instance, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } else { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } } llfn diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs new file mode 100644 index 00000000000..665d195790c --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -0,0 +1,238 @@ +use std::borrow::Cow; +use std::ffi::CString; +use std::path::Path; + +use crate::fluent_generated as fluent; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_errors::{ + DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed, Handler, IntoDiagnostic, +}; +use rustc_macros::{Diagnostic, Subdiagnostic}; +use rustc_span::Span; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_ctarget_feature_prefix)] +#[note] +pub(crate) struct UnknownCTargetFeaturePrefix<'a> { + pub feature: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_ctarget_feature)] +#[note] +pub(crate) struct UnknownCTargetFeature<'a> { + pub feature: &'a str, + #[subdiagnostic] + pub rust_feature: PossibleFeature<'a>, +} + +#[derive(Subdiagnostic)] +pub(crate) enum PossibleFeature<'a> { + #[help(codegen_llvm_possible_feature)] + Some { rust_feature: &'a str }, + #[help(codegen_llvm_consider_filing_feature_request)] + None, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_error_creating_import_library)] +pub(crate) struct ErrorCreatingImportLibrary<'a> { + pub lib_name: &'a str, + pub error: String, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_symbol_already_defined)] +pub(crate) struct SymbolAlreadyDefined<'a> { + #[primary_span] + pub span: Span, + pub symbol_name: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_minimum_alignment_not_power_of_two)] +pub(crate) struct InvalidMinimumAlignmentNotPowerOfTwo { + pub align: u64, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_minimum_alignment_too_large)] +pub(crate) struct InvalidMinimumAlignmentTooLarge { + pub align: u64, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_sanitizer_memtag_requires_mte)] +pub(crate) struct SanitizerMemtagRequiresMte; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_error_writing_def_file)] +pub(crate) struct ErrorWritingDEFFile { + pub error: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_error_calling_dlltool)] +pub(crate) struct ErrorCallingDllTool<'a> { + pub dlltool_path: Cow<'a, str>, + pub error: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_dlltool_fail_import_library)] +pub(crate) struct DlltoolFailImportLibrary<'a> { + pub dlltool_path: Cow<'a, str>, + pub dlltool_args: String, + pub stdout: Cow<'a, str>, + pub stderr: Cow<'a, str>, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_dynamic_linking_with_lto)] +#[note] +pub(crate) struct DynamicLinkingWithLTO; + +pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); + +impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for ParseTargetMachineConfig<'_> { + fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> { + let diag: DiagnosticBuilder<'_, EM> = self.0.into_diagnostic(sess); + let (message, _) = diag.styled_message().first().expect("`LlvmError` with no message"); + let message = sess.eagerly_translate_to_string(message.clone(), diag.args()); + + let mut diag = sess.struct_diagnostic(fluent::codegen_llvm_parse_target_machine_config); + diag.set_arg("error", message); + diag + } +} + +pub(crate) struct TargetFeatureDisableOrEnable<'a> { + pub features: &'a [&'a str], + pub span: Option<Span>, + pub missing_features: Option<MissingFeatures>, +} + +#[derive(Subdiagnostic)] +#[help(codegen_llvm_missing_features)] +pub(crate) struct MissingFeatures; + +impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> { + fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> { + let mut diag = sess.struct_err(fluent::codegen_llvm_target_feature_disable_or_enable); + if let Some(span) = self.span { + diag.set_span(span); + }; + if let Some(missing_features) = self.missing_features { + diag.subdiagnostic(missing_features); + } + diag.set_arg("features", self.features.join(", ")); + diag + } +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_disallowed)] +pub(crate) struct LtoDisallowed; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_dylib)] +pub(crate) struct LtoDylib; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_proc_macro)] +pub(crate) struct LtoProcMacro; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_bitcode_from_rlib)] +pub(crate) struct LtoBitcodeFromRlib { + pub llvm_err: String, +} + +#[derive(Diagnostic)] +pub enum LlvmError<'a> { + #[diag(codegen_llvm_write_output)] + WriteOutput { path: &'a Path }, + #[diag(codegen_llvm_target_machine)] + CreateTargetMachine { triple: SmallCStr }, + #[diag(codegen_llvm_run_passes)] + RunLlvmPasses, + #[diag(codegen_llvm_serialize_module)] + SerializeModule { name: &'a str }, + #[diag(codegen_llvm_write_ir)] + WriteIr { path: &'a Path }, + #[diag(codegen_llvm_prepare_thin_lto_context)] + PrepareThinLtoContext, + #[diag(codegen_llvm_load_bitcode)] + LoadBitcode { name: CString }, + #[diag(codegen_llvm_write_thinlto_key)] + WriteThinLtoKey { err: std::io::Error }, + #[diag(codegen_llvm_multiple_source_dicompileunit)] + MultipleSourceDiCompileUnit, + #[diag(codegen_llvm_prepare_thin_lto_module)] + PrepareThinLtoModule, + #[diag(codegen_llvm_parse_bitcode)] + ParseBitcode, +} + +pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String); + +impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> { + fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> { + use LlvmError::*; + let msg_with_llvm_err = match &self.0 { + WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err, + CreateTargetMachine { .. } => fluent::codegen_llvm_target_machine_with_llvm_err, + RunLlvmPasses => fluent::codegen_llvm_run_passes_with_llvm_err, + SerializeModule { .. } => fluent::codegen_llvm_serialize_module_with_llvm_err, + WriteIr { .. } => fluent::codegen_llvm_write_ir_with_llvm_err, + PrepareThinLtoContext => fluent::codegen_llvm_prepare_thin_lto_context_with_llvm_err, + LoadBitcode { .. } => fluent::codegen_llvm_load_bitcode_with_llvm_err, + WriteThinLtoKey { .. } => fluent::codegen_llvm_write_thinlto_key_with_llvm_err, + MultipleSourceDiCompileUnit => { + fluent::codegen_llvm_multiple_source_dicompileunit_with_llvm_err + } + PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, + ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, + }; + let mut diag = self.0.into_diagnostic(sess); + diag.set_primary_message(msg_with_llvm_err); + diag.set_arg("llvm_err", self.1); + diag + } +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_from_llvm_optimization_diag)] +pub(crate) struct FromLlvmOptimizationDiag<'a> { + pub filename: &'a str, + pub line: std::ffi::c_uint, + pub column: std::ffi::c_uint, + pub pass_name: &'a str, + pub kind: &'a str, + pub message: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_from_llvm_diag)] +pub(crate) struct FromLlvmDiag { + pub message: String, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_write_bytecode)] +pub(crate) struct WriteBytecode<'a> { + pub path: &'a Path, + pub err: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_copy_bitcode)] +pub(crate) struct CopyBitcode { + pub err: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_debuginfo_compression)] +pub struct UnknownCompression { + pub algorithm: &'static str, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 825011941a2..a97b803fc64 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -7,22 +7,21 @@ use crate::type_of::LayoutLlvmExt; use crate::va_arg::emit_va_arg; use crate::value::Value; -use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; -use rustc_codegen_ssa::common::span_invalid_monomorphization_error; +use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; +use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; -use std::iter; fn get_simple_intrinsic<'ll>( cx: &CodegenCx<'ll, '_>, @@ -72,6 +71,8 @@ fn get_simple_intrinsic<'ll>( sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", sym::ptr_mask => "llvm.ptrmask", + sym::roundevenf32 => "llvm.roundeven.f32", + sym::roundevenf64 => "llvm.roundeven.f64", _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -89,7 +90,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); - let ty::FnDef(def_id, substs) = *callee_ty.kind() else { + let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else { bug!("expected fn item type, found {}", callee_ty); }; @@ -109,6 +110,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call( simple_ty, None, + None, simple_fn, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -149,7 +151,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { emit_va_arg(self, args[0], ret_ty) } } - Primitive::F64 | Primitive::Pointer => { + Primitive::F64 | Primitive::Pointer(_) => { emit_va_arg(self, args[0], ret_ty) } // `va_arg` should never be used with the return type f32. @@ -161,11 +163,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::volatile_load | sym::unaligned_volatile_load => { - let tp_ty = substs.type_at(0); + let tp_ty = fn_args.type_at(0); let ptr = args[0].immediate(); - let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { + let load = if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode { let llty = ty.llvm_type(self); - let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); self.volatile_load(llty, ptr) } else { self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr) @@ -228,22 +229,22 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::ctlz | sym::cttz => { let y = self.const_bool(false); self.call_intrinsic( - &format!("llvm.{}.i{}", name, width), + &format!("llvm.{name}.i{width}"), &[args[0].immediate(), y], ) } sym::ctlz_nonzero => { let y = self.const_bool(true); - let llvm_name = &format!("llvm.ctlz.i{}", width); + let llvm_name = &format!("llvm.ctlz.i{width}"); self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) } sym::cttz_nonzero => { let y = self.const_bool(true); - let llvm_name = &format!("llvm.cttz.i{}", width); + let llvm_name = &format!("llvm.cttz.i{width}"); self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) } sym::ctpop => self.call_intrinsic( - &format!("llvm.ctpop.i{}", width), + &format!("llvm.ctpop.i{width}"), &[args[0].immediate()], ), sym::bswap => { @@ -251,13 +252,13 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args[0].immediate() // byte swap a u8/i8 is just a no-op } else { self.call_intrinsic( - &format!("llvm.bswap.i{}", width), + &format!("llvm.bswap.i{width}"), &[args[0].immediate()], ) } } sym::bitreverse => self.call_intrinsic( - &format!("llvm.bitreverse.i{}", width), + &format!("llvm.bitreverse.i{width}"), &[args[0].immediate()], ), sym::rotate_left | sym::rotate_right => { @@ -284,15 +285,11 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }, None => { - span_invalid_monomorphization_error( - tcx.sess, + tcx.sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", - name, ty - ), - ); + name, + ty, + }); return; } } @@ -300,7 +297,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::raw_eq => { use abi::Abi::*; - let tp_ty = substs.type_at(0); + let tp_ty = fn_args.type_at(0); let layout = self.layout_of(tp_ty).layout; let use_integer_compare = match layout.abi() { Scalar(_) | ScalarPair(_, _) => true, @@ -319,18 +316,12 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.const_bool(true) } else if use_integer_compare { let integer_ty = self.type_ix(layout.size().bits()); - let ptr_ty = self.type_ptr_to(integer_ty); - let a_ptr = self.bitcast(a, ptr_ty); - let a_val = self.load(integer_ty, a_ptr, layout.align().abi); - let b_ptr = self.bitcast(b, ptr_ty); - let b_val = self.load(integer_ty, b_ptr, layout.align().abi); + let a_val = self.load(integer_ty, a, layout.align().abi); + let b_val = self.load(integer_ty, b, layout.align().abi); self.icmp(IntPredicate::IntEQ, a_val, b_val) } else { - let i8p_ty = self.type_i8p(); - let a_ptr = self.bitcast(a, i8p_ty); - let b_ptr = self.bitcast(b, i8p_ty); let n = self.const_usize(layout.size().bytes()); - let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); + let cmp = self.call_intrinsic("memcmp", &[a, b, n]); match self.cx.sess().target.arch.as_ref() { "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), @@ -338,19 +329,38 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } + sym::compare_bytes => { + // Here we assume that the `memcmp` provided by the target is a NOP for size 0. + let cmp = self.call_intrinsic( + "memcmp", + &[args[0].immediate(), args[1].immediate(), args[2].immediate()], + ); + // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`. + self.sext(cmp, self.type_ix(32)) + } + sym::black_box => { args[0].val.store(self, result); - + let result_val_span = [result.llval]; // We need to "use" the argument in some way LLVM can't introspect, and on // targets that support it we can typically leverage inline assembly to do // this. LLVM's interpretation of inline assembly is that it's, well, a black // box. This isn't the greatest implementation since it probably deoptimizes // more than we want, but it's so far good enough. + // + // For zero-sized types, the location pointed to by the result may be + // uninitialized. Do not "use" the result in this case; instead just clobber + // the memory. + let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() { + ("~{memory}", &[]) + } else { + ("r,~{memory}", &result_val_span) + }; crate::asm::inline_asm_call( self, "", - "r,~{memory}", - &[result.llval], + constraint, + inputs, self.type_void(), true, false, @@ -366,20 +376,20 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } _ if name.as_str().starts_with("simd_") => { - match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { + match generic_simd_intrinsic( + self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, + ) { Ok(llval) => llval, Err(()) => return, } } - _ => bug!("unknown intrinsic '{}'", name), + _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), }; if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { - let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); - let ptr = self.pointercast(result.llval, ptr_llty); - self.store(llval, ptr, result.align); + if let PassMode::Cast { .. } = &fn_abi.ret.mode { + self.store(llval, result.llval, result.align); } else { OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) .val @@ -403,9 +413,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time // optimization pass replaces calls to this intrinsic with code to test type membership. - let i8p_ty = self.type_i8p(); - let bitcast = self.bitcast(pointer, i8p_ty); - self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) + self.call_intrinsic("llvm.type.test", &[pointer, typeid]) } fn type_checked_load( @@ -415,7 +423,9 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { typeid: &'ll Value, ) -> Self::Value { let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); - self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + let type_checked_load = + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]); + self.extract_value(type_checked_load, 0) } fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { @@ -435,14 +445,16 @@ fn try_intrinsic<'ll>( dest: &'ll Value, ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, None, try_func, &[data], None); + let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); + bx.call(try_func_ty, None, None, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; bx.store(bx.const_i32(0), dest, ret_align); } else if wants_msvc_seh(bx.sess()) { codegen_msvc_try(bx, try_func, data, catch_func, dest); + } else if wants_wasm_eh(bx.sess()) { + codegen_wasm_try(bx, try_func, data, catch_func, dest); } else if bx.sess().target.os == "emscripten" { codegen_emcc_try(bx, try_func, data, catch_func, dest); } else { @@ -533,9 +545,9 @@ fn codegen_msvc_try<'ll>( // // More information can be found in libstd's seh.rs implementation. let ptr_align = bx.tcx().data_layout.pointer_align.abi; - let slot = bx.alloca(bx.type_i8p(), ptr_align); - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], normal, catchswitch, None); + let slot = bx.alloca(bx.type_ptr(), ptr_align); + let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); bx.switch_to_block(normal); bx.ret(bx.const_i32(0)); @@ -556,11 +568,11 @@ fn codegen_msvc_try<'ll>( // module. // // When modifying, make sure that the type_name string exactly matches - // the one used in src/libpanic_unwind/seh.rs. - let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); + // the one used in library/panic_unwind/src/seh.rs. + let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr()); let type_name = bx.const_bytes(b"rust_panic\0"); let type_info = - bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false); + bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false); let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); unsafe { llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); @@ -577,17 +589,17 @@ fn codegen_msvc_try<'ll>( bx.switch_to_block(catchpad_rust); let flags = bx.const_i32(8); let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); - let ptr = bx.load(bx.type_i8p(), slot, ptr_align); - let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], Some(&funclet)); + let ptr = bx.load(bx.type_ptr(), slot, ptr_align); + let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". bx.switch_to_block(catchpad_foreign); let flags = bx.const_i32(64); - let null = bx.const_null(bx.type_i8p()); + let null = bx.const_null(bx.type_ptr()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, catch_func, &[data, null], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet)); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -596,7 +608,81 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// WASM's definition of the `rust_try` function. +fn codegen_wasm_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + bx.set_personality_fn(bx.eh_personality()); + + let normal = bx.append_sibling_block("normal"); + let catchswitch = bx.append_sibling_block("catchswitch"); + let catchpad = bx.append_sibling_block("catchpad"); + let caught = bx.append_sibling_block("caught"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + + // We're generating an IR snippet that looks like: + // + // declare i32 @rust_try(%try_func, %data, %catch_func) { + // %slot = alloca i8* + // invoke %try_func(%data) to label %normal unwind label %catchswitch + // + // normal: + // ret i32 0 + // + // catchswitch: + // %cs = catchswitch within none [%catchpad] unwind to caller + // + // catchpad: + // %tok = catchpad within %cs [null] + // %ptr = call @llvm.wasm.get.exception(token %tok) + // %sel = call @llvm.wasm.get.ehselector(token %tok) + // call %catch_func(%data, %ptr) + // catchret from %tok to label %caught + // + // caught: + // ret i32 1 + // } + // + let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); + + bx.switch_to_block(normal); + bx.ret(bx.const_i32(0)); + + bx.switch_to_block(catchswitch); + let cs = bx.catch_switch(None, None, &[catchpad]); + + bx.switch_to_block(catchpad); + let null = bx.const_null(bx.type_ptr()); + let funclet = bx.catch_pad(cs, &[null]); + + let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]); + let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]); + + let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); + bx.catch_ret(&funclet, caught); + + bx.switch_to_block(caught); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -638,8 +724,8 @@ fn codegen_gnu_try<'ll>( let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -647,23 +733,23 @@ fn codegen_gnu_try<'ll>( // Type indicator for the exception being thrown. // // The first value in this tuple is a pointer to the exception object - // being thrown. The second value is a "selector" indicating which of + // being thrown. The second value is a "selector" indicating which of // the landing pad clauses the exception's type had been matched to. // rust_try ignores the selector. bx.switch_to_block(catch); - let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); + let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false); let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1); - let tydesc = bx.const_null(bx.type_i8p()); + let tydesc = bx.const_null(bx.type_ptr()); bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); - let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], None); + let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -702,8 +788,8 @@ fn codegen_emcc_try<'ll>( let try_func = llvm::get_param(bx.llfn(), 0); let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); - let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -711,14 +797,14 @@ fn codegen_emcc_try<'ll>( // Type indicator for the exception being thrown. // // The first value in this tuple is a pointer to the exception object - // being thrown. The second value is a "selector" indicating which of + // being thrown. The second value is a "selector" indicating which of // the landing pad clauses the exception's type had been matched to. bx.switch_to_block(catch); let tydesc = bx.eh_catch_typeinfo(); - let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); + let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false); let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2); bx.add_clause(vals, tydesc); - bx.add_clause(vals, bx.const_null(bx.type_i8p())); + bx.add_clause(vals, bx.const_null(bx.type_ptr())); let ptr = bx.extract_value(vals, 0); let selector = bx.extract_value(vals, 1); @@ -731,7 +817,7 @@ fn codegen_emcc_try<'ll>( // create an alloca and pass a pointer to that. let ptr_align = bx.tcx().data_layout.pointer_align.abi; let i8_align = bx.tcx().data_layout.i8_align.abi; - let catch_data_type = bx.type_struct(&[bx.type_i8p(), bx.type_bool()], false); + let catch_data_type = bx.type_struct(&[bx.type_ptr(), bx.type_bool()], false); let catch_data = bx.alloca(catch_data_type, ptr_align); let catch_data_0 = bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); @@ -739,16 +825,15 @@ fn codegen_emcc_try<'ll>( let catch_data_1 = bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); bx.store(is_rust_panic, catch_data_1, i8_align); - let catch_data = bx.bitcast(catch_data, bx.type_i8p()); - let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, catch_data], None); + let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); + bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -763,7 +848,7 @@ fn gen_fn<'ll, 'tcx>( ) -> (&'ll Type, &'ll Value) { let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, fn_abi); + let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. @@ -788,26 +873,32 @@ fn get_rust_try_fn<'ll, 'tcx>( // Define the type up front for the signature of the rust_try function. let tcx = cx.tcx; - let i8p = tcx.mk_mut_ptr(tcx.types.i8); + let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8); // `unsafe fn(*mut i8) -> ()` - let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - iter::once(i8p), - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - ))); + let try_fn_ty = Ty::new_fn_ptr( + tcx, + ty::Binder::dummy(tcx.mk_fn_sig( + [i8p], + Ty::new_unit(tcx), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + ); // `unsafe fn(*mut i8, *mut i8) -> ()` - let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - [i8p, i8p].iter().cloned(), - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - ))); + let catch_fn_ty = Ty::new_fn_ptr( + tcx, + ty::Binder::dummy(tcx.mk_fn_sig( + [i8p, i8p], + Ty::new_unit(tcx), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + ); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( - [try_fn_ty, i8p, catch_fn_ty].into_iter(), + [try_fn_ty, i8p, catch_fn_ty], tcx.types.i32, false, hir::Unsafety::Unsafe, @@ -822,45 +913,30 @@ fn generic_simd_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, + fn_args: GenericArgsRef<'tcx>, args: &[OperandRef<'tcx, &'ll Value>], ret_ty: Ty<'tcx>, llret_ty: &'ll Type, span: Span, ) -> Result<&'ll Value, ()> { - // macros for error handling: - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } - macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } + ($diag: expr) => {{ + bx.sess().emit_err($diag); + return Err(()); + }}; } macro_rules! require { - ($cond: expr, $($fmt: tt)*) => { + ($cond: expr, $diag: expr) => { if !$cond { - return_error!($($fmt)*); + return_error!($diag); } }; } macro_rules! require_simd { - ($ty: expr, $position: expr) => { - require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) + ($ty: expr, $diag: expr) => { + require!($ty.is_simd(), $diag) }; } @@ -870,7 +946,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - require_simd!(arg_tys[1], "argument"); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let expected_int_bits = (len.max(8) - 1).next_power_of_two(); @@ -882,21 +962,21 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) == Some(expected_bytes) => { let place = PlaceRef::alloca(bx, args[0].layout); args[0].val.store(bx, place); let int_ty = bx.type_ix(expected_bytes * 8); - let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); - bx.load(int_ty, ptr, Align::ONE) + bx.load(int_ty, place.llval, Align::ONE) } - _ => return_error!( - "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + _ => return_error!(InvalidMonomorphization::InvalidBitmask { + span, + name, mask_ty, expected_int_bits, expected_bytes - ), + }), }; let i1 = bx.type_i1(); @@ -908,7 +988,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // every intrinsic below takes a SIMD vector as its first argument - require_simd!(arg_tys[0], "input"); + require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] }); let in_ty = arg_tys[0]; let comparison = match name { @@ -923,23 +1003,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); if let Some(cmp_op) = comparison { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); require!( bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, - "expected return type with integer elements, found `{}` with non-integer `{}`", - ret_ty, - out_ty + InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty } ); return Ok(compare_simd_types( @@ -952,46 +1033,81 @@ fn generic_simd_intrinsic<'ll, 'tcx>( )); } - if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { - // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer. - // If there is no suffix, use the index array length. - let n: u64 = if stripped.is_empty() { - // Make sure this is actually an array, since typeck only checks the length-suffixed - // version of this intrinsic. - match args[2].layout.ty.kind() { - ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { - span_bug!(span, "could not evaluate shuffle index array length") - }) + if name == sym::simd_shuffle_generic { + let idx = fn_args[2] + .expect_const() + .eval(tcx, ty::ParamEnv::reveal_all(), Some(span)) + .unwrap() + .unwrap_branch(); + let n = idx.len() as u64; + + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + out_len == n, + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } + ); + require!( + in_elem == out_ty, + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } + ); + + let total_len = in_len * 2; + + let indices: Option<Vec<_>> = idx + .iter() + .enumerate() + .map(|(arg_idx, val)| { + let idx = val.unwrap_leaf().try_to_i32().unwrap(); + if idx >= i32::try_from(total_len).unwrap() { + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + span, + name, + arg_idx: arg_idx as u64, + total_len: total_len.into(), + }); + None + } else { + Some(bx.const_i32(idx)) } - _ => return_error!( - "simd_shuffle index must be an array of `u32`, got `{}`", - args[2].layout.ty - ), - } - } else { - stripped.parse().unwrap_or_else(|_| { - span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") }) + .collect(); + let Some(indices) = indices else { + return Ok(bx.const_null(llret_ty)); }; - require_simd!(ret_ty, "return"); + return Ok(bx.shuffle_vector( + args[0].immediate(), + args[1].immediate(), + bx.const_vector(&indices), + )); + } + + if name == sym::simd_shuffle { + // Make sure this is actually an array, since typeck only checks the length-suffixed + // version of this intrinsic. + let n: u64 = match args[2].layout.ty.kind() { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { + len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( + || span_bug!(span, "could not evaluate shuffle index array length"), + ) + } + _ => return_error!(InvalidMonomorphization::SimdShuffle { + span, + name, + ty: args[2].layout.ty + }), + }; + + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); require!( out_len == n, - "expected return type of length {}, found `{}` with length {}", - n, - ret_ty, - out_len + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } ); require!( in_elem == out_ty, - "expected return element type `{}` (element of input `{}`), \ - found `{}` with element type `{}`", - in_elem, - in_ty, - ret_ty, - out_ty + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } ); let total_len = u128::from(in_len) * 2; @@ -1004,15 +1120,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - emit_error!("shuffle index #{} is not a constant", arg_idx); + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexNotConstant { + span, + name, + arg_idx, + }); None } Some(idx) if idx >= total_len => { - emit_error!( - "shuffle index #{} is out of bounds (limit {})", + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + span, + name, arg_idx, - total_len - ); + total_len, + }); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1033,10 +1154,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_insert { require!( in_elem == arg_tys[2], - "expected inserted type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - arg_tys[2] + InvalidMonomorphization::InsertedType { + span, + name, + in_elem, + in_ty, + out_ty: arg_tys[2] + } ); return Ok(bx.insert_element( args[0].immediate(), @@ -1047,10 +1171,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_extract { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); } @@ -1058,17 +1179,18 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_select { let m_elem_ty = in_elem; let m_len = in_len; - require_simd!(arg_tys[1], "argument"); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); require!( m_len == v_len, - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len + InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } ); match m_elem_ty.kind() { ty::Int(_) => {} - _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), + _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }), } // truncate the mask to a vector of i1s let i1 = bx.type_i1(); @@ -1100,11 +1222,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( args[0].immediate(), i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), ), - _ => return_error!( - "vector argument `{}`'s element type `{}`, expected integer element type", + _ => return_error!(InvalidMonomorphization::VectorArgument { + span, + name, in_ty, in_elem - ), + }), }; // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. @@ -1126,7 +1249,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) == Some(expected_bytes) => { // Zero-extend iN to the array length: @@ -1136,15 +1259,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); bx.store(ze, ptr, Align::ONE); let array_ty = bx.type_array(bx.type_i8(), expected_bytes); - let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); return Ok(bx.load(array_ty, ptr, Align::ONE)); } - _ => return_error!( - "cannot return `{}`, expected `u{}` or `[u8; {}]`", + _ => return_error!(InvalidMonomorphization::CannotReturn { + span, + name, ret_ty, expected_int_bits, expected_bytes - ), + }), } } @@ -1157,25 +1280,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } + ($diag: expr) => {{ + bx.sess().emit_err($diag); + return Err(()); + }}; } let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { @@ -1183,16 +1292,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match f.bit_width() { 32 => ("f32", elem_ty), 64 => ("f64", elem_ty), - _ => { - return_error!( - "unsupported element type `{}` of floating-point vector `{}`", - f.name_str(), - in_ty - ); - } + _ => return_error!(InvalidMonomorphization::FloatingPointVector { + span, + name, + f_ty: *f, + in_ty, + }), } } else { - return_error!("`{}` is not a floating-point type", in_ty); + return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty }); }; let vec_ty = bx.type_vector(elem_ty, in_len); @@ -1214,13 +1322,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), - _ => return_error!("unrecognized intrinsic `{}`", name), + _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }), }; - let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); + let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}"); let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); let c = bx.call( fn_ty, None, + None, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -1253,50 +1362,34 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // FIXME: use: // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 - fn llvm_vector_str( - elem_ty: Ty<'_>, - vec_len: u64, - no_pointers: usize, - bx: &Builder<'_, '_, '_>, - ) -> String { - let p0s: String = "p0".repeat(no_pointers); + fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String { match *elem_ty.kind() { ty::Int(v) => format!( - "v{}{}i{}", + "v{}i{}", vec_len, - p0s, // Normalize to prevent crash if v: IntTy::Isize v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() ), ty::Uint(v) => format!( - "v{}{}i{}", + "v{}i{}", vec_len, - p0s, // Normalize to prevent crash if v: UIntTy::Usize v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() ), - ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()), + ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()), + ty::RawPtr(_) => format!("v{}p0", vec_len), _ => unreachable!(), } } - fn llvm_vector_ty<'ll>( - cx: &CodegenCx<'ll, '_>, - elem_ty: Ty<'_>, - vec_len: u64, - mut no_pointers: usize, - ) -> &'ll Type { - // FIXME: use cx.layout_of(ty).llvm_type() ? - let mut elem_ty = match *elem_ty.kind() { + fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type { + let elem_ty = match *elem_ty.kind() { ty::Int(v) => cx.type_int_from_ty(v), ty::Uint(v) => cx.type_uint_from_ty(v), ty::Float(v) => cx.type_float_from_ty(v), + ty::RawPtr(_) => cx.type_ptr(), _ => unreachable!(), }; - while no_pointers > 0 { - elem_ty = cx.type_ptr_to(elem_ty); - no_pointers -= 1; - } cx.type_vector(elem_ty, vec_len) } @@ -1308,79 +1401,69 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); - require_simd!(ret_ty, "return"); + require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); // Of the same length: let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - out_len + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len + } ); require!( in_len == out_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - out_len2 + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: out_len2 + } ); // The return type must match the first argument type - require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); - - // This counts how many pointers - fn ptr_count(t: Ty<'_>) -> usize { - match t.kind() { - ty::RawPtr(p) => 1 + ptr_count(p.ty), - _ => 0, - } - } - - // Non-ptr type - fn non_ptr(t: Ty<'_>) -> Ty<'_> { - match t.kind() { - ty::RawPtr(p) => non_ptr(p.ty), - _ => t, - } - } + require!( + ret_ty == in_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } + ); // The second argument must be a simd vector with an element type that's a pointer // to the element type of the first argument let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (pointer_count, underlying_ty) = match element_ty1.kind() { - ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)), - _ => { - require!( - false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*_ {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem - ); - unreachable!(); + + require!( + matches!( + element_ty1.kind(), + ty::RawPtr(p) if p.ty == in_elem && p.ty.kind() == element_ty0.kind() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Not, } - }; - assert!(pointer_count > 0); - assert_eq!(pointer_count - 1, ptr_count(element_ty0)); - assert_eq!(underlying_ty, non_ptr(element_ty0)); + ); // The element type of the third argument must be a signed integer type of any width: let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); @@ -1389,10 +1472,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of third argument `{}` \ - to be a signed integer type", - element_ty2, - arg_tys[2] + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } ); } } @@ -1409,15 +1494,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; // Type of the vector of pointers: - let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); - let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); + let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len); + let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len); // Type of the vector of elements: - let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); - let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); + let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); + let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len); let llvm_intrinsic = - format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}"); let fn_ty = bx.type_func( &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty, @@ -1426,6 +1511,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None, @@ -1441,78 +1527,64 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); + require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); // Of the same length: let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == element_len1, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - element_len1 + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len: element_len1 + } ); require!( in_len == element_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - element_len2 - ); - - // This counts how many pointers - fn ptr_count(t: Ty<'_>) -> usize { - match t.kind() { - ty::RawPtr(p) => 1 + ptr_count(p.ty), - _ => 0, - } - } - - // Non-ptr type - fn non_ptr(t: Ty<'_>) -> Ty<'_> { - match t.kind() { - ty::RawPtr(p) => non_ptr(p.ty), - _ => t, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: element_len2 } - } + ); // The second argument must be a simd vector with an element type that's a pointer // to the element type of the first argument let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); - let (pointer_count, underlying_ty) = match element_ty1.kind() { - ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { - (ptr_count(element_ty1), non_ptr(element_ty1)) - } - _ => { - require!( - false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*mut {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem - ); - unreachable!(); + + require!( + matches!( + element_ty1.kind(), + ty::RawPtr(p) + if p.ty == in_elem && p.mutbl.is_mut() && p.ty.kind() == element_ty0.kind() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Mut, } - }; - assert!(pointer_count > 0); - assert_eq!(pointer_count - 1, ptr_count(element_ty0)); - assert_eq!(underlying_ty, non_ptr(element_ty0)); + ); // The element type of the third argument must be a signed integer type of any width: match element_ty2.kind() { @@ -1520,10 +1592,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of third argument `{}` \ - be a signed integer type", - element_ty2, - arg_tys[2] + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } ); } } @@ -1542,21 +1616,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ret_t = bx.type_void(); // Type of the vector of pointers: - let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); - let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); + let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len); + let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len); // Type of the vector of elements: - let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); - let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); + let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); + let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len); let llvm_intrinsic = - format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}"); let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); let v = bx.call( fn_ty, None, + None, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None, @@ -1570,10 +1645,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::$name { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { ty::Int(_) | ty::Uint(_) => { @@ -1596,25 +1668,28 @@ fn generic_simd_intrinsic<'ll, 'tcx>( 32 => bx.const_real(bx.type_f32(), $identity), 64 => bx.const_real(bx.type_f64(), $identity), v => return_error!( - r#" -unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, - sym::$name, - in_ty, - in_elem, - v, - ret_ty + InvalidMonomorphization::UnsupportedSymbolOfSize { + span, + name, + symbol: sym::$name, + in_ty, + in_elem, + size: v, + ret_ty + } ), } }; Ok(bx.$float_reduce(acc, args[0].immediate())) } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1642,22 +1717,20 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, if name == sym::$name { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1675,22 +1748,20 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let input = if !$boolean { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); args[0].immediate() } else { match in_elem.kind() { ty::Int(_) | ty::Uint(_) => {} - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), } // boolean reductions operate on vectors of i1s: @@ -1703,13 +1774,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let r = bx.$red(input); Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1722,16 +1794,18 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bitwise_red!(simd_reduce_any: vector_reduce_or, true); if name == sym::simd_cast_ptr { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); match in_elem.kind() { @@ -1740,9 +1814,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); assert!(!check_sized); // we are in codegen, so we shouldn't see these types - require!(metadata.is_unit(), "cannot cast fat pointer `{}`", in_elem) + require!( + metadata.is_unit(), + InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } + ); + } + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) } - _ => return_error!("expected pointer, got `{}`", in_elem), } match out_elem.kind() { ty::RawPtr(p) => { @@ -1750,79 +1829,90 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); assert!(!check_sized); // we are in codegen, so we shouldn't see these types - require!(metadata.is_unit(), "cannot cast to fat pointer `{}`", out_elem) + require!( + metadata.is_unit(), + InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } + ); + } + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) } - _ => return_error!("expected pointer, got `{}`", out_elem), } - if in_elem == out_elem { - return Ok(args[0].immediate()); - } else { - return Ok(bx.pointercast(args[0].immediate(), llret_ty)); - } + return Ok(args[0].immediate()); } if name == sym::simd_expose_addr { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); match in_elem.kind() { ty::RawPtr(_) => {} - _ => return_error!("expected pointer, got `{}`", in_elem), + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) + } } match out_elem.kind() { ty::Uint(ty::UintTy::Usize) => {} - _ => return_error!("expected `usize`, got `{}`", out_elem), + _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }), } return Ok(bx.ptrtoint(args[0].immediate(), llret_ty)); } if name == sym::simd_from_exposed_addr { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); match in_elem.kind() { ty::Uint(ty::UintTy::Usize) => {} - _ => return_error!("expected `usize`, got `{}`", in_elem), + _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }), } match out_elem.kind() { ty::RawPtr(_) => {} - _ => return_error!("expected pointer, got `{}`", out_elem), + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) + } } return Ok(bx.inttoptr(args[0].immediate(), llret_ty)); } if name == sym::simd_cast || name == sym::simd_as { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); // casting cares about nominal type, not just structural type if in_elem == out_elem { @@ -1901,11 +1991,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, } require!( false, - "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", - in_ty, - in_elem, - ret_ty, - out_elem + InvalidMonomorphization::UnsupportedCast { + span, + name, + in_ty, + in_elem, + ret_ty, + out_elem + } ); } macro_rules! arith_binary { @@ -1917,10 +2010,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* _ => {}, } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) + require!( + false, + InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } + ); })* } } @@ -1948,10 +2041,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* _ => {}, } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) + require!( + false, + InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } + ); })* } } @@ -1959,6 +2052,52 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, simd_neg: Int => neg, Float => fneg; } + // Unary integer intrinsics + if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) { + let vec_ty = bx.cx.type_vector( + match *in_elem.kind() { + ty::Int(i) => bx.cx.type_int_from_ty(i), + ty::Uint(i) => bx.cx.type_uint_from_ty(i), + _ => return_error!(InvalidMonomorphization::UnsupportedOperation { + span, + name, + in_ty, + in_elem + }), + }, + in_len as u64, + ); + let intrinsic_name = match name { + sym::simd_bswap => "bswap", + sym::simd_bitreverse => "bitreverse", + sym::simd_ctlz => "ctlz", + sym::simd_cttz => "cttz", + _ => unreachable!(), + }; + let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits(); + let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,); + + return if name == sym::simd_bswap && int_size == 8 { + // byte swap is no-op for i8/u8 + Ok(args[0].immediate()) + } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) { + let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call( + fn_ty, + None, + None, + f, + &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], + None, + )) + } else { + let fn_ty = bx.type_func(&[vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None)) + }; + } + if name == sym::simd_arith_offset { // This also checks that the first operand is a ptr type. let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { @@ -1989,12 +2128,12 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), _ => { - return_error!( - "expected element type `{}` of vector type `{}` \ - to be a signed or unsigned integer type", - arg_tys[0].simd_size_and_type(bx.tcx()).1, - arg_tys[0] - ); + return_error!(InvalidMonomorphization::ExpectedVectorElementType { + span, + name, + expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1, + vector_type: arg_tys[0] + }); } }; let llvm_intrinsic = &format!( @@ -2008,7 +2147,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, None, f, &[lhs, rhs], None); + let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index d51aced85df..59d1ea05d8a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,46 +5,55 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(hash_raw_entry)] -#![feature(let_chains)] #![feature(extern_types)] -#![feature(once_cell)] +#![feature(hash_raw_entry)] #![feature(iter_intersperse)] +#![feature(let_chains)] +#![feature(never_type)] +#![feature(slice_group_by)] +#![feature(impl_trait_in_assoc_type)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] +#![deny(rustc::untranslatable_diagnostic)] +#![deny(rustc::diagnostic_outside_of_impl)] #[macro_use] extern crate rustc_macros; #[macro_use] extern crate tracing; +use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; +use errors::ParseTargetMachineConfig; pub use llvm_util::target_features; use rustc_ast::expand::allocator::AllocatorKind; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLTOInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, + CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, }; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; -use rustc_data_structures::fx::FxHashMap; -use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; +use rustc_data_structures::fx::FxIndexMap; +use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage}; +use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::ty::query::Providers; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; +use rustc_middle::util::Providers; +use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; use std::any::Any; use std::ffi::CStr; +use std::io::Write; mod back { pub mod archive; pub mod lto; + pub mod owned_target_machine; mod profiling; pub mod write; } @@ -62,9 +71,10 @@ mod context; mod coverageinfo; mod debuginfo; mod declare; +mod errors; mod intrinsic; -// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. +// The following is a workaround that replaces `pub mod llvm;` and that fixes issue 53912. #[path = "llvm/mod.rs"] mod llvm_; pub mod llvm { @@ -78,6 +88,8 @@ mod type_of; mod va_arg; mod value; +fluent_messages! { "../messages.ftl" } + #[derive(Clone)] pub struct LlvmCodegenBackend(()); @@ -132,18 +144,6 @@ impl ExtraBackendMethods for LlvmCodegenBackend { back::write::target_machine_factory(sess, optlvl, target_features) } - fn spawn_thread<F, T>(time_trace: bool, f: F) -> std::thread::JoinHandle<T> - where - F: FnOnce() -> T, - F: Send + 'static, - T: Send + 'static, - { - std::thread::spawn(move || { - let _profiler = TimeTraceProfiler::new(time_trace); - f() - }) - } - fn spawn_named_thread<F, T>( time_trace: bool, name: String, @@ -164,12 +164,34 @@ impl ExtraBackendMethods for LlvmCodegenBackend { impl WriteBackendMethods for LlvmCodegenBackend { type Module = ModuleLlvm; type ModuleBuffer = back::lto::ModuleBuffer; - type TargetMachine = &'static mut llvm::TargetMachine; + type TargetMachine = OwnedTargetMachine; + type TargetMachineError = crate::errors::LlvmError<'static>; type ThinData = back::lto::ThinData; type ThinBuffer = back::lto::ThinBuffer; fn print_pass_timings(&self) { unsafe { - llvm::LLVMRustPrintPassTimings(); + let mut size = 0; + let cstr = llvm::LLVMRustPrintPassTimings(&mut size as *mut usize); + if cstr.is_null() { + println!("failed to get pass timings"); + } else { + let timings = std::slice::from_raw_parts(cstr as *const u8, size); + std::io::stdout().write_all(timings).unwrap(); + libc::free(cstr as *mut _); + } + } + } + fn print_statistics(&self) { + unsafe { + let mut size = 0; + let cstr = llvm::LLVMRustPrintStatistics(&mut size as *mut usize); + if cstr.is_null() { + println!("failed to get pass stats"); + } else { + let stats = std::slice::from_raw_parts(cstr as *const u8, size); + std::io::stdout().write_all(stats).unwrap(); + libc::free(cstr as *mut _); + } } } fn run_link( @@ -181,7 +203,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { } fn run_fat_lto( cgcx: &CodegenContext<Self>, - modules: Vec<FatLTOInput<Self>>, + modules: Vec<FatLtoInput<Self>>, cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>, ) -> Result<LtoModuleCodegen<Self>, FatalError> { back::lto::run_fat(cgcx, modules, cached_modules) @@ -240,6 +262,10 @@ impl LlvmCodegenBackend { } impl CodegenBackend for LlvmCodegenBackend { + fn locale_resource(&self) -> &'static str { + crate::DEFAULT_LOCALE_RESOURCE + } + fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited } @@ -249,10 +275,10 @@ impl CodegenBackend for LlvmCodegenBackend { |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) } - fn print(&self, req: PrintRequest, sess: &Session) { - match req { - PrintRequest::RelocationModels => { - println!("Available relocation models:"); + fn print(&self, req: &PrintRequest, out: &mut dyn PrintBackendInfo, sess: &Session) { + match req.kind { + PrintKind::RelocationModels => { + writeln!(out, "Available relocation models:"); for name in &[ "static", "pic", @@ -263,26 +289,27 @@ impl CodegenBackend for LlvmCodegenBackend { "ropi-rwpi", "default", ] { - println!(" {}", name); + writeln!(out, " {name}"); } - println!(); + writeln!(out); } - PrintRequest::CodeModels => { - println!("Available code models:"); + PrintKind::CodeModels => { + writeln!(out, "Available code models:"); for name in &["tiny", "small", "kernel", "medium", "large"] { - println!(" {}", name); + writeln!(out, " {name}"); } - println!(); + writeln!(out); } - PrintRequest::TlsModels => { - println!("Available TLS models:"); + PrintKind::TlsModels => { + writeln!(out, "Available TLS models:"); for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { - println!(" {}", name); + writeln!(out, " {name}"); } - println!(); + writeln!(out); } - PrintRequest::StackProtectorStrategies => { - println!( + PrintKind::StackProtectorStrategies => { + writeln!( + out, r#"Available stack protector strategies: all Generate stack canaries in all functions. @@ -306,7 +333,7 @@ impl CodegenBackend for LlvmCodegenBackend { "# ); } - req => llvm_util::print(req, sess), + _other => llvm_util::print(req, out, sess), } } @@ -342,18 +369,18 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") .join(sess); - sess.time("llvm_dump_timing_file", || { - if sess.opts.unstable_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { + sess.time("llvm_dump_timing_file", || { let file_name = outputs.with_extension("llvm_timings.json"); llvm_util::time_trace_profiler_finish(&file_name); - } - }); + }); + } Ok((codegen_results, work_products)) } @@ -376,7 +403,9 @@ impl CodegenBackend for LlvmCodegenBackend { pub struct ModuleLlvm { llcx: &'static mut llvm::Context, llmod_raw: *const llvm::Module, - tm: &'static mut llvm::TargetMachine, + + // independent from llcx and llmod_raw, resources get disposed by drop impl + tm: OwnedTargetMachine, } unsafe impl Send for ModuleLlvm {} @@ -412,8 +441,7 @@ impl ModuleLlvm { let tm = match (cgcx.tm_factory)(tm_factory_config) { Ok(m) => m, Err(e) => { - handler.struct_err(&e).emit(); - return Err(FatalError); + return Err(handler.emit_almost_fatal(ParseTargetMachineConfig(e))); } }; @@ -429,7 +457,6 @@ impl ModuleLlvm { impl Drop for ModuleLlvm { fn drop(&mut self) { unsafe { - llvm::LLVMRustDisposeTargetMachine(&mut *(self.tm as *mut _)); llvm::LLVMContextDispose(&mut *(self.llcx as *mut _)); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index 45de284d22a..06e846a2b45 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -9,7 +9,7 @@ use libc::c_uint; use super::{DiagnosticInfo, SMDiagnostic}; use rustc_span::InnerSpan; -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] pub enum OptimizationDiagnosticKind { OptimizationRemark, OptimizationMissed, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e2d0390821d..a038b3af03d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,8 +1,6 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] -use rustc_codegen_ssa::coverageinfo::map as coverage_map; - use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, @@ -35,7 +33,7 @@ pub enum LLVMRustResult { pub struct LLVMRustCOFFShortExport { pub name: *const c_char, pub ordinal_present: bool, - // value of `ordinal` only important when `ordinal_present` is true + /// value of `ordinal` only important when `ordinal_present` is true pub ordinal: u16, } @@ -79,17 +77,23 @@ pub enum LLVMModFlagBehavior { Append = 5, AppendUnique = 6, Max = 7, + Min = 8, } // Consts for the LLVM CallConv type, pre-cast to usize. /// LLVM CallingConv::ID. Should we wrap this? +/// +/// See <https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/CallingConv.h> #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] pub enum CallConv { CCallConv = 0, FastCallConv = 8, ColdCallConv = 9, + PreserveMost = 14, + PreserveAll = 15, + Tail = 18, X86StdcallCallConv = 64, X86FastcallCallConv = 65, ArmAapcsCallConv = 67, @@ -195,6 +199,7 @@ pub enum AttributeKind { AllocSize = 37, AllocatedPointer = 38, AllocAlign = 39, + SanitizeSafeStack = 40, } /// LLVMIntPredicate @@ -427,6 +432,7 @@ pub enum MetadataType { MD_type = 19, MD_vcall_visibility = 28, MD_noundef = 29, + MD_kcfi_type = 36, } /// LLVMRustAsmDialect @@ -474,12 +480,16 @@ pub enum OptStage { pub struct SanitizerOptions { pub sanitize_address: bool, pub sanitize_address_recover: bool, + pub sanitize_cfi: bool, + pub sanitize_kcfi: bool, pub sanitize_memory: bool, pub sanitize_memory_recover: bool, pub sanitize_memory_track_origins: c_int, pub sanitize_thread: bool, pub sanitize_hwaddress: bool, pub sanitize_hwaddress_recover: bool, + pub sanitize_kernel_address: bool, + pub sanitize_kernel_address_recover: bool, } /// LLVMRelocMode @@ -548,6 +558,7 @@ pub enum ArchiveKind { K_BSD, K_DARWIN, K_COFF, + K_AIXBIG, } // LLVMRustThinLTOData @@ -579,6 +590,16 @@ pub enum ThreadLocalMode { LocalExec, } +/// LLVMRustTailCallKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum TailCallKind { + None, + Tail, + MustTail, + NoTail, +} + /// LLVMRustChecksumKind #[derive(Copy, Clone)] #[repr(C)] @@ -637,9 +658,6 @@ pub struct Builder<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct PassManager<'a>(InvariantOpaque<'a>); extern "C" { - pub type PassManagerBuilder; -} -extern "C" { pub type Pass; } extern "C" { @@ -675,200 +693,6 @@ extern "C" { pub type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint); -pub mod coverageinfo { - use super::coverage_map; - - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) - #[derive(Copy, Clone, Debug)] - #[repr(C)] - pub enum RegionKind { - /// A CodeRegion associates some code with a counter - CodeRegion = 0, - - /// An ExpansionRegion represents a file expansion region that associates - /// a source range with the expansion of a virtual source file, such as - /// for a macro instantiation or #include file. - ExpansionRegion = 1, - - /// A SkippedRegion represents a source range with code that was skipped - /// by a preprocessor or similar means. - SkippedRegion = 2, - - /// A GapRegion is like a CodeRegion, but its count is only set as the - /// line execution count when its the only region in the line. - GapRegion = 3, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion = 4, - } - - /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the - /// coverage map, in accordance with the - /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). - /// The struct composes fields representing the `Counter` type and value(s) (injected counter - /// ID, or expression type and operands), the source file (an indirect index into a "filenames - /// array", encoded separately), and source location (start and end positions of the represented - /// code region). - /// - /// Matches LLVMRustCounterMappingRegion. - #[derive(Copy, Clone, Debug)] - #[repr(C)] - pub struct CounterMappingRegion { - /// The counter type and type-dependent counter data, if any. - counter: coverage_map::Counter, - - /// If the `RegionKind` is a `BranchRegion`, this represents the counter - /// for the false branch of the region. - false_counter: coverage_map::Counter, - - /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the - /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes - /// that, in turn, are used to look up the filename for this region. - file_id: u32, - - /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find - /// the mapping regions created as a result of macro expansion, by checking if their file id - /// matches the expanded file id. - expanded_file_id: u32, - - /// 1-based starting line of the mapping region. - start_line: u32, - - /// 1-based starting column of the mapping region. - start_col: u32, - - /// 1-based ending line of the mapping region. - end_line: u32, - - /// 1-based ending column of the mapping region. If the high bit is set, the current - /// mapping region is a gap area. - end_col: u32, - - kind: RegionKind, - } - - impl CounterMappingRegion { - pub(crate) fn code_region( - counter: coverage_map::Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::CodeRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn branch_region( - counter: coverage_map::Counter, - false_counter: coverage_map::Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::BranchRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn expansion_region( - file_id: u32, - expanded_file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: coverage_map::Counter::zero(), - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::ExpansionRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn skipped_region( - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: coverage_map::Counter::zero(), - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::SkippedRegion, - } - } - - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn gap_region( - counter: coverage_map::Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: coverage_map::Counter::zero(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col: (1_u32 << 31) | end_col, - kind: RegionKind::GapRegion, - } - } - } -} - pub mod debuginfo { use super::{InvariantOpaque, Metadata}; use bitflags::bitflags; @@ -945,15 +769,27 @@ pub mod debuginfo { NoDebug, FullDebug, LineTablesOnly, + DebugDirectivesOnly, } impl DebugEmissionKind { pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + // We should be setting LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we are + // instead adding a new debuginfo option "line-tables-only" so as to + // not break anything and to allow users to have 'limited' debug info. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. use rustc_session::config::DebugInfo; match kind { DebugInfo::None => DebugEmissionKind::NoDebug, - DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, - DebugInfo::Full => DebugEmissionKind::FullDebug, + DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly, + DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly, + DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug, } } } @@ -983,6 +819,9 @@ pub type SelfProfileBeforePassCallback = unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); +pub type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; +pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; + extern "C" { pub fn LLVMRustInstallFatalErrorHandler(); pub fn LLVMRustDisableSystemDialogsOnCrash(); @@ -1002,7 +841,7 @@ extern "C" { pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; @@ -1041,7 +880,7 @@ extern "C" { // Operations on array, pointer, and vector types (sequence types) pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; - pub fn LLVMPointerType(ElementType: &Type, AddressSpace: c_uint) -> &Type; + pub fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; pub fn LLVMGetElementType(Ty: &Type) -> &Type; @@ -1049,7 +888,8 @@ extern "C" { // Operations on other types pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + pub fn LLVMTokenTypeInContext(C: &Context) -> &Type; + pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values pub fn LLVMTypeOf(Val: &Value) -> &Type; @@ -1060,13 +900,21 @@ extern "C" { pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>; + pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; // Operations on constants of any type pub fn LLVMConstNull(Ty: &Type) -> &Value; pub fn LLVMGetUndef(Ty: &Type) -> &Value; + pub fn LLVMGetPoison(Ty: &Type) -> &Value; // Operations on metadata + // FIXME: deprecated, replace with LLVMMDStringInContext2 pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + + pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata; + + // FIXME: deprecated, replace with LLVMMDNodeInContext2 pub fn LLVMMDNodeInContext<'a>( C: &'a Context, Vals: *const &'a Value, @@ -1105,6 +953,8 @@ extern "C" { Packed: Bool, ) -> &'a Value; + // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 + // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc pub fn LLVMConstArray<'a>( ElementTy: &'a Type, ConstantVals: *const &'a Value, @@ -1113,7 +963,7 @@ extern "C" { pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions - pub fn LLVMRustConstInBoundsGEP2<'a>( + pub fn LLVMConstInBoundsGEP2<'a>( ty: &'a Type, ConstantVal: &'a Value, ConstantIndices: *const &'a Value, @@ -1164,6 +1014,7 @@ extern "C" { NameLen: size_t, ) -> Option<&Value>; pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub fn LLVMRustSetTailCallKind(CallInst: &Value, TKC: TailCallKind); // Operations on attributes pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; @@ -1244,7 +1095,7 @@ extern "C" { pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); + pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata); // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; @@ -1270,7 +1121,8 @@ extern "C" { NumArgs: c_uint, Then: &'a BasicBlock, Catch: &'a BasicBlock, - Bundle: Option<&OperandBundleDef<'a>>, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, Name: *const c_char, ) -> &'a Value; pub fn LLVMBuildLandingPad<'a>( @@ -1283,38 +1135,38 @@ extern "C" { pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMRustBuildCleanupPad<'a>( + pub fn LLVMBuildCleanupPad<'a>( B: &Builder<'a>, ParentPad: Option<&'a Value>, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCleanupRet<'a>( + pub fn LLVMBuildCleanupRet<'a>( B: &Builder<'a>, CleanupPad: &'a Value, - UnwindBB: Option<&'a BasicBlock>, + BB: Option<&'a BasicBlock>, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchPad<'a>( + pub fn LLVMBuildCatchPad<'a>( B: &Builder<'a>, ParentPad: &'a Value, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchRet<'a>( + pub fn LLVMBuildCatchRet<'a>( B: &Builder<'a>, - Pad: &'a Value, + CatchPad: &'a Value, BB: &'a BasicBlock, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchSwitch<'a>( + pub fn LLVMBuildCatchSwitch<'a>( Builder: &Builder<'a>, ParentPad: Option<&'a Value>, - BB: Option<&'a BasicBlock>, + UnwindBB: Option<&'a BasicBlock>, NumHandlers: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); + pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); // Add a case to the switch instruction @@ -1608,11 +1460,12 @@ extern "C" { DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMRustBuildIntCast<'a>( + pub fn LLVMBuildIntCast2<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, - IsSigned: bool, + IsSigned: Bool, + Name: *const c_char, ) -> &'a Value; // Comparisons @@ -1640,7 +1493,8 @@ extern "C" { Fn: &'a Value, Args: *const &'a Value, NumArgs: c_uint, - Bundle: Option<&OperandBundleDef<'a>>, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, ) -> &'a Value; pub fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, @@ -1804,8 +1658,6 @@ extern "C" { /// Creates a legacy pass manager -- only used for final codegen. pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; - pub fn LLVMInitializePasses(); - pub fn LLVMTimeTraceProfilerInitialize(); pub fn LLVMTimeTraceProfilerFinishThread(); @@ -1824,7 +1676,10 @@ extern "C" { pub fn LLVMRustGetLastError() -> *const c_char; /// Print the pass timings since static dtors aren't picking them up. - pub fn LLVMRustPrintPassTimings(); + pub fn LLVMRustPrintPassTimings(size: *const size_t) -> *const c_char; + + /// Print the statistics since static dtors aren't picking them up. + pub fn LLVMRustPrintStatistics(size: *const size_t) -> *const c_char; pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; @@ -1857,6 +1712,8 @@ extern "C" { pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( Filenames: *const *const c_char, FilenamesLen: size_t, + Lengths: *const size_t, + LengthsLen: size_t, BufferOut: &RustString, ); @@ -1864,15 +1721,18 @@ extern "C" { pub fn LLVMRustCoverageWriteMappingToBuffer( VirtualFileMappingIDs: *const c_uint, NumVirtualFileMappingIDs: c_uint, - Expressions: *const coverage_map::CounterExpression, + Expressions: *const crate::coverageinfo::ffi::CounterExpression, NumExpressions: c_uint, - MappingRegions: *const coverageinfo::CounterMappingRegion, + MappingRegions: *const crate::coverageinfo::ffi::CounterMappingRegion, NumMappingRegions: c_uint, BufferOut: &RustString, ); - pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; - pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; + pub fn LLVMRustCoverageCreatePGOFuncNameVar( + F: &Value, + FuncName: *const c_char, + FuncNameLen: size_t, + ) -> &Value; pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] @@ -1902,7 +1762,7 @@ extern "C" { ); pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; @@ -1960,6 +1820,21 @@ extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateMethod<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + SPFlags: DISPFlags, + TParam: &'a DIArray, + ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2127,7 +2002,8 @@ extern "C" { Builder: &DIBuilder<'a>, Name: *const c_char, NameLen: size_t, - Value: i64, + Value: *const u64, + SizeInBits: c_uint, IsUnsigned: bool, ) -> &'a DIEnumerator; @@ -2210,6 +2086,7 @@ extern "C" { ) -> &'a DILocation; pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; + pub fn LLVMRustDIBuilderCreateOpLLVMFragment() -> u64; #[allow(improper_ctypes)] pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); @@ -2220,7 +2097,12 @@ extern "C" { pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustPrintTargetCPUs( + T: &TargetMachine, + cpu: *const c_char, + print: unsafe extern "C" fn(out: *mut c_void, string: *const c_char, len: usize), + out: *mut c_void, + ); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2230,6 +2112,8 @@ extern "C" { ); pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; + + // This function makes copies of pointed to data, so the data's lifetime may end after this function returns pub fn LLVMRustCreateTargetMachine( Triple: *const c_char, CPU: *const c_char, @@ -2249,8 +2133,14 @@ extern "C" { RelaxELFRelocations: bool, UseInitArray: bool, SplitDwarfFile: *const c_char, - ) -> Option<&'static mut TargetMachine>; - pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); + OutputObjFile: *const c_char, + DebugInfoCompression: *const c_char, + ForceEmulatedTls: bool, + ArgsCstrBuff: *const c_char, + ArgsCstrBuffLen: usize, + ) -> *mut TargetMachine; + + pub fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, @@ -2269,6 +2159,7 @@ extern "C" { TM: &'a TargetMachine, OptLevel: PassBuilderOptLevel, OptStage: OptStage, + IsLinkerPluginLTO: bool, NoPrepopulatePasses: bool, VerifyIR: bool, UseThinLTOBuffers: bool, @@ -2380,11 +2271,11 @@ extern "C" { pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustBuildOperandBundleDef<'a>( + pub fn LLVMRustBuildOperandBundleDef( Name: *const c_char, - Inputs: *const &'a Value, + Inputs: *const &'_ Value, NumInputs: c_uint, - ) -> &'a mut OperandBundleDef<'a>; + ) -> &mut OperandBundleDef<'_>; pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); @@ -2398,6 +2289,8 @@ extern "C" { pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub fn LLVMRustModuleCost(M: &Module) -> u64; + #[allow(improper_ctypes)] + pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); @@ -2433,12 +2326,12 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustThinLTOGetDICompileUnit( - M: &Module, - CU1: &mut *mut c_void, - CU2: &mut *mut c_void, - ); - pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + pub fn LLVMRustGetSliceFromObjectDataByName( + data: *const u8, + len: usize, + name: *const u8, + out_len: &mut usize, + ) -> *const u8; pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( @@ -2466,10 +2359,26 @@ extern "C" { remark_all_passes: bool, remark_passes: *const *const c_char, remark_passes_len: usize, + remark_file: *const c_char, + pgo_available: bool, ); #[allow(improper_ctypes)] pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; + + pub fn LLVMRustIsBitcode(ptr: *const u8, len: usize) -> bool; + + pub fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; + + pub fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; + + pub fn LLVMRustGetSymbols( + buf_ptr: *const u8, + buf_len: usize, + state: *mut c_void, + callback: GetSymbolsCallback, + error_callback: GetSymbolsErrorCallback, + ) -> *mut c_void; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index f820e752371..4f5cc575da6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -137,6 +137,7 @@ impl FromStr for ArchiveKind { "bsd" => Ok(ArchiveKind::K_BSD), "darwin" => Ok(ArchiveKind::K_DARWIN), "coff" => Ok(ArchiveKind::K_COFF), + "aix_big" => Ok(ArchiveKind::K_AIXBIG), _ => Err(()), } } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 2fd58567c48..7c8ef67ffd1 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,20 +1,24 @@ use crate::back::write::create_informational_target_machine; +use crate::errors::{ + PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, + UnknownCTargetFeaturePrefix, +}; use crate::llvm; use libc::c_int; use rustc_codegen_ssa::target_features::{ supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, }; +use rustc_codegen_ssa::traits::PrintBackendInfo; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_fs_util::path_to_c_string; use rustc_middle::bug; -use rustc_session::config::PrintRequest; +use rustc_session::config::{PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; -use smallvec::{smallvec, SmallVec}; -use std::ffi::{CStr, CString}; +use std::ffi::{c_char, c_void, CStr, CString}; use std::path::Path; use std::ptr; use std::slice; @@ -77,10 +81,10 @@ unsafe fn configure_llvm(sess: &Session) { }; // Set the llvm "program name" to make usage and invalid argument messages more clear. add("rustc -Cllvm-args=\"...\" with", true); - if sess.time_llvm_passes() { + if sess.opts.unstable_opts.time_llvm_passes { add("-time-passes", false); } - if sess.print_llvm_passes() { + if sess.opts.unstable_opts.print_llvm_passes { add("-debug-pass=Structure", false); } if sess.target.generate_arange_section @@ -107,6 +111,10 @@ unsafe fn configure_llvm(sess: &Session) { // Use non-zero `import-instr-limit` multiplier for cold callsites. add("-import-cold-multiplier=0.1", false); + if sess.print_llvm_stats() { + add("-stats", false); + } + for arg in sess_args { add(&(*arg), true); } @@ -116,8 +124,6 @@ unsafe fn configure_llvm(sess: &Session) { llvm::LLVMTimeTraceProfilerInitialize(); } - llvm::LLVMInitializePasses(); - rustc_llvm::initialize_available_targets(); llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); @@ -130,6 +136,60 @@ pub fn time_trace_profiler_finish(file_name: &Path) { } } +pub enum TargetFeatureFoldStrength<'a> { + // The feature is only tied when enabling the feature, disabling + // this feature shouldn't disable the tied feature. + EnableOnly(&'a str), + // The feature is tied for both enabling and disabling this feature. + Both(&'a str), +} + +impl<'a> TargetFeatureFoldStrength<'a> { + fn as_str(&self) -> &'a str { + match self { + TargetFeatureFoldStrength::EnableOnly(feat) => feat, + TargetFeatureFoldStrength::Both(feat) => feat, + } + } +} + +pub struct LLVMFeature<'a> { + pub llvm_feature_name: &'a str, + pub dependency: Option<TargetFeatureFoldStrength<'a>>, +} + +impl<'a> LLVMFeature<'a> { + pub fn new(llvm_feature_name: &'a str) -> Self { + Self { llvm_feature_name, dependency: None } + } + + pub fn with_dependency( + llvm_feature_name: &'a str, + dependency: TargetFeatureFoldStrength<'a>, + ) -> Self { + Self { llvm_feature_name, dependency: Some(dependency) } + } + + pub fn contains(&self, feat: &str) -> bool { + self.iter().any(|dep| dep == feat) + } + + pub fn iter(&'a self) -> impl Iterator<Item = &'a str> { + let dependencies = self.dependency.iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + +impl<'a> IntoIterator for LLVMFeature<'a> { + type Item = &'a str; + type IntoIter = impl Iterator<Item = &'a str>; + + fn into_iter(self) -> Self::IntoIter { + let dependencies = self.dependency.into_iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + // WARNING: the features after applying `to_llvm_features` must be known // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. @@ -145,50 +205,70 @@ pub fn time_trace_profiler_finish(file_name: &Path) { // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { ("x86", "sse4.2") => { - if get_version() >= (14, 0, 0) { - smallvec!["sse4.2", "crc32"] - } else { - smallvec!["sse4.2"] - } + LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32")) } - ("x86", "pclmulqdq") => smallvec!["pclmul"], - ("x86", "rdrand") => smallvec!["rdrnd"], - ("x86", "bmi1") => smallvec!["bmi"], - ("x86", "cmpxchg16b") => smallvec!["cx16"], - ("x86", "avx512vaes") => smallvec!["vaes"], - ("x86", "avx512gfni") => smallvec!["gfni"], - ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], - ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], - ("aarch64", "dpb") => smallvec!["ccpp"], - ("aarch64", "dpb2") => smallvec!["ccdp"], - ("aarch64", "frintts") => smallvec!["fptoint"], - ("aarch64", "fcma") => smallvec!["complxnum"], - ("aarch64", "pmuv3") => smallvec!["perfmon"], - ("aarch64", "paca") => smallvec!["pauth"], - ("aarch64", "pacg") => smallvec!["pauth"], - // Rust ties fp and neon together. In LLVM neon implicitly enables fp, - // but we manually enable neon when a feature only implicitly enables fp - ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], - ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], - ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], - ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], - ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], - ("aarch64", "sve") => smallvec!["sve", "neon"], - ("aarch64", "sve2") => smallvec!["sve2", "neon"], - ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], - ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], - ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], - ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], - (_, s) => smallvec![s], + ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"), + ("x86", "rdrand") => LLVMFeature::new("rdrnd"), + ("x86", "bmi1") => LLVMFeature::new("bmi"), + ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), + ("aarch64", "dpb") => LLVMFeature::new("ccpp"), + ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), + ("aarch64", "frintts") => LLVMFeature::new("fptoint"), + ("aarch64", "fcma") => LLVMFeature::new("complxnum"), + ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"), + ("aarch64", "paca") => LLVMFeature::new("pauth"), + ("aarch64", "pacg") => LLVMFeature::new("pauth"), + // Rust ties fp and neon together. + ("aarch64", "neon") => { + LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")) + } + // In LLVM neon implicitly enables fp, but we manually enable + // neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => { + LLVMFeature::with_dependency("f32mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "f64mm") => { + LLVMFeature::with_dependency("f64mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fhm") => { + LLVMFeature::with_dependency("fp16fml", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fp16") => { + LLVMFeature::with_dependency("fullfp16", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "jsconv") => { + LLVMFeature::with_dependency("jsconv", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve") => { + LLVMFeature::with_dependency("sve", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2") => { + LLVMFeature::with_dependency("sve2", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-aes") => { + LLVMFeature::with_dependency("sve2-aes", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sm4") => { + LLVMFeature::with_dependency("sve2-sm4", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sha3") => { + LLVMFeature::with_dependency("sve2-sha3", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-bitperm") => LLVMFeature::with_dependency( + "sve2-bitperm", + TargetFeatureFoldStrength::EnableOnly("neon"), + ), + (_, s) => LLVMFeature::new(s), } } -// Given a map from target_features to whether they are enabled or disabled, -// ensure only valid combinations are allowed. +/// Given a map from target_features to whether they are enabled or disabled, +/// ensure only valid combinations are allowed. pub fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, @@ -206,11 +286,11 @@ pub fn check_tied_features( return None; } -// Used to generate cfg variables and apply features -// Must express features in the way Rust understands them +/// Used to generate cfg variables and apply features +/// Must express features in the way Rust understands them pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); - let mut features: Vec<Symbol> = supported_target_features(sess) + supported_target_features(sess) .iter() .filter_map(|&(feature, gate)| { if sess.is_nightly_build() || allow_unstable || gate.is_none() { @@ -223,28 +303,19 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { // check that all features in a given smallvec are enabled for llvm_feature in to_llvm_features(sess, feature) { let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { return false; } } true }) .map(|feature| Symbol::intern(feature)) - .collect(); - - // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 - // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use - // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI. - // The target feature can be dropped once we no longer support older LLVM versions. - if sess.is_nightly_build() && get_version() >= (14, 0, 0) { - features.push(Symbol::intern("llvm14-builtins-abi")); - } - features + .collect() } pub fn print_version() { let (major, minor, patch) = get_version(); - println!("LLVM version: {}.{}.{}", major, minor, patch); + println!("LLVM version: {major}.{minor}.{patch}"); } pub fn get_version() -> (u32, u32, u32) { @@ -284,60 +355,81 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { ret } -fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { - let mut target_features = llvm_target_features(tm); +fn print_target_features(out: &mut dyn PrintBackendInfo, sess: &Session, tm: &llvm::TargetMachine) { + let mut llvm_target_features = llvm_target_features(tm); + let mut known_llvm_target_features = FxHashSet::<&'static str>::default(); let mut rustc_target_features = supported_target_features(sess) .iter() - .filter_map(|(feature, _gate)| { - for llvm_feature in to_llvm_features(sess, *feature) { - // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. - match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok().map( - |index| { - let (_f, desc) = target_features.remove(index); - (*feature, desc) - }, - ) { - Some(v) => return Some(v), - None => {} - } - } - None + .map(|(feature, _gate)| { + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name; + let desc = + match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() { + Some(index) => { + known_llvm_target_features.insert(llvm_feature); + llvm_target_features[index].1 + } + None => "", + }; + + (*feature, desc) }) .collect::<Vec<_>>(); rustc_target_features.extend_from_slice(&[( "crt-static", "Enables C Run-time Libraries to be statically linked", )]); - let max_feature_len = target_features + llvm_target_features.retain(|(f, _d)| !known_llvm_target_features.contains(f)); + + let max_feature_len = llvm_target_features .iter() .chain(rustc_target_features.iter()) .map(|(feature, _desc)| feature.len()) .max() .unwrap_or(0); - println!("Features supported by rustc for this target:"); + writeln!(out, "Features supported by rustc for this target:"); for (feature, desc) in &rustc_target_features { - println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + writeln!(out, " {feature:max_feature_len$} - {desc}."); } - println!("\nCode-generation features supported by LLVM for this target:"); - for (feature, desc) in &target_features { - println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + writeln!(out, "\nCode-generation features supported by LLVM for this target:"); + for (feature, desc) in &llvm_target_features { + writeln!(out, " {feature:max_feature_len$} - {desc}."); } - if target_features.is_empty() { - println!(" Target features listing is not supported by this LLVM version."); + if llvm_target_features.is_empty() { + writeln!(out, " Target features listing is not supported by this LLVM version."); } - println!("\nUse +feature to enable a feature, or -feature to disable it."); - println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); - println!("Code-generation features cannot be used in cfg or #[target_feature],"); - println!("and may be renamed or removed in a future version of LLVM or rustc.\n"); + writeln!(out, "\nUse +feature to enable a feature, or -feature to disable it."); + writeln!(out, "For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); + writeln!(out, "Code-generation features cannot be used in cfg or #[target_feature],"); + writeln!(out, "and may be renamed or removed in a future version of LLVM or rustc.\n"); } -pub(crate) fn print(req: PrintRequest, sess: &Session) { +pub(crate) fn print(req: &PrintRequest, mut out: &mut dyn PrintBackendInfo, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess); - match req { - PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, - PrintRequest::TargetFeatures => print_target_features(sess, tm), + match req.kind { + PrintKind::TargetCPUs => { + // SAFETY generate a C compatible string from a byte slice to pass + // the target CPU name into LLVM, the lifetime of the reference is + // at least as long as the C function + let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) + .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); + unsafe extern "C" fn callback(out: *mut c_void, string: *const c_char, len: usize) { + let out = &mut *(out as *mut &mut dyn PrintBackendInfo); + let bytes = slice::from_raw_parts(string as *const u8, len); + write!(out, "{}", String::from_utf8_lossy(bytes)); + } + unsafe { + llvm::LLVMRustPrintTargetCPUs( + &tm, + cpu_cstring.as_ptr(), + callback, + &mut out as *mut &mut dyn PrintBackendInfo as *mut c_void, + ); + } + } + PrintKind::TargetFeatures => print_target_features(out, sess, &tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } } @@ -415,8 +507,6 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) - // Drop +atomics-32 feature introduced in LLVM 15. - .filter(|v| *v != "+atomics-32" || get_version() >= (15, 0, 0)) .map(String::from), ); @@ -431,15 +521,10 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .filter_map(|s| { let enable_disable = match s.chars().next() { None => return None, - Some(c @ '+' | c @ '-') => c, + Some(c @ ('+' | '-')) => c, Some(_) => { if diagnostics { - let mut diag = sess.struct_warn(&format!( - "unknown feature specified for `-Ctarget-feature`: `{}`", - s - )); - diag.note("features must begin with a `+` to enable or `-` to disable it"); - diag.emit(); + sess.emit_warning(UnknownCTargetFeaturePrefix { feature: s }); } return None; } @@ -456,17 +541,15 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str None } }); - let mut diag = sess.struct_warn(&format!( - "unknown feature specified for `-Ctarget-feature`: `{}`", - feature - )); - diag.note("it is still passed through to the codegen backend"); - if let Some(rust_feature) = rust_feature { - diag.help(&format!("you might have meant: `{}`", rust_feature)); + let unknown_feature = if let Some(rust_feature) = rust_feature { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::Some { rust_feature }, + } } else { - diag.note("consider filing a feature request"); - } - diag.emit(); + UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } + }; + sess.emit_warning(unknown_feature); } if diagnostics { @@ -482,20 +565,30 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. + let llvm_feature = to_llvm_features(sess, feature); + Some( - to_llvm_features(sess, feature) - .into_iter() - .map(move |f| format!("{}{}", enable_disable, f)), + std::iter::once(format!("{}{}", enable_disable, llvm_feature.llvm_feature_name)) + .chain(llvm_feature.dependency.into_iter().filter_map(move |feat| { + match (enable_disable, feat) { + ('-' | '+', TargetFeatureFoldStrength::Both(f)) + | ('+', TargetFeatureFoldStrength::EnableOnly(f)) => { + Some(format!("{enable_disable}{f}")) + } + _ => None, + } + })), ) }) .flatten(); features.extend(feats); if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); + sess.emit_err(TargetFeatureDisableOrEnable { + features: f, + span: None, + missing_features: None, + }); } features diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 1eceb7f5c87..38e8220569a 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,6 +1,7 @@ use crate::attributes; use crate::base; use crate::context::CodegenCx; +use crate::errors::SymbolAlreadyDefined; use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; @@ -8,7 +9,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; -use rustc_middle::ty::{self, Instance, TypeVisitable}; +use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; @@ -25,10 +26,8 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { let llty = self.layout_of(ty).llvm_type(self); let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { - self.sess().span_fatal( - self.tcx.def_span(def_id), - &format!("symbol `{}` is already defined", symbol_name), - ) + self.sess() + .emit_fatal(SymbolAlreadyDefined { span: self.tcx.def_span(def_id), symbol_name }) }); unsafe { @@ -49,10 +48,10 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { visibility: Visibility, symbol_name: &str, ) { - assert!(!instance.substs.needs_infer()); + assert!(!instance.args.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let lldecl = self.declare_fn(symbol_name, fn_abi); + let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); @@ -112,7 +111,7 @@ impl CodegenCx<'_, '_> { } // Symbols from executables can't really be imported any further. - let all_exe = self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable); + let all_exe = self.tcx.crate_types().iter().all(|ty| *ty == CrateType::Executable); let is_declaration_for_linker = is_declaration || linkage == llvm::Linkage::AvailableExternallyLinkage; if all_exe && !is_declaration_for_linker { @@ -126,8 +125,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) - .unwrap_or(false); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 5eec7dc6130..06b7703672f 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -52,8 +52,12 @@ impl<'ll> CodegenCx<'ll, '_> { unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } } + pub(crate) fn type_token(&self) -> &'ll Type { + unsafe { llvm::LLVMTokenTypeInContext(self.llcx) } + } + pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 @@ -108,13 +112,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { - // FIXME(eddyb) We could find a better approximation if ity.align < align. - let ity = Integer::approximate_align(self, align); - self.type_from_integer(ity) - } - - /// Return a LLVM type that has at most the required alignment, + /// Return an LLVM type that has at most the required alignment, /// and exactly the required size, as a best-effort padding array. pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { let unit = Integer::approximate_align(self, align); @@ -185,17 +183,12 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } } - fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type { - assert_ne!( - self.type_kind(ty), - TypeKind::Function, - "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense" - ); - ty.ptr_to(AddressSpace::DATA) + fn type_ptr(&self) -> &'ll Type { + self.type_ptr_ext(AddressSpace::DATA) } - fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type { - ty.ptr_to(address_space) + fn type_ptr_ext(&self, address_space: AddressSpace) -> &'ll Type { + unsafe { llvm::LLVMPointerTypeInContext(self.llcx, address_space.0) } } fn element_type(&self, ty: &'ll Type) -> &'ll Type { @@ -238,17 +231,13 @@ impl Type { unsafe { llvm::LLVMInt8TypeInContext(llcx) } } - // Creates an integer type with the given number of bits, e.g., i24 + /// Creates an integer type with the given number of bits, e.g., i24 pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } } - pub fn i8p_llcx(llcx: &llvm::Context) -> &Type { - Type::i8_llcx(llcx).ptr_to(AddressSpace::DATA) - } - - fn ptr_to(&self, address_space: AddressSpace) -> &Type { - unsafe { llvm::LLVMPointerType(self, address_space.0) } + pub fn ptr_llcx(llcx: &llvm::Context) -> &Type { + unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::DATA.0) } } } @@ -288,11 +277,30 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { ty.llvm_type(self) } + fn scalar_copy_backend_type(&self, layout: TyAndLayout<'tcx>) -> Option<Self::Type> { + layout.scalar_copy_llvm_type(self) + } } impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn add_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); let v = [self.const_usize(0), typeid_metadata]; unsafe { llvm::LLVMGlobalSetMetadata( @@ -307,13 +315,43 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - fn typeid_metadata(&self, typeid: String) -> &'ll Value { - unsafe { + fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> { + Some(unsafe { llvm::LLVMMDStringInContext( self.llcx, typeid.as_ptr() as *const c_char, typeid.len() as c_uint, ) + }) + } + + fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) + } + } + + fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) } } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index dc1165835e7..dcc62d314ff 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,13 +1,13 @@ use crate::common::*; use crate::context::TypeLowering; -use crate::llvm_util::get_version; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; -use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Ty, TypeVisitable}; -use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; +use rustc_middle::ty::{self, Ty, TypeVisitableExt}; +use rustc_target::abi::HasDataLayout; +use rustc_target::abi::{Abi, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; use smallvec::{smallvec, SmallVec}; @@ -23,7 +23,7 @@ fn uncached_llvm_type<'a, 'tcx>( match layout.abi { Abi::Scalar(_) => bug!("handled elsewhere"), Abi::Vector { element, count } => { - let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO); + let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } Abi::ScalarPair(..) => { @@ -43,10 +43,8 @@ fn uncached_llvm_type<'a, 'tcx>( // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str - // For performance reasons we use names only when emitting LLVM IR. Unless we are on - // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213, - // #79564, and #79246. - if get_version() < (14, 0, 0) || !cx.sess().fewer_names() => + // For performance reasons we use names only when emitting LLVM IR. + if !cx.sess().fewer_names() => { let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string())); if let (&ty::Adt(def, _), &Variants::Single { index }) = @@ -59,13 +57,10 @@ fn uncached_llvm_type<'a, 'tcx>( if let (&ty::Generator(_, _, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { - write!(&mut name, "::{}", ty::GeneratorSubsts::variant_name(index)).unwrap(); + write!(&mut name, "::{}", ty::GeneratorArgs::variant_name(index)).unwrap(); } Some(name) } - // Use identified structure types for ADT. Due to pointee types in LLVM IR their definition - // might be recursive. Other cases are non-recursive and we can use literal structure types. - ty::Adt(..) => Some(String::new()), _ => None, }; @@ -140,7 +135,7 @@ fn struct_llfields<'a, 'tcx>( prev_effective_align = effective_field_align; } let padding_used = result.len() > field_count; - if !layout.is_unsized() && field_count > 0 { + if layout.is_sized() && field_count > 0 { if offset > layout.size { bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); } @@ -157,7 +152,7 @@ fn struct_llfields<'a, 'tcx>( } else { debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); } - let field_remapping = if padding_used { Some(field_remapping) } else { None }; + let field_remapping = padding_used.then_some(field_remapping); (result, packed, field_remapping) } @@ -181,12 +176,7 @@ pub trait LayoutLlvmExt<'tcx> { fn is_llvm_scalar_pair(&self) -> bool; fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; - fn scalar_llvm_type_at<'a>( - &self, - cx: &CodegenCx<'a, 'tcx>, - scalar: Scalar, - offset: Size, - ) -> &'a Type; + fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type; fn scalar_pair_element_llvm_type<'a>( &self, cx: &CodegenCx<'a, 'tcx>, @@ -195,14 +185,14 @@ pub trait LayoutLlvmExt<'tcx> { ) -> &'a Type; fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo>; + fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type>; } impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { match self.abi { Abi::Scalar(_) | Abi::Vector { .. } => true, - Abi::ScalarPair(..) => false, - Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(), + Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, } } @@ -225,24 +215,16 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + // This must produce the same result for `repr(transparent)` wrappers as for the inner type! + // In other words, this should generally not look at the type at all, but only at the + // layout. if let Abi::Scalar(scalar) = self.abi { // Use a different cache for scalars because pointers to DSTs // can be either fat or thin (data pointers of fat pointers). if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { return llty; } - let llty = match *self.ty.kind() { - ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => { - cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx)) - } - ty::Adt(def, _) if def.is_box() => { - cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx)) - } - ty::FnPtr(sig) => { - cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty())) - } - _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO), - }; + let llty = self.scalar_llvm_type_at(cx, scalar); cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); return llty; } @@ -302,26 +284,12 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { self.llvm_type(cx) } - fn scalar_llvm_type_at<'a>( - &self, - cx: &CodegenCx<'a, 'tcx>, - scalar: Scalar, - offset: Size, - ) -> &'a Type { + fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type { match scalar.primitive() { Int(i, _) => cx.type_from_integer(i), F32 => cx.type_f32(), F64 => cx.type_f64(), - Pointer => { - // If we know the alignment, pick something better than i8. - let (pointee, address_space) = - if let Some(pointee) = self.pointee_info_at(cx, offset) { - (cx.type_pointee_for_align(pointee.align), pointee.address_space) - } else { - (cx.type_i8(), AddressSpace::DATA) - }; - cx.type_ptr_to_ext(pointee, address_space) - } + Pointer(address_space) => cx.type_ptr_ext(address_space), } } @@ -331,38 +299,25 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { index: usize, immediate: bool, ) -> &'a Type { - // HACK(eddyb) special-case fat pointers until LLVM removes - // pointee types, to avoid bitcasting every `OperandRef::deref`. - match self.ty.kind() { - ty::Ref(..) | ty::RawPtr(_) => { - return self.field(cx, index).llvm_type(cx); - } - // only wide pointer boxes are handled as pointers - // thin pointer boxes with scalar allocators are handled by the general logic below - ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { - let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); - return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); - } - _ => {} - } - + // This must produce the same result for `repr(transparent)` wrappers as for the inner type! + // In other words, this should generally not look at the type at all, but only at the + // layout. let Abi::ScalarPair(a, b) = self.abi else { bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); }; let scalar = [a, b][index]; // Make sure to return the same type `immediate_llvm_type` would when - // dealing with an immediate pair. This means that `(bool, bool)` is + // dealing with an immediate pair. This means that `(bool, bool)` is // effectively represented as `{i8, i8}` in memory and two `i1`s as an // immediate, just like `bool` is typically `i8` in memory and only `i1` - // when immediate. We need to load/store `bool` as `i8` to avoid + // when immediate. We need to load/store `bool` as `i8` to avoid // crippling LLVM optimizations or triggering other LLVM bugs with `i1`. if immediate && scalar.is_bool() { return cx.type_i1(); } - let offset = if index == 0 { Size::ZERO } else { a.size(cx).align_to(b.align(cx).abi) }; - self.scalar_llvm_type_at(cx, scalar, offset) + self.scalar_llvm_type_at(cx, scalar) } fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64 { @@ -414,4 +369,39 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); result } + + fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type> { + debug_assert!(self.is_sized()); + + // FIXME: this is a fairly arbitrary choice, but 128 bits on WASM + // (matching the 128-bit SIMD types proposal) and 256 bits on x64 + // (like AVX2 registers) seems at least like a tolerable starting point. + let threshold = cx.data_layout().pointer_size * 4; + if self.layout.size() > threshold { + return None; + } + + // Vectors, even for non-power-of-two sizes, have the same layout as + // arrays but don't count as aggregate types + // While LLVM theoretically supports non-power-of-two sizes, and they + // often work fine, sometimes x86-isel deals with them horribly + // (see #115212) so for now only use power-of-two ones. + if let FieldsShape::Array { count, .. } = self.layout.fields() + && count.is_power_of_two() + && let element = self.field(cx, 0) + && element.ty.is_integral() + { + // `cx.type_ix(bits)` is tempting here, but while that works great + // for things that *stay* as memory-to-memory copies, it also ends + // up suppressing vectorization as it introduces shifts when it + // extracts all the individual values. + + let ety = element.llvm_type(cx); + return Some(cx.type_vector(ety, *count)); + } + + // FIXME: The above only handled integer arrays; surely more things + // would also be possible. Be careful about provenance, though! + None + } } diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ceb3d5a84ab..172c66a7af1 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -5,7 +5,7 @@ use crate::value::Value; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::{ common::IntPredicate, - traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods}, + traits::{BaseTypeMethods, BuilderMethods, ConstMethods}, }; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::ty::Ty; @@ -26,24 +26,18 @@ fn round_pointer_up_to_alignment<'ll>( fn emit_direct_ptr_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, list: OperandRef<'tcx, &'ll Value>, - llty: &'ll Type, size: Size, align: Align, slot_size: Align, allow_higher_align: bool, ) -> (&'ll Value, Align) { - let va_list_ty = bx.type_i8p(); - let va_list_ptr_ty = bx.type_ptr_to(va_list_ty); - let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty { - bx.bitcast(list.immediate(), va_list_ptr_ty) - } else { - list.immediate() - }; + let va_list_ty = bx.type_ptr(); + let va_list_addr = list.immediate(); let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi); let (addr, addr_align) = if allow_higher_align && align > slot_size { - (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align) + (round_pointer_up_to_alignment(bx, ptr, align, bx.type_ptr()), align) } else { (ptr, slot_size) }; @@ -56,9 +50,9 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>( if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); - (bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align) + (adjusted, addr_align) } else { - (bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align) + (addr, addr_align) } } @@ -73,7 +67,7 @@ fn emit_ptr_va_arg<'ll, 'tcx>( let layout = bx.cx.layout_of(target_ty); let (llty, size, align) = if indirect { ( - bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx), + bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx), bx.cx.data_layout().pointer_size, bx.cx.data_layout().pointer_align, ) @@ -81,7 +75,7 @@ fn emit_ptr_va_arg<'ll, 'tcx>( (layout.llvm_type(bx.cx), layout.size, layout.align) }; let (addr, addr_align) = - emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align); + emit_direct_ptr_va_arg(bx, list, size, align.abi, slot_size, allow_higher_align); if indirect { let tmp_ret = bx.load(llty, addr, addr_align); bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi) @@ -146,7 +140,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( bx.cond_br(use_stack, on_stack, in_reg); bx.switch_to_block(in_reg); - let top_type = bx.type_i8p(); + let top_type = bx.type_ptr(); let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); @@ -158,7 +152,6 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); } let reg_type = layout.llvm_type(bx); - let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); bx.br(end); @@ -175,6 +168,87 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( val } +fn emit_s390x_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of the s390x ELF ABI calling convention for va_args see + // https://github.com/IBM/s390x-abi (chapter 1.2.4) + let va_list_addr = list.immediate(); + let va_list_layout = list.deref(bx.cx).layout; + let va_list_ty = va_list_layout.llvm_type(bx); + let layout = bx.cx.layout_of(target_ty); + + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let in_mem = bx.append_sibling_block("va_arg.in_mem"); + let end = bx.append_sibling_block("va_arg.end"); + + // FIXME: vector ABI not yet supported. + let target_ty_size = bx.cx.size_of(target_ty).bytes(); + let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); + let unpadded_size = if indirect { 8 } else { target_ty_size }; + let padded_size = 8; + let padding = padded_size - unpadded_size; + + let gpr_type = indirect || !layout.is_single_fp_element(bx.cx); + let (max_regs, reg_count_field, reg_save_index, reg_padding) = + if gpr_type { (5, 0, 2, padding) } else { (4, 1, 16, 0) }; + + // Check whether the value was passed in a register or in memory. + let reg_count = bx.struct_gep( + va_list_ty, + va_list_addr, + va_list_layout.llvm_field_index(bx.cx, reg_count_field), + ); + let reg_count_v = bx.load(bx.type_i64(), reg_count, Align::from_bytes(8).unwrap()); + let use_regs = bx.icmp(IntPredicate::IntULT, reg_count_v, bx.const_u64(max_regs)); + bx.cond_br(use_regs, in_reg, in_mem); + + // Emit code to load the value if it was passed in a register. + bx.switch_to_block(in_reg); + + // Work out the address of the value in the register save area. + let reg_ptr = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let reg_ptr_v = bx.load(bx.type_ptr(), reg_ptr, bx.tcx().data_layout.pointer_align.abi); + let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8)); + let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding)); + let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]); + + // Update the register count. + let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1)); + bx.store(new_reg_count_v, reg_count, Align::from_bytes(8).unwrap()); + bx.br(end); + + // Emit code to load the value if it was passed in memory. + bx.switch_to_block(in_mem); + + // Work out the address of the value in the argument overflow area. + let arg_ptr = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 2)); + let arg_ptr_v = bx.load(bx.type_ptr(), arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let arg_off = bx.const_u64(padding); + let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]); + + // Update the argument overflow area pointer. + let arg_size = bx.cx().const_u64(padded_size); + let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]); + bx.store(new_arg_ptr_v, arg_ptr, bx.tcx().data_layout.pointer_align.abi); + bx.br(end); + + // Return the appropriate result. + bx.switch_to_block(end); + let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]); + let val_type = layout.llvm_type(bx); + let val_addr = if indirect { + bx.load(bx.cx.type_ptr(), val_addr, bx.tcx().data_layout.pointer_align.abi) + } else { + val_addr + }; + bx.load(val_type, val_addr, layout.align.abi) +} + pub(super) fn emit_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, addr: OperandRef<'tcx, &'ll Value>, @@ -200,6 +274,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) } "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + "s390x" => emit_s390x_va_arg(bx, addr, target_ty), // Windows x86_64 "x86_64" if target.is_like_windows => { let target_ty_size = bx.cx.size_of(target_ty).bytes(); |