diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
37 files changed, 3539 insertions, 2358 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index d4437bd449d..28be6d033f8 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -19,6 +19,7 @@ use rustc_target::abi::call::ArgAbi; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int}; pub use rustc_target::spec::abi::Abi; +use rustc_target::spec::SanitizerSet; use libc::c_uint; use smallvec::SmallVec; @@ -33,13 +34,6 @@ pub trait ArgAttributesExt { ); } -fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // LLVM prior to version 12 had known miscompiles in the presence of - // noalias attributes (see #54878), but we don't support earlier - // versions at all anymore. We now enable mutable noalias by default. - cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) -} - const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; @@ -87,8 +81,12 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' attrs.push(llattr.create_attr(cx.llcx)); } } - if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { - attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); + } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { + // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects + // memory sanitizer's behavior. + + if regular.contains(ArgAttribute::NoUndef) { + attrs.push(llvm::AttributeKind::NoUndef.create_attr(cx.llcx)); } } @@ -213,7 +211,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) } else if self.is_unsized_indirect() { bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast(cast) = self.mode { + } else if let PassMode::Cast(cast, _) = &self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. let can_store_through_cast_ptr = false; @@ -223,7 +221,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { bx.store(val, cast_dst, self.layout.align.abi); } else { // The actual return type is a struct, but the ABI - // adaptation code has cast it into some scalar type. The + // adaptation code has cast it into some scalar type. The // code that follows is the only reliable way I have // found to do a transform like i64 -> {i32,i32}. // Basically we dump the data onto the stack then memcpy it. @@ -283,7 +281,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(_) | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } - | PassMode::Cast(_) => { + | PassMode::Cast(..) => { let next_arg = next(); self.store(bx, next_arg, dst); } @@ -325,20 +323,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { // Ignore "extra" args from the call site for C variadic functions. // Only the "fixed" args are part of the LLVM function signature. - let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + let args = + if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args }; - let args_capacity: usize = args.iter().map(|arg| - if arg.pad.is_some() { 1 } else { 0 } + - if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } - ).sum(); + // This capacity calculation is approximate. let mut llargument_tys = Vec::with_capacity( - if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, + self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }, ); - let llreturn_ty = match self.ret.mode { + let llreturn_ty = match &self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, _) => cast.llvm_type(cx), PassMode::Indirect { .. } => { llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); cx.type_void() @@ -346,12 +342,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { }; for arg in args { - // add padding - if let Some(ty) = arg.pad { - llargument_tys.push(ty.llvm_type(cx)); - } - - let llarg_ty = match arg.mode { + let llarg_ty = match &arg.mode { PassMode::Ignore => continue, PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), PassMode::Pair(..) => { @@ -366,7 +357,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Cast(cast, pad_i32) => { + // add padding + if *pad_i32 { + llargument_tys.push(Reg::i32().llvm_type(cx)); + } + cast.llvm_type(cx) + } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { cx.type_ptr_to(arg.memory_ty(cx)) } @@ -391,23 +388,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn llvm_cconv(&self) -> llvm::CallConv { - match self.conv { - Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, - Conv::RustCold => llvm::ColdCallConv, - Conv::AmdGpuKernel => llvm::AmdGpuKernel, - Conv::AvrInterrupt => llvm::AvrInterrupt, - Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, - Conv::ArmAapcs => llvm::ArmAapcsCallConv, - Conv::Msp430Intr => llvm::Msp430Intr, - Conv::PtxKernel => llvm::PtxKernel, - Conv::X86Fastcall => llvm::X86FastcallCallConv, - Conv::X86Intr => llvm::X86_Intr, - Conv::X86Stdcall => llvm::X86StdcallCallConv, - Conv::X86ThisCall => llvm::X86_ThisCall, - Conv::X86VectorCall => llvm::X86_VectorCall, - Conv::X86_64SysV => llvm::X86_64_SysV, - Conv::X86_64Win64 => llvm::X86_64_Win64, - } + self.conv.into() } fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { @@ -426,46 +407,46 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } _ => {} } - for arg in &self.args { - if arg.pad.is_some() { - apply(&ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(attrs); let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(attrs); } - PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { assert!(!on_stack); apply(attrs); apply(extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(a); apply(b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(&ArgAttributes::new()); + } apply(&cast.attrs); } } @@ -488,17 +469,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; - match self.ret.mode { - PassMode::Direct(ref attrs) => { + match &self.ret.mode { + PassMode::Direct(attrs) => { attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); } - PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + PassMode::Indirect { attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, _) => { cast.attrs.apply_attrs_to_callsite( llvm::AttributePlace::ReturnValue, &bx.cx, @@ -517,13 +498,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } } - for arg in &self.args { - if arg.pad.is_some() { - apply(bx.cx, &ArgAttributes::new()); - } - match arg.mode { + for arg in self.args.iter() { + match &arg.mode { PassMode::Ignore => {} - PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + PassMode::Indirect { attrs, extra_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); attributes::apply_to_callsite( @@ -532,23 +510,22 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[byval], ); } - PassMode::Direct(ref attrs) - | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) + | PassMode::Indirect { attrs, extra_attrs: None, on_stack: false } => { apply(bx.cx, attrs); } - PassMode::Indirect { - ref attrs, - extra_attrs: Some(ref extra_attrs), - on_stack: _, - } => { + PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack: _ } => { apply(bx.cx, attrs); apply(bx.cx, extra_attrs); } - PassMode::Pair(ref a, ref b) => { + PassMode::Pair(a, b) => { apply(bx.cx, a); apply(bx.cx, b); } - PassMode::Cast(cast) => { + PassMode::Cast(cast, pad_i32) => { + if *pad_i32 { + apply(bx.cx, &ArgAttributes::new()); + } apply(bx.cx, &cast.attrs); } } @@ -569,15 +546,49 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { &[cmse_nonsecure_call], ); } + + // Some intrinsics require that an elementtype attribute (with the pointee type of a + // pointer argument) is added to the callsite. + let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; + if element_type_index >= 0 { + let arg_ty = self.args[element_type_index as usize].layout.ty; + let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; + let element_type_attr = unsafe { + llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) + }; + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(element_type_index as u32), + &[element_type_attr], + ); + } } } impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { - fn_abi.apply_attrs_callsite(self, callsite) - } - fn get_param(&mut self, index: usize) -> Self::Value { llvm::get_param(self.llfn(), index as c_uint) } } + +impl From<Conv> for llvm::CallConv { + fn from(conv: Conv) -> Self { + match conv { + Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, + Conv::AmdGpuKernel => llvm::AmdGpuKernel, + Conv::AvrInterrupt => llvm::AvrInterrupt, + Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, + Conv::ArmAapcs => llvm::ArmAapcsCallConv, + Conv::Msp430Intr => llvm::Msp430Intr, + Conv::PtxKernel => llvm::PtxKernel, + Conv::X86Fastcall => llvm::X86FastcallCallConv, + Conv::X86Intr => llvm::X86_Intr, + Conv::X86Stdcall => llvm::X86StdcallCallConv, + Conv::X86ThisCall => llvm::X86_ThisCall, + Conv::X86VectorCall => llvm::X86_VectorCall, + Conv::X86_64SysV => llvm::X86_64_SysV, + Conv::X86_64Win64 => llvm::X86_64_Win64, + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 72961ae888e..ad0636894b7 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -1,10 +1,12 @@ use crate::attributes; use libc::c_uint; -use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_ast::expand::allocator::{ + alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy, + ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, +}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; -use rustc_span::symbol::sym; use crate::debuginfo; use crate::llvm::{self, False, True}; @@ -15,7 +17,7 @@ pub(crate) unsafe fn codegen( module_llvm: &mut ModuleLlvm, module_name: &str, kind: AllocatorKind, - has_alloc_error_handler: bool, + alloc_error_handler_kind: AllocatorKind, ) { let llcx = &*module_llvm.llcx; let llmod = module_llvm.llmod(); @@ -29,74 +31,78 @@ pub(crate) unsafe fn codegen( let i8p = llvm::LLVMPointerType(i8, 0); let void = llvm::LLVMVoidTypeInContext(llcx); - for method in ALLOCATOR_METHODS { - let mut args = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match *ty { - AllocatorTy::Layout => { - args.push(usize); // size - args.push(usize); // align + if kind == AllocatorKind::Default { + for method in ALLOCATOR_METHODS { + let mut args = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match *ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), } - AllocatorTy::Ptr => args.push(i8p), - AllocatorTy::Usize => args.push(usize), - - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), } - } - let output = match method.output { - AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + let ty = llvm::LLVMFunctionType( + output.unwrap_or(void), + args.as_ptr(), + args.len() as c_uint, + False, + ); + let name = global_fn_name(method.name); + let llfn = + llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } - }; - let ty = llvm::LLVMFunctionType( - output.unwrap_or(void), - args.as_ptr(), - args.len() as c_uint, - False, - ); - let name = format!("__rust_{}", method.name); - let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); - - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let callee = kind.fn_name(method.name); - let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); - - let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); - llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); - let args = args - .iter() - .enumerate() - .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) - .collect::<Vec<_>>(); - let ret = llvm::LLVMRustBuildCall( - llbuilder, - ty, - callee, - args.as_ptr(), - args.len() as c_uint, - None, - ); - llvm::LLVMSetTailCall(ret, True); - if output.is_some() { - llvm::LLVMBuildRet(llbuilder, ret); - } else { - llvm::LLVMBuildRetVoid(llbuilder); + let callee = default_fn_name(method.name); + let callee = + llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::<Vec<_>>(); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + llvm::LLVMSetTailCall(ret, True); + if output.is_some() { + llvm::LLVMBuildRet(llbuilder, ret); + } else { + llvm::LLVMBuildRetVoid(llbuilder); + } + llvm::LLVMDisposeBuilder(llbuilder); } - llvm::LLVMDisposeBuilder(llbuilder); } // rust alloc error handler @@ -117,14 +123,13 @@ pub(crate) unsafe fn codegen( attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } - let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default }; - let callee = kind.fn_name(sym::oom); + let callee = alloc_error_handler_name(alloc_error_handler_kind); let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); // -> ! DIFlagNoReturn attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); @@ -133,8 +138,15 @@ pub(crate) unsafe fn codegen( .enumerate() .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) .collect::<Vec<_>>(); - let ret = - llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); llvm::LLVMSetTailCall(ret, True); llvm::LLVMBuildRetVoid(llbuilder); llvm::LLVMDisposeBuilder(llbuilder); @@ -149,6 +161,14 @@ pub(crate) unsafe fn codegen( let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); + let name = NO_ALLOC_SHIM_IS_UNSTABLE; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let llval = llvm::LLVMConstInt(i8, 0, False); + llvm::LLVMSetInitializer(ll_g, llval); + if tcx.sess.opts.debuginfo != DebugInfo::None { let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a53946995ee..2a6ad1be763 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -3,7 +3,6 @@ use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -20,7 +19,6 @@ use rustc_target::asm::*; use libc::{c_char, c_uint}; use smallvec::SmallVec; -use tracing::debug; impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_inline_asm( @@ -132,7 +130,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { op_idx.insert(idx, constraints.len()); constraints.push(reg_to_llvm(reg, Some(&value.layout))); } - InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => { + InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => { let value = llvm_fixup_input( self, in_value.immediate(), @@ -140,7 +138,16 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &in_value.layout, ); inputs.push(value); - constraints.push(format!("{}", op_idx[&idx])); + + // In the case of fixed registers, we have the choice of + // either using a tied operand or duplicating the constraint. + // We prefer the latter because it matches the behavior of + // Clang. + if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) { + constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string()); + } else { + constraints.push(format!("{}", op_idx[&idx])); + } } InlineAsmOperandRef::SymFn { instance } => { inputs.push(self.cx.get_fn(instance)); @@ -229,14 +236,31 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::Nvptx64 => {} InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} InlineAsmArch::Hexagon => {} + InlineAsmArch::LoongArch64 => { + constraints.extend_from_slice(&[ + "~{$fcc0}".to_string(), + "~{$fcc1}".to_string(), + "~{$fcc2}".to_string(), + "~{$fcc3}".to_string(), + "~{$fcc4}".to_string(), + "~{$fcc5}".to_string(), + "~{$fcc6}".to_string(), + "~{$fcc7}".to_string(), + ]); + } InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} - InlineAsmArch::S390x => {} + InlineAsmArch::S390x => { + constraints.push("~{cc}".to_string()); + } InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} InlineAsmArch::Msp430 => { constraints.push("~{sr}".to_string()); } + InlineAsmArch::M68k => { + constraints.push("~{ccr}".to_string()); + } } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -278,13 +302,13 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let mut attrs = SmallVec::<[_; 2]>::new(); if options.contains(InlineAsmOptions::PURE) { if options.contains(InlineAsmOptions::NOMEM) { - attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx)); + attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::READONLY) { - attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx)); + attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx)); } attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::NOMEM) { - attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx)); + attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); } else { // LLVM doesn't have an attribute to represent ReadOnly + SideEffect } @@ -374,7 +398,7 @@ impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { } unsafe { - llvm::LLVMRustAppendModuleInlineAsm( + llvm::LLVMAppendModuleInlineAsm( self.llmod, template_str.as_ptr().cast(), template_str.len(), @@ -419,13 +443,6 @@ pub(crate) fn inline_asm_call<'ll>( let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { - if unwind && llvm_util::get_version() < (13, 0, 0) { - bx.cx.sess().span_fatal( - line_spans[0], - "unwinding from inline assembly is only supported on llvm >= 13.", - ); - } - let v = llvm::LLVMRustInlineAsm( fty, asm.as_ptr().cast(), @@ -439,13 +456,13 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, v, inputs, dest, catch, funclet) + bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) } else { - bx.call(fty, v, inputs, None) + bx.call(fty, None, None, v, inputs, None) }; // Store mark in a metadata node so we can map LLVM errors - // back to source locations. See #17552. + // back to source locations. See #17552. let key = "srcloc"; let kind = llvm::LLVMGetMDKindIDInContext( bx.llcx, @@ -505,6 +522,44 @@ fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> { } } +/// If the register is an AArch64 integer register then return its index. +fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> { + match reg { + InlineAsmReg::AArch64(AArch64InlineAsmReg::x0) => Some(0), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x1) => Some(1), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x2) => Some(2), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x3) => Some(3), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x4) => Some(4), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x5) => Some(5), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x6) => Some(6), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x7) => Some(7), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x8) => Some(8), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x9) => Some(9), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x10) => Some(10), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x11) => Some(11), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x12) => Some(12), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x13) => Some(13), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x14) => Some(14), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x15) => Some(15), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x16) => Some(16), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x17) => Some(17), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x18) => Some(18), + // x19 is reserved + InlineAsmReg::AArch64(AArch64InlineAsmReg::x20) => Some(20), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x21) => Some(21), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x22) => Some(22), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x23) => Some(23), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x24) => Some(24), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x25) => Some(25), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x26) => Some(26), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x27) => Some(27), + InlineAsmReg::AArch64(AArch64InlineAsmReg::x28) => Some(28), + // x29 is reserved + InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) => Some(30), + _ => None, + } +} + /// If the register is an AArch64 vector register then return its index. fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> { match reg { @@ -535,6 +590,22 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> 'x' }; format!("{{{}mm{}}}", class, idx) + } else if let Some(idx) = a64_reg_index(reg) { + let class = if let Some(layout) = layout { + match layout.size.bytes() { + 8 => 'x', + _ => 'w', + } + } else { + // We use i32 as the type for discarded outputs + 'w' + }; + if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { + // LLVM doesn't recognize x30. use lr instead. + "{lr}".to_string() + } else { + format!("{{{}{}}}", class, idx) + } } else if let Some(idx) = a64_vreg_index(reg) { let class = if let Some(layout) = layout { match layout.size.bytes() { @@ -550,9 +621,6 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> 'q' }; format!("{{{}{}}}", class, idx) - } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { - // LLVM doesn't recognize x30 - "{lr}".to_string() } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { // LLVM doesn't recognize r14 "{lr}".to_string() @@ -560,6 +628,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> format!("{{{}}}", reg.name()) } } + // The constraints can be retrieved from + // https://llvm.org/docs/LangRef.html#supported-constraint-code-list InlineAsmRegOrRegClass::RegClass(reg) => match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", @@ -577,6 +647,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", @@ -618,6 +690,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -633,6 +708,8 @@ fn modifier_to_llvm( reg: InlineAsmRegClass, modifier: Option<char>, ) -> Option<char> { + // The modifiers can be retrieved from + // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers match reg { InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) @@ -658,6 +735,7 @@ fn modifier_to_llvm( } } InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::LoongArch(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, InlineAsmRegClass::PowerPC(_) => None, @@ -713,6 +791,7 @@ fn modifier_to_llvm( InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::M68k(_) => None, InlineAsmRegClass::Err => unreachable!(), } } @@ -741,6 +820,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' cx.type_vector(cx.type_i64(), 2) } InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), @@ -784,6 +865,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -794,6 +878,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' /// Helper function to get the LLVM type for a Scalar. Pointers are returned as /// the equivalent integer type. fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { + let dl = &cx.tcx.data_layout; match scalar.primitive() { Primitive::Int(Integer::I8, _) => cx.type_i8(), Primitive::Int(Integer::I16, _) => cx.type_i16(), @@ -801,7 +886,8 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty Primitive::Int(Integer::I64, _) => cx.type_i64(), Primitive::F32 => cx.type_f32(), Primitive::F64 => cx.type_f64(), - Primitive::Pointer => cx.type_isize(), + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()), _ => unreachable!(), } } @@ -813,6 +899,7 @@ fn llvm_fixup_input<'ll, 'tcx>( reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value { + let dl = &bx.tcx.data_layout; match (reg, layout.abi) { (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { @@ -826,8 +913,10 @@ fn llvm_fixup_input<'ll, 'tcx>( let elem_ty = llvm_asm_scalar_type(bx.cx, s); let count = 16 / layout.size.bytes(); let vec_ty = bx.cx.type_vector(elem_ty, count); - if let Primitive::Pointer = s.primitive() { - value = bx.ptrtoint(value, bx.cx.type_isize()); + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + if let Primitive::Pointer(_) = s.primitive() { + let t = bx.type_from_integer(dl.ptr_sized_integer()); + value = bx.ptrtoint(value, t); } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } @@ -903,7 +992,7 @@ fn llvm_fixup_output<'ll, 'tcx>( } (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { value = bx.extract_element(value, bx.const_i32(0)); - if let Primitive::Pointer = s.primitive() { + if let Primitive::Pointer(_) = s.primitive() { value = bx.inttoptr(value, layout.llvm_type(bx.cx)); } value diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 32bfa5094c3..6d00464e0a0 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -12,8 +12,9 @@ use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtec use smallvec::SmallVec; use crate::attributes; +use crate::errors::{MissingFeatures, SanitizerMemtagRequiresMte, TargetFeatureDisableOrEnable}; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, Attribute, AttributeKind, AttributePlace}; +use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; use crate::llvm_util; pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; @@ -35,6 +36,10 @@ pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attrib /// Get LLVM attribute for the provided inline heuristic. #[inline] fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { + if !cx.tcx.sess.opts.unstable_opts.inline_llvm { + // disable LLVM inlining + return Some(AttributeKind::NoInline.create_attr(cx.llcx)); + } match inline { InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), @@ -57,7 +62,7 @@ pub fn sanitize_attrs<'ll>( ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; - if enabled.contains(SanitizerSet::ADDRESS) { + if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } if enabled.contains(SanitizerSet::MEMORY) { @@ -69,17 +74,23 @@ pub fn sanitize_attrs<'ll>( if enabled.contains(SanitizerSet::HWADDRESS) { attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); } + if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { + attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); + } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. let features = cx.tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.sess.err("`-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`"); + cx.tcx.sess.emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); } + if enabled.contains(SanitizerSet::SAFESTACK) { + attrs.push(llvm::AttributeKind::SanitizeSafeStack.create_attr(cx.llcx)); + } attrs } @@ -94,10 +105,10 @@ pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { let mut fp = cx.sess().target.frame_pointer; + let opts = &cx.sess().opts; // "mcount" function relies on stack pointer. // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. - if cx.sess().instrument_mcount() || matches!(cx.sess().opts.cg.force_frame_pointers, Some(true)) - { + if opts.unstable_opts.instrument_mcount || matches!(opts.cg.force_frame_pointers, Some(true)) { fp = FramePointer::Always; } let attr_value = match fp { @@ -110,8 +121,9 @@ pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attr /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().instrument_mcount() { +fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { + let mut attrs = SmallVec::new(); + if cx.sess().opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. @@ -119,14 +131,49 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribu // See test/CodeGen/mcount.c in clang. let mcount_name = cx.sess().target.mcount.as_ref(); - Some(llvm::CreateAttrStringValue( + attrs.push(llvm::CreateAttrStringValue( cx.llcx, "instrument-function-entry-inlined", &mcount_name, - )) - } else { - None + )); + } + if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. + // Function prologue and epilogue are instrumented with NOP sleds, + // a runtime library later replaces them with detours into tracing code. + if options.always { + attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-always")); + } + if options.never { + attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-never")); + } + if options.ignore_loops { + attrs.push(llvm::CreateAttrString(cx.llcx, "xray-ignore-loops")); + } + // LLVM will not choose the default for us, but rather requires specific + // threshold in absence of "xray-always". Use the same default as Clang. + let threshold = options.instruction_threshold.unwrap_or(200); + attrs.push(llvm::CreateAttrStringValue( + cx.llcx, + "xray-instruction-threshold", + &threshold.to_string(), + )); + if options.skip_entry { + attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-entry")); + } + if options.skip_exit { + attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-exit")); + } + } + attrs +} + +fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + if !cx.sess().opts.unstable_opts.no_jump_tables { + return None; } + + Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { @@ -224,6 +271,10 @@ pub(crate) fn default_optimisation_attrs<'ll>( attrs } +fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { + llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") +} + /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub fn from_fn_attrs<'ll, 'tcx>( @@ -246,13 +297,12 @@ pub fn from_fn_attrs<'ll, 'tcx>( OptimizeAttr::Speed => {} } - let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - InlineAttr::Never - } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { - InlineAttr::Hint - } else { - codegen_fn_attrs.inline - }; + let inline = + if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { + InlineAttr::Hint + } else { + codegen_fn_attrs.inline + }; to_add.extend(inline_attr(cx, inline)); // The `uwtable` attribute according to LLVM is: @@ -282,6 +332,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // FIXME: none of these three functions interact with source level attributes. to_add.extend(frame_pointer_type_attr(cx)); to_add.extend(instrument_function_attr(cx)); + to_add.extend(nojumptables_attr(cx)); to_add.extend(probestack_attr(cx)); to_add.extend(stackprotector_attr(cx)); @@ -292,19 +343,68 @@ pub fn from_fn_attrs<'ll, 'tcx>( to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { - to_add.push(AttributeKind::ReadOnly.create_attr(cx.llcx)); + to_add.push(MemoryEffects::ReadOnly.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) { - to_add.push(AttributeKind::ReadNone.create_attr(cx.llcx)); + to_add.push(MemoryEffects::None.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); + // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. + // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. + // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 + to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); + // Need this for AArch64. + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) + || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) + { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; + } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + } // apply to return place instead of function (unlike all other attributes applied in this function) let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); + } + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + } + } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); } @@ -330,15 +430,17 @@ pub fn from_fn_attrs<'ll, 'tcx>( ) { let span = cx .tcx - .get_attr(instance.def_id(), sym::target_feature) + .get_attrs(instance.def_id(), sym::target_feature) + .next() .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); - let msg = format!( - "the target features {} must all be either enabled or disabled together", - f.join(", ") - ); - let mut err = cx.tcx.sess.struct_span_err(span, &msg); - err.help("add the missing features in a `target_feature` attribute"); - err.emit(); + cx.tcx + .sess + .create_err(TargetFeatureDisableOrEnable { + features: f, + span: Some(span), + missing_features: Some(MissingFeatures), + }) + .emit(); return; } @@ -370,7 +472,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // the WebAssembly specification, which has this feature. This won't be // needed when LLVM enables this `multivalue` feature by default. if !cx.tcx.is_closure(instance.def_id()) { - let abi = cx.tcx.fn_sig(instance.def_id()).abi(); + let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); if abi == Abi::Wasm { function_features.push("+multivalue".to_string()); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index bccc2a995a3..a6416e9540c 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,25 +1,31 @@ //! A helper class for dealing with static archives use std::env; -use std::ffi::{CStr, CString, OsString}; +use std::ffi::{c_char, c_void, CStr, CString, OsString}; use std::io; use std::mem; use std::path::{Path, PathBuf}; use std::ptr; use std::str; +use crate::common; +use crate::errors::{ + DlltoolFailImportLibrary, ErrorCallingDllTool, ErrorCreatingImportLibrary, ErrorWritingDEFFile, +}; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; -use rustc_codegen_ssa::back::archive::ArchiveBuilder; -use rustc_data_structures::temp_dir::MaybeTempDir; -use rustc_session::cstore::{DllCallingConvention, DllImport}; +use rustc_codegen_ssa::back::archive::{ + get_native_object_symbols, try_extract_macho_fat_archive, ArArchiveBuilder, + ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, UnknownArchiveKind, +}; + +use rustc_session::cstore::DllImport; use rustc_session::Session; /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] -pub struct LlvmArchiveBuilder<'a> { +pub(crate) struct LlvmArchiveBuilder<'a> { sess: &'a Session, - dst: PathBuf, additions: Vec<Addition>, } @@ -55,17 +61,18 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { } impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { - /// Creates a new static archive, ready for modifying the archive specified - /// by `config`. - fn new(sess: &'a Session, output: &Path) -> LlvmArchiveBuilder<'a> { - LlvmArchiveBuilder { sess, dst: output.to_path_buf(), additions: Vec::new() } - } - - fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()> - where - F: FnMut(&str) -> bool + 'static, - { - let archive_ro = match ArchiveRO::open(archive) { + fn add_archive( + &mut self, + archive: &Path, + skip: Box<dyn FnMut(&str) -> bool + 'static>, + ) -> io::Result<()> { + let mut archive = archive.to_path_buf(); + if self.sess.target.llvm_target.contains("-apple-macosx") { + if let Some(new_archive) = try_extract_macho_fat_archive(&self.sess, &archive)? { + archive = new_archive + } + } + let archive_ro = match ArchiveRO::open(&archive) { Ok(ar) => ar, Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), }; @@ -73,7 +80,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { return Ok(()); } self.additions.push(Addition::Archive { - path: archive.to_path_buf(), + path: archive, archive: archive_ro, skip: Box::new(skip), }); @@ -89,41 +96,55 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. - fn build(mut self) -> bool { - match self.build_with_llvm() { + fn build(mut self: Box<Self>, output: &Path) -> bool { + match self.build_with_llvm(output) { Ok(any_members) => any_members, - Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), + Err(e) => self.sess.emit_fatal(ArchiveBuildFailure { error: e }), } } +} - fn inject_dll_import_lib( - &mut self, +pub struct LlvmArchiveBuilderBuilder; + +impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { + // FIXME use ArArchiveBuilder on most targets again once reading thin archives is + // implemented + if true { + Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + } else { + Box::new(ArArchiveBuilder::new(sess, get_llvm_object_symbols)) + } + } + + fn create_dll_import_lib( + &self, + sess: &Session, lib_name: &str, dll_imports: &[DllImport], - tmpdir: &MaybeTempDir, - ) { + tmpdir: &Path, + is_direct_dependency: bool, + ) -> PathBuf { + let name_suffix = if is_direct_dependency { "_imports" } else { "_imports_indirect" }; let output_path = { - let mut output_path: PathBuf = tmpdir.as_ref().to_path_buf(); - output_path.push(format!("{}_imports", lib_name)); + let mut output_path: PathBuf = tmpdir.to_path_buf(); + output_path.push(format!("{}{}", lib_name, name_suffix)); output_path.with_extension("lib") }; - let target = &self.sess.target; - let mingw_gnu_toolchain = target.vendor == "pc" - && target.os == "windows" - && target.env == "gnu" - && target.abi.is_empty(); + let target = &sess.target; + let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(target); let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { - if self.sess.target.arch == "x86" { + if sess.target.arch == "x86" { ( - LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), - import.ordinal, + common::i686_decorated_name(import, mingw_gnu_toolchain, false), + import.ordinal(), ) } else { - (import.name.to_string(), import.ordinal) + (import.name.to_string(), import.ordinal()) } }) .collect(); @@ -132,10 +153,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { // The binutils linker used on -windows-gnu targets cannot read the import // libraries generated by LLVM: in our attempts, the linker produced an .EXE // that loaded but crashed with an AV upon calling one of the imported - // functions. Therefore, use binutils to create the import library instead, + // functions. Therefore, use binutils to create the import library instead, // by writing a .DEF file to the temp dir and calling binutils's dlltool. let def_file_path = - tmpdir.as_ref().join(format!("{}_imports", lib_name)).with_extension("def"); + tmpdir.join(format!("{}{}", lib_name, name_suffix)).with_extension("def"); let def_file_content = format!( "EXPORTS\n{}", @@ -154,12 +175,30 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - self.sess.fatal(&format!("Error writing .DEF file: {}", e)); + sess.emit_fatal(ErrorWritingDEFFile { error: e }); } }; - let dlltool = find_binutils_dlltool(self.sess); - let result = std::process::Command::new(dlltool) + // --no-leading-underscore: For the `import_name_type` feature to work, we need to be + // able to control the *exact* spelling of each of the symbols that are being imported: + // hence we don't want `dlltool` adding leading underscores automatically. + let dlltool = find_binutils_dlltool(sess); + let temp_prefix = { + let mut path = PathBuf::from(&output_path); + path.pop(); + path.push(lib_name); + path + }; + // dlltool target architecture args from: + // https://github.com/llvm/llvm-project-release-prs/blob/llvmorg-15.0.6/llvm/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp#L69 + let (dlltool_target_arch, dlltool_target_bitness) = match sess.target.arch.as_ref() { + "x86_64" => ("i386:x86-64", "--64"), + "x86" => ("i386", "--32"), + "aarch64" => ("arm64", "--64"), + "arm" => ("arm", "--32"), + _ => panic!("unsupported arch {}", sess.target.arch), + }; + let result = std::process::Command::new(&dlltool) .args([ "-d", def_file_path.to_str().unwrap(), @@ -167,18 +206,30 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { lib_name, "-l", output_path.to_str().unwrap(), + "-m", + dlltool_target_arch, + "-f", + dlltool_target_bitness, + "--no-leading-underscore", + "--temp-prefix", + temp_prefix.to_str().unwrap(), ]) .output(); match result { Err(e) => { - self.sess.fatal(&format!("Error calling dlltool: {}", e)); + sess.emit_fatal(ErrorCallingDllTool { + dlltool_path: dlltool.to_string_lossy(), + error: e, + }); + } + // dlltool returns '0' on failure, so check for error output instead. + Ok(output) if !output.stderr.is_empty() => { + sess.emit_fatal(DlltoolFailImportLibrary { + stdout: String::from_utf8_lossy(&output.stdout), + stderr: String::from_utf8_lossy(&output.stderr), + }) } - Ok(output) if !output.status.success() => self.sess.fatal(&format!( - "Dlltool could not create import library: {}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )), _ => {} } } else { @@ -187,10 +238,10 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { let output_path_z = rustc_fs_util::path_to_c_string(&output_path); - tracing::trace!("invoking LLVMRustWriteImportLibrary"); - tracing::trace!(" dll_name {:#?}", dll_name_z); - tracing::trace!(" output_path {}", output_path.display()); - tracing::trace!( + trace!("invoking LLVMRustWriteImportLibrary"); + trace!(" dll_name {:#?}", dll_name_z); + trace!(" output_path {}", output_path.display()); + trace!( " import names: {}", dll_imports .iter() @@ -201,7 +252,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { // All import names are Rust identifiers and therefore cannot contain \0 characters. // FIXME: when support for #[link_name] is implemented, ensure that the import names - // still don't contain any \0 characters. Also need to check that the names don't + // still don't contain any \0 characters. Also need to check that the names don't // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved // in definition files. let cstring_import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = @@ -220,42 +271,91 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { output_path_z.as_ptr(), ffi_exports.as_ptr(), ffi_exports.len(), - llvm_machine_type(&self.sess.target.arch) as u16, - !self.sess.target.is_like_msvc, + llvm_machine_type(&sess.target.arch) as u16, + !sess.target.is_like_msvc, ) }; if result == crate::llvm::LLVMRustResult::Failure { - self.sess.fatal(&format!( - "Error creating import library for {}: {}", + sess.emit_fatal(ErrorCreatingImportLibrary { lib_name, - llvm::last_error().unwrap_or("unknown LLVM error".to_string()) - )); + error: llvm::last_error().unwrap_or("unknown LLVM error".to_string()), + }); } }; - self.add_archive(&output_path, |_| false).unwrap_or_else(|e| { - self.sess.fatal(&format!( - "failed to add native library {}: {}", - output_path.display(), - e - )); - }); + output_path + } +} + +// The object crate doesn't know how to get symbols for LLVM bitcode and COFF bigobj files. +// As such we need to use LLVM for them. +#[deny(unsafe_op_in_unsafe_fn)] +fn get_llvm_object_symbols( + buf: &[u8], + f: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> io::Result<bool> { + let is_bitcode = unsafe { llvm::LLVMRustIsBitcode(buf.as_ptr(), buf.len()) }; + + // COFF bigobj file, msvc LTO file or import library. See + // https://github.com/llvm/llvm-project/blob/453f27bc9/llvm/lib/BinaryFormat/Magic.cpp#L38-L51 + let is_unsupported_windows_obj_file = buf.get(0..4) == Some(b"\0\0\xFF\xFF"); + + if is_bitcode || is_unsupported_windows_obj_file { + let mut state = Box::new(f); + + let err = unsafe { + llvm::LLVMRustGetSymbols( + buf.as_ptr(), + buf.len(), + &mut *state as *mut &mut _ as *mut c_void, + callback, + error_callback, + ) + }; + + if err.is_null() { + return Ok(true); + } else { + return Err(unsafe { *Box::from_raw(err as *mut io::Error) }); + } + + unsafe extern "C" fn callback( + state: *mut c_void, + symbol_name: *const c_char, + ) -> *mut c_void { + let f = unsafe { &mut *(state as *mut &mut dyn FnMut(&[u8]) -> io::Result<()>) }; + match f(unsafe { CStr::from_ptr(symbol_name) }.to_bytes()) { + Ok(()) => std::ptr::null_mut(), + Err(err) => Box::into_raw(Box::new(err)) as *mut c_void, + } + } + + unsafe extern "C" fn error_callback(error: *const c_char) -> *mut c_void { + let error = unsafe { CStr::from_ptr(error) }; + Box::into_raw(Box::new(io::Error::new( + io::ErrorKind::Other, + format!("LLVM error: {}", error.to_string_lossy()), + ))) as *mut c_void + } + } else { + get_native_object_symbols(buf, f) } } impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self) -> io::Result<bool> { + fn build_with_llvm(&mut self, output: &Path) -> io::Result<bool> { let kind = &*self.sess.target.archive_format; - let kind = kind.parse::<ArchiveKind>().map_err(|_| kind).unwrap_or_else(|kind| { - self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) - }); + let kind = kind + .parse::<ArchiveKind>() + .map_err(|_| kind) + .unwrap_or_else(|kind| self.sess.emit_fatal(UnknownArchiveKind { kind })); let mut additions = mem::take(&mut self.additions); let mut strings = Vec::new(); let mut members = Vec::new(); - let dst = CString::new(self.dst.to_str().unwrap())?; + let dst = CString::new(output.to_str().unwrap())?; unsafe { for addition in &mut additions { @@ -327,22 +427,6 @@ impl<'a> LlvmArchiveBuilder<'a> { ret } } - - fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { - let name = import.name; - let prefix = if mingw { "" } else { "_" }; - - match import.calling_convention { - DllCallingConvention::C => format!("{}{}", prefix, name), - DllCallingConvention::Stdcall(arg_list_size) => { - format!("{}{}@{}", prefix, name, arg_list_size) - } - DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), - DllCallingConvention::Vectorcall(arg_list_size) => { - format!("{}@@{}", name, arg_list_size) - } - } - } } fn string_to_io_error(s: String) -> io::Error { @@ -351,28 +435,26 @@ fn string_to_io_error(s: String) -> io::Error { fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.cg.dlltool { return dlltool_path.clone().into_os_string(); } - let mut tool_name: OsString = if sess.host.arch != sess.target.arch { - // We are cross-compiling, so we need the tool with the prefix matching our target - if sess.target.arch == "x86" { - "i686-w64-mingw32-dlltool" - } else { - "x86_64-w64-mingw32-dlltool" - } + let tool_name: OsString = if sess.host.options.is_like_windows { + // If we're compiling on Windows, always use "dlltool.exe". + "dlltool.exe" } else { - // We are not cross-compiling, so we just want `dlltool` - "dlltool" + // On other platforms, use the architecture-specific name. + match sess.target.arch.as_ref() { + "x86_64" => "x86_64-w64-mingw32-dlltool", + "x86" => "i686-w64-mingw32-dlltool", + "aarch64" => "aarch64-w64-mingw32-dlltool", + + // For non-standard architectures (e.g., aarch32) fallback to "dlltool". + _ => "dlltool", + } } .into(); - if sess.host.options.is_like_windows { - // If we're compiling on Windows, add the .exe suffix - tool_name.push(".exe"); - } - // NOTE: it's not clear how useful it is to explicitly search PATH. for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { let full_path = dir.join(&tool_name); @@ -382,7 +464,7 @@ fn find_binutils_dlltool(sess: &Session) -> OsString { } // The user didn't specify the location of the dlltool binary, and we weren't able - // to find the appropriate one on the PATH. Just return the name of the tool + // to find the appropriate one on the PATH. Just return the name of the tool // and let the invocation fail with a hopefully useful error message. tool_name } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index be539499b56..8b05af7bed9 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,15 +1,17 @@ -use crate::back::write::{ - self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, +use crate::back::write::{self, save_temp_bitcode, DiagnosticHandlers}; +use crate::errors::{ + DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, }; -use crate::llvm::archive_ro::ArchiveRO; -use crate::llvm::{self, build_string, False, True}; -use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; +use crate::llvm::{self, build_string}; +use crate::{LlvmCodegenBackend, ModuleLlvm}; +use object::read::archive::ArchiveFile; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::memmap::Mmap; use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; @@ -17,14 +19,12 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; -use tracing::{debug, info}; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; use std::iter; use std::path::Path; -use std::ptr; use std::slice; use std::sync::Arc; @@ -34,8 +34,8 @@ pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { - CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, - CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, + CrateType::Executable | CrateType::Dylib | CrateType::Staticlib | CrateType::Cdylib => true, + CrateType::Rlib | CrateType::ProcMacro => false, } } @@ -75,28 +75,24 @@ fn prepare_lto( // with either fat or thin LTO let mut upstream_modules = Vec::new(); if cgcx.lto != Lto::ThinLocal { - if cgcx.opts.cg.prefer_dynamic { - diag_handler - .struct_err("cannot prefer dynamic linking when performing LTO") - .note( - "only 'staticlib', 'bin', and 'cdylib' outputs are \ - supported with LTO", - ) - .emit(); - return Err(FatalError); - } - // Make sure we actually can run LTO for crate_type in cgcx.crate_types.iter() { if !crate_type_allows_lto(*crate_type) { - let e = diag_handler.fatal( - "lto can only be run for executables, cdylibs and \ - static library outputs", - ); - return Err(e); + diag_handler.emit_err(LtoDisallowed); + return Err(FatalError); + } else if *crate_type == CrateType::Dylib { + if !cgcx.opts.unstable_opts.dylib_lto { + diag_handler.emit_err(LtoDylib); + return Err(FatalError); + } } } + if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto { + diag_handler.emit_err(DynamicLinkingWithLTO); + return Err(FatalError); + } + for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); @@ -107,28 +103,45 @@ fn prepare_lto( .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); } - let archive = ArchiveRO::open(path).expect("wanted an rlib"); + let archive_data = unsafe { + Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib")) + .expect("couldn't map rlib") + }; + let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib"); let obj_files = archive - .iter() - .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) + .members() + .filter_map(|child| { + child.ok().and_then(|c| { + std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c)) + }) + }) .filter(|&(name, _)| looks_like_rust_object_file(name)); for (name, child) in obj_files { info!("adding bitcode from {}", name); - match get_bitcode_slice_from_object_data(child.data()) { + match get_bitcode_slice_from_object_data( + child.data(&*archive_data).expect("corrupt rlib"), + ) { Ok(data) => { let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); } - Err(msg) => return Err(diag_handler.fatal(&msg)), + Err(e) => { + diag_handler.emit_err(e); + return Err(FatalError); + } } } } } + // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to + // __llvm_profile_runtime, therefore we won't know until link time if this symbol + // should have default visibility. + symbols_below_threshold.push(CString::new("__llvm_profile_counter_bias").unwrap()); Ok((symbols_below_threshold, upstream_modules)) } -fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { +fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], LtoBitcodeFromRlib> { let mut len = 0; let data = unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; @@ -143,8 +156,9 @@ fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { Ok(bc) } else { assert!(len == 0); - let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); - Err(format!("failed to get bitcode from object file for LTO ({})", msg)) + Err(LtoBitcodeFromRlib { + llvm_err: llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()), + }) } } @@ -198,8 +212,8 @@ pub(crate) fn run_thin( } pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { - let name = module.name.clone(); - let buffer = ThinBuffer::new(module.module_llvm.llmod()); + let name = module.name; + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -316,10 +330,9 @@ fn fat_lto( }); info!("linking {:?}", name); let data = bc_decoded.data(); - linker.add(data).map_err(|()| { - let msg = format!("failed to load bitcode of module {:?}", name); - write::llvm_err(diag_handler, &msg) - })?; + linker + .add(data) + .map_err(|()| write::llvm_err(diag_handler, LlvmError::LoadBitcode { name }))?; serialized_bitcode.push(bc_decoded); } drop(linker); @@ -413,7 +426,7 @@ fn thin_lto( info!("going for that thin, thin LTO"); let green_modules: FxHashMap<_, _> = - cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect(); + cached_modules.iter().map(|(_, wp)| (wp.cgu_name.clone(), wp.clone())).collect(); let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len(); let mut thin_buffers = Vec::with_capacity(modules.len()); @@ -477,7 +490,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len() as u32, ) - .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?; + .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::PrepareThinLtoContext))?; let data = ThinData(data); @@ -550,8 +563,7 @@ fn thin_lto( // session, overwriting the previous serialized data (if any). if let Some(path) = key_map_path { if let Err(err) = curr_key_map.save_to_file(&path) { - let msg = format!("Error while writing ThinLTO key data: {}", err); - return Err(write::llvm_err(diag_handler, &msg)); + return Err(write::llvm_err(diag_handler, LlvmError::WriteThinLtoKey { err })); } } @@ -565,7 +577,7 @@ pub(crate) fn run_pass_manager( module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, ) -> Result<(), FatalError> { - let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); + let _timer = cgcx.prof.verbose_generic_activity_with_arg("LLVM_lto_optimize", &*module.name); let config = cgcx.config(module.kind); // Now we have one massive module inside of llmod. Time to run the @@ -583,65 +595,13 @@ pub(crate) fn run_pass_manager( llvm::LLVMRustAddModuleFlag( module.module_llvm.llmod(), llvm::LLVMModFlagBehavior::Error, - "LTOPostLink\0".as_ptr().cast(), + c"LTOPostLink".as_ptr().cast(), 1, ); } - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; - let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - )?; - debug!("lto done"); - return Ok(()); - } - - let pm = llvm::LLVMCreatePassManager(); - llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); - - if config.verify_llvm_ir { - let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - let opt_level = config - .opt_level - .map(|x| to_llvm_opt_settings(x).0) - .unwrap_or(llvm::CodeGenOptLevel::None); - with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { - if thin { - llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); - } else { - llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( - b, pm, /* Internalize = */ False, /* RunInliner = */ True, - ); - } - }); - - // We always generate bitcode through ThinLTOBuffers, - // which do not support anonymous globals - if config.bitcode_needed() { - let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - if config.verify_llvm_ir { - let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); - llvm::LLVMRustAddPass(pm, pass.unwrap()); - } - - llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()); - - llvm::LLVMDisposePassManager(pm); + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + write::llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage)?; } debug!("lto done"); Ok(()) @@ -695,9 +655,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub fn new(m: &llvm::Module) -> ThinBuffer { + pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); ThinBuffer(buffer) } } @@ -729,8 +689,7 @@ pub unsafe fn optimize_thin_module( let module_name = &thin_module.shared.module_names[thin_module.idx]; let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); - let tm = - (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?; + let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, e))?; // Right now the implementation we've got only works over serialized // modules, so we create a fresh new LLVM context and parse the module @@ -749,18 +708,6 @@ pub unsafe fn optimize_thin_module( let llmod = module.module_llvm.llmod(); save_temp_bitcode(cgcx, &module, "thin-lto-input"); - // Before we do much else find the "main" `DICompileUnit` that we'll be - // using below. If we find more than one though then rustc has changed - // in a way we're not ready for, so generate an ICE by returning - // an error. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); - if !cu2.is_null() { - let msg = "multiple source DICompileUnits found"; - return Err(write::llvm_err(&diag_handler, msg)); - } - // Up next comes the per-module local analyses that we do for Thin LTO. // Each of these functions is basically copied from the LLVM // implementation and then tailored to suit this implementation. Ideally @@ -773,8 +720,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); } @@ -784,8 +730,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); } @@ -795,8 +740,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); } @@ -805,49 +749,11 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)); + return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } - // Ok now this is a bit unfortunate. This is also something you won't - // find upstream in LLVM's ThinLTO passes! This is a hack for now to - // work around bugs in LLVM. - // - // First discovered in #45511 it was found that as part of ThinLTO - // importing passes LLVM will import `DICompileUnit` metadata - // information across modules. This means that we'll be working with one - // LLVM module that has multiple `DICompileUnit` instances in it (a - // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of - // bugs in LLVM's backend which generates invalid DWARF in a situation - // like this: - // - // https://bugs.llvm.org/show_bug.cgi?id=35212 - // https://bugs.llvm.org/show_bug.cgi?id=35562 - // - // While the first bug there is fixed the second ended up causing #46346 - // which was basically a resurgence of #45511 after LLVM's bug 35212 was - // fixed. - // - // This function below is a huge hack around this problem. The function - // below is defined in `PassWrapper.cpp` and will basically "merge" - // all `DICompileUnit` instances in a module. Basically it'll take all - // the objects, rewrite all pointers of `DISubprogram` to point to the - // first `DICompileUnit`, and then delete all the other units. - // - // This is probably mangling to the debug info slightly (but hopefully - // not too much) but for now at least gets LLVM to emit valid DWARF (or - // so it appears). Hopefully we can remove this once upstream bugs are - // fixed in LLVM. - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); - save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); - } - // Alright now that we've done everything related to the ThinLTO // analysis it's time to run some optimizations! Here we use the same // `run_pass_manager` as the "fat" LTO above except that we tell it to @@ -926,11 +832,7 @@ pub fn parse_module<'a>( diag_handler: &Handler, ) -> Result<&'a llvm::Module, FatalError> { unsafe { - llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( - || { - let msg = "failed to parse bitcode for LTO module"; - write::llvm_err(diag_handler, msg) - }, - ) + llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()) + .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::ParseBitcode)) } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 2b465ce40e7..53b4296802e 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -5,7 +5,10 @@ use crate::back::profiling::{ use crate::base; use crate::common; use crate::consts; -use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; +use crate::errors::{ + CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, WithLlvmError, WriteBytecode, +}; +use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; @@ -21,15 +24,14 @@ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; -use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; -use tracing::debug; +use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; @@ -39,10 +41,10 @@ use std::slice; use std::str; use std::sync::Arc; -pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { +pub fn llvm_err<'a>(handler: &rustc_errors::Handler, err: LlvmError<'a>) -> FatalError { match llvm::last_error() { - Some(err) => handler.fatal(&format!("{}: {}", msg, err)), - None => handler.fatal(msg), + Some(llvm_err) => handler.emit_almost_fatal(WithLlvmError(err, llvm_err)), + None => handler.emit_almost_fatal(err), } } @@ -87,10 +89,9 @@ pub fn write_output_file<'ll>( } } - result.into_result().map_err(|()| { - let msg = format!("could not write output to {}", output.display()); - llvm_err(handler, &msg) - }) + result + .into_result() + .map_err(|()| llvm_err(handler, LlvmError::WriteOutput { path: output })) } } @@ -100,7 +101,7 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm: // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) + .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise()) } pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { @@ -119,7 +120,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), err).raise()) } pub fn to_llvm_opt_settings( @@ -205,7 +206,7 @@ pub fn target_machine_factory( sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; - let asm_comments = sess.asm_comments(); + let asm_comments = sess.opts.unstable_opts.asm_comments; let relax_elf_relocations = sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); @@ -214,6 +215,8 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); + let force_emulated_tls = sess.target.force_emulated_tls; + Arc::new(move |config: TargetMachineFactoryConfig| { let split_dwarf_file = path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; @@ -239,12 +242,11 @@ pub fn target_machine_factory( relax_elf_relocations, use_init_array, split_dwarf_file.as_ptr(), + force_emulated_tls, ) }; - tm.ok_or_else(|| { - format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) - }) + tm.ok_or_else(|| LlvmError::CreateTargetMachine { triple: triple.clone() }) }) } @@ -304,7 +306,6 @@ impl<'a> DiagnosticHandlers<'a> { remark_passes.as_ptr(), remark_passes.len(), ); - llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); DiagnosticHandlers { data, llcx, old_handler } } } @@ -312,9 +313,7 @@ impl<'a> DiagnosticHandlers<'a> { impl<'a> Drop for DiagnosticHandlers<'a> { fn drop(&mut self) { - use std::ptr::null_mut; unsafe { - llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); drop(Box::from_raw(self.data)); } @@ -342,16 +341,6 @@ fn report_inline_asm( cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); } -unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { - if user.is_null() { - return; - } - let (cgcx, _) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); - - let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); - report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); -} - unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { if user.is_null() { return; @@ -370,25 +359,37 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void }; if enabled { - diag_handler.note_without_error(&format!( - "{}:{}:{}: {}: {}", - opt.filename, opt.line, opt.column, opt.pass_name, opt.message, - )); + diag_handler.emit_note(FromLlvmOptimizationDiag { + filename: &opt.filename, + line: opt.line, + column: opt.column, + pass_name: &opt.pass_name, + kind: match opt.kind { + OptimizationDiagnosticKind::OptimizationRemark => "success", + OptimizationDiagnosticKind::OptimizationMissed + | OptimizationDiagnosticKind::OptimizationFailure => "missed", + OptimizationDiagnosticKind::OptimizationAnalysis + | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute + | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis", + OptimizationDiagnosticKind::OptimizationRemarkOther => "other", + }, + message: &opt.message, + }); } } llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { - let msg = llvm::build_string(|s| { + let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.warn(&msg); + diag_handler.emit_warning(FromLlvmDiag { message }); } llvm::diagnostic::Unsupported(diagnostic_ref) => { - let msg = llvm::build_string(|s| { + let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.err(&msg); + diag_handler.emit_err(FromLlvmDiag { message }); } llvm::diagnostic::UnknownDiagnostic(..) => {} } @@ -423,7 +424,11 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } -pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( +fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { + config.instrument_coverage.then(|| CString::new("default_%m_%p.profraw").unwrap()) +} + +pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, module: &ModuleCodegen<ModuleLlvm>, @@ -438,6 +443,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + let instr_profile_output_path = get_instr_profile_output_path(config); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { @@ -449,16 +455,19 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), + sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS), + sanitize_kernel_address_recover: config + .sanitizer_recover + .contains(SanitizerSet::KERNELADDRESS), }) } else { None }; - let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { - Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) - } else { - None - }; + let mut llvm_profiler = cgcx + .prof + .llvm_recording_enabled() + .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())); let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); @@ -470,7 +479,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. - let result = llvm::LLVMRustOptimizeWithNewPassManager( + let result = llvm::LLVMRustOptimize( module.module_llvm.llmod(), &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), @@ -488,6 +497,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, + instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, @@ -499,7 +509,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( llvm_plugins.as_ptr().cast(), llvm_plugins.len(), ); - result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) + result.into_result().map_err(|()| llvm_err(diag_handler, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. @@ -513,18 +523,11 @@ pub(crate) unsafe fn optimize( let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; - let tm = &*module.module_llvm.tm; let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); - if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { - diag_handler.warn( - "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", - ); - } - if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); let out = path_to_c_string(&out); @@ -532,191 +535,24 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = match cgcx.lto { - Lto::Fat => llvm::OptStage::PreLinkFatLTO, - Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, - _ => llvm::OptStage::PreLinkNoLTO, - }; - return optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - ); - } - - if cgcx.prof.llvm_recording_enabled() { - diag_handler - .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); - } - - // Create the two optimizing pass managers. These mirror what clang - // does, and are by populated by LLVM's default PassManagerBuilder. - // Each manager has a different set of passes, but they also share - // some common passes. - let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); - let mpm = llvm::LLVMCreatePassManager(); - - { - let find_pass = |pass_name: &str| { - let pass_name = SmallCStr::new(pass_name); - llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) - }; - - if config.verify_llvm_ir { - // Verification should run as the very first pass. - llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); - } - - let mut extra_passes = Vec::new(); - let mut have_name_anon_globals_pass = false; - - for pass_name in &config.passes { - if pass_name == "lint" { - // Linting should also be performed early, directly on the generated IR. - llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); - continue; - } - - if let Some(pass) = find_pass(pass_name) { - extra_passes.push(pass); - } else { - diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); - } - - if pass_name == "name-anon-globals" { - have_name_anon_globals_pass = true; - } - } - - // Instrumentation must be inserted before optimization, - // otherwise LLVM may optimize some functions away which - // breaks llvm-cov. - // - // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. - if config.instrument_gcov { - llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); - } - if config.instrument_coverage { - llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); - } - if config.debug_info_for_profiling { - llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); - } - - add_sanitizer_passes(config, &mut extra_passes); - - // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need - // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise - // we'll get errors in LLVM. - let using_thin_buffers = config.bitcode_needed(); - if !config.no_prepopulate_passes { - llvm::LLVMAddAnalysisPasses(tm, fpm); - llvm::LLVMAddAnalysisPasses(tm, mpm); - let opt_level = to_llvm_opt_settings(opt_level).0; - let prepare_for_thin_lto = cgcx.lto == Lto::Thin - || cgcx.lto == Lto::ThinLocal - || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); - with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { - llvm::LLVMRustAddLastExtensionPasses( - b, - extra_passes.as_ptr(), - extra_passes.len() as size_t, - ); - llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); - llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); - }); - - have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; - if using_thin_buffers && !prepare_for_thin_lto { - llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); - have_name_anon_globals_pass = true; - } - } else { - // If we don't use the standard pipeline, directly populate the MPM - // with the extra passes. - for pass in extra_passes { - llvm::LLVMRustAddPass(mpm, pass); - } - } - - if using_thin_buffers && !have_name_anon_globals_pass { - // As described above, this will probably cause an error in LLVM - if config.no_prepopulate_passes { - diag_handler.err( - "The current compilation is going to use thin LTO buffers \ - without running LLVM's NameAnonGlobals pass. \ - This will likely cause errors in LLVM. Consider adding \ - -C passes=name-anon-globals to the compiler command line.", - ); - } else { - bug!( - "We are using thin LTO buffers without running the NameAnonGlobals pass. \ - This will likely cause errors in LLVM and should never happen." - ); - } - } - } - - diag_handler.abort_if_errors(); - - // Finally, run the actual optimization passes - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_function_passes", - &*module.name, - ); - llvm::LLVMRustRunFunctionPassManager(fpm, llmod); - } - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_module_passes", - &*module.name, - ); - llvm::LLVMRunPassManager(mpm, llmod); - } - - // Deallocate managers that we're now done with - llvm::LLVMDisposePassManager(fpm); - llvm::LLVMDisposePassManager(mpm); + let opt_stage = match cgcx.lto { + Lto::Fat => llvm::OptStage::PreLinkFatLTO, + Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ => llvm::OptStage::PreLinkNoLTO, + }; + return llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage); } Ok(()) } -unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { - if config.sanitizer.contains(SanitizerSet::ADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); - passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); - passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); - } - if config.sanitizer.contains(SanitizerSet::MEMORY) { - let track_origins = config.sanitizer_memory_track_origins as c_int; - let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); - passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); - } - if config.sanitizer.contains(SanitizerSet::THREAD) { - passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); - } - if config.sanitizer.contains(SanitizerSet::HWADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); - passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); - } -} - pub(crate) fn link( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, mut modules: Vec<ModuleCodegen<ModuleLlvm>>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { use super::lto::{Linker, ModuleBuffer}; - // Sort the modules by name to ensure to ensure deterministic behavior. + // Sort the modules by name to ensure deterministic behavior. modules.sort_by(|a, b| a.name.cmp(&b.name)); let (first, elements) = modules.split_first().expect("Bug! modules must contain at least one module."); @@ -726,8 +562,7 @@ pub(crate) fn link( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); linker.add(buffer.data()).map_err(|()| { - let msg = format!("failed to serialize module {:?}", module.name); - llvm_err(diag_handler, &msg) + llvm_err(diag_handler, LlvmError::SerializeModule { name: &module.name }) })?; } drop(linker); @@ -790,7 +625,7 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto); let data = thin.data(); if let Some(bitcode_filename) = bc_out.file_name() { @@ -805,9 +640,8 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); - if let Err(e) = fs::write(&bc_out, data) { - let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); - diag_handler.err(&msg); + if let Err(err) = fs::write(&bc_out, data) { + diag_handler.emit_err(WriteBytecode { path: &bc_out, err }); } } @@ -857,10 +691,9 @@ pub(crate) unsafe fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result.into_result().map_err(|()| { - let msg = format!("failed to write LLVM IR to {}", out.display()); - llvm_err(diag_handler, &msg) - })?; + result + .into_result() + .map_err(|()| llvm_err(diag_handler, LlvmError::WriteIr { path: &out }))?; } if config.emit_asm { @@ -928,8 +761,8 @@ pub(crate) unsafe fn codegen( EmitObj::Bitcode => { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); - if let Err(e) = link_or_copy(&bc_out, &obj_out) { - diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); + if let Err(err) = link_or_copy(&bc_out, &obj_out) { + diag_handler.emit_err(CopyBitcode { err }); } if !config.emit_bc { @@ -941,14 +774,25 @@ pub(crate) unsafe fn codegen( EmitObj::None => {} } + record_llvm_cgu_instructions_stats(&cgcx.prof, llmod); drop(handlers); } + // `.dwo` files are only emitted if: + // + // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not + // produce dwarf objects, even if otherwise enabled) + // - Target supports Split DWARF + // - Split debuginfo is enabled + // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different + // sections in the `.o` files). + let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_)) + && cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split; Ok(module.into_compiled_module( config.emit_obj != EmitObj::None, - cgcx.target_can_use_split_dwarf - && cgcx.split_debuginfo != SplitDebuginfo::Off - && cgcx.split_dwarf_kind == SplitDwarfKind::Split, + dwarf_object_emitted, config.emit_bc, &cgcx.output_filenames, )) @@ -1047,11 +891,11 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.module\0".as_ptr().cast(), + c"rustc.embedded.module".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; + let section = if is_apple { c"__LLVM,__bitcode" } else { c".llvmbc" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); @@ -1060,91 +904,25 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.cmdline\0".as_ptr().cast(), + c"rustc.embedded.cmdline".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; + let section = if is_apple { c"__LLVM,__cmdline" } else { c".llvmcmd" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); - } -} - -pub unsafe fn with_llvm_pmb( - llmod: &llvm::Module, - config: &ModuleConfig, - opt_level: llvm::CodeGenOptLevel, - prepare_for_thin_lto: bool, - f: &mut dyn FnMut(&llvm::PassManagerBuilder), -) { - use std::ptr; - - // Create the PassManagerBuilder for LLVM. We configure it with - // reasonable defaults and prepare it to actually populate the pass - // manager. - let builder = llvm::LLVMRustPassManagerBuilderCreate(); - let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); - let inline_threshold = config.inline_threshold; - let pgo_gen_path = get_pgo_gen_path(config); - let pgo_use_path = get_pgo_use_path(config); - let pgo_sample_use_path = get_pgo_sample_use_path(config); - - llvm::LLVMRustConfigurePassManagerBuilder( - builder, - opt_level, - config.merge_functions, - config.vectorize_slp, - config.vectorize_loop, - prepare_for_thin_lto, - pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - opt_size as c_int, - ); - - llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); - - // Here we match what clang does (kinda). For O0 we only inline - // always-inline functions (but don't add lifetime intrinsics), at O1 we - // inline with lifetime intrinsics, and O2+ we add an inliner with a - // thresholds copied from clang. - match (opt_level, opt_size, inline_threshold) { - (.., Some(t)) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); - } - (llvm::CodeGenOptLevel::Aggressive, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); - } - (_, llvm::CodeGenOptSizeDefault, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); - } - (_, llvm::CodeGenOptSizeAggressive, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); - } - (llvm::CodeGenOptLevel::None, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Less, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Default, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); - } + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } - - f(builder); - llvm::LLVMRustPassManagerBuilderDispose(builder); } // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. // This is required to satisfy `dllimport` references to static data in .rlibs -// when using MSVC linker. We do this only for data, as linker can fix up +// when using MSVC linker. We do this only for data, as linker can fix up // code references on its own. // See #26591, #27438 fn create_msvc_imps( @@ -1210,3 +988,23 @@ fn record_artifact_size( self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); } } + +fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) { + if !prof.enabled() { + return; + } + + let raw_stats = + llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(&llmod, s) }) + .expect("cannot get module instruction stats"); + + #[derive(serde::Deserialize)] + struct InstructionsStats { + module: String, + total: u64, + } + + let InstructionsStats { module, total } = + serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats"); + prof.artifact_size("cgu_instructions", module, total); +} diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 86f92dc0239..2f7eb08ad3d 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -107,11 +107,14 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen } // Create the llvm.used and llvm.compiler.used variables. - if !cx.used_statics().borrow().is_empty() { - cx.create_used_variable() + if !cx.used_statics.borrow().is_empty() { + cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow()); } - if !cx.compiler_used_statics().borrow().is_empty() { - cx.create_compiler_used_variable() + if !cx.compiler_used_statics.borrow().is_empty() { + cx.create_used_variable_impl( + c"llvm.compiler.used", + &*cx.compiler_used_statics.borrow(), + ); } // Run replace-all-uses-with for statics that need it. This must diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 4a4cccb490d..b4aa001547c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1,34 +1,32 @@ +use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; -use crate::llvm::{self, BasicBlock, False}; -use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; -use crate::llvm_util; +use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use cstr::cstr; use libc::{c_char, c_uint}; -use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind}; +use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; -use rustc_target::spec::{HasTargetSpec, Target}; +use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use std::borrow::Cow; -use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; -use tracing::{debug, instrument}; // All Builders must have an llfn associated with them #[must_use] @@ -45,13 +43,10 @@ impl Drop for Builder<'_, '_, '_> { } } -// FIXME(eddyb) use a checked constructor when they become `const fn`. -const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; - /// Empty string, to be used where LLVM expects an instruction name, indicating /// that the instruction is to be left unnamed (i.e. numbered, in textual IR). // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. -const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); +const UNNAMED: *const c_char = c"".as_ptr(); impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value; @@ -217,6 +212,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], then: &'ll BasicBlock, @@ -226,10 +223,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("invoke {:?} with args ({:?})", llfn, args); let args = self.check_call("invoke", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles = vec![funclet_bundle]; - unsafe { + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); + + bundles.retain(|bundle| bundle.is_some()); + let invoke = unsafe { llvm::LLVMRustBuildInvoke( self.llbuilder, llty, @@ -238,10 +245,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { args.len() as c_uint, then, catch, - bundle, + bundles.as_ptr(), + bundles.len() as c_uint, UNNAMED, ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, invoke); } + invoke } fn unreachable(&mut self) { @@ -362,11 +374,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { Int(I64) => "llvm.ssub.with.overflow.i64", Int(I128) => "llvm.ssub.with.overflow.i128", - Uint(U8) => "llvm.usub.with.overflow.i8", - Uint(U16) => "llvm.usub.with.overflow.i16", - Uint(U32) => "llvm.usub.with.overflow.i32", - Uint(U64) => "llvm.usub.with.overflow.i64", - Uint(U128) => "llvm.usub.with.overflow.i128", + Uint(_) => { + // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these + // to be the canonical form. It will attempt to reform llvm.usub.with.overflow + // in the backend if profitable. + let sub = self.sub(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); + return (sub, cmp); + } _ => unreachable!(), }, @@ -408,20 +423,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { let mut bx = Builder::with_cx(self.cx); bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); - bx.dynamic_alloca(ty, align) - } - - fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { unsafe { - let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED); + let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); alloca } } - fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value { + fn byte_array_alloca(&mut self, len: &'ll Value, align: Align) -> &'ll Value { unsafe { - let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED); + let alloca = + llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), len, UNNAMED); llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); alloca } @@ -469,7 +481,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); if place.layout.is_zst() { - return OperandRef::new_zst(self, place.layout); + return OperandRef::zero_sized(place.layout); } #[instrument(level = "trace", skip(bx))] @@ -480,7 +492,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { layout: TyAndLayout<'tcx>, offset: Size, ) { - if !scalar.is_always_valid(bx) { + if !scalar.is_uninit_valid() { bx.noundef_metadata(load); } @@ -490,7 +502,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { bx.range_metadata(load, scalar.valid_range(bx)); } } - abi::Pointer => { + abi::Pointer(_) => { if !scalar.valid_range(bx).contains(0) { bx.nonnull_metadata(load); } @@ -553,15 +565,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn write_operand_repeatedly( - mut self, + &mut self, cg_elem: OperandRef<'tcx, &'ll Value>, count: u64, dest: PlaceRef<'tcx, &'ll Value>, - ) -> Self { + ) { let zero = self.const_usize(0); let count = self.const_usize(count); - let start = dest.project_index(&mut self, zero).llval; - let end = dest.project_index(&mut self, count).llval; + let start = dest.project_index(self, zero).llval; + let end = dest.project_index(self, count).llval; let header_bb = self.append_sibling_block("repeat_loop_header"); let body_bb = self.append_sibling_block("repeat_loop_body"); @@ -589,7 +601,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { body_bx.br(header_bb); header_bx.add_incoming_to_phi(current, next, body_bb); - Self::build(self.cx, next_bb) + *self = Self::build(self.cx, next_bb); } fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { @@ -626,32 +638,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn type_metadata(&mut self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); - let v = [self.const_usize(0), typeid_metadata]; - unsafe { - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( - self.cx.llcx, - v.as_ptr(), - v.len() as c_uint, - )), - ) - } - } - - fn typeid_metadata(&mut self, typeid: String) -> Self::Value { - unsafe { - llvm::LLVMMDStringInContext( - self.cx.llcx, - typeid.as_ptr() as *const c_char, - typeid.as_bytes().len() as c_uint, - ) - } - } - fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { self.store_with_flags(val, ptr, align, MemFlags::empty()) } @@ -752,11 +738,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(false, val, dest_ty) } - fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { self.fptoint_sat(true, val, dest_ty) } @@ -846,7 +832,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + unsafe { + llvm::LLVMBuildIntCast2( + self.llbuilder, + val, + dest_ty, + if is_signed { True } else { False }, + UNNAMED, + ) + } } fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -984,29 +978,40 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn cleanup_landing_pad(&mut self, ty: &'ll Type, pers_fn: &'ll Value) -> &'ll Value { - let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { llvm::LLVMSetCleanup(landing_pad, llvm::True); } - landing_pad + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } - fn resume(&mut self, exn: &'ll Value) { + fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 1); + self.add_clause(landing_pad, self.const_array(self.type_i8p(), &[])); + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) + } + + fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) { + let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); + let mut exn = self.const_poison(ty); + exn = self.insert_value(exn, exn0, 0); + exn = self.insert_value(exn, exn1, 1); unsafe { llvm::LLVMBuildResume(self.llbuilder, exn); } } fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("cleanuppad"); let ret = unsafe { - llvm::LLVMRustBuildCleanupPad( + llvm::LLVMBuildCleanupPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), - name.as_ptr(), + args.len() as c_uint, + c"cleanuppad".as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) @@ -1014,20 +1019,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { unsafe { - llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) .expect("LLVM does not have support for cleanupret"); } } fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("catchpad"); let ret = unsafe { - llvm::LLVMRustBuildCatchPad( + llvm::LLVMBuildCatchPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), - name.as_ptr(), + args.len() as c_uint, + c"catchpad".as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for catchpad")) @@ -1039,20 +1043,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unwind: Option<&'ll BasicBlock>, handlers: &[&'ll BasicBlock], ) -> &'ll Value { - let name = cstr!("catchswitch"); let ret = unsafe { - llvm::LLVMRustBuildCatchSwitch( + llvm::LLVMBuildCatchSwitch( self.llbuilder, parent, unwind, handlers.len() as c_uint, - name.as_ptr(), + c"catchswitch".as_ptr(), ) }; let ret = ret.expect("LLVM does not have support for catchswitch"); for handler in handlers { unsafe { - llvm::LLVMRustAddHandler(ret, handler); + llvm::LLVMAddHandler(ret, handler); } } ret @@ -1064,35 +1067,23 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst: &'ll Value, cmp: &'ll Value, src: &'ll Value, - mut order: rustc_codegen_ssa::common::AtomicOrdering, + order: rustc_codegen_ssa::common::AtomicOrdering, failure_order: rustc_codegen_ssa::common::AtomicOrdering, weak: bool, ) -> &'ll Value { let weak = if weak { llvm::True } else { llvm::False }; - if llvm_util::get_version() < (13, 0, 0) { - use rustc_codegen_ssa::common::AtomicOrdering::*; - // Older llvm has the pre-C++17 restriction on - // success and failure memory ordering, - // requiring the former to be at least as strong as the latter. - // So, for llvm 12, we upgrade the success ordering to a stronger - // one if necessary. - match (order, failure_order) { - (Relaxed, Acquire) => order = Acquire, - (Release, Acquire) => order = AcquireRelease, - (_, SequentiallyConsistent) => order = SequentiallyConsistent, - _ => {} - } - } unsafe { - llvm::LLVMRustBuildAtomicCmpXchg( + let value = llvm::LLVMBuildAtomicCmpXchg( self.llbuilder, dst, cmp, src, AtomicOrdering::from_generic(order), AtomicOrdering::from_generic(failure_order), - weak, - ) + llvm::False, // SingleThreaded + ); + llvm::LLVMSetWeak(value, weak); + value } } fn atomic_rmw( @@ -1109,7 +1100,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst, src, AtomicOrdering::from_generic(order), - False, + llvm::False, // SingleThreaded ) } } @@ -1117,13 +1108,18 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn atomic_fence( &mut self, order: rustc_codegen_ssa::common::AtomicOrdering, - scope: rustc_codegen_ssa::common::SynchronizationScope, + scope: SynchronizationScope, ) { + let single_threaded = match scope { + SynchronizationScope::SingleThread => llvm::True, + SynchronizationScope::CrossThread => llvm::False, + }; unsafe { - llvm::LLVMRustBuildAtomicFence( + llvm::LLVMBuildFence( self.llbuilder, AtomicOrdering::from_generic(order), - SynchronizationScope::from_generic(scope), + single_threaded, + UNNAMED, ); } } @@ -1173,7 +1169,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, - None, + [].as_ptr(), + 0 as c_uint, ); } } @@ -1181,6 +1178,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, @@ -1188,19 +1187,34 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("call {:?} with args ({:?})", llfn, args); let args = self.check_call("call", llty, llfn, args); - let bundle = funclet.map(|funclet| funclet.bundle()); - let bundle = bundle.as_ref().map(|b| &*b.raw); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles = vec![funclet_bundle]; - unsafe { + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); + + bundles.retain(|bundle| bundle.is_some()); + let call = unsafe { llvm::LLVMRustBuildCall( self.llbuilder, llty, llfn, args.as_ptr() as *const &llvm::Value, args.len() as c_uint, - bundle, + bundles.as_ptr(), + bundles.len() as c_uint, ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, call); } + call } fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -1360,8 +1374,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { - let ret = - unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; + let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; ret.expect("LLVM does not have support for catchret") } @@ -1433,7 +1446,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, f, args, None) + self.call(ty, None, None, f, args, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { @@ -1470,51 +1483,32 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - fn fptoint_sat_broken_in_llvm(&self) -> bool { - match self.tcx.sess.target.arch.as_ref() { - // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 - "riscv64" => llvm_util::get_version() < (13, 0, 0), - _ => false, - } - } - - fn fptoint_sat( - &mut self, - signed: bool, - val: &'ll Value, - dest_ty: &'ll Type, - ) -> Option<&'ll Value> { - if !self.fptoint_sat_broken_in_llvm() { - let src_ty = self.cx.val_ty(val); - let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector - { - assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); - ( - self.cx.element_type(src_ty), - self.cx.element_type(dest_ty), - Some(self.cx.vector_length(src_ty)), - ) - } else { - (src_ty, dest_ty, None) - }; - let float_width = self.cx.float_width(float_ty); - let int_width = self.cx.int_width(int_ty); - - let instr = if signed { "fptosi" } else { "fptoui" }; - let name = if let Some(vector_length) = vector_length { - format!( - "llvm.{}.sat.v{}i{}.v{}f{}", - instr, vector_length, int_width, vector_length, float_width - ) - } else { - format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) - }; - let f = - self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + ( + self.cx.element_type(src_ty), + self.cx.element_type(dest_ty), + Some(self.cx.vector_length(src_ty)), + ) } else { - None - } + (src_ty, dest_ty, None) + }; + let float_width = self.cx.float_width(float_ty); + let int_width = self.cx.int_width(int_ty); + + let instr = if signed { "fptosi" } else { "fptoui" }; + let name = if let Some(vector_length) = vector_length { + format!( + "llvm.{}.sat.v{}i{}.v{}f{}", + instr, vector_length, int_width, vector_length, float_width + ) + } else { + format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + }; + let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); + self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None) } pub(crate) fn landing_pad( @@ -1531,4 +1525,71 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) } } + + // Emits CFI pointer type membership tests. + fn cfi_type_test( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + if is_indirect_call && fn_abi.is_some() && self.tcx.sess.is_sanitizer_cfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::CFI) { + return; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let typeid = typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + + // Test whether the function pointer is associated with the type identifier. + let cond = self.type_test(llfn, typeid_metadata); + let bb_pass = self.append_sibling_block("type_test.pass"); + let bb_fail = self.append_sibling_block("type_test.fail"); + self.cond_br(cond, bb_pass, bb_fail); + + self.switch_to_block(bb_fail); + self.abort(); + self.unreachable(); + + self.switch_to_block(bb_pass); + } + } + + // Emits KCFI operand bundles. + fn kcfi_operand_bundle( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) -> Option<llvm::OperandBundleDef<'ll>> { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + let kcfi_bundle = if is_indirect_call && self.tcx.sess.is_sanitizer_kcfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::KCFI) { + return None; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + kcfi_bundle + } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 72155d874a2..4b9ca2e7d19 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -6,14 +6,14 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; +use crate::common; use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; use rustc_codegen_ssa::traits::*; -use tracing::debug; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::{self, Instance, TypeVisitable}; +use rustc_middle::ty::{self, Instance, TypeVisitableExt}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -27,7 +27,7 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> debug!("get_fn(instance={:?})", instance); - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); assert!(!instance.substs.has_escaping_bound_vars()); if let Some(&llfn) = cx.instances.borrow().get(&instance) { @@ -49,8 +49,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> let llptrty = fn_abi.ptr_to_llvm_type(cx); // This is subtle and surprising, but sometimes we have to bitcast - // the resulting fn pointer. The reason has to do with external - // functions. If you have two crates that both bind the same C + // the resulting fn pointer. The reason has to do with external + // functions. If you have two crates that both bind the same C // library, they may not use precisely the same types: for // example, they will probably each declare their own structs, // which are distinct types from LLVM's point of view (nominal @@ -79,13 +79,31 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> llfn } } else { - let llfn = cx.declare_fn(sym, fn_abi); + let instance_def_id = instance.def_id(); + let llfn = if tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) + { + // Fix for https://github.com/rust-lang/rust/issues/104453 + // On x86 Windows, LLVM uses 'L' as the prefix for any private + // global symbols, so when we create an undecorated function symbol + // that begins with an 'L' LLVM misinterprets that as a private + // global symbol that it created and so fails the compilation at a + // later stage since such a symbol must have a definition. + // + // To avoid this, we set the Storage Class to "DllImport" so that + // LLVM will prefix the name with `__imp_`. Ideally, we'd like the + // existing logic below to set the Storage Class, but it has an + // exemption for MinGW for backwards compatability. + let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance)); + unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } + llfn + } else { + cx.declare_fn(sym, fn_abi, Some(instance)) + }; debug!("get_fn: not casting pointer!"); attributes::from_fn_attrs(cx, llfn, instance); - let instance_def_id = instance.def_id(); - // Apply an appropriate linkage/visibility value to our item that we // just declared. // @@ -174,7 +192,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> // MinGW: For backward compatibility we rely on the linker to decide whether it // should use dllimport for functions. if cx.use_dll_storage_attrs - && tcx.is_dllimport_foreign_item(instance_def_id) + && let Some(library) = tcx.native_library(instance_def_id) + && library.kind.is_dllimport() && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc") { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 77cbbf4c6ca..a3910fef954 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -8,15 +8,19 @@ use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_ast::Mutability; -use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; +use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; +use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::TyCtxt; +use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; +use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; +use rustc_target::spec::Target; use libc::{c_char, c_uint}; -use tracing::debug; +use std::fmt::Write; /* * A note on nomenclature of linking: "extern", "foreign", and "upcall". @@ -125,6 +129,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMGetUndef(t) } } + fn const_poison(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMGetPoison(t) } + } + fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { unsafe { llvm::LLVMConstInt(t, i as u64, True) } } @@ -211,7 +219,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn const_to_opt_uint(&self, v: &'ll Value) -> Option<u64> { - try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) }) + try_as_const_integral(v).and_then(|v| unsafe { + let mut i = 0u64; + let success = llvm::LLVMRustConstIntGetZExtValue(v, &mut i); + success.then_some(i) + }) } fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option<u128> { @@ -222,17 +234,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { }) } - fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { - self.const_undef(self.type_ix(0)) - } - fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { Scalar::Int(int) => { let data = int.assert_bits(layout.size(self)); let llval = self.const_uint_big(self.type_ix(bitsize), data); - if layout.primitive() == Pointer { + if matches!(layout.primitive(), Pointer(_)) { unsafe { llvm::LLVMConstIntToPtr(llval, llty) } } else { self.const_bitcast(llval, llty) @@ -248,8 +256,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), _ => self.static_addr_of(init, alloc.align, None), }; - if !self.sess().fewer_names() { - llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes()); + if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() { + let hash = self.tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + alloc.hash_stable(&mut hcx, &mut hasher); + hasher.finish::<Hash128>() + }); + llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes()); } (value, AddressSpace::DATA) } @@ -257,6 +270,15 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.get_fn_addr(fn_instance.polymorphize(self.tcx)), self.data_layout().instruction_address_space, ), + GlobalAlloc::VTable(ty, trait_ref) => { + let alloc = self + .tcx + .global_alloc(self.tcx.vtable_allocation((ty, trait_ref))) + .unwrap_memory(); + let init = const_alloc_to_llvm(self, alloc); + let value = self.static_addr_of(init, alloc.inner().align, None); + (value, AddressSpace::DATA) + } GlobalAlloc::Static(def_id) => { assert!(self.tcx.is_static(def_id)); assert!(!self.tcx.is_thread_local_static(def_id)); @@ -271,7 +293,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { 1, ) }; - if layout.primitive() != Pointer { + if !matches!(layout.primitive(), Pointer(_)) { unsafe { llvm::LLVMConstPtrToInt(llval, llty) } } else { self.const_bitcast(llval, llty) @@ -284,38 +306,24 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { const_alloc_to_llvm(self, alloc) } - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, &'ll Value> { - let alloc_align = alloc.inner().align; - assert_eq!(alloc_align, layout.align.abi); - let llty = self.type_ptr_to(layout.llvm_type(self)); - let llval = if layout.size == Size::ZERO { - let llval = self.const_usize(alloc_align.bytes()); - unsafe { llvm::LLVMConstIntToPtr(llval, llty) } - } else { - let init = const_alloc_to_llvm(self, alloc); - let base_addr = self.static_addr_of(init, alloc_align, None); - - let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( - self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p()), - &self.const_usize(offset.bytes()), - 1, - ) - }; - self.const_bitcast(llval, llty) - }; - PlaceRef::new_sized(llval, layout) - } - fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { consts::ptrcast(val, ty) } + + fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + self.const_bitcast(val, ty) + } + + fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { + unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p()), + &self.const_usize(offset.bytes()), + 1, + ) + } + } } /// Get the [LLVM type][Type] of a [`Value`]. @@ -348,3 +356,73 @@ fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { unsafe { llvm::LLVMIsAConstantInt(v) } } + +pub(crate) fn get_dllimport<'tcx>( + tcx: TyCtxt<'tcx>, + id: DefId, + name: &str, +) -> Option<&'tcx DllImport> { + tcx.native_library(id) + .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) +} + +pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { + target.vendor == "pc" && target.os == "windows" && target.env == "gnu" && target.abi.is_empty() +} + +pub(crate) fn i686_decorated_name( + dll_import: &DllImport, + mingw: bool, + disable_name_mangling: bool, +) -> String { + let name = dll_import.name.as_str(); + + let (add_prefix, add_suffix) = match dll_import.import_name_type { + Some(PeImportNameType::NoPrefix) => (false, true), + Some(PeImportNameType::Undecorated) => (false, false), + _ => (true, true), + }; + + // Worst case: +1 for disable name mangling, +1 for prefix, +4 for suffix (@@__). + let mut decorated_name = String::with_capacity(name.len() + 6); + + if disable_name_mangling { + // LLVM uses a binary 1 ('\x01') prefix to a name to indicate that mangling needs to be disabled. + decorated_name.push('\x01'); + } + + let prefix = if add_prefix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C | DllCallingConvention::Vectorcall(_) => None, + DllCallingConvention::Stdcall(_) => (!mingw + || dll_import.import_name_type == Some(PeImportNameType::Decorated)) + .then_some('_'), + DllCallingConvention::Fastcall(_) => Some('@'), + } + } else if !dll_import.is_fn && !mingw { + // For static variables, prefix with '_' on MSVC. + Some('_') + } else { + None + }; + if let Some(prefix) = prefix { + decorated_name.push(prefix); + } + + decorated_name.push_str(name); + + if add_suffix && dll_import.is_fn { + match dll_import.calling_convention { + DllCallingConvention::C => {} + DllCallingConvention::Stdcall(arg_list_size) + | DllCallingConvention::Fastcall(arg_list_size) => { + write!(&mut decorated_name, "@{}", arg_list_size).unwrap(); + } + DllCallingConvention::Vectorcall(arg_list_size) => { + write!(&mut decorated_name, "@@{}", arg_list_size).unwrap(); + } + } + } + + decorated_name +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 2b16ae1a88d..2087754c66b 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,47 +1,45 @@ use crate::base; -use crate::common::CodegenCx; +use crate::common::{self, CodegenCx}; use crate::debuginfo; +use crate::errors::{ + InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, +}; use crate::llvm::{self, True}; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use cstr::cstr; -use libc::c_uint; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ - read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer, + read_target_uint, Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar, }; use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, span_bug}; +use rustc_session::config::Lto; use rustc_target::abi::{ - AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, + Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, }; use std::ops::Range; -use tracing::debug; pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); let dl = cx.data_layout(); let pointer_size = dl.pointer_size.bytes() as usize; - // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, - // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` + // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( llvals: &mut Vec<&'ll Value>, cx: &'a CodegenCx<'ll, 'b>, alloc: &'a Allocation, range: Range<usize>, ) { - let chunks = alloc - .init_mask() - .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end)); + let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); let chunk_to_llval = move |chunk| match chunk { InitChunk::Init(range) => { @@ -59,13 +57,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< // to avoid the cost of generating large complex const expressions. // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. - let max = if llvm_util::get_version() < (14, 0, 0) { - // Generating partially-uninit consts inhibits optimizations in LLVM < 14. - // See https://github.com/rust-lang/rust/issues/84565. - 1 - } else { - cx.sess().opts.unstable_opts.uninit_const_chunk_threshold - }; + let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold; let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; if allow_uninit_chunks { @@ -79,12 +71,12 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } let mut next_offset = 0; - for &(offset, alloc_id) in alloc.relocations().iter() { + for &(offset, alloc_id) in alloc.provenance().ptrs().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a relocation, it + // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); @@ -93,16 +85,13 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< dl.endian, // This `inspect` is okay since it is within the bounds of the allocation, it doesn't // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the relocation as a relocation pointer offset. + // and we properly interpret the provenance as a relocation pointer offset. alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), ) .expect("const_alloc_to_llvm: could not read relocation pointer") as u64; - let address_space = match cx.tcx.global_alloc(alloc_id) { - GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, - GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) => AddressSpace::DATA, - }; + let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); llvals.push(cx.scalar_to_backend( InterpScalar::from_pointer( @@ -110,7 +99,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< &cx.tcx, ), Scalar::Initialized { - value: Primitive::Pointer, + value: Primitive::Pointer(address_space), valid_range: WrappingRange::full(dl.pointer_size), }, cx.type_i8p_ext(address_space), @@ -119,7 +108,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } if alloc.len() >= next_offset { let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all relocations, it is + // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); @@ -139,13 +128,18 @@ pub fn codegen_static_initializer<'ll, 'tcx>( fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { // The target may require greater alignment for globals than the type does. // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, - // which can force it to be smaller. Rust doesn't support this yet. + // which can force it to be smaller. Rust doesn't support this yet. if let Some(min) = cx.sess().target.min_global_align { match Align::from_bits(min) { Ok(min) => align = align.max(min), - Err(err) => { - cx.sess().err(&format!("invalid minimum global alignment: {}", err)); - } + Err(err) => match err { + AlignFromBytesError::NotPowerOfTwo(align) => { + cx.sess().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align }); + } + AlignFromBytesError::TooLarge(align) => { + cx.sess().emit_err(InvalidMinimumAlignmentTooLarge { align }); + } + }, } } unsafe { @@ -158,32 +152,19 @@ fn check_and_apply_linkage<'ll, 'tcx>( attrs: &CodegenFnAttrs, ty: Ty<'tcx>, sym: &str, - span_def_id: DefId, + def_id: DefId, ) -> &'ll Value { let llty = cx.layout_of(ty).llvm_type(cx); - if let Some(linkage) = attrs.linkage { + if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); - // If this is a static with a linkage specified, then we need to handle - // it a little specially. The typesystem prevents things like &T and - // extern "C" fn() from being non-null, so we can't just declare a - // static and call it a day. Some linkages (like weak) will make it such - // that the static actually has a null value. - let llty2 = if let ty::RawPtr(ref mt) = ty.kind() { - cx.layout_of(mt.ty).llvm_type(cx) - } else { - cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), - "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", - ) - }; unsafe { // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(sym, llty2); + let g1 = cx.declare_global(sym, cx.type_i8()); llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); // Declare an internal global `extern_with_linkage_foo` which - // is initialized with the address of `foo`. If `foo` is + // is initialized with the address of `foo`. If `foo` is // discarded during linking (for example, if `foo` has weak // linkage and there are no definitions), then // `extern_with_linkage_foo` will instead be initialized to @@ -191,15 +172,19 @@ fn check_and_apply_linkage<'ll, 'tcx>( let mut real_name = "_rust_extern_with_linkage_".to_string(); real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().span_fatal( - cx.tcx.def_span(span_def_id), - &format!("symbol `{}` is already defined", &sym), - ) + cx.sess().emit_fatal(SymbolAlreadyDefined { + span: cx.tcx.def_span(def_id), + symbol_name: sym, + }) }); llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); - llvm::LLVMSetInitializer(g2, g1); + llvm::LLVMSetInitializer(g2, cx.const_ptrcast(g1, llty)); g2 } + } else if cx.tcx.sess.target.arch == "x86" && + let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) + { + cx.declare_global(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), true), llty) } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global @@ -293,12 +278,23 @@ impl<'ll> CodegenCx<'ll, '_> { llvm::set_thread_local_mode(g, self.tls_model); } + let dso_local = unsafe { self.should_assume_dso_local(g, true) }; + if dso_local { + unsafe { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + if !def_id.is_local() { let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && + // Local definitions can never be imported, so we must not apply + // the DLLImport annotation. + !dso_local && // ThinLTO can't handle this workaround in all cases, so we don't // emit the attrs. Instead we make them unnecessary by disallowing // dynamic linking when linker plugin based LTO is enabled. - !self.tcx.sess.opts.cg.linker_plugin_lto.enabled(); + !self.tcx.sess.opts.cg.linker_plugin_lto.enabled() && + self.tcx.sess.lto() != Lto::Thin; // If this assertion triggers, there's something wrong with commandline // argument validation. @@ -327,19 +323,16 @@ impl<'ll> CodegenCx<'ll, '_> { } } - if self.use_dll_storage_attrs && self.tcx.is_dllimport_foreign_item(def_id) { + if self.use_dll_storage_attrs + && let Some(library) = self.tcx.native_library(def_id) + && library.kind.is_dllimport() + { // For foreign (native) libs we know the exact storage type to use. unsafe { llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); } } - unsafe { - if self.should_assume_dso_local(g, true) { - llvm::LLVMRustSetDSOLocal(g, true); - } - } - self.instances.borrow_mut().insert(instance, g); g } @@ -473,7 +466,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // // We could remove this hack whenever we decide to drop macOS 10.10 support. if self.tcx.sess.target.is_like_osx { - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state // (not as part of the interpreter execution). // @@ -481,16 +474,16 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // happens to be zero. Instead, we should only check the value of defined bytes // and set all undefined bytes to zero if this allocation is headed for the // BSS. - let all_bytes_are_zero = alloc.relocations().is_empty() + let all_bytes_are_zero = alloc.provenance().ptrs().is_empty() && alloc .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) .iter() .all(|&byte| byte == 0); let sect_name = if all_bytes_are_zero { - cstr!("__DATA,__thread_bss") + c"__DATA,__thread_bss" } else { - cstr!("__DATA,__thread_data") + c"__DATA,__thread_data" }; llvm::LLVMSetSection(g, sect_name.as_ptr()); } @@ -500,29 +493,27 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // go into custom sections of the wasm executable. if self.tcx.sess.target.is_like_wasm { if let Some(section) = attrs.link_section { - let section = llvm::LLVMMDStringInContext( + let section = llvm::LLVMMDStringInContext2( self.llcx, section.as_str().as_ptr().cast(), - section.as_str().len() as c_uint, + section.as_str().len(), ); - assert!(alloc.relocations().is_empty()); + assert!(alloc.provenance().ptrs().is_empty()); - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state (not // as part of the interpreter execution). let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); - let alloc = llvm::LLVMMDStringInContext( - self.llcx, - bytes.as_ptr().cast(), - bytes.len() as c_uint, - ); + let alloc = + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); let data = [section, alloc]; - let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); + let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); + let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, - "wasm.custom_sections\0".as_ptr().cast(), - meta, + c"wasm.custom_sections".as_ptr().cast(), + val, ); } } else { @@ -535,10 +526,20 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // The semantics of #[used] in Rust only require the symbol to make it into the // object file. It is explicitly allowed for the linker to strip the symbol if it - // is dead. As such, use llvm.compiler.used instead of llvm.used. + // is dead, which means we are allowed to use `llvm.compiler.used` instead of + // `llvm.used` here. + // // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs - // in some versions of the gold linker. + // in the handling of `.init_array` (the static constructor list) in versions of + // the gold linker (prior to the one released with binutils 2.36). + // + // That said, we only ever emit these when compiling for ELF targets, unless + // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage + // on other targets, in particular MachO targets have *their* static constructor + // lists broken if `llvm.compiler.used` is emitted rather than `llvm.used`. However, + // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_hir_analysis`, + // so we don't need to take care of it here. self.add_compiler_used_global(g); } if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 5857b83f6c9..e8a7afcc632 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use crate::llvm_util; use crate::type_::Type; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; @@ -26,6 +25,7 @@ use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; +use rustc_span::source_map::Spanned; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; @@ -142,27 +142,16 @@ pub unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (13, 0, 0) { - if sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-S128", ""); - } - if sess.target.arch == "wasm32" { - target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); - } - if sess.target.arch == "wasm64" { - target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); - } - } - if llvm_version < (14, 0, 0) { - if sess.target.llvm_target == "i686-pc-windows-msvc" - || sess.target.llvm_target == "i586-pc-windows-msvc" - { - target_data_layout = - "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" - .to_string(); - } - if sess.target.arch == "wasm32" { - target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); + if llvm_version < (16, 0, 0) { + if sess.target.arch == "s390x" { + // LLVM 16 data layout changed to always set 64-bit vector alignment, + // which is conditional in earlier LLVM versions. + // https://reviews.llvm.org/D131158 for the discussion. + target_data_layout = target_data_layout.replace("-v128:64", ""); + } else if sess.target.arch == "riscv64" { + // LLVM 16 introduced this change so as to produce more efficient code. + // See https://reviews.llvm.org/D116735 for the discussion. + target_data_layout = target_data_layout.replace("-n32:64-", "-n64-"); } } @@ -192,7 +181,7 @@ pub unsafe fn create_module<'ll>( // // FIXME(#34960) let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = cfg_llvm_root.trim() != ""; + let custom_llvm_used = !cfg_llvm_root.trim().is_empty(); if !custom_llvm_used && target_data_layout != llvm_data_layout { bug!( @@ -234,18 +223,40 @@ pub unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); - llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + c"RtLibUseGOT".as_ptr().cast(), + 1, + ); } - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. - let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. + // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) + if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Warning, - canonical_jump_tables, + llvm::LLVMModFlagBehavior::Override, + c"CFI Canonical Jump Tables".as_ptr().cast(), + 1, + ); + } + + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) + if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"EnableSplitLTOUnit".as_ptr().cast(), + 1, + ); + } + + // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) + if sess.is_sanitizer_kcfi_enabled() { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"kcfi".as_ptr().cast(), 1, ); } @@ -259,7 +270,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 1, ) } @@ -268,7 +279,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 2, ) } @@ -276,34 +287,43 @@ pub unsafe fn create_module<'ll>( } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { - if sess.target.arch != "aarch64" { - sess.err("-Zbranch-protection is only supported on aarch64"); + let behavior = if llvm_version >= (15, 0, 0) { + llvm::LLVMModFlagBehavior::Min } else { + llvm::LLVMModFlagBehavior::Error + }; + + if sess.target.arch == "aarch64" { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "branch-target-enforcement\0".as_ptr().cast(), + behavior, + c"branch-target-enforcement".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address\0".as_ptr().cast(), + behavior, + c"sign-return-address".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address-all\0".as_ptr().cast(), + behavior, + c"sign-return-address-all".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address-with-bkey\0".as_ptr().cast(), + behavior, + c"sign-return-address-with-bkey".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); + } else { + bug!( + "branch-protection used on non-AArch64 target; \ + this should be checked in rustc_session." + ); } } @@ -312,7 +332,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-branch\0".as_ptr().cast(), + c"cf-protection-branch".as_ptr().cast(), 1, ) } @@ -320,7 +340,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-return\0".as_ptr().cast(), + c"cf-protection-return".as_ptr().cast(), 1, ) } @@ -329,7 +349,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Error, - "Virtual Function Elim\0".as_ptr().cast(), + c"Virtual Function Elim".as_ptr().cast(), 1, ); } @@ -403,12 +423,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); - let coverage_cx = if tcx.sess.instrument_coverage() { - let covctx = coverageinfo::CrateCoverageContext::new(); - Some(covctx) - } else { - None - }; + let coverage_cx = + tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new); let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); @@ -464,15 +480,14 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { self.coverage_cx.as_ref() } - fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { - let section = cstr!("llvm.metadata"); + pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let array = self.const_array(self.type_ptr_to(self.type_i8()), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, section.as_ptr()); + llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); } } } @@ -520,14 +535,9 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let llfn = match tcx.lang_items().eh_personality() { Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( - ty::Instance::resolve( - tcx, - ty::ParamEnv::reveal_all(), - def_id, - tcx.intern_substs(&[]), - ) - .unwrap() - .unwrap(), + ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty()) + .unwrap() + .unwrap(), ), _ => { let name = if wants_msvc_seh(self.sess()) { @@ -562,14 +572,6 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.codegen_unit } - fn used_statics(&self) -> &RefCell<Vec<&'ll Value>> { - &self.used_statics - } - - fn compiler_used_statics(&self) -> &RefCell<Vec<&'ll Value>> { - &self.compiler_used_statics - } - fn set_frame_pointer_type(&self, llfn: &'ll Value) { if let Some(attr) = attributes::frame_pointer_type_attr(self) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); @@ -583,20 +585,15 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } - fn create_used_variable(&self) { - self.create_used_variable_impl(cstr!("llvm.used"), &*self.used_statics.borrow()); - } - - fn create_compiler_used_variable(&self) { - self.create_used_variable_impl( - cstr!("llvm.compiler.used"), - &*self.compiler_used_statics.borrow(), - ); - } - fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { - if self.get_declared_value("main").is_none() { - Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) + let entry_name = self.sess().target.entry_name.as_ref(); + if self.get_declared_value(entry_name).is_none() { + Some(self.declare_entry_fn( + entry_name, + self.sess().target.entry_abi.into(), + llvm::UnnamedAddr::Global, + fn_type, + )) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} @@ -753,9 +750,13 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.round.f32", fn(t_f32) -> t_f32); ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32); + ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64); + ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); @@ -897,6 +898,9 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } + + ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); + None } @@ -969,9 +973,9 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { if let LayoutError::SizeOverflow(_) = err { - self.sess().span_fatal(span, &err.to_string()) + self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { - span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + span_bug!(span, "failed to get layout for `{ty}`: {err:?}") } } } @@ -987,25 +991,16 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.sess().span_fatal(span, &err.to_string()) + self.sess().emit_fatal(Spanned { span, node: err }) } else { match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { - span_bug!( - span, - "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", - sig, - extra_args, - err - ); + span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } FnAbiRequest::OfInstance { instance, extra_args } => { span_bug!( span, - "`fn_abi_of_instance({}, {:?})` failed: {}", - instance, - extra_args, - err + "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 58f391692c4..21a1ac34844 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -7,7 +7,7 @@ use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; use rustc_data_structures::fx::FxIndexSet; use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefIdSet; +use rustc_hir::def_id::DefId; use rustc_llvm::RustString; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -16,12 +16,10 @@ use rustc_middle::ty::TyCtxt; use std::ffi::CString; -use tracing::debug; - /// Generates and exports the Coverage Map. /// -/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions -/// 5 (LLVM 12, only) and 6 (zero-based encoded as 4 and 5, respectively), as defined at +/// Rust Coverage Map generation supports LLVM Coverage Mapping Format version +/// 6 (zero-based encoded as 5), as defined at /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). /// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) /// bundled with Rust's fork of LLVM. @@ -31,16 +29,13 @@ use tracing::debug; /// implementing this Rust version, and though the format documentation is very explicit and /// detailed, some undocumented details in Clang's implementation (that may or may not be important) /// were also replicated for Rust's Coverage Map. -pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { +pub fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; - // Ensure the installed version of LLVM supports at least Coverage Map - // Version 5 (encoded as a zero-based value: 4), which was introduced with - // LLVM 12. + // Ensure the installed version of LLVM supports Coverage Map Version 6 + // (encoded as a zero-based value: 5), which was introduced with LLVM 13. let version = coverageinfo::mapping_version(); - if version < 4 { - tcx.sess.fatal("rustc option `-C instrument-coverage` requires LLVM 12 or higher."); - } + assert_eq!(version, 5, "The `CoverageMappingVersion` exposed by `llvm-wrapper` is out of sync"); debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); @@ -62,7 +57,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { return; } - let mut mapgen = CoverageMapGenerator::new(tcx, version); + let mut mapgen = CoverageMapGenerator::new(tcx); // Encode coverage mappings and generate function records let mut function_data = Vec::new(); @@ -125,25 +120,18 @@ struct CoverageMapGenerator { } impl CoverageMapGenerator { - fn new(tcx: TyCtxt<'_>, version: u32) -> Self { + fn new(tcx: TyCtxt<'_>) -> Self { let mut filenames = FxIndexSet::default(); - if version >= 5 { - // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) - // requires setting the first filename to the compilation directory. - // Since rustc generates coverage maps with relative paths, the - // compilation directory can be combined with the the relative paths - // to get absolute paths, if needed. - let working_dir = tcx - .sess - .opts - .working_dir - .remapped_path_if_available() - .to_string_lossy() - .to_string(); - let c_filename = - CString::new(working_dir).expect("null error converting filename to C string"); - filenames.insert(c_filename); - } + // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) + // requires setting the first filename to the compilation directory. + // Since rustc generates coverage maps with relative paths, the + // compilation directory can be combined with the relative paths + // to get absolute paths, if needed. + let working_dir = + tcx.sess.opts.working_dir.remapped_path_if_available().to_string_lossy().to_string(); + let c_filename = + CString::new(working_dir).expect("null error converting filename to C string"); + filenames.insert(c_filename); Self { filenames } } @@ -175,7 +163,7 @@ impl CoverageMapGenerator { counter_regions.sort_unstable_by_key(|(_counter, region)| *region); for (counter, region) in counter_regions { let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; - let same_file = current_file_name.as_ref().map_or(false, |p| *p == file_name); + let same_file = current_file_name.is_some_and(|p| p == file_name); if !same_file { if current_file_name.is_some() { current_file_id += 1; @@ -285,14 +273,14 @@ fn save_function_record( /// "code coverage dead code cgu" during the partitioning process. This prevents us from generating /// code regions for the same function more than once which can lead to linker errors regarding /// duplicate symbols. -fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { +fn add_unused_functions(cx: &CodegenCx<'_, '_>) { assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); let tcx = cx.tcx; let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let eligible_def_ids: DefIdSet = tcx + let eligible_def_ids: Vec<DefId> = tcx .mir_keys(()) .iter() .filter_map(|local_def_id| { @@ -307,9 +295,8 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator ) { return None; - } else if ignore_unused_generics - && tcx.generics_of(def_id).requires_monomorphization(tcx) - { + } + if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { return None; } Some(local_def_id.to_def_id()) @@ -318,7 +305,9 @@ fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { let codegenned_def_ids = tcx.codegened_and_inlined_items(()); - for &non_codegenned_def_id in eligible_def_ids.difference(codegenned_def_ids) { + for non_codegenned_def_id in + eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) + { let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); // If a function is marked `#[no_coverage]`, then skip generating a diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 98ba38356a4..cd261293e9b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -27,9 +27,6 @@ use rustc_middle::ty::Instance; use std::cell::RefCell; use std::ffi::CString; -use std::iter; -use tracing::debug; - pub mod mapgen; const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; @@ -38,7 +35,7 @@ const VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'ll, 'tcx> { - // Coverage data for each instrumented function identified by DefId. + /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -202,7 +199,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< tcx.symbol_name(instance).name, cx.fn_abi_of_fn_ptr( ty::Binder::dummy(tcx.mk_fn_sig( - iter::once(tcx.mk_unit()), + [tcx.mk_unit()], tcx.mk_unit(), false, hir::Unsafety::Unsafe, @@ -210,6 +207,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< )), ty::List::empty(), ), + None, ); llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 99e4ded62f1..64961baf272 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; +use rustc_index::Idx; /// Produces DIScope DIEs for each MIR Scope which has variables defined in it. // FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. @@ -93,7 +93,7 @@ fn make_mir_scope<'ll, 'tcx>( let callee = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - callee, + ty::EarlyBinder::bind(callee), ); let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); cx.dbg_scope_fn(callee, callee_fn_abi, None) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 5186aee57fb..8be54b7eb71 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -5,14 +5,13 @@ use crate::llvm; use crate::builder::Builder; use crate::common::CodegenCx; use crate::value::Value; +use rustc_ast::attr; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; +use rustc_middle::{bug, middle::debugger_visualizer::DebuggerVisualizerType}; use rustc_session::config::{CrateType, DebugInfo}; - use rustc_span::symbol::sym; -use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -22,9 +21,9 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); // Load just the first byte as that's all that's necessary to force // LLVM to keep around the reference to the global. - let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); + let volatile_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); unsafe { - llvm::LLVMSetAlignment(volative_load_instruction, 1); + llvm::LLVMSetAlignment(volatile_load_instruction, 1); } } } @@ -39,7 +38,6 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; section_var.unwrap_or_else(|| { - let section_name = b".debug_gdb_scripts\0"; let mut section_contents = Vec::new(); // Add the pretty printers for the standard library first. @@ -55,7 +53,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' // The initial byte `4` instructs GDB that the following pretty printer // is defined inline as opposed to in a standalone file. section_contents.extend_from_slice(b"\x04"); - let vis_name = format!("pretty-printer-{}-{}\n", crate_name.as_str(), index); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); section_contents.extend_from_slice(vis_name.as_bytes()); section_contents.extend_from_slice(&visualizer.src); @@ -72,7 +70,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); + llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr().cast()); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); @@ -87,7 +85,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = - cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index f8bd2d234f3..166454d3ae7 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -20,18 +20,14 @@ use crate::llvm::debuginfo::{ }; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; -use rustc_middle::mir::{self, GeneratorLayout}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; @@ -42,7 +38,6 @@ use rustc_span::{self, FileNameDisplayPreference, SourceFile}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; -use tracing::debug; use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; @@ -51,7 +46,6 @@ use std::hash::{Hash, Hasher}; use std::iter; use std::path::{Path, PathBuf}; use std::ptr; -use tracing::instrument; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -114,7 +108,8 @@ macro_rules! return_if_di_node_created_in_meantime { } /// Extract size and alignment from a TyAndLayout. -fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { +#[inline] +fn size_and_align_of(ty_and_layout: TyAndLayout<'_>) -> (Size, Align) { (ty_and_layout.size, ty_and_layout.align.abi) } @@ -135,7 +130,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( let (size, align) = cx.size_and_align_of(array_type); - let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; + let upper_bound = len.eval_target_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; let subrange = unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; @@ -179,15 +174,14 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( return_if_di_node_created_in_meantime!(cx, unique_type_id); - let (thin_pointer_size, thin_pointer_align) = - cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); + let data_layout = &cx.tcx.data_layout; let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); match fat_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. debug_assert_eq!( - (thin_pointer_size, thin_pointer_align), + (data_layout.pointer_size, data_layout.pointer_align.abi), cx.size_and_align_of(ptr_type), "ptr_type={}, pointee_type={}", ptr_type, @@ -198,8 +192,8 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), pointee_type_di_node, - thin_pointer_size.bits(), - thin_pointer_align.bits() as u32, + data_layout.pointer_size.bits(), + data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. ptr_type_debuginfo_name.as_ptr().cast(), ptr_type_debuginfo_name.len(), @@ -785,10 +779,10 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( codegen_unit_name: &str, debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, ) -> &'ll DIDescriptor { - let mut name_in_debuginfo = match tcx.sess.local_crate_source_file { - Some(ref path) => path.clone(), - None => PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str()), - }; + let mut name_in_debuginfo = tcx + .sess + .local_crate_source_file() + .unwrap_or_else(|| PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str())); // To avoid breaking split DWARF, we need to ensure that each codegen unit // has a unique `DW_AT_name`. This is because there's a remote chance that @@ -811,14 +805,12 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( name_in_debuginfo.push(codegen_unit_name); debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); - let rustc_producer = - format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); + let rustc_producer = format!("rustc version {}", tcx.sess.cfg_version); // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. let producer = format!("clang LLVM ({})", rustc_producer); let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); - let flags = "\0"; let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { output_filenames @@ -834,24 +826,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( } .unwrap_or_default(); let split_name = split_name.to_str().unwrap(); - - // FIXME(#60020): - // - // This should actually be - // - // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); - // - // That is, we should set LLVM's emission kind to `LineTablesOnly` if - // we are compiling with "limited" debuginfo. However, some of the - // existing tools relied on slightly more debuginfo being generated than - // would be the case with `LineTablesOnly`, and we did not want to break - // these tools in a "drive-by fix", without a good idea or plan about - // what limited debuginfo should exactly look like. So for now we keep - // the emission kind as `FullDebug`. - // - // See https://github.com/rust-lang/rust/issues/60020 for details. - let kind = DebugEmissionKind::FullDebug; - assert!(tcx.sess.opts.debuginfo != DebugInfo::None); + let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( @@ -872,7 +847,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( producer.as_ptr().cast(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, - flags.as_ptr().cast(), + c"".as_ptr().cast(), 0, // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final @@ -885,8 +860,6 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); if tcx.sess.opts.unstable_opts.profile { - let cu_desc_metadata = - llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); @@ -894,20 +867,16 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), path_to_mdstring(debug_context.llcontext, gcda_path), - cu_desc_metadata, + unit_metadata, ]; - let gcov_metadata = llvm::LLVMMDNodeInContext( + let gcov_metadata = llvm::LLVMMDNodeInContext2( debug_context.llcontext, gcov_cu_info.as_ptr(), - gcov_cu_info.len() as c_uint, + gcov_cu_info.len(), ); + let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); - let llvm_gcov_ident = cstr!("llvm.gcov"); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - llvm_gcov_ident.as_ptr(), - gcov_metadata, - ); + llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val); } // Insert `llvm.ident` metadata on the wasm targets since that will @@ -920,7 +889,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); llvm::LLVMAddNamedMetadataOperand( debug_context.llmod, - cstr!("llvm.ident").as_ptr(), + c"llvm.ident".as_ptr(), llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), ); } @@ -928,15 +897,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( return unit_metadata; }; - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { + fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata { let path_str = path_to_c_string(path); - unsafe { - llvm::LLVMMDStringInContext( - llcx, - path_str.as_ptr(), - path_str.as_bytes().len() as c_uint, - ) - } + unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) } } } @@ -999,7 +962,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( .iter() .enumerate() .map(|(i, f)| { - let field_name = if variant_def.ctor_kind == CtorKind::Fn { + let field_name = if variant_def.ctor_kind() == Some(CtorKind::Fn) { // This is a tuple struct tuple_field_name(i) } else { @@ -1027,33 +990,6 @@ fn build_struct_type_di_node<'ll, 'tcx>( // Tuples //=----------------------------------------------------------------------------- -/// Returns names of captured upvars for closures and generators. -/// -/// Here are some examples: -/// - `name__field1__field2` when the upvar is captured by value. -/// - `_ref__name__field` when the upvar is captured by reference. -/// -/// For generators this only contains upvars that are shared by all states. -fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec<String> { - let body = tcx.optimized_mir(def_id); - - body.var_debug_info - .iter() - .filter_map(|var| { - let is_ref = match var.value { - mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => { - // The projection is either `[.., Field, Deref]` or `[.., Field]`. It - // implies whether the variable is captured by value or by reference. - matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) - } - _ => return None, - }; - let prefix = if is_ref { "_ref__" } else { "" }; - Some(prefix.to_owned() + var.name.as_str()) - }) - .collect() -} - /// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. /// For a generator, this will handle upvars shared by all states. fn build_upvar_field_di_nodes<'ll, 'tcx>( @@ -1084,7 +1020,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) ); - let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); + let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); let layout = cx.layout_of(closure_or_generator_ty); up_var_tys @@ -1230,43 +1166,6 @@ fn build_union_type_di_node<'ll, 'tcx>( ) } -// FIXME(eddyb) maybe precompute this? Right now it's computed once -// per generator monomorphization, but it doesn't depend on substs. -fn generator_layout_and_saved_local_names<'tcx>( - tcx: TyCtxt<'tcx>, - def_id: DefId, -) -> (&'tcx GeneratorLayout<'tcx>, IndexVec<mir::GeneratorSavedLocal, Option<Symbol>>) { - let body = tcx.optimized_mir(def_id); - let generator_layout = body.generator_layout().unwrap(); - let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); - - let state_arg = mir::Local::new(1); - for var in &body.var_debug_info { - let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; - if place.local != state_arg { - continue; - } - match place.projection[..] { - [ - // Deref of the `Pin<&mut Self>` state argument. - mir::ProjectionElem::Field(..), - mir::ProjectionElem::Deref, - // Field of a variant of the state. - mir::ProjectionElem::Downcast(_, variant), - mir::ProjectionElem::Field(field, _), - ] => { - let name = &mut generator_saved_local_names - [generator_layout.variant_fields[variant][field]]; - if name.is_none() { - name.replace(var.name); - } - } - _ => {} - } - } - (generator_layout, generator_saved_local_names) -} - /// Computes the type parameters for a type, if any, for the given metadata. fn build_generic_type_param_di_nodes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, @@ -1278,12 +1177,12 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( let names = get_parameter_names(cx, generics); let template_params: SmallVec<_> = iter::zip(substs, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_di_node = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -1291,10 +1190,8 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( name.len(), actual_type_di_node, ) - }) - } else { - None - } + } + }) }) .collect(); @@ -1420,7 +1317,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( cx, type_map::stub( cx, - Stub::VtableTy { vtable_holder }, + Stub::VTableTy { vtable_holder }, unique_type_id, &vtable_type_name, (size, pointer_align), @@ -1499,24 +1396,18 @@ fn vcall_visibility_metadata<'ll, 'tcx>( // If there is not LTO and the visibility in public, we have to assume that the vtable can // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. (Lto::No | Lto::ThinLocal, Visibility::Public, _) - | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { - VCallVisibility::Public - } + | (Lto::No, Visibility::Restricted(_), false) => VCallVisibility::Public, // With LTO and a quasi-public visibility, the usages of the functions of the vtable are // all known by the `LinkageUnit`. // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. (Lto::Fat | Lto::Thin, Visibility::Public, _) - | ( - Lto::ThinLocal | Lto::Thin | Lto::Fat, - Visibility::Restricted(_) | Visibility::Invisible, - false, - ) => VCallVisibility::LinkageUnit, + | (Lto::ThinLocal | Lto::Thin | Lto::Fat, Visibility::Restricted(_), false) => { + VCallVisibility::LinkageUnit + } // If there is only one CGU, private vtables can only be seen by that CGU/translation unit // and therefore we know of all usages of functions in the vtable. - (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { - VCallVisibility::TranslationUnit - } + (_, Visibility::Restricted(_), true) => VCallVisibility::TranslationUnit, }; let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); @@ -1572,6 +1463,11 @@ pub fn create_vtable_di_node<'ll, 'tcx>( return; } + // When full debuginfo is enabled, we want to try and prevent vtables from being + // merged. Otherwise debuggers will have a hard time mapping from dyn pointer + // to concrete type. + llvm::SetUnnamedAddress(vtable, llvm::UnnamedAddr::No); + let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index d6e2c8ccdf4..ecb0912d328 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -1,29 +1,31 @@ use std::borrow::Cow; use libc::c_uint; -use rustc_codegen_ssa::debuginfo::{ - type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +use rustc_codegen_ssa::{ + debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, + traits::ConstMethods, }; + +use rustc_index::IndexVec; use rustc_middle::{ bug, ty::{ self, layout::{LayoutOf, TyAndLayout}, - util::Discr, - AdtDef, GeneratorSubsts, + AdtDef, GeneratorSubsts, Ty, }, }; -use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::{ common::CodegenCx, debuginfo::{ metadata::{ - build_field_di_node, closure_saved_names_of_captured_variables, - enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, - type_map::{self, UniqueTypeId}, + build_field_di_node, + enums::{tag_base_type, DiscrResult}, + file_metadata, size_and_align_of, type_di_node, + type_map::{self, Stub, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER, }, @@ -35,59 +37,161 @@ use crate::{ }, }; -/// In CPP-like mode, we generate a union of structs for each variant and an -/// explicit discriminant field roughly equivalent to the following C/C++ code: +// The names of the associated constants in each variant wrapper struct. +// These have to match up with the names being used in `intrinsic.natvis`. +const ASSOC_CONST_DISCR_NAME: &str = "NAME"; +const ASSOC_CONST_DISCR_EXACT: &str = "DISCR_EXACT"; +const ASSOC_CONST_DISCR_BEGIN: &str = "DISCR_BEGIN"; +const ASSOC_CONST_DISCR_END: &str = "DISCR_END"; + +const ASSOC_CONST_DISCR128_EXACT_LO: &str = "DISCR128_EXACT_LO"; +const ASSOC_CONST_DISCR128_EXACT_HI: &str = "DISCR128_EXACT_HI"; +const ASSOC_CONST_DISCR128_BEGIN_LO: &str = "DISCR128_BEGIN_LO"; +const ASSOC_CONST_DISCR128_BEGIN_HI: &str = "DISCR128_BEGIN_HI"; +const ASSOC_CONST_DISCR128_END_LO: &str = "DISCR128_END_LO"; +const ASSOC_CONST_DISCR128_END_HI: &str = "DISCR128_END_HI"; + +// The name of the tag field in the top-level union +const TAG_FIELD_NAME: &str = "tag"; +const TAG_FIELD_NAME_128_LO: &str = "tag128_lo"; +const TAG_FIELD_NAME_128_HI: &str = "tag128_hi"; + +// We assign a "virtual" discriminant value to the sole variant of +// a single-variant enum. +const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; + +/// In CPP-like mode, we generate a union with a field for each variant and an +/// explicit tag field. The field of each variant has a struct type +/// that encodes the discriminant of the variant and it's data layout. +/// The union also has a nested enumeration type that is only used for encoding +/// variant names in an efficient way. Its enumerator values do _not_ correspond +/// to the enum's discriminant values. +/// It's roughly equivalent to the following C/C++ code: /// /// ```c -/// union enum$<{fully-qualified-name}> { -/// struct {variant 0 name} { -/// <variant 0 fields> +/// union enum2$<{fully-qualified-name}> { +/// struct Variant0 { +/// struct {name-of-variant-0} { +/// <variant 0 fields> +/// } value; +/// +/// static VariantNames NAME = {name-of-variant-0}; +/// static int_type DISCR_EXACT = {discriminant-of-variant-0}; /// } variant0; +/// /// <other variant structs> -/// {name} discriminant; +/// +/// int_type tag; +/// +/// enum VariantNames { +/// <name-of-variant-0> = 0, // The numeric values are variant index, +/// <name-of-variant-1> = 1, // not discriminant values. +/// <name-of-variant-2> = 2, +/// ... +/// } /// } /// ``` /// -/// As you can see, the type name is wrapped `enum$`. This way we can have a -/// single NatVis rule for handling all enums. +/// As you can see, the type name is wrapped in `enum2$<_>`. This way we can +/// have a single NatVis rule for handling all enums. The `2` in `enum2$<_>` +/// is an encoding version tag, so that debuggers can decide to decode this +/// differently than the previous `enum$<_>` encoding emitted by earlier +/// compiler versions. /// -/// At the LLVM IR level this looks like +/// Niche-tag enums have one special variant, usually called the +/// "untagged variant". This variant has a field that +/// doubles as the tag of the enum. The variant is active when the value of +/// that field is within a pre-defined range. Therefore the variant struct +/// has a `DISCR_BEGIN` and `DISCR_END` field instead of `DISCR_EXACT` in +/// that case. Both `DISCR_BEGIN` and `DISCR_END` are inclusive bounds. +/// Note that these ranges can wrap around, so that `DISCR_END < DISCR_BEGIN`. /// -/// ```txt -/// DW_TAG_union_type (top-level type for enum) -/// DW_TAG_member (member for variant 1) -/// DW_TAG_member (member for variant 2) -/// DW_TAG_member (member for variant 3) -/// DW_TAG_structure_type (type of variant 1) -/// DW_TAG_structure_type (type of variant 2) -/// DW_TAG_structure_type (type of variant 3) -/// DW_TAG_enumeration_type (type of tag) -/// ``` +/// Single-variant enums don't actually have a tag field. In this case we +/// emit a static tag field (that always has the value 0) so we can use the +/// same representation (and NatVis). /// -/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things -/// differently in order to allow a NatVis visualizer to extract all the information needed: -/// We generate a union of two fields, one for the dataful variant -/// and one that just points to the discriminant (which is some field within the dataful variant). -/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful -/// variants and make the discriminant field that type. We then use NatVis to render the enum type -/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: +/// For niche-layout enums it's possible to have a 128-bit tag. NatVis, VS, and +/// WinDbg (the main targets for CPP-like debuginfo at the moment) don't support +/// 128-bit integers, so all values involved get split into two 64-bit fields. +/// Instead of the `tag` field, we generate two fields `tag128_lo` and `tag128_hi`, +/// Instead of `DISCR_EXACT`, we generate `DISCR128_EXACT_LO` and `DISCR128_EXACT_HI`, +/// and so on. /// -/// ```c -/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { -/// struct <dataful variant name> { -/// <fields in dataful variant> -/// } dataful_variant; -/// enum Discriminant$ { -/// <non-dataful variants> -/// } discriminant; +/// +/// The following pseudocode shows how to decode an enum value in a debugger: +/// +/// ```text +/// +/// fn find_active_variant(enum_value) -> (VariantName, VariantValue) { +/// let is_128_bit = enum_value.has_field("tag128_lo"); +/// +/// if !is_128_bit { +/// // Note: `tag` can be a static field for enums with only one +/// // inhabited variant. +/// let tag = enum_value.field("tag").value; +/// +/// // For each variant, check if it is a match. Only one of them will match, +/// // so if we find it we can return it immediately. +/// for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) { +/// if variant_field.has_field("DISCR_EXACT") { +/// // This variant corresponds to a single tag value +/// if variant_field.field("DISCR_EXACT").value == tag { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } else { +/// // This is a range variant +/// let begin = variant_field.field("DISCR_BEGIN"); +/// let end = variant_field.field("DISCR_END"); +/// +/// if is_in_range(tag, begin, end) { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } +/// } +/// } else { +/// // Basically the same as with smaller tags, we just have to +/// // stitch the values together. +/// let tag: u128 = (enum_value.field("tag128_lo").value as u128) | +/// (enum_value.field("tag128_hi").value as u128 << 64); +/// +/// for variant_field in enum_value.fields().filter(|f| f.name.starts_with("variant")) { +/// if variant_field.has_field("DISCR128_EXACT_LO") { +/// let discr_exact = (variant_field.field("DISCR128_EXACT_LO" as u128) | +/// (variant_field.field("DISCR128_EXACT_HI") as u128 << 64); +/// +/// // This variant corresponds to a single tag value +/// if discr_exact.value == tag { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } else { +/// // This is a range variant +/// let begin = (variant_field.field("DISCR128_BEGIN_LO").value as u128) | +/// (variant_field.field("DISCR128_BEGIN_HI").value as u128 << 64); +/// let end = (variant_field.field("DISCR128_END_LO").value as u128) | +/// (variant_field.field("DISCR128_END_HI").value as u128 << 64); +/// +/// if is_in_range(tag, begin, end) { +/// return (variant_field.field("NAME"), variant_field.value); +/// } +/// } +/// } +/// } +/// +/// // We should have found an active variant at this point. +/// unreachable!(); /// } -/// ``` /// -/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` -/// and evaluates `this.discriminant`. If the value is between the min niche and max -/// niche, then the enum is in the dataful variant and `this.dataful_variant` is -/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that -/// case, we just need to render the name of the `this.discriminant` enum. +/// // Check if a value is within the given range +/// // (where the range might wrap around the value space) +/// fn is_in_range(value, start, end) -> bool { +/// if start < end { +/// value >= start && value <= end +/// } else { +/// value >= start || value <= end +/// } +/// } +/// +/// ``` pub(super) fn build_enum_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, @@ -135,27 +239,28 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ref variants, tag_field, .. - } => build_union_fields_for_direct_tag_enum( + } => build_union_fields_for_enum( cx, enum_adt_def, enum_type_and_layout, enum_type_di_node, - &mut variants.indices(), + variants.indices(), tag_field, + None, ), Variants::Multiple { - tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + tag_encoding: TagEncoding::Niche { untagged_variant, .. }, ref variants, tag_field, .. - } => build_union_fields_for_niche_tag_enum( + } => build_union_fields_for_enum( cx, enum_adt_def, enum_type_and_layout, enum_type_di_node, - dataful_variant, - &mut variants.indices(), + variants.indices(), tag_field, + Some(untagged_variant), ), } }, @@ -217,137 +322,344 @@ fn build_single_variant_union_fields<'ll, 'tcx>( let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( cx, - enum_type_and_layout.ty, + enum_type_and_layout, enum_type_di_node, variant_index, enum_adt_def.variant(variant_index), variant_layout, ); - // NOTE: The field name of the union is the same as the variant name, not "variant0". - let variant_name = enum_adt_def.variant(variant_index).name.as_str(); + let tag_base_type = cx.tcx.types.u32; + let tag_base_type_di_node = type_di_node(cx, tag_base_type); + let tag_base_type_align = cx.align_of(tag_base_type); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + enum_type_di_node, + std::iter::once(( + variant_index, + Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + )), + ); - smallvec![build_field_di_node( + let variant_struct_type_wrapper_di_node = build_variant_struct_wrapper_type_di_node( cx, + enum_type_and_layout, enum_type_di_node, - variant_name, - // NOTE: We use the size and align of the entire type, not from variant_layout - // since the later is sometimes smaller (if it has fewer fields). - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, + variant_index, + None, variant_struct_type_di_node, - )] + variant_names_type_di_node, + tag_base_type_di_node, + tag_base_type, + DiscrResult::NoDiscriminant, + ); + + smallvec![ + build_field_di_node( + cx, + enum_type_di_node, + &variant_union_field_name(variant_index), + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_wrapper_di_node, + ), + unsafe { + llvm::LLVMRustDIBuilderCreateStaticMemberType( + DIB(cx), + enum_type_di_node, + TAG_FIELD_NAME.as_ptr().cast(), + TAG_FIELD_NAME.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + DIFlags::FlagZero, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align.bits() as u32, + ) + } + ] } -fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( +fn build_union_fields_for_enum<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_adt_def: AdtDef<'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, - variant_indices: &mut dyn Iterator<Item = VariantIdx>, + variant_indices: impl Iterator<Item = VariantIdx> + Clone, tag_field: usize, + untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + enum_type_di_node, + variant_indices.clone().map(|variant_index| { + let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + (variant_index, variant_name) + }), + ); + let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices .map(|variant_index| { let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let variant_def = enum_adt_def.variant(variant_index); + + let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_index, + variant_def, + variant_layout, + ); + VariantFieldInfo { variant_index, - variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( - cx, - enum_type_and_layout.ty, - enum_type_di_node, - variant_index, - enum_adt_def.variant(variant_index), - variant_layout, - ), + variant_struct_type_di_node, source_info: None, + discr: super::compute_discriminant_value(cx, enum_type_and_layout, variant_index), } }) .collect(); - let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name.as_str(), - tag_base_type, - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) - }), - enum_type_di_node, - ); - build_union_fields_for_direct_tag_enum_or_generator( cx, enum_type_and_layout, enum_type_di_node, &variant_field_infos, - discr_type_di_node, + variant_names_type_di_node, + tag_base_type, tag_field, + untagged_variant_index, ) } -fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( +// The base type of the VariantNames DW_AT_enumeration_type is always the same. +// It has nothing to do with the tag of the enum and just has to be big enough +// to hold all variant names. +fn variant_names_enum_base_type<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> Ty<'tcx> { + cx.tcx.types.u32 +} + +/// This function builds a DW_AT_enumeration_type that contains an entry for +/// each variant. Note that this has nothing to do with the discriminant. The +/// numeric value of each enumerator corresponds to the variant index. The +/// type is only used for efficiently encoding the name of each variant in +/// debuginfo. +fn build_variant_names_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_adt_def: AdtDef<'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, - enum_type_di_node: &'ll DIType, - dataful_variant_index: VariantIdx, - variant_indices: &mut dyn Iterator<Item = VariantIdx>, - tag_field: usize, -) -> SmallVec<&'ll DIType> { - let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + containing_scope: &'ll DIType, + variants: impl Iterator<Item = (VariantIdx, Cow<'tcx, str>)>, +) -> &'ll DIType { + // Create an enumerator for each variant. + super::build_enumeration_type_di_node( cx, - enum_type_and_layout.ty, - enum_type_di_node, - dataful_variant_index, - &enum_adt_def.variant(dataful_variant_index), - enum_type_and_layout.for_variant(cx, dataful_variant_index), - ); + "VariantNames", + variant_names_enum_base_type(cx), + variants.map(|(variant_index, variant_name)| (variant_name, variant_index.as_u32().into())), + containing_scope, + ) +} - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); - // Create an DW_TAG_enumerator for each variant except the dataful one. - let discr_type_di_node = super::build_enumeration_type_di_node( +fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_generator_type_and_layout: TyAndLayout<'tcx>, + enum_or_generator_type_di_node: &'ll DIType, + variant_index: VariantIdx, + untagged_variant_index: Option<VariantIdx>, + variant_struct_type_di_node: &'ll DIType, + variant_names_type_di_node: &'ll DIType, + tag_base_type_di_node: &'ll DIType, + tag_base_type: Ty<'tcx>, + discr: DiscrResult, +) -> &'ll DIType { + type_map::build_type_with_children( cx, - "Discriminant$", - tag_base_type, - &mut variant_indices.filter_map(|variant_index| { - if let Some(discr_val) = - super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) - { - let discr = Discr { val: discr_val as u128, ty: tag_base_type }; - let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); - Some((discr, variant_name)) - } else { - debug_assert_eq!(variant_index, dataful_variant_index); - None - } - }), - enum_type_di_node, - ); - - smallvec![ - build_field_di_node( - cx, - enum_type_di_node, - "dataful_variant", - size_and_align_of(enum_type_and_layout), - Size::ZERO, - DIFlags::FlagZero, - dataful_variant_struct_type_di_node, - ), - build_field_di_node( + type_map::stub( cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(tag_base_type), - enum_type_and_layout.fields.offset(tag_field), + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type_wrapper( + cx.tcx, + enum_or_generator_type_and_layout.ty, + variant_index, + ), + &variant_struct_wrapper_type_name(variant_index), + // NOTE: We use size and align of enum_type, not from variant_layout: + size_and_align_of(enum_or_generator_type_and_layout), + Some(enum_or_generator_type_di_node), DIFlags::FlagZero, - discr_type_di_node, ), - ] + |cx, wrapper_struct_type_di_node| { + enum DiscrKind { + Exact(u64), + Exact128(u128), + Range(u64, u64), + Range128(u128, u128), + } + + let (tag_base_type_size, tag_base_type_align) = cx.size_and_align_of(tag_base_type); + let is_128_bits = tag_base_type_size.bits() > 64; + + let discr = match discr { + DiscrResult::NoDiscriminant => DiscrKind::Exact(SINGLE_VARIANT_VIRTUAL_DISR), + DiscrResult::Value(discr_val) => { + if is_128_bits { + DiscrKind::Exact128(discr_val) + } else { + debug_assert_eq!(discr_val, discr_val as u64 as u128); + DiscrKind::Exact(discr_val as u64) + } + } + DiscrResult::Range(min, max) => { + assert_eq!(Some(variant_index), untagged_variant_index); + if is_128_bits { + DiscrKind::Range128(min, max) + } else { + debug_assert_eq!(min, min as u64 as u128); + debug_assert_eq!(max, max as u64 as u128); + DiscrKind::Range(min as u64, max as u64) + } + } + }; + + let mut fields = SmallVec::new(); + + // We always have a field for the value + fields.push(build_field_di_node( + cx, + wrapper_struct_type_di_node, + "value", + size_and_align_of(enum_or_generator_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_di_node, + )); + + let build_assoc_const = + |name: &str, type_di_node: &'ll DIType, value: u64, align: Align| unsafe { + llvm::LLVMRustDIBuilderCreateStaticMemberType( + DIB(cx), + wrapper_struct_type_di_node, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + type_di_node, + DIFlags::FlagZero, + Some(cx.const_u64(value)), + align.bits() as u32, + ) + }; + + // We also always have an associated constant for the discriminant value + // of the variant. + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_NAME, + variant_names_type_di_node, + variant_index.as_u32() as u64, + cx.align_of(variant_names_enum_base_type(cx)), + )); + + // Emit the discriminant value (or range) corresponding to the variant. + match discr { + DiscrKind::Exact(discr_val) => { + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_EXACT, + tag_base_type_di_node, + discr_val, + tag_base_type_align, + )); + } + DiscrKind::Exact128(discr_val) => { + let align = cx.align_of(cx.tcx.types.u64); + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let Split128 { hi, lo } = split_128(discr_val); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_EXACT_LO, + type_di_node, + lo, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_EXACT_HI, + type_di_node, + hi, + align, + )); + } + DiscrKind::Range(begin, end) => { + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_BEGIN, + tag_base_type_di_node, + begin, + tag_base_type_align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR_END, + tag_base_type_di_node, + end, + tag_base_type_align, + )); + } + DiscrKind::Range128(begin, end) => { + let align = cx.align_of(cx.tcx.types.u64); + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let Split128 { hi: begin_hi, lo: begin_lo } = split_128(begin); + let Split128 { hi: end_hi, lo: end_lo } = split_128(end); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_BEGIN_HI, + type_di_node, + begin_hi, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_BEGIN_LO, + type_di_node, + begin_lo, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_END_HI, + type_di_node, + end_hi, + align, + )); + + fields.push(build_assoc_const( + ASSOC_CONST_DISCR128_END_LO, + type_di_node, + end_lo, + align, + )); + } + } + + fields + }, + NO_GENERICS, + ) + .di_node +} + +struct Split128 { + hi: u64, + lo: u64, +} + +fn split_128(value: u128) -> Split128 { + Split128 { hi: (value >> 64) as u64, lo: value as u64 } } fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( @@ -365,10 +677,33 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( }; let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + cx.tcx.generator_layout_and_saved_local_names(generator_def_id); - let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); + + let tag_base_type = tag_base_type(cx, generator_type_and_layout); + + let variant_names_type_di_node = build_variant_names_type_di_node( + cx, + generator_type_di_node, + variant_range + .clone() + .map(|variant_index| (variant_index, GeneratorSubsts::variant_name(variant_index))), + ); + + let discriminants: IndexVec<VariantIdx, DiscrResult> = { + let discriminants_iter = generator_substs.discriminants(generator_def_id, cx.tcx); + let mut discriminants: IndexVec<VariantIdx, DiscrResult> = + IndexVec::with_capacity(variant_count); + for (variant_index, discr) in discriminants_iter { + // Assert that the index in the IndexMap matches up with the given VariantIdx. + assert_eq!(variant_index, discriminants.next_index()); + discriminants.push(DiscrResult::Value(discr.val)); + } + discriminants + }; // Build the type node for each field. let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range @@ -391,29 +726,24 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( None }; - VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } + VariantFieldInfo { + variant_index, + variant_struct_type_di_node, + source_info, + discr: discriminants[variant_index], + } }) .collect(); - let tag_base_type = tag_base_type(cx, generator_type_and_layout); - let discr_type_name = "Discriminant$"; - let discr_type_di_node = super::build_enumeration_type_di_node( - cx, - discr_type_name, - tag_base_type, - &mut generator_substs - .discriminants(generator_def_id, cx.tcx) - .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), - generator_type_di_node, - ); - build_union_fields_for_direct_tag_enum_or_generator( cx, generator_type_and_layout, generator_type_di_node, &variant_field_infos[..], - discr_type_di_node, + variant_names_type_di_node, + tag_base_type, tag_field, + None, ) } @@ -425,8 +755,11 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( enum_type_di_node: &'ll DIType, variant_field_infos: &[VariantFieldInfo<'ll>], discr_type_di_node: &'ll DIType, + tag_base_type: Ty<'tcx>, tag_field: usize, + untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { + let tag_base_type_di_node = type_di_node(cx, tag_base_type); let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); // We create a field in the union for each variant ... @@ -438,6 +771,19 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( let field_name = variant_union_field_name(variant_member_info.variant_index); let (size, align) = size_and_align_of(enum_type_and_layout); + let variant_struct_type_wrapper = build_variant_struct_wrapper_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_member_info.variant_index, + untagged_variant_index, + variant_member_info.variant_struct_type_di_node, + discr_type_di_node, + tag_base_type_di_node, + tag_base_type, + variant_member_info.discr, + ); + // We use LLVMRustDIBuilderCreateMemberType() member type directly because // the build_field_di_node() function does not support specifying a source location, // which is something that we don't do anywhere else. @@ -456,7 +802,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( // Union fields are always at offset zero Size::ZERO.bits(), DIFlags::FlagZero, - variant_member_info.variant_struct_type_di_node, + variant_struct_type_wrapper, ) } })); @@ -466,16 +812,53 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) ); - // ... and a field for the discriminant. - unions_fields.push(build_field_di_node( - cx, - enum_type_di_node, - "discriminant", - cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - enum_type_and_layout.fields.offset(tag_field), - DIFlags::FlagZero, - discr_type_di_node, - )); + // ... and a field for the tag. If the tag is 128 bits wide, this will actually + // be two 64-bit fields. + let is_128_bits = cx.size_of(tag_base_type).bits() > 64; + + if is_128_bits { + let type_di_node = type_di_node(cx, cx.tcx.types.u64); + let size_and_align = cx.size_and_align_of(cx.tcx.types.u64); + + let (lo_offset, hi_offset) = match cx.tcx.data_layout.endian { + Endian::Little => (0, 8), + Endian::Big => (8, 0), + }; + + let tag_field_offset = enum_type_and_layout.fields.offset(tag_field).bytes(); + let lo_offset = Size::from_bytes(tag_field_offset + lo_offset); + let hi_offset = Size::from_bytes(tag_field_offset + hi_offset); + + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME_128_LO, + size_and_align, + lo_offset, + DIFlags::FlagZero, + type_di_node, + )); + + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME_128_HI, + size_and_align, + hi_offset, + DIFlags::FlagZero, + type_di_node, + )); + } else { + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + tag_base_type_di_node, + )); + } unions_fields } @@ -485,6 +868,7 @@ struct VariantFieldInfo<'ll> { variant_index: VariantIdx, variant_struct_type_di_node: &'ll DIType, source_info: Option<(&'ll DIFile, c_uint)>, + discr: DiscrResult, } fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { @@ -512,3 +896,29 @@ fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) } + +fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, str> { + const PRE_ALLOCATED: [&str; 16] = [ + "Variant0", + "Variant1", + "Variant2", + "Variant3", + "Variant4", + "Variant5", + "Variant6", + "Variant7", + "Variant8", + "Variant9", + "Variant10", + "Variant11", + "Variant12", + "Variant13", + "Variant14", + "Variant15", + ]; + + PRE_ALLOCATED + .get(variant_index.as_usize()) + .map(|&s| Cow::from(s)) + .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 73e01d0453b..9e0e847a155 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -3,19 +3,20 @@ use rustc_codegen_ssa::debuginfo::{ wants_c_like_enum_debuginfo, }; use rustc_hir::def::CtorKind; -use rustc_index::vec::IndexVec; +use rustc_index::IndexSlice; use rustc_middle::{ bug, - mir::{Field, GeneratorLayout, GeneratorSavedLocal}, + mir::{GeneratorLayout, GeneratorSavedLocal}, ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - util::Discr, AdtDef, GeneratorSubsts, Ty, VariantDef, }, }; use rustc_span::Symbol; -use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{ + FieldIdx, HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants, +}; use std::borrow::Cow; use crate::{ @@ -90,8 +91,9 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( cx, &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), tag_base_type(cx, enum_type_and_layout), - &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { - (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + (name, discr.val) }), containing_scope, ), @@ -122,7 +124,8 @@ fn tag_base_type<'ll, 'tcx>( Primitive::Int(t, _) => t, Primitive::F32 => Integer::I32, Primitive::F64 => Integer::I64, - Primitive::Pointer => { + // FIXME(erikdesjardins): handle non-default addrspace ptr sizes + Primitive::Pointer(_) => { // If the niche is the NULL value of a reference, then `discr_enum_ty` will be // a RawPtr. CodeView doesn't know what to do with enums whose base type is a // pointer so we fix this up to just be `usize`. @@ -145,14 +148,11 @@ fn tag_base_type<'ll, 'tcx>( /// This is a helper function and does not register anything in the type map by itself. /// /// `variants` is an iterator of (discr-value, variant-name). -/// -// NOTE: Handling of discriminant values is somewhat inconsistent. They can appear as u128, -// u64, and i64. Here everything gets mapped to i64 because that's what LLVM's API expects. fn build_enumeration_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, type_name: &str, base_type: Ty<'tcx>, - variants: &mut dyn Iterator<Item = (Discr<'tcx>, Cow<'tcx, str>)>, + enumerators: impl Iterator<Item = (Cow<'tcx, str>, u128)>, containing_scope: &'ll DIType, ) -> &'ll DIType { let is_unsigned = match base_type.kind() { @@ -160,24 +160,22 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( ty::Uint(_) => true, _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), }; + let (size, align) = cx.size_and_align_of(base_type); - let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = variants - .map(|(discr, variant_name)| { - unsafe { - Some(llvm::LLVMRustDIBuilderCreateEnumerator( - DIB(cx), - variant_name.as_ptr().cast(), - variant_name.len(), - // FIXME: what if enumeration has i128 discriminant? - discr.val as i64, - is_unsigned, - )) - } + let enumerator_di_nodes: SmallVec<Option<&'ll DIType>> = enumerators + .map(|(name, value)| unsafe { + let value = [value as u64, (value >> 64) as u64]; + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + name.as_ptr().cast(), + name.len(), + value.as_ptr(), + size.bits() as libc::c_uint, + is_unsigned, + )) }) .collect(); - let (size, align) = cx.size_and_align_of(base_type); - unsafe { llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), @@ -247,32 +245,37 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( /// and a DW_TAG_member for each field (but not the discriminant). fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_type: Ty<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, variant_index: VariantIdx, variant_def: &VariantDef, variant_layout: TyAndLayout<'tcx>, ) -> &'ll DIType { - debug_assert_eq!(variant_layout.ty, enum_type); + debug_assert_eq!(variant_layout.ty, enum_type_and_layout.ty); type_map::build_type_with_children( cx, type_map::stub( cx, Stub::Struct, - UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), + UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + enum_type_and_layout.ty, + variant_index, + ), variant_def.name.as_str(), // NOTE: We use size and align of enum_type, not from variant_layout: - cx.size_and_align_of(enum_type), + size_and_align_of(enum_type_and_layout), Some(enum_type_di_node), DIFlags::FlagZero, ), |cx, struct_type_di_node| { (0..variant_layout.fields.count()) .map(|field_index| { - let field_name = if variant_def.ctor_kind != CtorKind::Fn { + let field_name = if variant_def.ctor_kind() != Some(CtorKind::Fn) { // Fields have names - Cow::from(variant_def.fields[field_index].name.as_str()) + let field = &variant_def.fields[FieldIdx::from_usize(field_index)]; + Cow::from(field.name.as_str()) } else { // Tuple-like super::tuple_field_name(field_index) @@ -290,9 +293,9 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( type_di_node(cx, field_layout.ty), ) }) - .collect() + .collect::<SmallVec<_>>() }, - |cx| build_generic_type_param_di_nodes(cx, enum_type), + |cx| build_generic_type_param_di_nodes(cx, enum_type_and_layout.ty), ) .di_node } @@ -320,7 +323,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, generator_layout: &GeneratorLayout<'tcx>, - state_specific_upvar_names: &IndexVec<GeneratorSavedLocal, Option<Symbol>>, + state_specific_upvar_names: &IndexSlice<GeneratorSavedLocal, Option<Symbol>>, common_upvar_names: &[String], ) -> &'ll DIType { let variant_name = GeneratorSubsts::variant_name(variant_index); @@ -353,7 +356,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) .map(|field_index| { let generator_saved_local = generator_layout.variant_fields[variant_index] - [Field::from_usize(field_index)]; + [FieldIdx::from_usize(field_index)]; let field_name_maybe = state_specific_upvar_names[generator_saved_local]; let field_name = field_name_maybe .as_ref() @@ -398,39 +401,60 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .di_node } +#[derive(Copy, Clone)] +enum DiscrResult { + NoDiscriminant, + Value(u128), + Range(u128, u128), +} + +impl DiscrResult { + fn opt_single_val(&self) -> Option<u128> { + if let Self::Value(d) = *self { Some(d) } else { None } + } +} + /// Returns the discriminant value corresponding to the variant index. /// /// Will return `None` if there is less than two variants (because then the enum won't have) -/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no +/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no /// single discriminant value). fn compute_discriminant_value<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, variant_index: VariantIdx, -) -> Option<u64> { +) -> DiscrResult { match enum_type_and_layout.layout.variants() { - &Variants::Single { .. } => None, - &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( - enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val - as u64, + &Variants::Single { .. } => DiscrResult::NoDiscriminant, + &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => DiscrResult::Value( + enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val, ), &Variants::Multiple { - tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, + tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant }, tag, .. } => { - if variant_index == dataful_variant { - None + if variant_index == untagged_variant { + let valid_range = enum_type_and_layout + .for_variant(cx, variant_index) + .largest_niche + .as_ref() + .unwrap() + .valid_range; + + let min = valid_range.start.min(valid_range.end); + let min = tag.size(cx).truncate(min); + + let max = valid_range.start.max(valid_range.end); + let max = tag.size(cx).truncate(max); + + DiscrResult::Range(min, max) } else { let value = (variant_index.as_u32() as u128) .wrapping_sub(niche_variants.start().as_u32() as u128) .wrapping_add(niche_start); let value = tag.size(cx).truncate(value); - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - Some(value as u64) + DiscrResult::Value(value) } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index f1935e0ec31..978141917c6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -4,9 +4,8 @@ use crate::{ common::CodegenCx, debuginfo::{ metadata::{ - closure_saved_names_of_captured_variables, enums::tag_base_type, - file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, StubInfo, UniqueTypeId}, unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, UNKNOWN_LINE_NUMBER, @@ -88,7 +87,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( cx, - enum_type, + enum_type_and_layout, enum_type_di_node, variant_index, enum_adt_def.variant(variant_index), @@ -157,7 +156,7 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( ), |cx, generator_type_di_node| { let (generator_layout, state_specific_upvar_names) = - generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + cx.tcx.generator_layout_and_saved_local_names(generator_def_id); let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { bug!( @@ -167,7 +166,7 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( }; let common_upvar_names = - closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants @@ -378,7 +377,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( /// /// The DW_AT_discr_value is optional, and is omitted if /// - This is the only variant of a univariant enum (i.e. their is no discriminant) -/// - This is the "dataful" variant of a niche-layout enum +/// - This is the "untagged" variant of a niche-layout enum /// (where only the other variants are identified by a single value) /// /// There is only ever a single member, the type of which is a struct that describes the @@ -413,7 +412,13 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.map(|v| cx.const_u64(v)), + discr_value.opt_single_val().map(|value| { + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + cx.const_u64(value as u64) + }), DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) @@ -433,6 +438,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( /// DW_TAG_structure_type (type of variant 1) /// DW_TAG_structure_type (type of variant 2) /// DW_TAG_structure_type (type of variant 3) +/// ``` struct VariantMemberInfo<'a, 'll> { variant_index: VariantIdx, variant_name: Cow<'a, str>, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 8fc8118849b..e30622cbdce 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -47,6 +47,8 @@ pub(super) enum UniqueTypeId<'tcx> { VariantPart(Ty<'tcx>, private::HiddenZst), /// The ID for the artificial struct type describing a single enum variant. VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode. + VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst), /// The ID of the artificial type we create for VTables. VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst), } @@ -71,6 +73,15 @@ impl<'tcx> UniqueTypeId<'tcx> { UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) } + pub fn for_enum_variant_struct_type_wrapper( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantStructTypeCppLikeWrapper(enum_ty, variant_idx, private::HiddenZst) + } + pub fn for_vtable_ty( tcx: TyCtxt<'tcx>, self_type: Ty<'tcx>, @@ -146,7 +157,7 @@ impl<'ll> DINodeCreationResult<'ll> { pub enum Stub<'ll> { Struct, Union, - VtableTy { vtable_holder: &'ll DIType }, + VTableTy { vtable_holder: &'ll DIType }, } pub struct StubInfo<'ll, 'tcx> { @@ -180,9 +191,9 @@ pub(super) fn stub<'ll, 'tcx>( let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); let metadata = match kind { - Stub::Struct | Stub::VtableTy { .. } => { + Stub::Struct | Stub::VTableTy { .. } => { let vtable_holder = match kind { - Stub::VtableTy { vtable_holder } => Some(vtable_holder), + Stub::VTableTy { vtable_holder } => Some(vtable_holder), _ => None, }; unsafe { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index cf591295b84..aa7ae9355bc 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -21,13 +21,14 @@ use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::Hash128; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; -use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; +use rustc_middle::ty::subst::SubstsRef; +use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -39,7 +40,7 @@ use smallvec::SmallVec; use std::cell::OnceCell; use std::cell::RefCell; use std::iter; -use tracing::debug; +use std::ops::Range; mod create_scope_map; pub mod gdb; @@ -61,7 +62,7 @@ pub struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell<FxHashMap<Option<(u128, SourceFileHash)>, &'ll DIFile>>, + created_files: RefCell<FxHashMap<Option<(Hash128, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, @@ -112,7 +113,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - "Dwarf Version\0".as_ptr().cast(), + c"Dwarf Version".as_ptr().cast(), dwarf_version, ); } else { @@ -120,17 +121,16 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - "CodeView\0".as_ptr().cast(), + c"CodeView".as_ptr().cast(), 1, ) } // Prevent bitcode readers from deleting the debug info. - let ptr = "Debug Info Version\0".as_ptr(); llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - ptr.cast(), + c"Debug Info Version".as_ptr().cast(), llvm::LLVMRustDebugMetadataVersion(), ); } @@ -164,12 +164,14 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { variable_alloca: Self::Value, direct_offset: Size, indirect_offsets: &[Size], + fragment: Option<Range<Size>>, ) { - // Convert the direct and indirect offsets to address ops. + // Convert the direct and indirect offsets and fragment byte range to address ops. // FIXME(eddyb) use `const`s instead of getting the values via FFI, // the values should match the ones in the DWARF standard anyway. let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; + let op_llvm_fragment = || unsafe { llvm::LLVMRustDIBuilderCreateOpLLVMFragment() }; let mut addr_ops = SmallVec::<[u64; 8]>::new(); if direct_offset.bytes() > 0 { @@ -183,6 +185,13 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { addr_ops.push(offset.bytes() as u64); } } + if let Some(fragment) = fragment { + // `DW_OP_LLVM_fragment` takes as arguments the fragment's + // offset and size, both of them in bits. + addr_ops.push(op_llvm_fragment()); + addr_ops.push(fragment.start.bits() as u64); + addr_ops.push((fragment.end - fragment.start).bits() as u64); + } unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. @@ -200,8 +209,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { unsafe { - let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc); } } @@ -313,7 +321,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let def_id = instance.def_id(); - let containing_scope = get_containing_scope(self, instance); + let (containing_scope, is_method) = get_containing_scope(self, instance); let span = tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); @@ -369,8 +377,29 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - unsafe { - return llvm::LLVMRustDIBuilderCreateFunction( + // When we're adding a method to a type DIE, we only want a DW_AT_declaration there, because + // LLVM LTO can't unify type definitions when a child DIE is a full subprogram definition. + // When we use this `decl` below, the subprogram definition gets created at the CU level + // with a DW_AT_specification pointing back to the type's declaration. + let decl = is_method.then(|| unsafe { + llvm::LLVMRustDIBuilderCreateMethod( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + flags, + spflags & !DISPFlags::SPFlagDefinition, + template_parameters, + ) + }); + + return unsafe { + llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, name.as_ptr().cast(), @@ -385,15 +414,15 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { spflags, maybe_definition_llfn, template_parameters, - None, - ); - } + decl, + ) + }; fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> &'ll DIArray { - if cx.sess().opts.debuginfo == DebugInfo::Limited { + if cx.sess().opts.debuginfo != DebugInfo::Full { return create_DIArray(DIB(cx), &[]); } @@ -452,12 +481,12 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let names = get_parameter_names(cx, generics); iter::zip(substs, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_metadata = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -465,10 +494,8 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { name.len(), actual_type_metadata, )) - }) - } else { - None - } + } + }) }) .collect() } else { @@ -486,14 +513,16 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { names } + /// Returns a scope, plus `true` if that's a type scope for "class" methods, + /// otherwise `false` for plain namespace scopes. fn get_containing_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, - ) -> &'ll DIScope { + ) -> (&'ll DIScope, bool) { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( @@ -504,39 +533,33 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // Only "class" methods are generally understood by LLVM, // so avoid methods on other types (e.g., `<*mut T>::null`). - match impl_self_ty.kind() { - ty::Adt(def, ..) if !def.is_box() => { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.needs_subst() - { - Some(type_di_node(cx, impl_self_ty)) - } else { - Some(namespace::item_namespace(cx, def.did())) - } + if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() + { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - _ => None, } } else { // For trait method impls we still use the "parallel namespace" // strategy - None } - }); + } - self_type.unwrap_or_else(|| { - namespace::item_namespace( - cx, - DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }, - ) - }) + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); + (scope, false) } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 8f243673907..6bcd3e5bf58 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -5,8 +5,8 @@ use super::CodegenUnitDebugContext; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; -use rustc_middle::ty::{self, DefIdTree, Ty}; -use tracing::trace; +use rustc_middle::ty::{self, Ty}; +use trace; use crate::common::CodegenCx; use crate::llvm; @@ -72,7 +72,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( layout.is_unsized() ); - if !layout.is_unsized() { + if layout.is_sized() { return None; } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 5a5c4f7f860..164b12cf8d4 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -18,9 +18,13 @@ use crate::llvm; use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; -use rustc_middle::ty::Ty; +use rustc_codegen_ssa::traits::TypeMembershipMethods; +use rustc_middle::ty::{Instance, Ty}; +use rustc_symbol_mangling::typeid::{ + kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance, + TypeIdOptions, +}; use smallvec::SmallVec; -use tracing::debug; /// Declare a function. /// @@ -31,6 +35,7 @@ fn declare_raw_fn<'ll>( name: &str, callconv: llvm::CallConv, unnamed: llvm::UnnamedAddr, + visibility: llvm::Visibility, ty: &'ll Type, ) -> &'ll Value { debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); @@ -40,6 +45,7 @@ fn declare_raw_fn<'ll>( llvm::SetFunctionCallConv(llfn, callconv); llvm::SetUnnamedAddress(llfn, unnamed); + llvm::set_visibility(llfn, visibility); let mut attrs = SmallVec::<[_; 4]>::new(); @@ -77,14 +83,48 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - declare_raw_fn(self, name, llvm::CCallConv, unnamed, fn_type) + // Declare C ABI functions with the visibility used by C by default. + let visibility = if self.tcx.sess.target.default_hidden_visibility { + llvm::Visibility::Hidden + } else { + llvm::Visibility::Default + }; + + declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type) + } + + /// Declare an entry Function + /// + /// The ABI of this function can change depending on the target (although for now the same as + /// `declare_cfn`) + /// + /// If there’s a value with the same name already declared, the function will + /// update the declaration and return existing Value instead. + pub fn declare_entry_fn( + &self, + name: &str, + callconv: llvm::CallConv, + unnamed: llvm::UnnamedAddr, + fn_type: &'ll Type, + ) -> &'ll Value { + let visibility = if self.tcx.sess.target.default_hidden_visibility { + llvm::Visibility::Hidden + } else { + llvm::Visibility::Default + }; + declare_raw_fn(self, name, callconv, unnamed, visibility, fn_type) } /// Declare a Rust function. /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + pub fn declare_fn( + &self, + name: &str, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + instance: Option<Instance<'tcx>>, + ) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); // Function addresses in Rust are never significant, allowing functions to @@ -94,9 +134,62 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { name, fn_abi.llvm_cconv(), llvm::UnnamedAddr::Global, + llvm::Visibility::Default, fn_abi.llvm_type(self), ); fn_abi.apply_attrs_llfn(self, llfn); + + if self.tcx.sess.is_sanitizer_cfi_enabled() { + if let Some(instance) = instance { + let typeid = typeid_for_instance(self.tcx, &instance, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_instance( + self.tcx, + &instance, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } else { + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi( + self.tcx, + fn_abi, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } + } + + if self.tcx.sess.is_sanitizer_kcfi_enabled() { + // LLVM KCFI does not support multiple !kcfi_type attachments + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + if let Some(instance) = instance { + let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, &instance, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } else { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } + } + llfn } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs new file mode 100644 index 00000000000..44869ced1ae --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -0,0 +1,226 @@ +use std::borrow::Cow; +use std::ffi::CString; +use std::path::Path; + +use crate::fluent_generated as fluent; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_errors::{ + DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed, Handler, IntoDiagnostic, +}; +use rustc_macros::{Diagnostic, Subdiagnostic}; +use rustc_span::Span; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_ctarget_feature_prefix)] +#[note] +pub(crate) struct UnknownCTargetFeaturePrefix<'a> { + pub feature: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_ctarget_feature)] +#[note] +pub(crate) struct UnknownCTargetFeature<'a> { + pub feature: &'a str, + #[subdiagnostic] + pub rust_feature: PossibleFeature<'a>, +} + +#[derive(Subdiagnostic)] +pub(crate) enum PossibleFeature<'a> { + #[help(codegen_llvm_possible_feature)] + Some { rust_feature: &'a str }, + #[help(codegen_llvm_consider_filing_feature_request)] + None, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_error_creating_import_library)] +pub(crate) struct ErrorCreatingImportLibrary<'a> { + pub lib_name: &'a str, + pub error: String, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_symbol_already_defined)] +pub(crate) struct SymbolAlreadyDefined<'a> { + #[primary_span] + pub span: Span, + pub symbol_name: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_minimum_alignment_not_power_of_two)] +pub(crate) struct InvalidMinimumAlignmentNotPowerOfTwo { + pub align: u64, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_minimum_alignment_too_large)] +pub(crate) struct InvalidMinimumAlignmentTooLarge { + pub align: u64, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_sanitizer_memtag_requires_mte)] +pub(crate) struct SanitizerMemtagRequiresMte; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_error_writing_def_file)] +pub(crate) struct ErrorWritingDEFFile { + pub error: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_error_calling_dlltool)] +pub(crate) struct ErrorCallingDllTool<'a> { + pub dlltool_path: Cow<'a, str>, + pub error: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_dlltool_fail_import_library)] +pub(crate) struct DlltoolFailImportLibrary<'a> { + pub stdout: Cow<'a, str>, + pub stderr: Cow<'a, str>, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_dynamic_linking_with_lto)] +#[note] +pub(crate) struct DynamicLinkingWithLTO; + +pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); + +impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for ParseTargetMachineConfig<'_> { + fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> { + let diag: DiagnosticBuilder<'_, EM> = self.0.into_diagnostic(sess); + let (message, _) = diag.styled_message().first().expect("`LlvmError` with no message"); + let message = sess.eagerly_translate_to_string(message.clone(), diag.args()); + + let mut diag = sess.struct_diagnostic(fluent::codegen_llvm_parse_target_machine_config); + diag.set_arg("error", message); + diag + } +} + +pub(crate) struct TargetFeatureDisableOrEnable<'a> { + pub features: &'a [&'a str], + pub span: Option<Span>, + pub missing_features: Option<MissingFeatures>, +} + +#[derive(Subdiagnostic)] +#[help(codegen_llvm_missing_features)] +pub(crate) struct MissingFeatures; + +impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> { + fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> { + let mut diag = sess.struct_err(fluent::codegen_llvm_target_feature_disable_or_enable); + if let Some(span) = self.span { + diag.set_span(span); + }; + if let Some(missing_features) = self.missing_features { + diag.subdiagnostic(missing_features); + } + diag.set_arg("features", self.features.join(", ")); + diag + } +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_disallowed)] +pub(crate) struct LtoDisallowed; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_dylib)] +pub(crate) struct LtoDylib; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_lto_bitcode_from_rlib)] +pub(crate) struct LtoBitcodeFromRlib { + pub llvm_err: String, +} + +#[derive(Diagnostic)] +pub enum LlvmError<'a> { + #[diag(codegen_llvm_write_output)] + WriteOutput { path: &'a Path }, + #[diag(codegen_llvm_target_machine)] + CreateTargetMachine { triple: SmallCStr }, + #[diag(codegen_llvm_run_passes)] + RunLlvmPasses, + #[diag(codegen_llvm_serialize_module)] + SerializeModule { name: &'a str }, + #[diag(codegen_llvm_write_ir)] + WriteIr { path: &'a Path }, + #[diag(codegen_llvm_prepare_thin_lto_context)] + PrepareThinLtoContext, + #[diag(codegen_llvm_load_bitcode)] + LoadBitcode { name: CString }, + #[diag(codegen_llvm_write_thinlto_key)] + WriteThinLtoKey { err: std::io::Error }, + #[diag(codegen_llvm_multiple_source_dicompileunit)] + MultipleSourceDiCompileUnit, + #[diag(codegen_llvm_prepare_thin_lto_module)] + PrepareThinLtoModule, + #[diag(codegen_llvm_parse_bitcode)] + ParseBitcode, +} + +pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String); + +impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> { + fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> { + use LlvmError::*; + let msg_with_llvm_err = match &self.0 { + WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err, + CreateTargetMachine { .. } => fluent::codegen_llvm_target_machine_with_llvm_err, + RunLlvmPasses => fluent::codegen_llvm_run_passes_with_llvm_err, + SerializeModule { .. } => fluent::codegen_llvm_serialize_module_with_llvm_err, + WriteIr { .. } => fluent::codegen_llvm_write_ir_with_llvm_err, + PrepareThinLtoContext => fluent::codegen_llvm_prepare_thin_lto_context_with_llvm_err, + LoadBitcode { .. } => fluent::codegen_llvm_load_bitcode_with_llvm_err, + WriteThinLtoKey { .. } => fluent::codegen_llvm_write_thinlto_key_with_llvm_err, + MultipleSourceDiCompileUnit => { + fluent::codegen_llvm_multiple_source_dicompileunit_with_llvm_err + } + PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, + ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, + }; + let mut diag = self.0.into_diagnostic(sess); + diag.set_primary_message(msg_with_llvm_err); + diag.set_arg("llvm_err", self.1); + diag + } +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_from_llvm_optimization_diag)] +pub(crate) struct FromLlvmOptimizationDiag<'a> { + pub filename: &'a str, + pub line: std::ffi::c_uint, + pub column: std::ffi::c_uint, + pub pass_name: &'a str, + pub kind: &'a str, + pub message: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_from_llvm_diag)] +pub(crate) struct FromLlvmDiag { + pub message: String, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_write_bytecode)] +pub(crate) struct WriteBytecode<'a> { + pub path: &'a Path, + pub err: std::io::Error, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_copy_bitcode)] +pub(crate) struct CopyBitcode { + pub err: std::io::Error, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9f364749287..4e28034a850 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -8,8 +8,8 @@ use crate::va_arg::emit_va_arg; use crate::value::Value; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; -use rustc_codegen_ssa::common::span_invalid_monomorphization_error; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; +use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::OperandRef; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; @@ -22,7 +22,6 @@ use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; -use std::iter; fn get_simple_intrinsic<'ll>( cx: &CodegenCx<'ll, '_>, @@ -71,6 +70,9 @@ fn get_simple_intrinsic<'ll>( sym::nearbyintf64 => "llvm.nearbyint.f64", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::ptr_mask => "llvm.ptrmask", + sym::roundevenf32 => "llvm.roundeven.f32", + sym::roundevenf64 => "llvm.roundeven.f64", _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -107,6 +109,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let (simple_ty, simple_fn) = simple.unwrap(); self.call( simple_ty, + None, + None, simple_fn, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -147,7 +151,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { emit_va_arg(self, args[0], ret_ty) } } - Primitive::F64 | Primitive::Pointer => { + Primitive::F64 | Primitive::Pointer(_) => { emit_va_arg(self, args[0], ret_ty) } // `va_arg` should never be used with the return type f32. @@ -161,7 +165,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::volatile_load | sym::unaligned_volatile_load => { let tp_ty = substs.type_at(0); let ptr = args[0].immediate(); - let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { + let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let llty = ty.llvm_type(self); let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); self.volatile_load(llty, ptr) @@ -282,15 +286,11 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }, None => { - span_invalid_monomorphization_error( - tcx.sess, + tcx.sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, - &format!( - "invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", - name, ty - ), - ); + name, + ty, + }); return; } } @@ -338,17 +338,26 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::black_box => { args[0].val.store(self, result); - + let result_val_span = [result.llval]; // We need to "use" the argument in some way LLVM can't introspect, and on // targets that support it we can typically leverage inline assembly to do // this. LLVM's interpretation of inline assembly is that it's, well, a black // box. This isn't the greatest implementation since it probably deoptimizes // more than we want, but it's so far good enough. + // + // For zero-sized types, the location pointed to by the result may be + // uninitialized. Do not "use" the result in this case; instead just clobber + // the memory. + let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() { + ("~{memory}", &[]) + } else { + ("r,~{memory}", &result_val_span) + }; crate::asm::inline_asm_call( self, "", - "r,~{memory}", - &[result.llval], + constraint, + inputs, self.type_void(), true, false, @@ -370,11 +379,11 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } - _ => bug!("unknown intrinsic '{}'", name), + _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), }; if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { + if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); let ptr = self.pointercast(result.llval, ptr_llty); self.store(llval, ptr, result.align); @@ -413,7 +422,9 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { typeid: &'ll Value, ) -> Self::Value { let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); - self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + let type_checked_load = + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]); + self.extract_value(type_checked_load, 0) } fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { @@ -434,7 +445,7 @@ fn try_intrinsic<'ll>( ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, try_func, &[data], None); + bx.call(try_func_ty, None, None, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; @@ -533,7 +544,7 @@ fn codegen_msvc_try<'ll>( let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], normal, catchswitch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); bx.switch_to_block(normal); bx.ret(bx.const_i32(0)); @@ -554,7 +565,7 @@ fn codegen_msvc_try<'ll>( // module. // // When modifying, make sure that the type_name string exactly matches - // the one used in src/libpanic_unwind/seh.rs. + // the one used in library/panic_unwind/src/seh.rs. let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); let type_name = bx.const_bytes(b"rust_panic\0"); let type_info = @@ -577,7 +588,7 @@ fn codegen_msvc_try<'ll>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_i8p(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -585,7 +596,7 @@ fn codegen_msvc_try<'ll>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, catch_func, &[data, null], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet)); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -594,7 +605,7 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -637,7 +648,7 @@ fn codegen_gnu_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -645,7 +656,7 @@ fn codegen_gnu_try<'ll>( // Type indicator for the exception being thrown. // // The first value in this tuple is a pointer to the exception object - // being thrown. The second value is a "selector" indicating which of + // being thrown. The second value is a "selector" indicating which of // the landing pad clauses the exception's type had been matched to. // rust_try ignores the selector. bx.switch_to_block(catch); @@ -655,13 +666,13 @@ fn codegen_gnu_try<'ll>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, ptr], None); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -701,7 +712,7 @@ fn codegen_emcc_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -709,7 +720,7 @@ fn codegen_emcc_try<'ll>( // Type indicator for the exception being thrown. // // The first value in this tuple is a pointer to the exception object - // being thrown. The second value is a "selector" indicating which of + // being thrown. The second value is a "selector" indicating which of // the landing pad clauses the exception's type had been matched to. bx.switch_to_block(catch); let tydesc = bx.eh_catch_typeinfo(); @@ -740,13 +751,13 @@ fn codegen_emcc_try<'ll>( let catch_data = bx.bitcast(catch_data, bx.type_i8p()); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, catch_func, &[data, catch_data], None); + bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -761,7 +772,7 @@ fn gen_fn<'ll, 'tcx>( ) -> (&'ll Type, &'ll Value) { let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, fn_abi); + let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. @@ -789,7 +800,7 @@ fn get_rust_try_fn<'ll, 'tcx>( let i8p = tcx.mk_mut_ptr(tcx.types.i8); // `unsafe fn(*mut i8) -> ()` let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - iter::once(i8p), + [i8p], tcx.mk_unit(), false, hir::Unsafety::Unsafe, @@ -797,7 +808,7 @@ fn get_rust_try_fn<'ll, 'tcx>( ))); // `unsafe fn(*mut i8, *mut i8) -> ()` let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - [i8p, i8p].iter().cloned(), + [i8p, i8p], tcx.mk_unit(), false, hir::Unsafety::Unsafe, @@ -805,7 +816,7 @@ fn get_rust_try_fn<'ll, 'tcx>( ))); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( - [try_fn_ty, i8p, catch_fn_ty].into_iter(), + [try_fn_ty, i8p, catch_fn_ty], tcx.types.i32, false, hir::Unsafety::Unsafe, @@ -825,40 +836,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( llret_ty: &'ll Type, span: Span, ) -> Result<&'ll Value, ()> { - // macros for error handling: - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } - macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } + ($diag: expr) => {{ + bx.sess().emit_err($diag); + return Err(()); + }}; } macro_rules! require { - ($cond: expr, $($fmt: tt)*) => { + ($cond: expr, $diag: expr) => { if !$cond { - return_error!($($fmt)*); + return_error!($diag); } }; } macro_rules! require_simd { - ($ty: expr, $position: expr) => { - require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) + ($ty: expr, $diag: expr) => { + require!($ty.is_simd(), $diag) }; } @@ -868,7 +863,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - require_simd!(arg_tys[1], "argument"); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let expected_int_bits = (len.max(8) - 1).next_power_of_two(); @@ -880,7 +879,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) == Some(expected_bytes) => { let place = PlaceRef::alloca(bx, args[0].layout); @@ -889,12 +888,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); bx.load(int_ty, ptr, Align::ONE) } - _ => return_error!( - "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + _ => return_error!(InvalidMonomorphization::InvalidBitmask { + span, + name, mask_ty, expected_int_bits, expected_bytes - ), + }), }; let i1 = bx.type_i1(); @@ -906,7 +906,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // every intrinsic below takes a SIMD vector as its first argument - require_simd!(arg_tys[0], "input"); + require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] }); let in_ty = arg_tys[0]; let comparison = match name { @@ -921,23 +921,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); if let Some(cmp_op) = comparison { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); require!( bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, - "expected return type with integer elements, found `{}` with non-integer `{}`", - ret_ty, - out_ty + InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty } ); return Ok(compare_simd_types( @@ -958,14 +959,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // version of this intrinsic. match args[2].layout.ty.kind() { ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { - span_bug!(span, "could not evaluate shuffle index array length") - }) + len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( + || span_bug!(span, "could not evaluate shuffle index array length"), + ) } - _ => return_error!( - "simd_shuffle index must be an array of `u32`, got `{}`", - args[2].layout.ty - ), + _ => return_error!(InvalidMonomorphization::SimdShuffle { + span, + name, + ty: args[2].layout.ty + }), } } else { stripped.parse().unwrap_or_else(|_| { @@ -973,23 +975,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }) }; - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); require!( out_len == n, - "expected return type of length {}, found `{}` with length {}", - n, - ret_ty, - out_len + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } ); require!( in_elem == out_ty, - "expected return element type `{}` (element of input `{}`), \ - found `{}` with element type `{}`", - in_elem, - in_ty, - ret_ty, - out_ty + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } ); let total_len = u128::from(in_len) * 2; @@ -1002,15 +996,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - emit_error!("shuffle index #{} is not a constant", arg_idx); + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexNotConstant { + span, + name, + arg_idx, + }); None } Some(idx) if idx >= total_len => { - emit_error!( - "shuffle index #{} is out of bounds (limit {})", + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + span, + name, arg_idx, - total_len - ); + total_len, + }); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1031,10 +1030,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_insert { require!( in_elem == arg_tys[2], - "expected inserted type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - arg_tys[2] + InvalidMonomorphization::InsertedType { + span, + name, + in_elem, + in_ty, + out_ty: arg_tys[2] + } ); return Ok(bx.insert_element( args[0].immediate(), @@ -1045,10 +1047,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_extract { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); } @@ -1056,17 +1055,18 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_select { let m_elem_ty = in_elem; let m_len = in_len; - require_simd!(arg_tys[1], "argument"); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); require!( m_len == v_len, - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len + InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } ); match m_elem_ty.kind() { ty::Int(_) => {} - _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), + _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }), } // truncate the mask to a vector of i1s let i1 = bx.type_i1(); @@ -1098,11 +1098,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( args[0].immediate(), i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), ), - _ => return_error!( - "vector argument `{}`'s element type `{}`, expected integer element type", + _ => return_error!(InvalidMonomorphization::VectorArgument { + span, + name, in_ty, in_elem - ), + }), }; // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. @@ -1124,7 +1125,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ty::Array(elem, len) if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) - && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) == Some(expected_bytes) => { // Zero-extend iN to the array length: @@ -1137,12 +1138,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); return Ok(bx.load(array_ty, ptr, Align::ONE)); } - _ => return_error!( - "cannot return `{}`, expected `u{}` or `[u8; {}]`", + _ => return_error!(InvalidMonomorphization::CannotReturn { + span, + name, ret_ty, expected_int_bits, expected_bytes - ), + }), } } @@ -1155,25 +1157,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { - #[allow(unused_macro_rules)] - macro_rules! emit_error { - ($msg: tt) => { - emit_error!($msg, ) - }; - ($msg: tt, $($fmt: tt)*) => { - span_invalid_monomorphization_error( - bx.sess(), span, - &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), - name, $($fmt)*)); - } - } macro_rules! return_error { - ($($fmt: tt)*) => { - { - emit_error!($($fmt)*); - return Err(()); - } - } + ($diag: expr) => {{ + bx.sess().emit_err($diag); + return Err(()); + }}; } let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { @@ -1181,16 +1169,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match f.bit_width() { 32 => ("f32", elem_ty), 64 => ("f64", elem_ty), - _ => { - return_error!( - "unsupported element type `{}` of floating-point vector `{}`", - f.name_str(), - in_ty - ); - } + _ => return_error!(InvalidMonomorphization::FloatingPointVector { + span, + name, + f_ty: *f, + in_ty, + }), } } else { - return_error!("`{}` is not a floating-point type", in_ty); + return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty }); }; let vec_ty = bx.type_vector(elem_ty, in_len); @@ -1212,12 +1199,18 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), - _ => return_error!("unrecognized intrinsic `{}`", name), + _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }), }; let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); - let c = - bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); + let c = bx.call( + fn_ty, + None, + None, + f, + &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), + None, + ); Ok(c) } @@ -1301,37 +1294,48 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); - require_simd!(ret_ty, "return"); + require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); // Of the same length: let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - out_len + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len + } ); require!( in_len == out_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - out_len2 + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: out_len2 + } ); // The return type must match the first argument type - require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); + require!( + ret_ty == in_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } + ); // This counts how many pointers fn ptr_count(t: Ty<'_>) -> usize { @@ -1358,15 +1362,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*_ {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Not, + } ); unreachable!(); } @@ -1382,10 +1386,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of third argument `{}` \ - to be a signed integer type", - element_ty2, - arg_tys[2] + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } ); } } @@ -1416,8 +1422,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( llvm_elem_vec_ty, ); let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = - bx.call(fn_ty, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[1].immediate(), alignment, mask, args[0].immediate()], + None, + ); return Ok(v); } @@ -1429,32 +1441,40 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, "first"); - require_simd!(arg_tys[1], "second"); - require_simd!(arg_tys[2], "third"); + require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); // Of the same length: let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == element_len1, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "second", - in_len, - in_ty, - arg_tys[1], - element_len1 + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len: element_len1 + } ); require!( in_len == element_len2, - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", - "third", - in_len, - in_ty, - arg_tys[2], - element_len2 + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: element_len2 + } ); // This counts how many pointers @@ -1479,21 +1499,21 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); let (pointer_count, underlying_ty) = match element_ty1.kind() { - ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { + ty::RawPtr(p) if p.ty == in_elem && p.mutbl.is_mut() => { (ptr_count(element_ty1), non_ptr(element_ty1)) } _ => { require!( false, - "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*mut {}`", - element_ty1, - arg_tys[1], - in_elem, - in_ty, - element_ty1, - in_elem + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Mut, + } ); unreachable!(); } @@ -1508,10 +1528,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => { require!( false, - "expected element type `{}` of third argument `{}` \ - be a signed integer type", - element_ty2, - arg_tys[2] + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } ); } } @@ -1542,8 +1564,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = - bx.call(fn_ty, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[0].immediate(), args[1].immediate(), alignment, mask], + None, + ); return Ok(v); } @@ -1553,10 +1581,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::$name { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { ty::Int(_) | ty::Uint(_) => { @@ -1579,25 +1604,28 @@ fn generic_simd_intrinsic<'ll, 'tcx>( 32 => bx.const_real(bx.type_f32(), $identity), 64 => bx.const_real(bx.type_f64(), $identity), v => return_error!( - r#" -unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, - sym::$name, - in_ty, - in_elem, - v, - ret_ty + InvalidMonomorphization::UnsupportedSymbolOfSize { + span, + name, + symbol: sym::$name, + in_ty, + in_elem, + size: v, + ret_ty + } ), } }; Ok(bx.$float_reduce(acc, args[0].immediate())) } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1625,22 +1653,20 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, if name == sym::$name { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); return match in_elem.kind() { ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1658,22 +1684,20 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let input = if !$boolean { require!( ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, - in_ty, - ret_ty + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); args[0].immediate() } else { match in_elem.kind() { ty::Int(_) | ty::Uint(_) => {} - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), } // boolean reductions operate on vectors of i1s: @@ -1686,13 +1710,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let r = bx.$red(input); Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) } - _ => return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - sym::$name, + _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { + span, + name, + symbol: sym::$name, in_ty, in_elem, ret_ty - ), + }), }; } }; @@ -1704,17 +1729,130 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bitwise_red!(simd_reduce_all: vector_reduce_and, true); bitwise_red!(simd_reduce_any: vector_reduce_or, true); + if name == sym::simd_cast_ptr { + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); + + match in_elem.kind() { + ty::RawPtr(p) => { + let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) + }); + assert!(!check_sized); // we are in codegen, so we shouldn't see these types + require!( + metadata.is_unit(), + InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } + ); + } + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) + } + } + match out_elem.kind() { + ty::RawPtr(p) => { + let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) + }); + assert!(!check_sized); // we are in codegen, so we shouldn't see these types + require!( + metadata.is_unit(), + InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } + ); + } + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) + } + } + + if in_elem == out_elem { + return Ok(args[0].immediate()); + } else { + return Ok(bx.pointercast(args[0].immediate(), llret_ty)); + } + } + + if name == sym::simd_expose_addr { + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); + + match in_elem.kind() { + ty::RawPtr(_) => {} + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) + } + } + match out_elem.kind() { + ty::Uint(ty::UintTy::Usize) => {} + _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }), + } + + return Ok(bx.ptrtoint(args[0].immediate(), llret_ty)); + } + + if name == sym::simd_from_exposed_addr { + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); + + match in_elem.kind() { + ty::Uint(ty::UintTy::Usize) => {} + _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }), + } + match out_elem.kind() { + ty::RawPtr(_) => {} + _ => { + return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) + } + } + + return Ok(bx.inttoptr(args[0].immediate(), llret_ty)); + } + if name == sym::simd_cast || name == sym::simd_as { - require_simd!(ret_ty, "return"); + require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", - in_len, - in_ty, - ret_ty, - out_len + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } ); // casting cares about nominal type, not just structural type if in_elem == out_elem { @@ -1793,11 +1931,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, } require!( false, - "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", - in_ty, - in_elem, - ret_ty, - out_elem + InvalidMonomorphization::UnsupportedCast { + span, + name, + in_ty, + in_elem, + ret_ty, + out_elem + } ); } macro_rules! arith_binary { @@ -1809,10 +1950,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* _ => {}, } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) + require!( + false, + InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } + ); })* } } @@ -1840,10 +1981,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* _ => {}, } - require!(false, - "unsupported operation on `{}` with element `{}`", - in_ty, - in_elem) + require!( + false, + InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } + ); })* } } @@ -1881,12 +2022,12 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), _ => { - return_error!( - "expected element type `{}` of vector type `{}` \ - to be a signed or unsigned integer type", - arg_tys[0].simd_size_and_type(bx.tcx()).1, - arg_tys[0] - ); + return_error!(InvalidMonomorphization::ExpectedVectorElementType { + span, + name, + expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1, + vector_type: arg_tys[0] + }); } }; let llvm_intrinsic = &format!( @@ -1900,7 +2041,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, f, &[lhs, rhs], None); + let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index e67568b7b8f..24968e00cc8 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,20 +5,26 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(hash_raw_entry)] -#![feature(let_chains)] -#![feature(let_else)] #![feature(extern_types)] -#![feature(once_cell)] +#![feature(hash_raw_entry)] #![feature(iter_intersperse)] +#![feature(let_chains)] +#![feature(never_type)] +#![feature(impl_trait_in_assoc_type)] +#![feature(c_str_literals)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] +#![deny(rustc::untranslatable_diagnostic)] +#![deny(rustc::diagnostic_outside_of_impl)] #[macro_use] extern crate rustc_macros; +#[macro_use] +extern crate tracing; use back::write::{create_informational_target_machine, create_target_machine}; +use errors::ParseTargetMachineConfig; pub use llvm_util::target_features; use rustc_ast::expand::allocator::AllocatorKind; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; @@ -28,11 +34,12 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; -use rustc_data_structures::fx::FxHashMap; -use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; +use rustc_data_structures::fx::FxIndexMap; +use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage}; +use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; use rustc_session::Session; @@ -61,9 +68,10 @@ mod context; mod coverageinfo; mod debuginfo; mod declare; +mod errors; mod intrinsic; -// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. +// The following is a workaround that replaces `pub mod llvm;` and that fixes issue 53912. #[path = "llvm/mod.rs"] mod llvm_; pub mod llvm { @@ -77,6 +85,8 @@ mod type_of; mod va_arg; mod value; +fluent_messages! { "../messages.ftl" } + #[derive(Clone)] pub struct LlvmCodegenBackend(()); @@ -107,11 +117,11 @@ impl ExtraBackendMethods for LlvmCodegenBackend { tcx: TyCtxt<'tcx>, module_name: &str, kind: AllocatorKind, - has_alloc_error_handler: bool, + alloc_error_handler_kind: AllocatorKind, ) -> ModuleLlvm { let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); unsafe { - allocator::codegen(tcx, &mut module_llvm, module_name, kind, has_alloc_error_handler); + allocator::codegen(tcx, &mut module_llvm, module_name, kind, alloc_error_handler_kind); } module_llvm } @@ -130,12 +140,6 @@ impl ExtraBackendMethods for LlvmCodegenBackend { ) -> TargetMachineFactoryFn<Self> { back::write::target_machine_factory(sess, optlvl, target_features) } - fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { - llvm_util::target_cpu(sess) - } - fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { - llvm_util::tune_cpu(sess) - } fn spawn_thread<F, T>(time_trace: bool, f: F) -> std::thread::JoinHandle<T> where @@ -169,8 +173,8 @@ impl ExtraBackendMethods for LlvmCodegenBackend { impl WriteBackendMethods for LlvmCodegenBackend { type Module = ModuleLlvm; type ModuleBuffer = back::lto::ModuleBuffer; - type Context = llvm::Context; type TargetMachine = &'static mut llvm::TargetMachine; + type TargetMachineError = crate::errors::LlvmError<'static>; type ThinData = back::lto::ThinData; type ThinBuffer = back::lto::ThinBuffer; fn print_pass_timings(&self) { @@ -246,6 +250,10 @@ impl LlvmCodegenBackend { } impl CodegenBackend for LlvmCodegenBackend { + fn locale_resource(&self) -> &'static str { + crate::DEFAULT_LOCALE_RESOURCE + } + fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited } @@ -348,18 +356,18 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") .join(sess); - sess.time("llvm_dump_timing_file", || { - if sess.opts.unstable_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { + sess.time("llvm_dump_timing_file", || { let file_name = outputs.with_extension("llvm_timings.json"); llvm_util::time_trace_profiler_finish(&file_name); - } - }); + }); + } Ok((codegen_results, work_products)) } @@ -370,12 +378,12 @@ impl CodegenBackend for LlvmCodegenBackend { codegen_results: CodegenResults, outputs: &OutputFilenames, ) -> Result<(), ErrorGuaranteed> { - use crate::back::archive::LlvmArchiveBuilder; + use crate::back::archive::LlvmArchiveBuilderBuilder; use rustc_codegen_ssa::back::link::link_binary; // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary::<LlvmArchiveBuilder<'_>>(sess, &codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) } } @@ -418,8 +426,7 @@ impl ModuleLlvm { let tm = match (cgcx.tm_factory)(tm_factory_config) { Ok(m) => m, Err(e) => { - handler.struct_err(&e).emit(); - return Err(FatalError); + return Err(handler.emit_almost_fatal(ParseTargetMachineConfig(e))); } }; diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs index 64db4f7462d..7d948970223 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -83,17 +83,6 @@ impl<'a> Child<'a> { } } } - - pub fn data(&self) -> &'a [u8] { - unsafe { - let mut data_len = 0; - let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); - if data_ptr.is_null() { - panic!("failed to read data from archive child"); - } - slice::from_raw_parts(data_ptr as *const u8, data_len as usize) - } - } } impl<'a> Drop for Child<'a> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index e2fa5e488ed..45de284d22a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -20,19 +20,6 @@ pub enum OptimizationDiagnosticKind { OptimizationRemarkOther, } -impl OptimizationDiagnosticKind { - pub fn describe(self) -> &'static str { - match self { - OptimizationRemark | OptimizationRemarkOther => "remark", - OptimizationMissed => "missed", - OptimizationAnalysis => "analysis", - OptimizationAnalysisFPCommute => "floating-point", - OptimizationAnalysisAliasing => "aliasing", - OptimizationFailure => "failure", - } - } -} - pub struct OptimizationDiagnostic<'ll> { pub kind: OptimizationDiagnosticKind, pub pass_name: String, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 0ad65e5d99b..6ef3418cc5f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -35,7 +35,7 @@ pub enum LLVMRustResult { pub struct LLVMRustCOFFShortExport { pub name: *const c_char, pub ordinal_present: bool, - // value of `ordinal` only important when `ordinal_present` is true + /// value of `ordinal` only important when `ordinal_present` is true pub ordinal: u16, } @@ -79,6 +79,7 @@ pub enum LLVMModFlagBehavior { Append = 5, AppendUnique = 6, Max = 7, + Min = 8, } // Consts for the LLVM CallConv type, pre-cast to usize. @@ -183,7 +184,6 @@ pub enum AttributeKind { OptimizeNone = 24, ReturnsTwice = 25, ReadNone = 26, - InaccessibleMemOnly = 27, SanitizeHWAddress = 28, WillReturn = 29, StackProtectReq = 30, @@ -191,6 +191,12 @@ pub enum AttributeKind { StackProtect = 32, NoUndef = 33, SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, + SanitizeSafeStack = 40, } /// LLVMIntPredicate @@ -395,27 +401,6 @@ impl AtomicOrdering { } } -/// LLVMRustSynchronizationScope -#[derive(Copy, Clone)] -#[repr(C)] -pub enum SynchronizationScope { - SingleThread, - CrossThread, -} - -impl SynchronizationScope { - pub fn from_generic(sc: rustc_codegen_ssa::common::SynchronizationScope) -> Self { - match sc { - rustc_codegen_ssa::common::SynchronizationScope::SingleThread => { - SynchronizationScope::SingleThread - } - rustc_codegen_ssa::common::SynchronizationScope::CrossThread => { - SynchronizationScope::CrossThread - } - } - } -} - /// LLVMRustFileType #[derive(Copy, Clone)] #[repr(C)] @@ -444,6 +429,7 @@ pub enum MetadataType { MD_type = 19, MD_vcall_visibility = 28, MD_noundef = 29, + MD_kcfi_type = 36, } /// LLVMRustAsmDialect @@ -497,6 +483,8 @@ pub struct SanitizerOptions { pub sanitize_thread: bool, pub sanitize_hwaddress: bool, pub sanitize_hwaddress_recover: bool, + pub sanitize_kernel_address: bool, + pub sanitize_kernel_address_recover: bool, } /// LLVMRelocMode @@ -565,16 +553,7 @@ pub enum ArchiveKind { K_BSD, K_DARWIN, K_COFF, -} - -/// LLVMRustPassKind -#[derive(Copy, Clone, PartialEq, Debug)] -#[repr(C)] -#[allow(dead_code)] // Variants constructed by C++. -pub enum PassKind { - Other, - Function, - Module, + K_AIXBIG, } // LLVMRustThinLTOData @@ -587,10 +566,6 @@ extern "C" { pub type ThinLTOBuffer; } -// LLVMRustModuleNameCallback -pub type ThinLTOModuleNameCallback = - unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); - /// LLVMRustThinLTOModule #[repr(C)] pub struct ThinLTOModule { @@ -620,6 +595,15 @@ pub enum ChecksumKind { SHA256, } +/// LLVMRustMemoryEffects +#[derive(Copy, Clone)] +#[repr(C)] +pub enum MemoryEffects { + None, + ReadOnly, + InaccessibleMemOnly, +} + extern "C" { type Opaque; } @@ -656,15 +640,9 @@ extern "C" { } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); -extern "C" { - pub type MemoryBuffer; -} #[repr(C)] pub struct PassManager<'a>(InvariantOpaque<'a>); extern "C" { - pub type PassManagerBuilder; -} -extern "C" { pub type Pass; } extern "C" { @@ -703,7 +681,9 @@ pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_v pub mod coverageinfo { use super::coverage_map; - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) + /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegionKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum RegionKind { @@ -737,7 +717,9 @@ pub mod coverageinfo { /// array", encoded separately), and source location (start and end positions of the represented /// code region). /// - /// Matches LLVMRustCounterMappingRegion. + /// Corresponds to struct `llvm::coverage::CounterMappingRegion`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegion`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct CounterMappingRegion { @@ -970,20 +952,48 @@ pub mod debuginfo { NoDebug, FullDebug, LineTablesOnly, + DebugDirectivesOnly, } impl DebugEmissionKind { pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + // We should be setting LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we are + // instead adding a new debuginfo option "line-tables-only" so as to + // not break anything and to allow users to have 'limited' debug info. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. use rustc_session::config::DebugInfo; match kind { DebugInfo::None => DebugEmissionKind::NoDebug, - DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, - DebugInfo::Full => DebugEmissionKind::FullDebug, + DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly, + DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly, + DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug, } } } } +use bitflags::bitflags; +// These values **must** match with LLVMRustAllocKindFlags +bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct AllocKindFlags : u64 { + const Unknown = 0; + const Alloc = 1; + const Realloc = 1 << 1; + const Free = 1 << 2; + const Uninitialized = 1 << 3; + const Zeroed = 1 << 4; + const Aligned = 1 << 5; + } +} + extern "C" { pub type ModuleBuffer; } @@ -992,6 +1002,9 @@ pub type SelfProfileBeforePassCallback = unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); +pub type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; +pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; + extern "C" { pub fn LLVMRustInstallFatalErrorHandler(); pub fn LLVMRustDisableSystemDialogsOnCrash(); @@ -1011,8 +1024,7 @@ extern "C" { pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMSetModuleInlineAsm2(M: &Module, Asm: *const c_char, AsmLen: size_t); - pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; @@ -1059,7 +1071,7 @@ extern "C" { // Operations on other types pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values pub fn LLVMTypeOf(Val: &Value) -> &Type; @@ -1070,13 +1082,20 @@ extern "C" { pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>; // Operations on constants of any type pub fn LLVMConstNull(Ty: &Type) -> &Value; pub fn LLVMGetUndef(Ty: &Type) -> &Value; + pub fn LLVMGetPoison(Ty: &Type) -> &Value; // Operations on metadata + // FIXME: deprecated, replace with LLVMMDStringInContext2 pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + + pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata; + + // FIXME: deprecated, replace with LLVMMDNodeInContext2 pub fn LLVMMDNodeInContext<'a>( C: &'a Context, Vals: *const &'a Value, @@ -1093,7 +1112,7 @@ extern "C" { pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; - pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; + pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool; pub fn LLVMRustConstInt128Get( ConstantVal: &ConstantInt, SExt: bool, @@ -1115,6 +1134,8 @@ extern "C" { Packed: Bool, ) -> &'a Value; + // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 + // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc pub fn LLVMConstArray<'a>( ElementTy: &'a Type, ConstantVals: *const &'a Value, @@ -1165,7 +1186,6 @@ extern "C" { pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; - pub fn LLVMSetThreadLocal(GlobalVar: &Value, IsThreadLocal: Bool); pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); @@ -1190,7 +1210,11 @@ extern "C" { pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; + pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute; // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( @@ -1251,7 +1275,7 @@ extern "C" { pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); + pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata); // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; @@ -1277,7 +1301,8 @@ extern "C" { NumArgs: c_uint, Then: &'a BasicBlock, Catch: &'a BasicBlock, - Bundle: Option<&OperandBundleDef<'a>>, + OpBundles: *const Option<&OperandBundleDef<'a>>, + NumOpBundles: c_uint, Name: *const c_char, ) -> &'a Value; pub fn LLVMBuildLandingPad<'a>( @@ -1290,38 +1315,38 @@ extern "C" { pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMRustBuildCleanupPad<'a>( + pub fn LLVMBuildCleanupPad<'a>( B: &Builder<'a>, ParentPad: Option<&'a Value>, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCleanupRet<'a>( + pub fn LLVMBuildCleanupRet<'a>( B: &Builder<'a>, CleanupPad: &'a Value, - UnwindBB: Option<&'a BasicBlock>, + BB: Option<&'a BasicBlock>, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchPad<'a>( + pub fn LLVMBuildCatchPad<'a>( B: &Builder<'a>, ParentPad: &'a Value, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchRet<'a>( + pub fn LLVMBuildCatchRet<'a>( B: &Builder<'a>, - Pad: &'a Value, + CatchPad: &'a Value, BB: &'a BasicBlock, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchSwitch<'a>( + pub fn LLVMBuildCatchSwitch<'a>( Builder: &Builder<'a>, ParentPad: Option<&'a Value>, - BB: Option<&'a BasicBlock>, + UnwindBB: Option<&'a BasicBlock>, NumHandlers: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); + pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); // Add a case to the switch instruction @@ -1615,11 +1640,12 @@ extern "C" { DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMRustBuildIntCast<'a>( + pub fn LLVMBuildIntCast2<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, - IsSigned: bool, + IsSigned: Bool, + Name: *const c_char, ) -> &'a Value; // Comparisons @@ -1647,7 +1673,8 @@ extern "C" { Fn: &'a Value, Args: *const &'a Value, NumArgs: c_uint, - Bundle: Option<&OperandBundleDef<'a>>, + OpBundles: *const Option<&OperandBundleDef<'a>>, + NumOpBundles: c_uint, ) -> &'a Value; pub fn LLVMRustBuildMemCpy<'a>( B: &Builder<'a>, @@ -1777,16 +1804,18 @@ extern "C" { Order: AtomicOrdering, ) -> &'a Value; - pub fn LLVMRustBuildAtomicCmpXchg<'a>( + pub fn LLVMBuildAtomicCmpXchg<'a>( B: &Builder<'a>, LHS: &'a Value, CMP: &'a Value, RHS: &'a Value, Order: AtomicOrdering, FailureOrder: AtomicOrdering, - Weak: Bool, + SingleThreaded: Bool, ) -> &'a Value; + pub fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool); + pub fn LLVMBuildAtomicRMW<'a>( B: &Builder<'a>, Op: AtomicRmwBinOp, @@ -1796,29 +1825,19 @@ extern "C" { SingleThreaded: Bool, ) -> &'a Value; - pub fn LLVMRustBuildAtomicFence( - B: &Builder<'_>, + pub fn LLVMBuildFence<'a>( + B: &Builder<'a>, Order: AtomicOrdering, - Scope: SynchronizationScope, - ); + SingleThreaded: Bool, + Name: *const c_char, + ) -> &'a Value; /// Writes a module to the specified path. Returns 0 on success. pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; - /// Creates a pass manager. + /// Creates a legacy pass manager -- only used for final codegen. pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; - /// Creates a function-by-function pass manager - pub fn LLVMCreateFunctionPassManagerForModule(M: &Module) -> &mut PassManager<'_>; - - /// Disposes a pass manager. - pub fn LLVMDisposePassManager<'a>(PM: &'a mut PassManager<'a>); - - /// Runs a pass manager on a module. - pub fn LLVMRunPassManager<'a>(PM: &PassManager<'a>, M: &'a Module) -> Bool; - - pub fn LLVMInitializePasses(); - pub fn LLVMTimeTraceProfilerInitialize(); pub fn LLVMTimeTraceProfilerFinishThread(); @@ -1827,32 +1846,6 @@ extern "C" { pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; - pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); - pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( - PMB: &PassManagerBuilder, - threshold: c_uint, - ); - pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - - pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - Internalize: Bool, - RunInliner: Bool, - ); - pub fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager( - PMB: &PassManagerBuilder, - PM: &PassManager<'_>, - ); - pub fn LLVMGetHostCPUFeatures() -> *mut c_char; pub fn LLVMDisposeMessage(message: *mut c_char); @@ -1941,7 +1934,7 @@ extern "C" { ); pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; @@ -1999,6 +1992,21 @@ extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateMethod<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + SPFlags: DISPFlags, + TParam: &'a DIArray, + ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2074,6 +2082,19 @@ extern "C" { Ty: &'a DIType, ) -> &'a DIType; + pub fn LLVMRustDIBuilderCreateStaticMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + val: Option<&'a Value>, + AlignInBits: u32, + ) -> &'a DIDerivedType; + pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2153,7 +2174,8 @@ extern "C" { Builder: &DIBuilder<'a>, Name: *const c_char, NameLen: size_t, - Value: i64, + Value: *const u64, + SizeInBits: c_uint, IsUnsigned: bool, ) -> &'a DIEnumerator; @@ -2236,6 +2258,7 @@ extern "C" { ) -> &'a DILocation; pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; + pub fn LLVMRustDIBuilderCreateOpLLVMFragment() -> u64; #[allow(improper_ctypes)] pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); @@ -2244,26 +2267,9 @@ extern "C" { pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; - pub fn LLVMRustPassKind(Pass: &Pass) -> PassKind; - pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; - pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustCreateMemorySanitizerPass( - TrackOrigins: c_int, - Recover: bool, - ) -> &'static mut Pass; - pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; - pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; - pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); - pub fn LLVMRustAddLastExtensionPasses( - PMB: &PassManagerBuilder, - Passes: *const &'static mut Pass, - NumPasses: size_t, - ); - pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine, cpu: *const c_char); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2292,31 +2298,14 @@ extern "C" { RelaxELFRelocations: bool, UseInitArray: bool, SplitDwarfFile: *const c_char, + ForceEmulatedTls: bool, ) -> Option<&'static mut TargetMachine>; pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); - pub fn LLVMRustAddBuilderLibraryInfo<'a>( - PMB: &'a PassManagerBuilder, - M: &'a Module, - DisableSimplifyLibCalls: bool, - ); - pub fn LLVMRustConfigurePassManagerBuilder( - PMB: &PassManagerBuilder, - OptLevel: CodeGenOptLevel, - MergeFunctions: bool, - SLPVectorize: bool, - LoopVectorize: bool, - PrepareForThinLTO: bool, - PGOGenPath: *const c_char, - PGOUsePath: *const c_char, - PGOSampleUsePath: *const c_char, - SizeLevel: c_int, - ); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, DisableSimplifyLibCalls: bool, ); - pub fn LLVMRustRunFunctionPassManager<'a>(PM: &PassManager<'a>, M: &'a Module); pub fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, PM: &PassManager<'a>, @@ -2325,7 +2314,7 @@ extern "C" { DwoOutput: *const c_char, FileType: FileType, ) -> LLVMRustResult; - pub fn LLVMRustOptimizeWithNewPassManager<'a>( + pub fn LLVMRustOptimize<'a>( M: &'a Module, TM: &'a TargetMachine, OptLevel: PassBuilderOptLevel, @@ -2343,6 +2332,7 @@ extern "C" { PGOGenPath: *const c_char, PGOUsePath: *const c_char, InstrumentCoverage: bool, + InstrProfileOutput: *const c_char, InstrumentGCOV: bool, PGOSampleUsePath: *const c_char, DebugInfoForProfiling: bool, @@ -2361,9 +2351,7 @@ extern "C" { ) -> LLVMRustResult; pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); pub fn LLVMRustPrintPasses(); - pub fn LLVMRustGetInstructionCount(M: &Module) -> u32; pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); - pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; @@ -2372,7 +2360,6 @@ extern "C" { AIR: &ArchiveIterator<'a>, ) -> Option<&'a mut ArchiveChild<'a>>; pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; - pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); @@ -2407,12 +2394,6 @@ extern "C" { cookie_out: &mut c_uint, ) -> &'a SMDiagnostic; - pub fn LLVMRustSetInlineAsmDiagnosticHandler( - C: &Context, - H: InlineAsmDiagHandlerTy, - CX: *mut c_void, - ); - #[allow(improper_ctypes)] pub fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, @@ -2449,17 +2430,16 @@ extern "C" { pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); - pub fn LLVMRustBuildOperandBundleDef<'a>( + pub fn LLVMRustBuildOperandBundleDef( Name: *const c_char, - Inputs: *const &'a Value, + Inputs: *const &'_ Value, NumInputs: c_uint, - ) -> &'a mut OperandBundleDef<'a>; + ) -> &mut OperandBundleDef<'_>; pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); - pub fn LLVMRustUnsetComdat(V: &Value); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); @@ -2468,8 +2448,10 @@ extern "C" { pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub fn LLVMRustModuleCost(M: &Module) -> u64; + #[allow(improper_ctypes)] + pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); - pub fn LLVMRustThinLTOBufferCreate(M: &Module) -> &'static mut ThinLTOBuffer; + pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; @@ -2491,11 +2473,6 @@ extern "C" { Module: &Module, Target: &TargetMachine, ) -> bool; - pub fn LLVMRustGetThinLTOModuleImports( - Data: *const ThinLTOData, - ModuleNameCallback: ThinLTOModuleNameCallback, - CallbackPayload: *mut c_void, - ); pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); pub fn LLVMRustParseBitcodeForLTO( Context: &Context, @@ -2508,12 +2485,6 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustThinLTOGetDICompileUnit( - M: &Module, - CU1: &mut *mut c_void, - CU2: &mut *mut c_void, - ); - pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( @@ -2545,4 +2516,16 @@ extern "C" { #[allow(improper_ctypes)] pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); + + pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; + + pub fn LLVMRustIsBitcode(ptr: *const u8, len: usize) -> bool; + + pub fn LLVMRustGetSymbols( + buf_ptr: *const u8, + buf_len: usize, + state: *mut c_void, + callback: GetSymbolsCallback, + error_callback: GetSymbolsErrorCallback, + ) -> *mut c_void; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 48fbc1de8ee..4f5cc575da6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -95,6 +95,14 @@ pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } } +pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { + unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } +} + +pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { + unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, @@ -129,6 +137,7 @@ impl FromStr for ArchiveKind { "bsd" => Ok(ArchiveKind::K_BSD), "darwin" => Ok(ArchiveKind::K_DARWIN), "coff" => Ok(ArchiveKind::K_COFF), + "aix_big" => Ok(ArchiveKind::K_AIXBIG), _ => Err(()), } } @@ -158,12 +167,6 @@ pub fn SetUniqueComdat(llmod: &Module, val: &Value) { } } -pub fn UnsetComdat(val: &Value) { - unsafe { - LLVMRustUnsetComdat(val); - } -} - pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { unsafe { LLVMSetUnnamedAddress(global, unnamed); @@ -183,6 +186,13 @@ impl AttributeKind { } } +impl MemoryEffects { + /// Create an LLVM Attribute with these memory effects. + pub fn create_attr(self, llcx: &Context) -> &Attribute { + unsafe { LLVMRustCreateMemoryEffectsAttr(llcx, self) } + } +} + pub fn set_section(llglobal: &Value, section_name: &str) { let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); unsafe { diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index a0a640473eb..03be0654b50 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,7 +1,10 @@ use crate::back::write::create_informational_target_machine; -use crate::{llvm, llvm_util}; +use crate::errors::{ + PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, + UnknownCTargetFeaturePrefix, +}; +use crate::llvm; use libc::c_int; -use libloading::Library; use rustc_codegen_ssa::target_features::{ supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, }; @@ -13,11 +16,8 @@ use rustc_session::config::PrintRequest; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; -use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; -use tracing::debug; -use std::mem; use std::path::Path; use std::ptr; use std::slice; @@ -80,10 +80,10 @@ unsafe fn configure_llvm(sess: &Session) { }; // Set the llvm "program name" to make usage and invalid argument messages more clear. add("rustc -Cllvm-args=\"...\" with", true); - if sess.time_llvm_passes() { + if sess.opts.unstable_opts.time_llvm_passes { add("-time-passes", false); } - if sess.print_llvm_passes() { + if sess.opts.unstable_opts.print_llvm_passes { add("-debug-pass=Structure", false); } if sess.target.generate_arange_section @@ -92,16 +92,6 @@ unsafe fn configure_llvm(sess: &Session) { add("-generate-arange-section", false); } - // Disable the machine outliner by default in LLVM versions 11 and LLVM - // version 12, where it leads to miscompilation. - // - // Ref: - // - https://github.com/rust-lang/rust/issues/85351 - // - https://reviews.llvm.org/D103167 - if llvm_util::get_version() < (13, 0, 0) { - add("-enable-machine-outliner=never", false); - } - match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { MergeFunctions::Disabled | MergeFunctions::Trampolines => {} MergeFunctions::Aliases => { @@ -129,24 +119,6 @@ unsafe fn configure_llvm(sess: &Session) { llvm::LLVMTimeTraceProfilerInitialize(); } - llvm::LLVMInitializePasses(); - - // Use the legacy plugin registration if we don't use the new pass manager - if !should_use_new_llvm_pass_manager( - &sess.opts.unstable_opts.new_llvm_pass_manager, - &sess.target.arch, - ) { - // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.unstable_opts.llvm_plugins { - let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); - debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); - - // Intentionally leak the dynamic library. We can't ever unload it - // since the library can make things that will live arbitrarily long. - mem::forget(lib); - } - } - rustc_llvm::initialize_available_targets(); llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); @@ -159,61 +131,139 @@ pub fn time_trace_profiler_finish(file_name: &Path) { } } +pub enum TargetFeatureFoldStrength<'a> { + // The feature is only tied when enabling the feature, disabling + // this feature shouldn't disable the tied feature. + EnableOnly(&'a str), + // The feature is tied for both enabling and disabling this feature. + Both(&'a str), +} + +impl<'a> TargetFeatureFoldStrength<'a> { + fn as_str(&self) -> &'a str { + match self { + TargetFeatureFoldStrength::EnableOnly(feat) => feat, + TargetFeatureFoldStrength::Both(feat) => feat, + } + } +} + +pub struct LLVMFeature<'a> { + pub llvm_feature_name: &'a str, + pub dependency: Option<TargetFeatureFoldStrength<'a>>, +} + +impl<'a> LLVMFeature<'a> { + pub fn new(llvm_feature_name: &'a str) -> Self { + Self { llvm_feature_name, dependency: None } + } + + pub fn with_dependency( + llvm_feature_name: &'a str, + dependency: TargetFeatureFoldStrength<'a>, + ) -> Self { + Self { llvm_feature_name, dependency: Some(dependency) } + } + + pub fn contains(&self, feat: &str) -> bool { + self.iter().any(|dep| dep == feat) + } + + pub fn iter(&'a self) -> impl Iterator<Item = &'a str> { + let dependencies = self.dependency.iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + +impl<'a> IntoIterator for LLVMFeature<'a> { + type Item = &'a str; + type IntoIter = impl Iterator<Item = &'a str>; + + fn into_iter(self) -> Self::IntoIter { + let dependencies = self.dependency.into_iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + // WARNING: the features after applying `to_llvm_features` must be known // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. // // To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def // where the * matches the architecture's name +// +// For targets not present in the above location, see llvm-project/llvm/lib/Target/{ARCH}/*.td +// where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`. +// // Beware to not use the llvm github project for this, but check the git submodule // found in src/llvm-project // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { ("x86", "sse4.2") => { - if get_version() >= (14, 0, 0) { - smallvec!["sse4.2", "crc32"] - } else { - smallvec!["sse4.2"] - } + LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32")) } - ("x86", "pclmulqdq") => smallvec!["pclmul"], - ("x86", "rdrand") => smallvec!["rdrnd"], - ("x86", "bmi1") => smallvec!["bmi"], - ("x86", "cmpxchg16b") => smallvec!["cx16"], - ("x86", "avx512vaes") => smallvec!["vaes"], - ("x86", "avx512gfni") => smallvec!["gfni"], - ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], - ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], - ("aarch64", "dpb") => smallvec!["ccpp"], - ("aarch64", "dpb2") => smallvec!["ccdp"], - ("aarch64", "frintts") => smallvec!["fptoint"], - ("aarch64", "fcma") => smallvec!["complxnum"], - ("aarch64", "pmuv3") => smallvec!["perfmon"], - ("aarch64", "paca") => smallvec!["pauth"], - ("aarch64", "pacg") => smallvec!["pauth"], - // Rust ties fp and neon together. In LLVM neon implicitly enables fp, - // but we manually enable neon when a feature only implicitly enables fp - ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], - ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], - ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], - ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], - ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], - ("aarch64", "sve") => smallvec!["sve", "neon"], - ("aarch64", "sve2") => smallvec!["sve2", "neon"], - ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], - ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], - ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], - ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], - (_, s) => smallvec![s], + ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"), + ("x86", "rdrand") => LLVMFeature::new("rdrnd"), + ("x86", "bmi1") => LLVMFeature::new("bmi"), + ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), + ("aarch64", "dpb") => LLVMFeature::new("ccpp"), + ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), + ("aarch64", "frintts") => LLVMFeature::new("fptoint"), + ("aarch64", "fcma") => LLVMFeature::new("complxnum"), + ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"), + ("aarch64", "paca") => LLVMFeature::new("pauth"), + ("aarch64", "pacg") => LLVMFeature::new("pauth"), + // Rust ties fp and neon together. + ("aarch64", "neon") => { + LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")) + } + // In LLVM neon implicitly enables fp, but we manually enable + // neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => { + LLVMFeature::with_dependency("f32mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "f64mm") => { + LLVMFeature::with_dependency("f64mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fhm") => { + LLVMFeature::with_dependency("fp16fml", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fp16") => { + LLVMFeature::with_dependency("fullfp16", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "jsconv") => { + LLVMFeature::with_dependency("jsconv", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve") => { + LLVMFeature::with_dependency("sve", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2") => { + LLVMFeature::with_dependency("sve2", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-aes") => { + LLVMFeature::with_dependency("sve2-aes", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sm4") => { + LLVMFeature::with_dependency("sve2-sm4", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sha3") => { + LLVMFeature::with_dependency("sve2-sha3", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-bitperm") => LLVMFeature::with_dependency( + "sve2-bitperm", + TargetFeatureFoldStrength::EnableOnly("neon"), + ), + (_, s) => LLVMFeature::new(s), } } -// Given a map from target_features to whether they are enabled or disabled, -// ensure only valid combinations are allowed. +/// Given a map from target_features to whether they are enabled or disabled, +/// ensure only valid combinations are allowed. pub fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, @@ -231,11 +281,11 @@ pub fn check_tied_features( return None; } -// Used to generate cfg variables and apply features -// Must express features in the way Rust understands them +/// Used to generate cfg variables and apply features +/// Must express features in the way Rust understands them pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); - let mut features: Vec<Symbol> = supported_target_features(sess) + supported_target_features(sess) .iter() .filter_map(|&(feature, gate)| { if sess.is_nightly_build() || allow_unstable || gate.is_none() { @@ -255,16 +305,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { true }) .map(|feature| Symbol::intern(feature)) - .collect(); - - // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 - // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use - // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI. - // The target feature can be dropped once we no longer support older LLVM versions. - if sess.is_nightly_build() && get_version() >= (14, 0, 0) { - features.push(Symbol::intern("llvm14-builtins-abi")); - } - features + .collect() } pub fn print_version() { @@ -310,30 +351,32 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { } fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { - let mut target_features = llvm_target_features(tm); + let mut llvm_target_features = llvm_target_features(tm); + let mut known_llvm_target_features = FxHashSet::<&'static str>::default(); let mut rustc_target_features = supported_target_features(sess) .iter() - .filter_map(|(feature, _gate)| { - for llvm_feature in to_llvm_features(sess, *feature) { - // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. - match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok().map( - |index| { - let (_f, desc) = target_features.remove(index); - (*feature, desc) - }, - ) { - Some(v) => return Some(v), - None => {} - } - } - None + .map(|(feature, _gate)| { + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name; + let desc = + match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() { + Some(index) => { + known_llvm_target_features.insert(llvm_feature); + llvm_target_features[index].1 + } + None => "", + }; + + (*feature, desc) }) .collect::<Vec<_>>(); rustc_target_features.extend_from_slice(&[( "crt-static", "Enables C Run-time Libraries to be statically linked", )]); - let max_feature_len = target_features + llvm_target_features.retain(|(f, _d)| !known_llvm_target_features.contains(f)); + + let max_feature_len = llvm_target_features .iter() .chain(rustc_target_features.iter()) .map(|(feature, _desc)| feature.len()) @@ -345,10 +388,10 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { println!(" {1:0$} - {2}.", max_feature_len, feature, desc); } println!("\nCode-generation features supported by LLVM for this target:"); - for (feature, desc) in &target_features { + for (feature, desc) in &llvm_target_features { println!(" {1:0$} - {2}.", max_feature_len, feature, desc); } - if target_features.is_empty() { + if llvm_target_features.is_empty() { println!(" Target features listing is not supported by this LLVM version."); } println!("\nUse +feature to enable a feature, or -feature to disable it."); @@ -361,7 +404,14 @@ pub(crate) fn print(req: PrintRequest, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess); match req { - PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, + PrintRequest::TargetCPUs => { + // SAFETY generate a C compatible string from a byte slice to pass + // the target CPU name into LLVM, the lifetime of the reference is + // at least as long as the C function + let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) + .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); + unsafe { llvm::LLVMRustPrintTargetCPUs(tm, cpu_cstring.as_ptr()) }; + } PrintRequest::TargetFeatures => print_target_features(sess, tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } @@ -440,6 +490,8 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + // Drop +atomics-32 feature introduced in LLVM 15. + .filter(|v| *v != "+atomics-32" || get_version() >= (15, 0, 0)) .map(String::from), ); @@ -454,15 +506,10 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .filter_map(|s| { let enable_disable = match s.chars().next() { None => return None, - Some(c @ '+' | c @ '-') => c, + Some(c @ ('+' | '-')) => c, Some(_) => { if diagnostics { - let mut diag = sess.struct_warn(&format!( - "unknown feature specified for `-Ctarget-feature`: `{}`", - s - )); - diag.note("features must begin with a `+` to enable or `-` to disable it"); - diag.emit(); + sess.emit_warning(UnknownCTargetFeaturePrefix { feature: s }); } return None; } @@ -479,17 +526,15 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str None } }); - let mut diag = sess.struct_warn(&format!( - "unknown feature specified for `-Ctarget-feature`: `{}`", - feature - )); - diag.note("it is still passed through to the codegen backend"); - if let Some(rust_feature) = rust_feature { - diag.help(&format!("you might have meant: `{}`", rust_feature)); + let unknown_feature = if let Some(rust_feature) = rust_feature { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::Some { rust_feature }, + } } else { - diag.note("consider filing a feature request"); - } - diag.emit(); + UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } + }; + sess.emit_warning(unknown_feature); } if diagnostics { @@ -505,20 +550,30 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. + let llvm_feature = to_llvm_features(sess, feature); + Some( - to_llvm_features(sess, feature) - .into_iter() - .map(move |f| format!("{}{}", enable_disable, f)), + std::iter::once(format!("{}{}", enable_disable, llvm_feature.llvm_feature_name)) + .chain(llvm_feature.dependency.into_iter().filter_map(move |feat| { + match (enable_disable, feat) { + ('-' | '+', TargetFeatureFoldStrength::Both(f)) + | ('+', TargetFeatureFoldStrength::EnableOnly(f)) => { + Some(format!("{}{}", enable_disable, f)) + } + _ => None, + } + })), ) }) .flatten(); features.extend(feats); if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); + sess.emit_err(TargetFeatureDisableOrEnable { + features: f, + span: None, + missing_features: None, + }); } features @@ -544,19 +599,3 @@ pub fn tune_cpu(sess: &Session) -> Option<&str> { let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } - -pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_arch: &str) -> bool { - // The new pass manager is enabled by default for LLVM >= 13. - // This matches Clang, which also enables it since Clang 13. - - // Since LLVM 15, the legacy pass manager is no longer supported. - if llvm_util::get_version() >= (15, 0, 0) { - return true; - } - - // There are some perf issues with the new pass manager when targeting - // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. - // See https://github.com/rust-lang/rust/issues/89609. - let min_version = if target_arch == "s390x" { 14 } else { 13 }; - user_opt.unwrap_or_else(|| llvm_util::get_version() >= (min_version, 0, 0)) -} diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 6e94284852f..c24854b27a0 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,6 +1,7 @@ use crate::attributes; use crate::base; use crate::context::CodegenCx; +use crate::errors::SymbolAlreadyDefined; use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; @@ -8,10 +9,9 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; -use rustc_middle::ty::{self, Instance, TypeVisitable}; +use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; use rustc_target::spec::RelocModel; -use tracing::debug; impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( @@ -26,10 +26,8 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { let llty = self.layout_of(ty).llvm_type(self); let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { - self.sess().span_fatal( - self.tcx.def_span(def_id), - &format!("symbol `{}` is already defined", symbol_name), - ) + self.sess() + .emit_fatal(SymbolAlreadyDefined { span: self.tcx.def_span(def_id), symbol_name }) }); unsafe { @@ -50,10 +48,10 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { visibility: Visibility, symbol_name: &str, ) { - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let lldecl = self.declare_fn(symbol_name, fn_abi); + let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); @@ -127,8 +125,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) - .unwrap_or(false); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index cf2d3c423c3..4ffa2b9c6a3 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -19,7 +19,7 @@ use rustc_target::abi::{AddressSpace, Align, Integer, Size}; use std::fmt; use std::ptr; -use libc::c_uint; +use libc::{c_char, c_uint}; impl PartialEq for Type { fn eq(&self, other: &Self) -> bool { @@ -53,7 +53,7 @@ impl<'ll> CodegenCx<'ll, '_> { } pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 @@ -127,10 +127,6 @@ impl<'ll> CodegenCx<'ll, '_> { pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } } - - pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { - unsafe { llvm::LLVMRustArrayType(ty, len) } - } } impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { @@ -231,6 +227,10 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn val_ty(&self, v: &'ll Value) -> &'ll Type { common::val_ty(v) } + + fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMRustArrayType(ty, len) } + } } impl Type { @@ -238,7 +238,7 @@ impl Type { unsafe { llvm::LLVMInt8TypeInContext(llcx) } } - // Creates an integer type with the given number of bits, e.g., i24 + /// Creates an integer type with the given number of bits, e.g., i24 pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } } @@ -288,4 +288,81 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { ty.llvm_type(self) } + fn scalar_copy_backend_type(&self, layout: TyAndLayout<'tcx>) -> Option<Self::Type> { + layout.scalar_copy_llvm_type(self) + } +} + +impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn add_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> { + Some(unsafe { + llvm::LLVMMDStringInContext( + self.llcx, + typeid.as_ptr() as *const c_char, + typeid.len() as c_uint, + ) + }) + } + + fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) + } + } + + fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) + } + } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 9f0e6c80b19..3339e4e07ed 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,17 +1,16 @@ use crate::common::*; use crate::context::TypeLowering; -use crate::llvm_util::get_version; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Ty, TypeVisitable}; -use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; +use rustc_middle::ty::{self, Ty, TypeVisitableExt}; +use rustc_target::abi::HasDataLayout; +use rustc_target::abi::{Abi, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; use smallvec::{smallvec, SmallVec}; -use tracing::debug; use std::fmt::Write; @@ -44,10 +43,8 @@ fn uncached_llvm_type<'a, 'tcx>( // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str - // For performance reasons we use names only when emitting LLVM IR. Unless we are on - // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213, - // #79564, and #79246. - if get_version() < (14, 0, 0) || !cx.sess().fewer_names() => + // For performance reasons we use names only when emitting LLVM IR. + if !cx.sess().fewer_names() => { let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string())); if let (&ty::Adt(def, _), &Variants::Single { index }) = @@ -141,7 +138,7 @@ fn struct_llfields<'a, 'tcx>( prev_effective_align = effective_field_align; } let padding_used = result.len() > field_count; - if !layout.is_unsized() && field_count > 0 { + if layout.is_sized() && field_count > 0 { if offset > layout.size { bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); } @@ -158,7 +155,7 @@ fn struct_llfields<'a, 'tcx>( } else { debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); } - let field_remapping = if padding_used { Some(field_remapping) } else { None }; + let field_remapping = padding_used.then_some(field_remapping); (result, packed, field_remapping) } @@ -196,14 +193,14 @@ pub trait LayoutLlvmExt<'tcx> { ) -> &'a Type; fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo>; + fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type>; } impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { match self.abi { Abi::Scalar(_) | Abi::Vector { .. } => true, - Abi::ScalarPair(..) => false, - Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(), + Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, } } @@ -313,14 +310,13 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { Int(i, _) => cx.type_from_integer(i), F32 => cx.type_f32(), F64 => cx.type_f64(), - Pointer => { + Pointer(address_space) => { // If we know the alignment, pick something better than i8. - let (pointee, address_space) = - if let Some(pointee) = self.pointee_info_at(cx, offset) { - (cx.type_pointee_for_align(pointee.align), pointee.address_space) - } else { - (cx.type_i8(), AddressSpace::DATA) - }; + let pointee = if let Some(pointee) = self.pointee_info_at(cx, offset) { + cx.type_pointee_for_align(pointee.align) + } else { + cx.type_i8() + }; cx.type_ptr_to_ext(pointee, address_space) } } @@ -334,7 +330,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { ) -> &'a Type { // HACK(eddyb) special-case fat pointers until LLVM removes // pointee types, to avoid bitcasting every `OperandRef::deref`. - match self.ty.kind() { + match *self.ty.kind() { ty::Ref(..) | ty::RawPtr(_) => { return self.field(cx, index).llvm_type(cx); } @@ -344,6 +340,11 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); } + // `dyn* Trait` has the same ABI as `*mut dyn Trait` + ty::Dynamic(bounds, region, ty::DynStar) => { + let ptr_ty = cx.tcx.mk_mut_ptr(cx.tcx.mk_dynamic(bounds, region, ty::Dyn)); + return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); + } _ => {} } @@ -353,10 +354,10 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { let scalar = [a, b][index]; // Make sure to return the same type `immediate_llvm_type` would when - // dealing with an immediate pair. This means that `(bool, bool)` is + // dealing with an immediate pair. This means that `(bool, bool)` is // effectively represented as `{i8, i8}` in memory and two `i1`s as an // immediate, just like `bool` is typically `i8` in memory and only `i1` - // when immediate. We need to load/store `bool` as `i8` to avoid + // when immediate. We need to load/store `bool` as `i8` to avoid // crippling LLVM optimizations or triggering other LLVM bugs with `i1`. if immediate && scalar.is_bool() { return cx.type_i1(); @@ -415,4 +416,35 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); result } + + fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type> { + debug_assert!(self.is_sized()); + + // FIXME: this is a fairly arbitrary choice, but 128 bits on WASM + // (matching the 128-bit SIMD types proposal) and 256 bits on x64 + // (like AVX2 registers) seems at least like a tolerable starting point. + let threshold = cx.data_layout().pointer_size * 4; + if self.layout.size() > threshold { + return None; + } + + // Vectors, even for non-power-of-two sizes, have the same layout as + // arrays but don't count as aggregate types + if let FieldsShape::Array { count, .. } = self.layout.fields() + && let element = self.field(cx, 0) + && element.ty.is_integral() + { + // `cx.type_ix(bits)` is tempting here, but while that works great + // for things that *stay* as memory-to-memory copies, it also ends + // up suppressing vectorization as it introduces shifts when it + // extracts all the individual values. + + let ety = element.llvm_type(cx); + return Some(cx.type_vector(ety, *count)); + } + + // FIXME: The above only handled integer arrays; surely more things + // would also be possible. Be careful about provenance, though! + None + } } diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ceb3d5a84ab..b19398e68c2 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -175,6 +175,89 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( val } +fn emit_s390x_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of the s390x ELF ABI calling convention for va_args see + // https://github.com/IBM/s390x-abi (chapter 1.2.4) + let va_list_addr = list.immediate(); + let va_list_layout = list.deref(bx.cx).layout; + let va_list_ty = va_list_layout.llvm_type(bx); + let layout = bx.cx.layout_of(target_ty); + + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let in_mem = bx.append_sibling_block("va_arg.in_mem"); + let end = bx.append_sibling_block("va_arg.end"); + + // FIXME: vector ABI not yet supported. + let target_ty_size = bx.cx.size_of(target_ty).bytes(); + let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); + let unpadded_size = if indirect { 8 } else { target_ty_size }; + let padded_size = 8; + let padding = padded_size - unpadded_size; + + let gpr_type = indirect || !layout.is_single_fp_element(bx.cx); + let (max_regs, reg_count_field, reg_save_index, reg_padding) = + if gpr_type { (5, 0, 2, padding) } else { (4, 1, 16, 0) }; + + // Check whether the value was passed in a register or in memory. + let reg_count = bx.struct_gep( + va_list_ty, + va_list_addr, + va_list_layout.llvm_field_index(bx.cx, reg_count_field), + ); + let reg_count_v = bx.load(bx.type_i64(), reg_count, Align::from_bytes(8).unwrap()); + let use_regs = bx.icmp(IntPredicate::IntULT, reg_count_v, bx.const_u64(max_regs)); + bx.cond_br(use_regs, in_reg, in_mem); + + // Emit code to load the value if it was passed in a register. + bx.switch_to_block(in_reg); + + // Work out the address of the value in the register save area. + let reg_ptr = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let reg_ptr_v = bx.load(bx.type_i8p(), reg_ptr, bx.tcx().data_layout.pointer_align.abi); + let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8)); + let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding)); + let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]); + + // Update the register count. + let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1)); + bx.store(new_reg_count_v, reg_count, Align::from_bytes(8).unwrap()); + bx.br(end); + + // Emit code to load the value if it was passed in memory. + bx.switch_to_block(in_mem); + + // Work out the address of the value in the argument overflow area. + let arg_ptr = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 2)); + let arg_ptr_v = bx.load(bx.type_i8p(), arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let arg_off = bx.const_u64(padding); + let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]); + + // Update the argument overflow area pointer. + let arg_size = bx.cx().const_u64(padded_size); + let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]); + bx.store(new_arg_ptr_v, arg_ptr, bx.tcx().data_layout.pointer_align.abi); + bx.br(end); + + // Return the appropriate result. + bx.switch_to_block(end); + let val_addr = bx.phi(bx.type_i8p(), &[reg_addr, mem_addr], &[in_reg, in_mem]); + let val_type = layout.llvm_type(bx); + let val_addr = if indirect { + let ptr_type = bx.cx.type_ptr_to(val_type); + let ptr_addr = bx.bitcast(val_addr, bx.cx.type_ptr_to(ptr_type)); + bx.load(ptr_type, ptr_addr, bx.tcx().data_layout.pointer_align.abi) + } else { + bx.bitcast(val_addr, bx.cx.type_ptr_to(val_type)) + }; + bx.load(val_type, val_addr, layout.align.abi) +} + pub(super) fn emit_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, addr: OperandRef<'tcx, &'ll Value>, @@ -200,6 +283,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) } "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + "s390x" => emit_s390x_va_arg(bx, addr, target_ty), // Windows x86_64 "x86_64" if target.is_like_windows => { let target_ty_size = bx.cx.size_of(target_ty).bytes(); |