diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
38 files changed, 565 insertions, 517 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 1277b7898c2..dea574a53cd 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -6,13 +6,13 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_middle::ty::layout::LayoutOf; -pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; +pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; use rustc_middle::ty::Ty; use rustc_middle::{bug, ty}; use rustc_session::config; -pub use rustc_target::abi::call::*; +pub(crate) use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int, Size}; -pub use rustc_target::spec::abi::Abi; +pub(crate) use rustc_target::spec::abi::Abi; use rustc_target::spec::SanitizerSet; use smallvec::SmallVec; @@ -25,7 +25,7 @@ use crate::type_of::LayoutLlvmExt; use crate::value::Value; use crate::{attributes, llvm_util}; -pub trait ArgAttributesExt { +trait ArgAttributesExt { fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); fn apply_attrs_to_callsite( &self, @@ -111,7 +111,7 @@ impl ArgAttributesExt for ArgAttributes { } } -pub trait LlvmType { +pub(crate) trait LlvmType { fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; } @@ -171,7 +171,7 @@ impl LlvmType for CastTarget { } } -pub trait ArgAbiExt<'ll, 'tcx> { +trait ArgAbiExt<'ll, 'tcx> { fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn store( &self, @@ -307,7 +307,7 @@ impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -pub trait FnAbiLlvmExt<'ll, 'tcx> { +pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn llvm_cconv(&self) -> llvm::CallConv; @@ -430,9 +430,32 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; + + let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| { + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (19, 0, 0) + && matches!(scalar.primitive(), Int(..)) + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + && !scalar.is_bool() + // LLVM also rejects full range. + && !scalar.is_always_valid(cx) + { + attributes::apply_to_llfn( + llfn, + idx, + &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))], + ); + } + }; + match &self.ret.mode { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); + } } PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); @@ -471,8 +494,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(attrs) - | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) => { + let i = apply(attrs); + if let abi::Abi::Scalar(scalar) = arg.layout.abi { + apply_range_attr(llvm::AttributePlace::Argument(i), scalar); + } + } + PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { apply(attrs); } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { @@ -481,8 +509,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { apply(meta_attrs); } PassMode::Pair(a, b) => { - apply(a); - apply(b); + let i = apply(a); + let ii = apply(b); + if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi { + apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a); + apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b); + } } PassMode::Cast { cast, pad_i32 } => { if *pad_i32 { @@ -537,15 +569,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } _ => {} } - if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { - // If the value is a boolean, the range is 0..2 and that ultimately - // become 0..0 when the type becomes i1, which would be rejected - // by the LLVM verifier. - if let Int(..) = scalar.primitive() { - if !scalar.is_bool() && !scalar.is_always_valid(bx) { - bx.range_metadata(callsite, scalar.valid_range(bx)); - } - } + if bx.cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() < (19, 0, 0) + && let abi::Abi::Scalar(scalar) = self.ret.layout.abi + && matches!(scalar.primitive(), Int(..)) + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + && !scalar.is_bool() + // LLVM also rejects full range. + && !scalar.is_always_valid(bx) + { + bx.range_metadata(callsite, scalar.valid_range(bx)); } for arg in self.args.iter() { match &arg.mode { diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 8fb31082793..b4f3784a31a 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -149,7 +149,7 @@ fn create_wrapper_function( } llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr()); let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index aea8395441a..1d91c3fb17d 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -1,3 +1,5 @@ +use std::assert_matches::assert_matches; + use libc::{c_char, c_uint}; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::mir::operand::OperandValue; @@ -89,7 +91,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // if the target feature needed by the register class is // disabled. This is necessary otherwise LLVM will try // to actually allocate a register for the dummy output. - assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); + assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_)); clobbers.push(format!("~{}", reg_to_llvm(reg, None))); continue; } else { @@ -911,8 +913,10 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty Primitive::Int(Integer::I16, _) => cx.type_i16(), Primitive::Int(Integer::I32, _) => cx.type_i32(), Primitive::Int(Integer::I64, _) => cx.type_i64(), + Primitive::Float(Float::F16) => cx.type_f16(), Primitive::Float(Float::F32) => cx.type_f32(), Primitive::Float(Float::F64) => cx.type_f64(), + Primitive::Float(Float::F128) => cx.type_f128(), // FIXME(erikdesjardins): handle non-default addrspace ptr sizes Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()), _ => unreachable!(), @@ -946,7 +950,9 @@ fn llvm_fixup_input<'ll, 'tcx>( value } } - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + if s.primitive() != Primitive::Float(Float::F128) => + { let elem_ty = llvm_asm_scalar_type(bx.cx, s); let count = 16 / layout.size.bytes(); let vec_ty = bx.cx.type_vector(elem_ty, count); @@ -1088,7 +1094,9 @@ fn llvm_fixup_output<'ll, 'tcx>( value } } - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + if s.primitive() != Primitive::Float(Float::F128) => + { value = bx.extract_element(value, bx.const_i32(0)); if let Primitive::Pointer(_) = s.primitive() { value = bx.inttoptr(value, layout.llvm_type(bx.cx)); @@ -1220,7 +1228,9 @@ fn llvm_fixup_output_type<'ll, 'tcx>( layout.llvm_type(cx) } } - (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) + if s.primitive() != Primitive::Float(Float::F128) => + { let elem_ty = llvm_asm_scalar_type(cx, s); let count = 16 / layout.size.bytes(); cx.type_vector(elem_ty, count) diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index fde95104093..92a857c2adc 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,6 +1,6 @@ //! Set and unset common attributes on LLVM values. -pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; +use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; @@ -17,13 +17,13 @@ use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace use crate::value::Value; use crate::{attributes, llvm_util}; -pub fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { +pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { llvm::AddFunctionAttributes(llfn, idx, attrs); } } -pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { +pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { llvm::AddCallSiteAttributes(callsite, idx, attrs); } @@ -80,7 +80,7 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub fn sanitize_attrs<'ll>( +pub(crate) fn sanitize_attrs<'ll>( cx: &CodegenCx<'ll, '_>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { @@ -120,7 +120,7 @@ pub fn sanitize_attrs<'ll>( /// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. #[inline] -pub fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) -> &Attribute { +pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) -> &Attribute { // NOTE: We should determine if we even need async unwind tables, as they // take have more overhead and if we can use sync unwind tables we // probably should. @@ -128,7 +128,7 @@ pub fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) -> &Att llvm::CreateUWTableAttr(llcx, async_unwind) } -pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { let mut fp = cx.sess().target.frame_pointer; let opts = &cx.sess().opts; // "mcount" function relies on stack pointer. @@ -280,19 +280,19 @@ fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { +pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { let target_cpu = llvm_util::target_cpu(cx.tcx.sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { llvm_util::tune_cpu(cx.tcx.sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary if !cx.sess().needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) @@ -327,7 +327,7 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Helper for `FnAbi::apply_attrs_llfn`: /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. -pub fn llfn_attrs_from_instance<'ll, 'tcx>( +pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::Instance<'tcx>, @@ -521,13 +521,20 @@ pub fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = function_features .iter() - .flat_map(|feat| { - llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{f}")) - }) + // Convert to LLVMFeatures and filter out unavailable ones + .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + // Convert LLVMFeatures & dependencies to +<feats>s + .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), })) + // HACK: LLVM versions 19+ do not have the FPMR feature and treat it as always enabled + // It only exists as a feature in LLVM 18, cannot be passed down for any other version + .chain(match &*cx.tcx.sess.target.arch { + "aarch64" if llvm_util::get_version().0 == 18 => vec!["+fpmr".to_string()], + _ => vec![], + }) .collect::<Vec<String>>(); if cx.tcx.sess.target.is_like_wasm { diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index a2ab19ac800..4f2c83634a8 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -5,17 +5,13 @@ use std::path::{Path, PathBuf}; use std::{io, mem, ptr, str}; use rustc_codegen_ssa::back::archive::{ - create_mingw_dll_import_lib, try_extract_macho_fat_archive, ArArchiveBuilder, - ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, ObjectReader, UnknownArchiveKind, - DEFAULT_OBJECT_READER, + try_extract_macho_fat_archive, ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, + ArchiveBuilderBuilder, ObjectReader, UnknownArchiveKind, DEFAULT_OBJECT_READER, }; -use rustc_codegen_ssa::common; use rustc_session::Session; -use tracing::trace; -use crate::errors::ErrorCreatingImportLibrary; use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; +use crate::llvm::{self, ArchiveKind}; /// Helper for adding many files to an archive. #[must_use = "must call build() to finish building the archive"] @@ -44,18 +40,6 @@ fn is_relevant_child(c: &Child<'_>) -> bool { } } -/// Map machine type strings to values of LLVM's MachineTypes enum. -fn llvm_machine_type(cpu: &str) -> LLVMMachineType { - match cpu { - "x86_64" => LLVMMachineType::AMD64, - "x86" => LLVMMachineType::I386, - "aarch64" => LLVMMachineType::ARM64, - "arm64ec" => LLVMMachineType::ARM64EC, - "arm" => LLVMMachineType::ARM, - _ => panic!("unsupported cpu type {cpu}"), - } -} - impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { fn add_archive( &mut self, @@ -102,90 +86,20 @@ impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { } } -pub struct LlvmArchiveBuilderBuilder; +pub(crate) struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { - // FIXME use ArArchiveBuilder on most targets again once reading thin archives is - // implemented - if true { + // Keeping LlvmArchiveBuilder around in case of a regression caused by using + // ArArchiveBuilder. + // FIXME(#128955) remove a couple of months after #128936 gets merged in case + // no regression is found. + if false { Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) } else { Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) } } - - fn create_dll_import_lib( - &self, - sess: &Session, - lib_name: &str, - import_name_and_ordinal_vector: Vec<(String, Option<u16>)>, - output_path: &Path, - ) { - if common::is_mingw_gnu_toolchain(&sess.target) { - // The binutils linker used on -windows-gnu targets cannot read the import - // libraries generated by LLVM: in our attempts, the linker produced an .EXE - // that loaded but crashed with an AV upon calling one of the imported - // functions. Therefore, use binutils to create the import library instead, - // by writing a .DEF file to the temp dir and calling binutils's dlltool. - create_mingw_dll_import_lib( - sess, - lib_name, - import_name_and_ordinal_vector, - output_path, - ); - } else { - // we've checked for \0 characters in the library name already - let dll_name_z = CString::new(lib_name).unwrap(); - - let output_path_z = rustc_fs_util::path_to_c_string(&output_path); - - trace!("invoking LLVMRustWriteImportLibrary"); - trace!(" dll_name {:#?}", dll_name_z); - trace!(" output_path {}", output_path.display()); - trace!( - " import names: {}", - import_name_and_ordinal_vector - .iter() - .map(|(name, _ordinal)| name.clone()) - .collect::<Vec<_>>() - .join(", "), - ); - - // All import names are Rust identifiers and therefore cannot contain \0 characters. - // FIXME: when support for #[link_name] is implemented, ensure that the import names - // still don't contain any \0 characters. Also need to check that the names don't - // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved - // in definition files. - let cstring_import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = - import_name_and_ordinal_vector - .into_iter() - .map(|(name, ordinal)| (CString::new(name).unwrap(), ordinal)) - .collect(); - - let ffi_exports: Vec<LLVMRustCOFFShortExport> = cstring_import_name_and_ordinal_vector - .iter() - .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) - .collect(); - let result = unsafe { - crate::llvm::LLVMRustWriteImportLibrary( - dll_name_z.as_ptr(), - output_path_z.as_ptr(), - ffi_exports.as_ptr(), - ffi_exports.len(), - llvm_machine_type(&sess.target.arch) as u16, - !sess.target.is_like_msvc, - ) - }; - - if result == crate::llvm::LLVMRustResult::Failure { - sess.dcx().emit_fatal(ErrorCreatingImportLibrary { - lib_name, - error: llvm::last_error().unwrap_or("unknown LLVM error".to_string()), - }); - } - } - } } // The object crate doesn't know how to get symbols for LLVM bitcode and COFF bigobj files. @@ -198,25 +112,11 @@ static LLVM_OBJECT_READER: ObjectReader = ObjectReader { get_xcoff_member_alignment: DEFAULT_OBJECT_READER.get_xcoff_member_alignment, }; -fn should_use_llvm_reader(buf: &[u8]) -> bool { - let is_bitcode = unsafe { llvm::LLVMRustIsBitcode(buf.as_ptr(), buf.len()) }; - - // COFF bigobj file, msvc LTO file or import library. See - // https://github.com/llvm/llvm-project/blob/453f27bc9/llvm/lib/BinaryFormat/Magic.cpp#L38-L51 - let is_unsupported_windows_obj_file = buf.get(0..4) == Some(b"\0\0\xFF\xFF"); - - is_bitcode || is_unsupported_windows_obj_file -} - #[deny(unsafe_op_in_unsafe_fn)] fn get_llvm_object_symbols( buf: &[u8], f: &mut dyn FnMut(&[u8]) -> io::Result<()>, ) -> io::Result<bool> { - if !should_use_llvm_reader(buf) { - return (DEFAULT_OBJECT_READER.get_symbols)(buf, f); - } - let mut state = Box::new(f); let err = unsafe { @@ -253,18 +153,10 @@ fn get_llvm_object_symbols( } fn llvm_is_64_bit_object_file(buf: &[u8]) -> bool { - if !should_use_llvm_reader(buf) { - return (DEFAULT_OBJECT_READER.is_64_bit_object_file)(buf); - } - unsafe { llvm::LLVMRustIs64BitSymbolicFile(buf.as_ptr(), buf.len()) } } fn llvm_is_ec_object_file(buf: &[u8]) -> bool { - if !should_use_llvm_reader(buf) { - return (DEFAULT_OBJECT_READER.is_ec_object_file)(buf); - } - unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index f68155f523a..aa6842c75ce 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -33,9 +33,9 @@ use crate::{LlvmCodegenBackend, ModuleLlvm}; /// We keep track of the computed LTO cache keys from the previous /// session to determine which CGUs we can reuse. -pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; +const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; -pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { +fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { CrateType::Executable | CrateType::Dylib @@ -616,7 +616,7 @@ pub(crate) fn run_pass_manager( llvm::LLVMRustAddModuleFlagU32( module.module_llvm.llmod(), llvm::LLVMModFlagBehavior::Error, - c"LTOPostLink".as_ptr().cast(), + c"LTOPostLink".as_ptr(), 1, ); } @@ -710,7 +710,7 @@ impl Drop for ThinBuffer { } } -pub unsafe fn optimize_thin_module( +pub(crate) unsafe fn optimize_thin_module( thin_module: ThinModule<LlvmCodegenBackend>, cgcx: &CodegenContext<LlvmCodegenBackend>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { @@ -806,7 +806,7 @@ pub unsafe fn optimize_thin_module( /// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys #[derive(Debug, Default)] -pub struct ThinLTOKeysMap { +struct ThinLTOKeysMap { // key = llvm name of importing module, value = LLVM cache key keys: BTreeMap<String, String>, } @@ -863,7 +863,7 @@ fn module_name_to_str(c_str: &CStr) -> &str { }) } -pub fn parse_module<'a>( +pub(crate) fn parse_module<'a>( cx: &'a llvm::Context, name: &CStr, data: &[u8], diff --git a/compiler/rustc_codegen_llvm/src/back/profiling.rs b/compiler/rustc_codegen_llvm/src/back/profiling.rs index 26fb4a96f84..79794775b7b 100644 --- a/compiler/rustc_codegen_llvm/src/back/profiling.rs +++ b/compiler/rustc_codegen_llvm/src/back/profiling.rs @@ -21,14 +21,14 @@ fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &st EventId::from_label(profiler.alloc_string(components.as_slice())) } -pub struct LlvmSelfProfiler<'a> { +pub(crate) struct LlvmSelfProfiler<'a> { profiler: Arc<SelfProfiler>, stack: Vec<TimingGuard<'a>>, llvm_pass_event_kind: StringId, } impl<'a> LlvmSelfProfiler<'a> { - pub fn new(profiler: Arc<SelfProfiler>) -> Self { + pub(crate) fn new(profiler: Arc<SelfProfiler>) -> Self { let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass"); Self { profiler, stack: Vec::default(), llvm_pass_event_kind } } @@ -43,7 +43,7 @@ impl<'a> LlvmSelfProfiler<'a> { } } -pub unsafe extern "C" fn selfprofile_before_pass_callback( +pub(crate) unsafe extern "C" fn selfprofile_before_pass_callback( llvm_self_profiler: *mut c_void, pass_name: *const c_char, ir_name: *const c_char, @@ -56,7 +56,7 @@ pub unsafe extern "C" fn selfprofile_before_pass_callback( } } -pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) { +pub(crate) unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) { let llvm_self_profiler = unsafe { &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>) }; llvm_self_profiler.after_pass_callback(); } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index f747f313926..a5c27d2282e 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -43,14 +43,14 @@ use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::type_::Type; use crate::{base, common, llvm_util, LlvmCodegenBackend, ModuleLlvm}; -pub fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError { +pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError { match llvm::last_error() { Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)), None => dcx.emit_almost_fatal(err), } } -pub fn write_output_file<'ll>( +fn write_output_file<'ll>( dcx: DiagCtxtHandle<'_>, target: &'ll llvm::TargetMachine, pm: &llvm::PassManager<'ll>, @@ -95,7 +95,7 @@ pub fn write_output_file<'ll>( } } -pub fn create_informational_target_machine( +pub(crate) fn create_informational_target_machine( sess: &Session, only_base_features: bool, ) -> OwnedTargetMachine { @@ -107,7 +107,7 @@ pub fn create_informational_target_machine( .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise()) } -pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { +pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { tcx.output_filenames(()).split_dwarf_path( tcx.sess.split_debuginfo(), @@ -130,9 +130,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMach .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise()) } -pub fn to_llvm_opt_settings( - cfg: config::OptLevel, -) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { +fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { use self::config::OptLevel::*; match cfg { No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), @@ -179,7 +177,7 @@ pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeMod } } -pub fn target_machine_factory( +pub(crate) fn target_machine_factory( sess: &Session, optlvl: config::OptLevel, target_features: &[String], @@ -320,7 +318,7 @@ pub(crate) fn save_temp_bitcode( } /// In what context is a dignostic handler being attached to a codegen unit? -pub enum CodegenDiagnosticsStage { +pub(crate) enum CodegenDiagnosticsStage { /// Prelink optimization stage. Opt, /// LTO/ThinLTO postlink optimization stage. @@ -329,14 +327,14 @@ pub enum CodegenDiagnosticsStage { Codegen, } -pub struct DiagnosticHandlers<'a> { +pub(crate) struct DiagnosticHandlers<'a> { data: *mut (&'a CodegenContext<LlvmCodegenBackend>, DiagCtxtHandle<'a>), llcx: &'a llvm::Context, old_handler: Option<&'a llvm::DiagnosticHandler>, } impl<'a> DiagnosticHandlers<'a> { - pub fn new( + pub(crate) fn new( cgcx: &'a CodegenContext<LlvmCodegenBackend>, dcx: DiagCtxtHandle<'a>, llcx: &'a llvm::Context, @@ -1032,7 +1030,7 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - c"rustc.embedded.module".as_ptr().cast(), + c"rustc.embedded.module".as_ptr(), ); llvm::LLVMSetInitializer(llglobal, llconst); @@ -1045,7 +1043,7 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - c"rustc.embedded.cmdline".as_ptr().cast(), + c"rustc.embedded.cmdline".as_ptr(), ); llvm::LLVMSetInitializer(llglobal, llconst); let section = if is_apple { @@ -1055,7 +1053,7 @@ unsafe fn embed_bitcode( } else { c".llvmcmd" }; - llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); + llvm::LLVMSetSection(llglobal, section.as_ptr()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. @@ -1108,7 +1106,7 @@ fn create_msvc_imps( .collect::<Vec<_>>(); for (imp_name, val) in globals { - let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr().cast()); + let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr()); llvm::LLVMSetInitializer(imp, val); llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index e8236b45c89..0ba8d82406a 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -32,7 +32,7 @@ use crate::context::CodegenCx; use crate::value::Value; use crate::{attributes, llvm}; -pub struct ValueIter<'ll> { +pub(crate) struct ValueIter<'ll> { cur: Option<&'ll Value>, step: unsafe extern "C" fn(&'ll Value) -> Option<&'ll Value>, } @@ -49,11 +49,14 @@ impl<'ll> Iterator for ValueIter<'ll> { } } -pub fn iter_globals(llmod: &llvm::Module) -> ValueIter<'_> { +pub(crate) fn iter_globals(llmod: &llvm::Module) -> ValueIter<'_> { unsafe { ValueIter { cur: llvm::LLVMGetFirstGlobal(llmod), step: llvm::LLVMGetNextGlobal } } } -pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen<ModuleLlvm>, u64) { +pub(crate) fn compile_codegen_unit( + tcx: TyCtxt<'_>, + cgu_name: Symbol, +) -> (ModuleCodegen<ModuleLlvm>, u64) { let start_time = Instant::now(); let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); @@ -140,7 +143,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen (module, cost) } -pub fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { +pub(crate) fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { let Some(sect) = attrs.link_section else { return }; unsafe { let buf = SmallCStr::new(sect.as_str()); @@ -148,7 +151,7 @@ pub fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { } } -pub fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { +pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { match linkage { Linkage::External => llvm::Linkage::ExternalLinkage, Linkage::AvailableExternally => llvm::Linkage::AvailableExternallyLinkage, @@ -164,7 +167,7 @@ pub fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { } } -pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { +pub(crate) fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { match linkage { Visibility::Default => llvm::Visibility::Default, Visibility::Hidden => llvm::Visibility::Hidden, diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0f44c5dba5e..40783825cae 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -36,7 +36,7 @@ use crate::{attributes, llvm_util}; // All Builders must have an llfn associated with them #[must_use] -pub struct Builder<'a, 'll, 'tcx> { +pub(crate) struct Builder<'a, 'll, 'tcx> { pub llbuilder: &'ll mut llvm::Builder<'ll>, pub cx: &'a CodegenCx<'ll, 'tcx>, } @@ -728,13 +728,32 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llvm::LLVMSetVolatile(store, llvm::True); } if flags.contains(MemFlags::NONTEMPORAL) { - // According to LLVM [1] building a nontemporal store must - // *always* point to a metadata value of the integer 1. - // - // [1]: https://llvm.org/docs/LangRef.html#store-instruction - let one = self.cx.const_i32(1); - let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); - llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); + // Make sure that the current target architectures supports "sane" non-temporal + // stores, i.e., non-temporal stores that are equivalent to regular stores except + // for performance. LLVM doesn't seem to care about this, and will happily treat + // `!nontemporal` stores as-if they were normal stores (for reordering optimizations + // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like + // regular stores but require special fences. + // So we keep a list of architectures where `!nontemporal` is known to be truly just + // a hint, and use regular stores everywhere else. + // (In the future, we could alternatively ensure that an sfence gets emitted after a sequence of movnt + // before any kind of synchronizing operation. But it's not clear how to do that with LLVM.) + // For more context, see <https://github.com/rust-lang/rust/issues/114582> and + // <https://github.com/llvm/llvm-project/issues/64521>. + const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] = + &["aarch64", "arm", "riscv32", "riscv64"]; + + let use_nontemporal = + WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch); + if use_nontemporal { + // According to LLVM [1] building a nontemporal store must + // *always* point to a metadata value of the integer 1. + // + // [1]: https://llvm.org/docs/LangRef.html#store-instruction + let one = self.cx.const_i32(1); + let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); + llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); + } } store } @@ -1324,7 +1343,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { Builder { llbuilder, cx } } - pub fn llfn(&self) -> &'ll Value { + pub(crate) fn llfn(&self) -> &'ll Value { unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } } @@ -1356,7 +1375,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - pub fn set_unpredictable(&mut self, inst: &'ll Value) { + pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) { unsafe { llvm::LLVMSetMetadata( inst, @@ -1366,15 +1385,15 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } - pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } } - pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) } } - pub fn insert_element( + pub(crate) fn insert_element( &mut self, vec: &'ll Value, elt: &'ll Value, @@ -1383,7 +1402,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) } } - pub fn shuffle_vector( + pub(crate) fn shuffle_vector( &mut self, v1: &'ll Value, v2: &'ll Value, @@ -1392,65 +1411,77 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) } } - pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) } } - pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_fadd_reassoc( + &mut self, + acc: &'ll Value, + src: &'ll Value, + ) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_fmul_reassoc( + &mut self, + acc: &'ll Value, + src: &'ll Value, + ) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) } } - pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) } } - pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) } } - pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) } } - pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) } } - pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { + pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { + pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } } - pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { + pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) } } - pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { + pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { unsafe { llvm::LLVMAddClause(landing_pad, clause); } } - pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { + pub(crate) fn catch_ret( + &mut self, + funclet: &Funclet<'ll>, + unwind: &'ll BasicBlock, + ) -> &'ll Value { let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; ret.expect("LLVM does not have support for catchret") } @@ -1496,7 +1527,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { Cow::Owned(casted_args) } - pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 663c5be46e5..949fd1bc124 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -20,7 +20,7 @@ use crate::value::Value; /// /// - `cx`: the crate context /// - `instance`: the instance to be instantiated -pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value { +pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value { let tcx = cx.tcx(); debug!("get_fn(instance={:?})", instance); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 65f974c5689..ef6560ecbe5 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -13,7 +13,7 @@ use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use tracing::debug; use crate::consts::const_alloc_to_llvm; -pub use crate::context::CodegenCx; +pub(crate) use crate::context::CodegenCx; use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True}; use crate::type_::Type; use crate::value::Value; @@ -58,21 +58,21 @@ use crate::value::Value; /// When inside of a landing pad, each function call in LLVM IR needs to be /// annotated with which landing pad it's a part of. This is accomplished via /// the `OperandBundleDef` value created for MSVC landing pads. -pub struct Funclet<'ll> { +pub(crate) struct Funclet<'ll> { cleanuppad: &'ll Value, operand: OperandBundleDef<'ll>, } impl<'ll> Funclet<'ll> { - pub fn new(cleanuppad: &'ll Value) -> Self { + pub(crate) fn new(cleanuppad: &'ll Value) -> Self { Funclet { cleanuppad, operand: OperandBundleDef::new("funclet", &[cleanuppad]) } } - pub fn cleanuppad(&self) -> &'ll Value { + pub(crate) fn cleanuppad(&self) -> &'ll Value { self.cleanuppad } - pub fn bundle(&self) -> &OperandBundleDef<'ll> { + pub(crate) fn bundle(&self) -> &OperandBundleDef<'ll> { &self.operand } } @@ -92,21 +92,16 @@ impl<'ll> BackendTypes for CodegenCx<'ll, '_> { } impl<'ll> CodegenCx<'ll, '_> { - pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { + pub(crate) fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { let len = u64::try_from(elts.len()).expect("LLVMConstArray2 elements len overflow"); unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } - pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); - unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } - } - - pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { + pub(crate) fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { bytes_in_context(self.llcx, bytes) } - pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { + pub(crate) fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); let r = llvm::LLVMGetAggregateElement(v, idx).unwrap(); @@ -221,6 +216,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { struct_in_context(self.llcx, elts, packed) } + fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } + } + fn const_to_opt_uint(&self, v: &'ll Value) -> Option<u64> { try_as_const_integral(v).and_then(|v| unsafe { let mut i = 0u64; @@ -339,18 +339,18 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } /// Get the [LLVM type][Type] of a [`Value`]. -pub fn val_ty(v: &Value) -> &Type { +pub(crate) fn val_ty(v: &Value) -> &Type { unsafe { llvm::LLVMTypeOf(v) } } -pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { +pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) } } -pub fn struct_in_context<'ll>( +fn struct_in_context<'ll>( llcx: &'ll llvm::Context, elts: &[&'ll Value], packed: bool, diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 75b298f14ca..b143e28c5f9 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -29,7 +29,7 @@ use crate::type_of::LayoutLlvmExt; use crate::value::Value; use crate::{base, debuginfo}; -pub fn const_alloc_to_llvm<'ll>( +pub(crate) fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>, is_static: bool, @@ -525,7 +525,7 @@ impl<'ll> CodegenCx<'ll, '_> { let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, - c"wasm.custom_sections".as_ptr().cast(), + c"wasm.custom_sections".as_ptr(), val, ); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index dd3f39ecead..01aae24ab56 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -11,6 +11,7 @@ use rustc_data_structures::base_n::{ToBaseN, ALPHANUMERIC_ONLY}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::PatchableFunctionEntry; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, @@ -39,7 +40,7 @@ use crate::{attributes, coverageinfo, debuginfo, llvm, llvm_util}; /// There is one `CodegenCx` per compilation unit. Each one has its own LLVM /// `llvm::Context` so that several compilation units may be optimized in parallel. /// All other LLVM data structures in the `CodegenCx` are tied to that `llvm::Context`. -pub struct CodegenCx<'ll, 'tcx> { +pub(crate) struct CodegenCx<'ll, 'tcx> { pub tcx: TyCtxt<'tcx>, pub use_dll_storage_attrs: bool, pub tls_model: llvm::ThreadLocalMode, @@ -110,7 +111,7 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { } } -pub unsafe fn create_module<'ll>( +pub(crate) unsafe fn create_module<'ll>( tcx: TyCtxt<'_>, llcx: &'ll llvm::Context, mod_name: &str, @@ -207,7 +208,7 @@ pub unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = c"RtLibUseGOT".as_ptr().cast(); + let avoid_plt = c"RtLibUseGOT".as_ptr(); unsafe { llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } @@ -215,7 +216,7 @@ pub unsafe fn create_module<'ll>( // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { - let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr().cast(); + let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr(); unsafe { llvm::LLVMRustAddModuleFlagU32( llmod, @@ -226,9 +227,23 @@ pub unsafe fn create_module<'ll>( } } + // If we're normalizing integers with CFI, ensure LLVM generated functions do the same. + // See https://github.com/llvm/llvm-project/pull/104826 + if sess.is_sanitizer_cfi_normalize_integers_enabled() { + let cfi_normalize_integers = c"cfi-normalize-integers".as_ptr().cast(); + unsafe { + llvm::LLVMRustAddModuleFlagU32( + llmod, + llvm::LLVMModFlagBehavior::Override, + cfi_normalize_integers, + 1, + ); + } + } + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { - let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr().cast(); + let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr(); unsafe { llvm::LLVMRustAddModuleFlagU32( llmod, @@ -241,10 +256,26 @@ pub unsafe fn create_module<'ll>( // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - let kcfi = c"kcfi".as_ptr().cast(); + let kcfi = c"kcfi".as_ptr(); unsafe { llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); } + + // Add "kcfi-offset" module flag with -Z patchable-function-entry (See + // https://reviews.llvm.org/D141172). + let pfe = + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry); + if pfe.prefix() > 0 { + let kcfi_offset = c"kcfi-offset".as_ptr().cast(); + unsafe { + llvm::LLVMRustAddModuleFlagU32( + llmod, + llvm::LLVMModFlagBehavior::Override, + kcfi_offset, + pfe.prefix().into(), + ); + } + } } // Control Flow Guard is currently only supported by the MSVC linker on Windows. @@ -280,26 +311,26 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Min, - c"branch-target-enforcement".as_ptr().cast(), + c"branch-target-enforcement".as_ptr(), bti.into(), ); llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Min, - c"sign-return-address".as_ptr().cast(), + c"sign-return-address".as_ptr(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-all".as_ptr().cast(), + c"sign-return-address-all".as_ptr(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-with-bkey".as_ptr().cast(), + c"sign-return-address-with-bkey".as_ptr(), u32::from(pac_opts.key == PAuthKey::B), ); } @@ -317,7 +348,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Override, - c"cf-protection-branch".as_ptr().cast(), + c"cf-protection-branch".as_ptr(), 1, ); } @@ -327,7 +358,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Override, - c"cf-protection-return".as_ptr().cast(), + c"cf-protection-return".as_ptr(), 1, ); } @@ -338,7 +369,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlagU32( llmod, llvm::LLVMModFlagBehavior::Error, - c"Virtual Function Elim".as_ptr().cast(), + c"Virtual Function Elim".as_ptr(), 1, ); } @@ -531,7 +562,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } #[inline] - pub fn coverage_context(&self) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { + pub(crate) fn coverage_context( + &self, + ) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { self.coverage_cx.as_ref() } @@ -1000,8 +1033,10 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.is.constant.i64", fn(t_i64) -> i1); ifn!("llvm.is.constant.i128", fn(t_i128) -> i1); ifn!("llvm.is.constant.isize", fn(t_isize) -> i1); + ifn!("llvm.is.constant.f16", fn(t_f16) -> i1); ifn!("llvm.is.constant.f32", fn(t_f32) -> i1); ifn!("llvm.is.constant.f64", fn(t_f64) -> i1); + ifn!("llvm.is.constant.f128", fn(t_f128) -> i1); ifn!("llvm.is.constant.ptr", fn(ptr) -> i1); ifn!("llvm.expect.i1", fn(i1, i1) -> i1); @@ -1064,7 +1099,7 @@ impl<'ll> CodegenCx<'ll, '_> { impl CodegenCx<'_, '_> { /// Generates a new symbol name with the given prefix. This symbol name must /// only be used for definitions with `internal` or `private` linkage. - pub fn generate_local_symbol_name(&self, prefix: &str) -> String { + pub(crate) fn generate_local_symbol_name(&self, prefix: &str) -> String { let idx = self.local_gen_sym_counter.get(); self.local_gen_sym_counter.set(idx + 1); // Include a '.' character, so there can be no accidental conflicts with diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 584d033d6bd..d7a4f105f3c 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,5 +1,5 @@ use rustc_middle::mir::coverage::{ - CodeRegion, ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, + ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, SourceRegion, }; /// Must match the layout of `LLVMRustCounterKind`. @@ -80,7 +80,7 @@ pub struct CounterExpression { /// Must match the layout of `LLVMRustCounterMappingRegionKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum RegionKind { +enum RegionKind { /// A CodeRegion associates some code with a counter CodeRegion = 0, @@ -110,13 +110,13 @@ pub enum RegionKind { MCDCBranchRegion = 6, } -pub mod mcdc { +mod mcdc { use rustc_middle::mir::coverage::{ConditionInfo, DecisionInfo}; /// Must match the layout of `LLVMRustMCDCDecisionParameters`. #[repr(C)] #[derive(Clone, Copy, Debug, Default)] - pub struct DecisionParameters { + pub(crate) struct DecisionParameters { bitmap_idx: u32, num_conditions: u16, } @@ -127,14 +127,14 @@ pub mod mcdc { /// Must match the layout of `LLVMRustMCDCBranchParameters`. #[repr(C)] #[derive(Clone, Copy, Debug, Default)] - pub struct BranchParameters { + pub(crate) struct BranchParameters { condition_id: LLVMConditionId, condition_ids: [LLVMConditionId; 2], } #[repr(C)] #[derive(Clone, Copy, Debug)] - pub enum ParameterTag { + enum ParameterTag { None = 0, Decision = 1, Branch = 2, @@ -142,24 +142,24 @@ pub mod mcdc { /// Same layout with `LLVMRustMCDCParameters` #[repr(C)] #[derive(Clone, Copy, Debug)] - pub struct Parameters { + pub(crate) struct Parameters { tag: ParameterTag, decision_params: DecisionParameters, branch_params: BranchParameters, } impl Parameters { - pub fn none() -> Self { + pub(crate) fn none() -> Self { Self { tag: ParameterTag::None, decision_params: Default::default(), branch_params: Default::default(), } } - pub fn decision(decision_params: DecisionParameters) -> Self { + pub(crate) fn decision(decision_params: DecisionParameters) -> Self { Self { tag: ParameterTag::Decision, decision_params, branch_params: Default::default() } } - pub fn branch(branch_params: BranchParameters) -> Self { + pub(crate) fn branch(branch_params: BranchParameters) -> Self { Self { tag: ParameterTag::Branch, decision_params: Default::default(), branch_params } } } @@ -236,9 +236,10 @@ impl CounterMappingRegion { pub(crate) fn from_mapping( mapping_kind: &MappingKind, local_file_id: u32, - code_region: &CodeRegion, + source_region: &SourceRegion, ) -> Self { - let &CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = + source_region; match *mapping_kind { MappingKind::Code(term) => Self::code_region( Counter::from_term(term), diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index 9433385c23a..5ed640b840e 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -2,8 +2,8 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxIndexSet; use rustc_index::bit_set::BitSet; use rustc_middle::mir::coverage::{ - CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, - MappingKind, Op, + CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, MappingKind, Op, + SourceRegion, }; use rustc_middle::ty::Instance; use rustc_span::Symbol; @@ -14,7 +14,7 @@ use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; /// Holds all of the coverage mapping data associated with a function instance, /// collected during traversal of `Coverage` statements in the function's MIR. #[derive(Debug)] -pub struct FunctionCoverageCollector<'tcx> { +pub(crate) struct FunctionCoverageCollector<'tcx> { /// Coverage info that was attached to this function by the instrumentor. function_coverage_info: &'tcx FunctionCoverageInfo, is_used: bool, @@ -32,7 +32,7 @@ pub struct FunctionCoverageCollector<'tcx> { impl<'tcx> FunctionCoverageCollector<'tcx> { /// Creates a new set of coverage data for a used (called) function. - pub fn new( + pub(crate) fn new( instance: Instance<'tcx>, function_coverage_info: &'tcx FunctionCoverageInfo, ) -> Self { @@ -40,7 +40,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { } /// Creates a new set of coverage data for an unused (never called) function. - pub fn unused( + pub(crate) fn unused( instance: Instance<'tcx>, function_coverage_info: &'tcx FunctionCoverageInfo, ) -> Self { @@ -195,13 +195,13 @@ impl<'tcx> FunctionCoverage<'tcx> { /// Return the source hash, generated from the HIR node structure, and used to indicate whether /// or not the source code structure changed between different compilations. - pub fn source_hash(&self) -> u64 { + pub(crate) fn source_hash(&self) -> u64 { if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } } /// Returns an iterator over all filenames used by this function's mappings. pub(crate) fn all_file_names(&self) -> impl Iterator<Item = Symbol> + Captures<'_> { - self.function_coverage_info.mappings.iter().map(|mapping| mapping.code_region.file_name) + self.function_coverage_info.mappings.iter().map(|mapping| mapping.source_region.file_name) } /// Convert this function's coverage expression data into a form that can be @@ -230,12 +230,12 @@ impl<'tcx> FunctionCoverage<'tcx> { /// that will be used by `mapgen` when preparing for FFI. pub(crate) fn counter_regions( &self, - ) -> impl Iterator<Item = (MappingKind, &CodeRegion)> + ExactSizeIterator { + ) -> impl Iterator<Item = (MappingKind, &SourceRegion)> + ExactSizeIterator { self.function_coverage_info.mappings.iter().map(move |mapping| { - let Mapping { kind, code_region } = mapping; + let Mapping { kind, source_region } = mapping; let kind = kind.map_terms(|term| if self.is_zero_term(term) { CovTerm::Zero } else { term }); - (kind, code_region) + (kind, source_region) }) } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index f8929a26011..069b62af5e7 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -27,7 +27,7 @@ use crate::{coverageinfo, llvm}; /// implementing this Rust version, and though the format documentation is very explicit and /// detailed, some undocumented details in Clang's implementation (that may or may not be important) /// were also replicated for Rust's Coverage Map. -pub fn finalize(cx: &CodegenCx<'_, '_>) { +pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure that LLVM is using a version of the coverage mapping format that diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 20a713b8564..69babc7c9cf 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -22,10 +22,10 @@ use crate::llvm; pub(crate) mod ffi; pub(crate) mod map_data; -pub mod mapgen; +mod mapgen; /// A context object for maintaining all state needed by the coverageinfo module. -pub struct CrateCoverageContext<'ll, 'tcx> { +pub(crate) struct CrateCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, @@ -34,7 +34,7 @@ pub struct CrateCoverageContext<'ll, 'tcx> { } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { function_coverage_map: Default::default(), pgo_func_name_var_map: Default::default(), @@ -42,7 +42,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } - pub fn take_function_coverage_map( + fn take_function_coverage_map( &self, ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { self.function_coverage_map.replace(FxIndexMap::default()) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index efe616838bf..c3087d8ec30 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -15,7 +15,7 @@ use crate::llvm::debuginfo::{DILocation, DIScope}; /// Produces DIScope DIEs for each MIR Scope which has variables defined in it. // FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. -pub fn compute_mir_scopes<'ll, 'tcx>( +pub(crate) fn compute_mir_scopes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, @@ -88,7 +88,7 @@ fn make_mir_scope<'ll, 'tcx>( let loc = cx.lookup_debug_loc(scope_data.span.lo()); let file_metadata = file_metadata(cx, &loc.file); - let parent_dbg_scope = match scope_data.inlined { + let dbg_scope = match scope_data.inlined { Some((callee, _)) => { // FIXME(eddyb) this would be `self.monomorphize(&callee)` // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. @@ -102,17 +102,15 @@ fn make_mir_scope<'ll, 'tcx>( cx.dbg_scope_fn(callee, callee_fn_abi, None) }) } - None => parent_scope.dbg_scope, - }; - - let dbg_scope = unsafe { - llvm::LLVMRustDIBuilderCreateLexicalBlock( - DIB(cx), - parent_dbg_scope, - file_metadata, - loc.line, - loc.col, - ) + None => unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_scope.dbg_scope, + file_metadata, + loc.line, + loc.col, + ) + }, }; let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 5a08f2f00e5..dc228e94811 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -16,7 +16,7 @@ use crate::value::Value; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. -pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, '_, '_>) { +pub(crate) fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, '_, '_>) { if needs_gdb_debug_scripts_section(bx) { let gdb_debug_scripts_section = get_or_insert_gdb_debug_scripts_section_global(bx); // Load just the first byte as that's all that's necessary to force @@ -30,12 +30,13 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, /// Allocates the global variable responsible for the .debug_gdb_scripts binary /// section. -pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Value { +pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( + cx: &CodegenCx<'ll, '_>, +) -> &'ll Value { let c_section_var_name = c"__rustc_debug_gdb_scripts_section__"; let section_var_name = c_section_var_name.to_str().unwrap(); - let section_var = - unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; + let section_var = unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr()) }; section_var.unwrap_or_else(|| { let mut section_contents = Vec::new(); @@ -70,7 +71,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr().cast()); + llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr()); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); @@ -83,7 +84,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' }) } -pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { +pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index ad638588612..dc59e9349b0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -85,7 +85,7 @@ const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = | // SmallVec is used quite a bit in this module, so create a shorthand. // The actual number of elements is not so important. -pub type SmallVec<T> = smallvec::SmallVec<[T; 16]>; +type SmallVec<T> = smallvec::SmallVec<[T; 16]>; mod enums; mod type_map; @@ -425,7 +425,7 @@ fn build_slice_type_di_node<'ll, 'tcx>( /// /// This function will look up the debuginfo node in the TypeMap. If it can't find it, it /// will create the node by dispatching to the corresponding `build_*_di_node()` function. -pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { +pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) @@ -456,7 +456,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D { build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } - ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), + ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), @@ -531,7 +531,7 @@ fn hex_encode(data: &[u8]) -> String { hex_string } -pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { +pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { let cache_key = Some((source_file.stable_id, source_file.src_hash)); return debug_context(cx) .created_files @@ -629,6 +629,9 @@ pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> }; let hash_value = hex_encode(source_file.src_hash.hash_bytes()); + let source = + cx.sess().opts.unstable_opts.embed_source.then_some(()).and(source_file.src.as_ref()); + unsafe { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), @@ -639,12 +642,14 @@ pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> hash_kind, hash_value.as_ptr().cast(), hash_value.len(), + source.map_or(ptr::null(), |x| x.as_ptr().cast()), + source.map_or(0, |x| x.len()), ) } } } -pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { +fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { debug_context(cx).created_files.borrow_mut().entry(None).or_insert_with(|| unsafe { let file_name = "<unknown>"; let directory = ""; @@ -659,6 +664,8 @@ pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { llvm::ChecksumKind::None, hash_value.as_ptr().cast(), hash_value.len(), + ptr::null(), + 0, ) }) } @@ -859,7 +866,7 @@ fn build_param_type_di_node<'ll, 'tcx>( } } -pub fn build_compile_unit_di_node<'ll, 'tcx>( +pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( tcx: TyCtxt<'tcx>, codegen_unit_name: &str, debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, @@ -943,6 +950,8 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( llvm::ChecksumKind::None, ptr::null(), 0, + ptr::null(), + 0, ); let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit( @@ -952,7 +961,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( producer.as_ptr().cast(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, - c"".as_ptr().cast(), + c"".as_ptr(), 0, // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final @@ -1319,7 +1328,11 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( /// Creates debug information for the given global variable. /// /// Adds the created debuginfo nodes directly to the crate's IR. -pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { +pub(crate) fn build_global_var_di_node<'ll>( + cx: &CodegenCx<'ll, '_>, + def_id: DefId, + global: &'ll Value, +) { if cx.dbg_cx.is_none() { return; } @@ -1559,7 +1572,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( /// given type. /// /// Adds the created metadata nodes directly to the crate's IR. -pub fn create_vtable_di_node<'ll, 'tcx>( +pub(crate) fn create_vtable_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, @@ -1604,7 +1617,7 @@ pub fn create_vtable_di_node<'ll, 'tcx>( } /// Creates an "extension" of an existing `DIScope` into another file. -pub fn extend_scope_to_file<'ll>( +pub(crate) fn extend_scope_to_file<'ll>( cx: &CodegenCx<'ll, '_>, scope_metadata: &'ll DIScope, file: &SourceFile, @@ -1613,7 +1626,7 @@ pub fn extend_scope_to_file<'ll>( unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } } -pub fn tuple_field_name(field_index: usize) -> Cow<'static, str> { +fn tuple_field_name(field_index: usize) -> Cow<'static, str> { const TUPLE_FIELD_NAMES: [&'static str; 16] = [ "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11", "__12", "__13", "__14", "__15", diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 13006638bb3..181022087f3 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; -use rustc_codegen_ssa::debuginfo::wants_c_like_enum_debuginfo; +use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstMethods; use rustc_index::IndexVec; use rustc_middle::bug; @@ -12,7 +12,7 @@ use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::common::CodegenCx; -use crate::debuginfo::metadata::enums::{tag_base_type, DiscrResult}; +use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ build_field_di_node, file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, @@ -190,7 +190,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type_and_layout = cx.layout_of(enum_type); let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); - assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); type_map::build_type_with_children( cx, @@ -265,7 +265,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( let coroutine_type_and_layout = cx.layout_of(coroutine_type); let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); - assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, coroutine_type_and_layout)); type_map::build_type_with_children( cx, @@ -381,7 +381,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( tag_field: usize, untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); let variant_names_type_di_node = build_variant_names_type_di_node( cx, @@ -676,7 +676,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); - let tag_base_type = tag_base_type(cx, coroutine_type_and_layout); + let tag_base_type = tag_base_type(cx.tcx, coroutine_type_and_layout); let variant_names_type_di_node = build_variant_names_type_di_node( cx, @@ -803,7 +803,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( assert_eq!( cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) + cx.size_and_align_of(self::tag_base_type(cx.tcx, enum_type_and_layout)) ); // ... and a field for the tag. If the tag is 128 bits wide, this will actually diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index fc3adaf0681..edaf73b74a2 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -1,17 +1,15 @@ use std::borrow::Cow; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; -use rustc_codegen_ssa::debuginfo::wants_c_like_enum_debuginfo; +use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; -use rustc_middle::ty::layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; use rustc_span::Symbol; -use rustc_target::abi::{ - FieldIdx, HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants, -}; +use rustc_target::abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{size_and_align_of, SmallVec}; @@ -39,7 +37,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type_and_layout = cx.layout_of(enum_type); - if wants_c_like_enum_debuginfo(enum_type_and_layout) { + if wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout) { return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); } @@ -74,7 +72,7 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( di_node: build_enumeration_type_di_node( cx, &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), - tag_base_type(cx, enum_type_and_layout), + tag_base_type(cx.tcx, enum_type_and_layout), enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); (name, discr.val) @@ -85,48 +83,6 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( } } -/// Extract the type with which we want to describe the tag of the given enum or coroutine. -fn tag_base_type<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, -) -> Ty<'tcx> { - assert!(match enum_type_and_layout.ty.kind() { - ty::Coroutine(..) => true, - ty::Adt(adt_def, _) => adt_def.is_enum(), - _ => false, - }); - - match enum_type_and_layout.layout.variants() { - // A single-variant enum has no discriminant. - Variants::Single { .. } => { - bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout) - } - - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => { - // Niche tags are always normalized to unsized integers of the correct size. - match tag.primitive() { - Primitive::Int(t, _) => t, - Primitive::Float(f) => Integer::from_size(f.size()).unwrap(), - // FIXME(erikdesjardins): handle non-default addrspace ptr sizes - Primitive::Pointer(_) => { - // If the niche is the NULL value of a reference, then `discr_enum_ty` will be - // a RawPtr. CodeView doesn't know what to do with enums whose base type is a - // pointer so we fix this up to just be `usize`. - // DWARF might be able to deal with this but with an integer type we are on - // the safe side there too. - cx.data_layout().ptr_sized_integer() - } - } - .to_ty(cx.tcx, false) - } - - Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { - // Direct tags preserve the sign. - tag.primitive().to_ty(cx.tcx) - } - } -} - /// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. /// This is a helper function and does not register anything in the type map by itself. /// @@ -301,7 +257,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( /// ---> DW_TAG_structure_type (type of variant 3) /// /// ``` -pub fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( +fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, variant_index: VariantIdx, coroutine_type_and_layout: TyAndLayout<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index d7e3b47e0bd..238fbad4dfd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; -use rustc_codegen_ssa::debuginfo::wants_c_like_enum_debuginfo; +use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstMethods; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -11,7 +11,6 @@ use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::common::CodegenCx; -use crate::debuginfo::metadata::enums::tag_base_type; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, visibility_di_flags, @@ -54,7 +53,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); - assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); type_map::build_type_with_children( cx, @@ -131,7 +130,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, coroutine_def_id); let coroutine_type_and_layout = cx.layout_of(coroutine_type); - assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, coroutine_type_and_layout)); let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); @@ -321,7 +320,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( &Variants::Single { .. } => None, &Variants::Multiple { tag_field, .. } => { - let tag_base_type = tag_base_type(cx, enum_or_coroutine_type_and_layout); + let tag_base_type = tag_base_type(cx.tcx, enum_or_coroutine_type_and_layout); let (size, align) = cx.size_and_align_of(tag_base_type); unsafe { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 25b2df9c52c..6d21f4204e3 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -22,7 +22,7 @@ mod private { // `UniqueTypeId` from being constructed directly, without asserting // the preconditions. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] - pub struct HiddenZst; + pub(crate) struct HiddenZst; } /// A unique identifier for anything that we create a debuginfo node for. @@ -48,17 +48,17 @@ pub(super) enum UniqueTypeId<'tcx> { } impl<'tcx> UniqueTypeId<'tcx> { - pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { + pub(crate) fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); UniqueTypeId::Ty(t, private::HiddenZst) } - pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { + pub(crate) fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) } - pub fn for_enum_variant_struct_type( + pub(crate) fn for_enum_variant_struct_type( tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>, variant_idx: VariantIdx, @@ -67,7 +67,7 @@ impl<'tcx> UniqueTypeId<'tcx> { UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) } - pub fn for_enum_variant_struct_type_wrapper( + pub(crate) fn for_enum_variant_struct_type_wrapper( tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>, variant_idx: VariantIdx, @@ -76,7 +76,7 @@ impl<'tcx> UniqueTypeId<'tcx> { UniqueTypeId::VariantStructTypeCppLikeWrapper(enum_ty, variant_idx, private::HiddenZst) } - pub fn for_vtable_ty( + pub(crate) fn for_vtable_ty( tcx: TyCtxt<'tcx>, self_type: Ty<'tcx>, implemented_trait: Option<PolyExistentialTraitRef<'tcx>>, @@ -93,7 +93,7 @@ impl<'tcx> UniqueTypeId<'tcx> { /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. /// /// Right now this takes the form of a hex-encoded opaque hash value. - pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { + fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { let mut hasher = StableHasher::new(); tcx.with_stable_hashing_context(|mut hcx| { hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) @@ -101,7 +101,7 @@ impl<'tcx> UniqueTypeId<'tcx> { hasher.finish::<Fingerprint>().to_hex() } - pub fn expect_ty(self) -> Ty<'tcx> { + pub(crate) fn expect_ty(self) -> Ty<'tcx> { match self { UniqueTypeId::Ty(ty, _) => ty, _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self), @@ -133,25 +133,25 @@ impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { } } -pub struct DINodeCreationResult<'ll> { +pub(crate) struct DINodeCreationResult<'ll> { pub di_node: &'ll DIType, pub already_stored_in_typemap: bool, } impl<'ll> DINodeCreationResult<'ll> { - pub fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { + pub(crate) fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { DINodeCreationResult { di_node, already_stored_in_typemap } } } #[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum Stub<'ll> { +pub(crate) enum Stub<'ll> { Struct, Union, VTableTy { vtable_holder: &'ll DIType }, } -pub struct StubInfo<'ll, 'tcx> { +pub(crate) struct StubInfo<'ll, 'tcx> { metadata: &'ll DIType, unique_type_id: UniqueTypeId<'tcx>, } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index b23e05182ca..f5d6fc6f080 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -40,13 +40,13 @@ use crate::llvm::debuginfo::{ use crate::value::Value; mod create_scope_map; -pub mod gdb; -pub mod metadata; +mod gdb; +pub(crate) mod metadata; mod namespace; mod utils; -pub use self::create_scope_map::compute_mir_scopes; -pub use self::metadata::build_global_var_di_node; +use self::create_scope_map::compute_mir_scopes; +pub(crate) use self::metadata::build_global_var_di_node; #[allow(non_upper_case_globals)] const DW_TAG_auto_variable: c_uint = 0x100; @@ -54,7 +54,7 @@ const DW_TAG_auto_variable: c_uint = 0x100; const DW_TAG_arg_variable: c_uint = 0x101; /// A context object for maintaining all state needed by the debuginfo module. -pub struct CodegenUnitDebugContext<'ll, 'tcx> { +pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, @@ -74,7 +74,7 @@ impl Drop for CodegenUnitDebugContext<'_, '_> { } impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { - pub fn new(llmod: &'ll llvm::Module) -> Self { + pub(crate) fn new(llmod: &'ll llvm::Module) -> Self { debug!("CodegenUnitDebugContext::new"); let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; // DIBuilder inherits context from the module, so we'd better use the same one @@ -90,7 +90,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { } } - pub fn finalize(&self, sess: &Session) { + pub(crate) fn finalize(&self, sess: &Session) { unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder); @@ -109,7 +109,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlagU32( self.llmod, llvm::LLVMModFlagBehavior::Warning, - c"Dwarf Version".as_ptr().cast(), + c"Dwarf Version".as_ptr(), dwarf_version, ); } else { @@ -117,7 +117,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlagU32( self.llmod, llvm::LLVMModFlagBehavior::Warning, - c"CodeView".as_ptr().cast(), + c"CodeView".as_ptr(), 1, ) } @@ -126,7 +126,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlagU32( self.llmod, llvm::LLVMModFlagBehavior::Warning, - c"Debug Info Version".as_ptr().cast(), + c"Debug Info Version".as_ptr(), llvm::LLVMRustDebugMetadataVersion(), ); } @@ -134,7 +134,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { } /// Creates any deferred debug metadata nodes -pub fn finalize(cx: &CodegenCx<'_, '_>) { +pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { if let Some(dbg_cx) = &cx.dbg_cx { debug!("finalize"); @@ -241,13 +241,13 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { // FIXME(eddyb) rename this to better indicate it's a duplicate of // `rustc_span::Loc` rather than `DILocation`, perhaps by making // `lookup_char_pos` return the right information instead. -pub struct DebugLoc { +struct DebugLoc { /// Information about the original source file. - pub file: Lrc<SourceFile>, + file: Lrc<SourceFile>, /// The (1-based) line number. - pub line: u32, + line: u32, /// The (1-based) column number. - pub col: u32, + col: u32, } impl CodegenCx<'_, '_> { @@ -255,7 +255,7 @@ impl CodegenCx<'_, '_> { // FIXME(eddyb) rename this to better indicate it's a duplicate of // `lookup_char_pos` rather than `dbg_loc`, perhaps by making // `lookup_char_pos` return the right information instead. - pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { + fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { let (file, line, col) = match self.sess().source_map().lookup_line(pos) { Ok(SourceFileAndLine { sf: file, line }) => { let line_pos = file.lines()[line]; @@ -570,7 +570,17 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { inlined_at: Option<&'ll DILocation>, span: Span, ) -> &'ll DILocation { - let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); + // When emitting debugging information, DWARF (i.e. everything but MSVC) + // treats line 0 as a magic value meaning that the code could not be + // attributed to any line in the source. That's also exactly what dummy + // spans are. Make that equivalence here, rather than passing dummy spans + // to lookup_debug_loc, which will return line 1 for them. + let (line, col) = if span.is_dummy() && !self.sess().target.is_like_msvc { + (0, 0) + } else { + let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); + (line, col) + }; unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index 5c4f8fe99e3..83d7a82dadc 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -9,7 +9,7 @@ use crate::common::CodegenCx; use crate::llvm; use crate::llvm::debuginfo::DIScope; -pub fn mangled_name_of_instance<'a, 'tcx>( +pub(crate) fn mangled_name_of_instance<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, instance: Instance<'tcx>, ) -> ty::SymbolName<'tcx> { @@ -17,7 +17,7 @@ pub fn mangled_name_of_instance<'a, 'tcx>( tcx.symbol_name(instance) } -pub fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { +pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { if let Some(&scope) = debug_context(cx).namespace_map.borrow().get(&def_id) { return scope; } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index e542aa96e8a..321553a3df0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -11,7 +11,7 @@ use crate::common::CodegenCx; use crate::llvm; use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope}; -pub fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { +pub(crate) fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { // The is_local_to_unit flag indicates whether a function is local to the // current compilation unit (i.e., if it is *static* in the C-sense). The // *reachable* set should provide a good approximation of this, as it @@ -24,7 +24,7 @@ pub fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { } #[allow(non_snake_case)] -pub fn create_DIArray<'ll>( +pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { @@ -32,7 +32,7 @@ pub fn create_DIArray<'ll>( } #[inline] -pub fn debug_context<'a, 'll, 'tcx>( +pub(crate) fn debug_context<'a, 'll, 'tcx>( cx: &'a CodegenCx<'ll, 'tcx>, ) -> &'a CodegenUnitDebugContext<'ll, 'tcx> { cx.dbg_cx.as_ref().unwrap() @@ -40,11 +40,11 @@ pub fn debug_context<'a, 'll, 'tcx>( #[inline] #[allow(non_snake_case)] -pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { +pub(crate) fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { cx.dbg_cx.as_ref().unwrap().builder } -pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { +pub(crate) fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { item_namespace(cx, cx.tcx.parent(def_id)) } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 2aa349b2782..4e4500b6373 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -65,7 +65,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// return its Value instead. - pub fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { + pub(crate) fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { debug!("declare_global(name={:?})", name); unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } } @@ -77,7 +77,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_cfn( + pub(crate) fn declare_cfn( &self, name: &str, unnamed: llvm::UnnamedAddr, @@ -100,7 +100,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_entry_fn( + pub(crate) fn declare_entry_fn( &self, name: &str, callconv: llvm::CallConv, @@ -119,7 +119,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_fn( + pub(crate) fn declare_fn( &self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, @@ -199,7 +199,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// return `None` if the name already has a definition associated with it. In that /// case an error should be reported to the user, because it usually happens due /// to user’s fault (e.g., misuse of `#[no_mangle]` or `#[export_name]` attributes). - pub fn define_global(&self, name: &str, ty: &'ll Type) -> Option<&'ll Value> { + pub(crate) fn define_global(&self, name: &str, ty: &'ll Type) -> Option<&'ll Value> { if self.get_defined_value(name).is_some() { None } else { @@ -210,19 +210,19 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// Declare a private global /// /// Use this function when you intend to define a global without a name. - pub fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { + pub(crate) fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod, ty) } } /// Gets declared value by name. - pub fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { + pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } } /// Gets defined or externally defined (AvailableExternally linkage) value by /// name. - pub fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { + pub(crate) fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { self.get_declared_value(name).and_then(|val| { let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; if !declaration { Some(val) } else { None } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 7e53d32ce8c..bb481d2a308 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -40,13 +40,6 @@ pub(crate) enum PossibleFeature<'a> { } #[derive(Diagnostic)] -#[diag(codegen_llvm_error_creating_import_library)] -pub(crate) struct ErrorCreatingImportLibrary<'a> { - pub lib_name: &'a str, - pub error: String, -} - -#[derive(Diagnostic)] #[diag(codegen_llvm_symbol_already_defined)] pub(crate) struct SymbolAlreadyDefined<'a> { #[primary_span] @@ -214,13 +207,13 @@ pub(crate) struct CopyBitcode { #[derive(Diagnostic)] #[diag(codegen_llvm_unknown_debuginfo_compression)] -pub struct UnknownCompression { +pub(crate) struct UnknownCompression { pub algorithm: &'static str, } #[derive(Diagnostic)] #[diag(codegen_llvm_mismatch_data_layout)] -pub struct MismatchedDataLayout<'a> { +pub(crate) struct MismatchedDataLayout<'a> { pub rustc_target: &'a str, pub rustc_layout: &'a str, pub llvm_target: &'a str, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 57d5f6fdf50..a300f5f707a 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,3 +1,4 @@ +use std::assert_matches::assert_matches; use std::cmp::Ordering; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; @@ -191,14 +192,22 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::is_val_statically_known => { let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); - match self.type_kind(intrinsic_type) { - TypeKind::Pointer | TypeKind::Integer | TypeKind::Float | TypeKind::Double => { - self.call_intrinsic( - &format!("llvm.is.constant.{:?}", intrinsic_type), - &[args[0].immediate()], - ) + let kind = self.type_kind(intrinsic_type); + let intrinsic_name = match kind { + TypeKind::Pointer | TypeKind::Integer => { + Some(format!("llvm.is.constant.{intrinsic_type:?}")) } - _ => self.const_bool(false), + // LLVM float types' intrinsic names differ from their type names. + TypeKind::Half => Some(format!("llvm.is.constant.f16")), + TypeKind::Float => Some(format!("llvm.is.constant.f32")), + TypeKind::Double => Some(format!("llvm.is.constant.f64")), + TypeKind::FP128 => Some(format!("llvm.is.constant.f128")), + _ => None, + }; + if let Some(intrinsic_name) = intrinsic_name { + self.call_intrinsic(&intrinsic_name, &[args[0].immediate()]) + } else { + self.const_bool(false) } } sym::unlikely => self @@ -1142,7 +1151,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if cfg!(debug_assertions) { for (ty, arg) in arg_tys.iter().zip(args) { if ty.is_simd() { - assert!(matches!(arg.val, OperandValue::Immediate(_))); + assert_matches!(arg.val, OperandValue::Immediate(_)); } } } @@ -1278,19 +1287,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle { - // Make sure this is actually an array, since typeck only checks the length-suffixed + // Make sure this is actually an array or SIMD vector, since typeck only checks the length-suffixed // version of this intrinsic. - let n: u64 = match args[2].layout.ty.kind() { + let idx_ty = args[2].layout.ty; + let n: u64 = match idx_ty.kind() { ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( || span_bug!(span, "could not evaluate shuffle index array length"), ) } - _ => return_error!(InvalidMonomorphization::SimdShuffle { - span, - name, - ty: args[2].layout.ty - }), + _ if idx_ty.is_simd() + && matches!( + idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), + ty::Uint(ty::UintTy::U32) + ) => + { + idx_ty.simd_size_and_type(bx.cx.tcx).0 + } + _ => return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }), }; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 518a86e0cb0..6a303e1e602 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -6,8 +6,10 @@ // tidy-alphabetical-start #![allow(internal_features)] +#![cfg_attr(bootstrap, feature(unsafe_extern_blocks))] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] +#![feature(assert_matches)] #![feature(exact_size_is_empty)] #![feature(extern_types)] #![feature(hash_raw_entry)] @@ -15,6 +17,7 @@ #![feature(iter_intersperse)] #![feature(let_chains)] #![feature(rustdoc_internals)] +#![warn(unreachable_pub)] // tidy-alphabetical-end use std::any::Any; @@ -44,11 +47,11 @@ use rustc_session::Session; use rustc_span::symbol::Symbol; mod back { - pub mod archive; - pub mod lto; - pub mod owned_target_machine; + pub(crate) mod archive; + pub(crate) mod lto; + pub(crate) mod owned_target_machine; mod profiling; - pub mod write; + pub(crate) mod write; } mod abi; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 579ec517cf0..138cc3219aa 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -564,12 +564,12 @@ pub enum ArchiveKind { } // LLVMRustThinLTOData -extern "C" { +unsafe extern "C" { pub type ThinLTOData; } // LLVMRustThinLTOBuffer -extern "C" { +unsafe extern "C" { pub type ThinLTOBuffer; } @@ -621,7 +621,7 @@ pub enum MemoryEffects { InaccessibleMemOnly, } -extern "C" { +unsafe extern "C" { type Opaque; } #[repr(C)] @@ -631,54 +631,54 @@ struct InvariantOpaque<'a> { } // Opaque pointer types -extern "C" { +unsafe extern "C" { pub type Module; } -extern "C" { +unsafe extern "C" { pub type Context; } -extern "C" { +unsafe extern "C" { pub type Type; } -extern "C" { +unsafe extern "C" { pub type Value; } -extern "C" { +unsafe extern "C" { pub type ConstantInt; } -extern "C" { +unsafe extern "C" { pub type Attribute; } -extern "C" { +unsafe extern "C" { pub type Metadata; } -extern "C" { +unsafe extern "C" { pub type BasicBlock; } #[repr(C)] pub struct Builder<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct PassManager<'a>(InvariantOpaque<'a>); -extern "C" { +unsafe extern "C" { pub type Pass; } -extern "C" { +unsafe extern "C" { pub type TargetMachine; } -extern "C" { +unsafe extern "C" { pub type Archive; } #[repr(C)] pub struct ArchiveIterator<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct ArchiveChild<'a>(InvariantOpaque<'a>); -extern "C" { +unsafe extern "C" { pub type Twine; } -extern "C" { +unsafe extern "C" { pub type DiagnosticInfo; } -extern "C" { +unsafe extern "C" { pub type SMDiagnostic; } #[repr(C)] @@ -688,7 +688,7 @@ pub struct OperandBundleDef<'a>(InvariantOpaque<'a>); #[repr(C)] pub struct Linker<'a>(InvariantOpaque<'a>); -extern "C" { +unsafe extern "C" { pub type DiagnosticHandler; } @@ -823,7 +823,7 @@ bitflags! { } } -extern "C" { +unsafe extern "C" { pub type ModuleBuffer; } @@ -834,7 +834,7 @@ pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); pub type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; -extern "C" { +unsafe extern "C" { // Create and destroy contexts. pub fn LLVMContextDispose(C: &'static mut Context); pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint; @@ -1518,7 +1518,7 @@ extern "C" { } #[link(name = "llvm-wrapper", kind = "static")] -extern "C" { +unsafe extern "C" { pub fn LLVMRustInstallErrorHandlers(); pub fn LLVMRustDisableSystemDialogsOnCrash(); @@ -1575,6 +1575,12 @@ extern "C" { pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute; + pub fn LLVMRustCreateRangeAttribute( + C: &Context, + num_bits: c_uint, + lower_words: *const u64, + upper_words: *const u64, + ) -> &Attribute; // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( @@ -1854,6 +1860,8 @@ extern "C" { CSKind: ChecksumKind, Checksum: *const c_char, ChecksumLen: size_t, + Source: *const c_char, + SourceLen: size_t, ) -> &'a DIFile; pub fn LLVMRustDIBuilderCreateSubroutineType<'a>( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 72691907c0d..d0db350a149 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -8,7 +8,7 @@ use std::string::FromUtf8Error; use libc::c_uint; use rustc_data_structures::small_c_str::SmallCStr; use rustc_llvm::RustString; -use rustc_target::abi::Align; +use rustc_target::abi::{Align, Size, WrappingRange}; pub use self::AtomicRmwBinOp::*; pub use self::CallConv::*; @@ -105,6 +105,21 @@ pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribu unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } } +pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { + let lower = range.start; + let upper = range.end.wrapping_add(1); + let lower_words = [lower as u64, (lower >> 64) as u64]; + let upper_words = [upper as u64, (upper >> 64) as u64]; + unsafe { + LLVMRustCreateRangeAttribute( + llcx, + size.bits().try_into().unwrap(), + lower_words.as_ptr(), + upper_words.as_ptr(), + ) + } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 9fd8ca43789..618602ed70f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -136,14 +136,14 @@ unsafe fn configure_llvm(sess: &Session) { unsafe { llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()) }; } -pub fn time_trace_profiler_finish(file_name: &Path) { +pub(crate) fn time_trace_profiler_finish(file_name: &Path) { unsafe { let file_name = path_to_c_string(file_name); llvm::LLVMRustTimeTraceProfilerFinish(file_name.as_ptr()); } } -pub enum TargetFeatureFoldStrength<'a> { +enum TargetFeatureFoldStrength<'a> { // The feature is only tied when enabling the feature, disabling // this feature shouldn't disable the tied feature. EnableOnly(&'a str), @@ -160,28 +160,28 @@ impl<'a> TargetFeatureFoldStrength<'a> { } } -pub struct LLVMFeature<'a> { - pub llvm_feature_name: &'a str, - pub dependency: Option<TargetFeatureFoldStrength<'a>>, +pub(crate) struct LLVMFeature<'a> { + llvm_feature_name: &'a str, + dependency: Option<TargetFeatureFoldStrength<'a>>, } impl<'a> LLVMFeature<'a> { - pub fn new(llvm_feature_name: &'a str) -> Self { + fn new(llvm_feature_name: &'a str) -> Self { Self { llvm_feature_name, dependency: None } } - pub fn with_dependency( + fn with_dependency( llvm_feature_name: &'a str, dependency: TargetFeatureFoldStrength<'a>, ) -> Self { Self { llvm_feature_name, dependency: Some(dependency) } } - pub fn contains(&self, feat: &str) -> bool { + fn contains(&self, feat: &str) -> bool { self.iter().any(|dep| dep == feat) } - pub fn iter(&'a self) -> impl Iterator<Item = &'a str> { + fn iter(&'a self) -> impl Iterator<Item = &'a str> { let dependencies = self.dependency.iter().map(|feat| feat.as_str()); std::iter::once(self.llvm_feature_name).chain(dependencies) } @@ -209,7 +209,7 @@ impl<'a> IntoIterator for LLVMFeature<'a> { // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { +pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> { let arch = if sess.target.arch == "x86_64" { "x86" } else if sess.target.arch == "arm64ec" { @@ -218,46 +218,65 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { &*sess.target.arch }; match (arch, s) { - ("x86", "sse4.2") => { - LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32")) - } - ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"), - ("x86", "rdrand") => LLVMFeature::new("rdrnd"), - ("x86", "bmi1") => LLVMFeature::new("bmi"), - ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), - ("x86", "lahfsahf") => LLVMFeature::new("sahf"), - ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), - ("aarch64", "dpb") => LLVMFeature::new("ccpp"), - ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), - ("aarch64", "frintts") => LLVMFeature::new("fptoint"), - ("aarch64", "fcma") => LLVMFeature::new("complxnum"), - ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"), - ("aarch64", "paca") => LLVMFeature::new("pauth"), - ("aarch64", "pacg") => LLVMFeature::new("pauth"), + ("x86", "sse4.2") => Some(LLVMFeature::with_dependency( + "sse4.2", + TargetFeatureFoldStrength::EnableOnly("crc32"), + )), + ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), + ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), + ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), + ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), + ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), + ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")), + ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")), + ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")), + ("aarch64", "frintts") => Some(LLVMFeature::new("fptoint")), + ("aarch64", "fcma") => Some(LLVMFeature::new("complxnum")), + ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), + ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), + ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), + ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")), + ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => { - LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")) + Some(LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8"))) } // In LLVM neon implicitly enables fp, but we manually enable // neon when a feature only implicitly enables fp - ("aarch64", "fhm") => LLVMFeature::new("fp16fml"), - ("aarch64", "fp16") => LLVMFeature::new("fullfp16"), + ("aarch64", "fhm") => Some(LLVMFeature::new("fp16fml")), + ("aarch64", "fp16") => Some(LLVMFeature::new("fullfp16")), + // Filter out features that are not supported by the current LLVM version + ("aarch64", "faminmax") if get_version().0 < 18 => None, + ("aarch64", "fp8") if get_version().0 < 18 => None, + ("aarch64", "fp8dot2") if get_version().0 < 18 => None, + ("aarch64", "fp8dot4") if get_version().0 < 18 => None, + ("aarch64", "fp8fma") if get_version().0 < 18 => None, + ("aarch64", "fpmr") if get_version().0 != 18 => None, + ("aarch64", "lut") if get_version().0 < 18 => None, + ("aarch64", "sme-f8f16") if get_version().0 < 18 => None, + ("aarch64", "sme-f8f32") if get_version().0 < 18 => None, + ("aarch64", "sme-fa64") if get_version().0 < 18 => None, + ("aarch64", "sme-lutv2") if get_version().0 < 18 => None, + ("aarch64", "ssve-fp8dot2") if get_version().0 < 18 => None, + ("aarch64", "ssve-fp8dot4") if get_version().0 < 18 => None, + ("aarch64", "ssve-fp8fma") if get_version().0 < 18 => None, + ("aarch64", "v9.5a") if get_version().0 < 18 => None, // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called // `fast-unaligned-access`. In LLVM 19, it was split back out. ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { - LLVMFeature::new("fast-unaligned-access") + Some(LLVMFeature::new("fast-unaligned-access")) } // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled. ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => { - LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")) + Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))) } - (_, s) => LLVMFeature::new(s), + (_, s) => Some(LLVMFeature::new(s)), } } /// Given a map from target_features to whether they are enabled or disabled, /// ensure only valid combinations are allowed. -pub fn check_tied_features( +pub(crate) fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, ) -> Option<&'static [&'static str]> { @@ -291,13 +310,17 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { return true; } // check that all features in a given smallvec are enabled - for llvm_feature in to_llvm_features(sess, feature) { - let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { - return false; + if let Some(feat) = to_llvm_features(sess, feature) { + for llvm_feature in feat { + let cstr = SmallCStr::new(llvm_feature); + if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { + return false; + } } + true + } else { + false } - true }) .map(|(feature, _, _)| Symbol::intern(feature)), ); @@ -337,19 +360,19 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { .collect() } -pub fn print_version() { +pub(crate) fn print_version() { let (major, minor, patch) = get_version(); println!("LLVM version: {major}.{minor}.{patch}"); } -pub fn get_version() -> (u32, u32, u32) { +pub(crate) fn get_version() -> (u32, u32, u32) { // Can be called without initializing LLVM unsafe { (llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor(), llvm::LLVMRustVersionPatch()) } } -pub fn print_passes() { +pub(crate) fn print_passes() { // Can be called without initializing LLVM unsafe { llvm::LLVMRustPrintPasses(); @@ -386,9 +409,9 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach .target .supported_target_features() .iter() - .map(|(feature, _gate, _implied)| { + .filter_map(|(feature, _gate, _implied)| { // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. - let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name; + let llvm_feature = to_llvm_features(sess, *feature)?.llvm_feature_name; let desc = match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() { Some(index) => { @@ -398,7 +421,7 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach None => "", }; - (*feature, desc) + Some((*feature, desc)) }) .collect::<Vec<_>>(); @@ -479,7 +502,7 @@ fn handle_native(name: &str) -> &str { } } -pub fn target_cpu(sess: &Session) -> &str { +pub(crate) fn target_cpu(sess: &Session) -> &str { match sess.opts.cg.target_cpu { Some(ref name) => handle_native(name), None => handle_native(sess.target.cpu.as_ref()), @@ -595,7 +618,7 @@ pub(crate) fn global_llvm_features( if feature_state.is_none() { let rust_feature = supported_features.iter().find_map(|&(rust_feature, _, _)| { - let llvm_features = to_llvm_features(sess, rust_feature); + let llvm_features = to_llvm_features(sess, rust_feature)?; if llvm_features.contains(feature) && !llvm_features.contains(rust_feature) { @@ -641,7 +664,7 @@ pub(crate) fn global_llvm_features( // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. - let llvm_feature = to_llvm_features(sess, feature); + let llvm_feature = to_llvm_features(sess, feature)?; Some( std::iter::once(format!( @@ -691,6 +714,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { let feature = s .strip_prefix(&['+', '-'][..]) .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); + if s.is_empty() { + return None; + } // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. if RUSTC_SPECIFIC_FEATURES.contains(&feature) { @@ -699,7 +725,7 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { Some(feature) } -pub fn tune_cpu(sess: &Session) -> Option<&str> { +pub(crate) fn tune_cpu(sess: &Session) -> Option<&str> { let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 7e3ab19898d..ec1e2cb8094 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -12,7 +12,7 @@ use rustc_target::abi::{AddressSpace, Align, Integer, Size}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::context::CodegenCx; -pub use crate::llvm::Type; +pub(crate) use crate::llvm::Type; use crate::llvm::{Bool, False, True}; use crate::type_of::LayoutLlvmExt; use crate::value::Value; diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 4755fa08afb..6e429a1674a 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -139,21 +139,21 @@ fn struct_llfields<'a, 'tcx>( } impl<'a, 'tcx> CodegenCx<'a, 'tcx> { - pub fn align_of(&self, ty: Ty<'tcx>) -> Align { + pub(crate) fn align_of(&self, ty: Ty<'tcx>) -> Align { self.layout_of(ty).align.abi } - pub fn size_of(&self, ty: Ty<'tcx>) -> Size { + pub(crate) fn size_of(&self, ty: Ty<'tcx>) -> Size { self.layout_of(ty).size } - pub fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { + pub(crate) fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { let layout = self.layout_of(ty); (layout.size, layout.align.abi) } } -pub trait LayoutLlvmExt<'tcx> { +pub(crate) trait LayoutLlvmExt<'tcx> { fn is_llvm_immediate(&self) -> bool; fn is_llvm_scalar_pair(&self) -> bool; fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; diff --git a/compiler/rustc_codegen_llvm/src/value.rs b/compiler/rustc_codegen_llvm/src/value.rs index 6295b0de356..2eabac3be8c 100644 --- a/compiler/rustc_codegen_llvm/src/value.rs +++ b/compiler/rustc_codegen_llvm/src/value.rs @@ -2,7 +2,7 @@ use std::hash::{Hash, Hasher}; use std::{fmt, ptr}; use crate::llvm; -pub use crate::llvm::Value; +pub(crate) use crate::llvm::Value; impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { |