diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 201 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 4 |
13 files changed, 252 insertions, 13 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b5f53f51838..97dc401251c 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -15,7 +15,6 @@ use rustc_middle::ty::layout::LayoutOf; pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; use rustc_middle::ty::Ty; use rustc_session::config; -use rustc_target::abi::call::ArgAbi; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int}; pub use rustc_target::spec::abi::Abi; diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index abc33a04598..97444f1a5bd 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -816,6 +816,9 @@ impl ThinLTOKeysMap { use std::io::Write; let file = File::create(path)?; let mut writer = io::BufWriter::new(file); + // The entries are loaded back into a hash map in `load_from_file()`, so + // the order in which we write them to file here does not matter. + #[allow(rustc::potential_query_instability)] for (module, key) in &self.keys { writeln!(writer, "{module} {key}")?; } diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 36484c3c3fc..28a88dd2efe 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -39,7 +39,7 @@ impl OwnedTargetMachine { split_dwarf_file: &CStr, output_obj_file: &CStr, debug_info_compression: &CStr, - force_emulated_tls: bool, + use_emulated_tls: bool, args_cstr_buff: &[u8], ) -> Result<Self, LlvmError<'static>> { assert!(args_cstr_buff.len() > 0); @@ -71,7 +71,7 @@ impl OwnedTargetMachine { split_dwarf_file.as_ptr(), output_obj_file.as_ptr(), debug_info_compression.as_ptr(), - force_emulated_tls, + use_emulated_tls, args_cstr_buff.as_ptr() as *const c_char, args_cstr_buff.len(), ) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 1a567c0fce8..bdabb9129a7 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -33,7 +33,7 @@ use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, Switc use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; -use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; @@ -223,7 +223,7 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); - let force_emulated_tls = sess.target.force_emulated_tls; + let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); // copy the exe path, followed by path all into one buffer // null terminating them so we can use them as null terminated strings @@ -297,7 +297,7 @@ pub fn target_machine_factory( &split_dwarf_file, &output_obj_file, &debuginfo_compression, - force_emulated_tls, + use_emulated_tls, &args_cstr_buff, ) }) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 92a8c00510b..ed0d5e68cbf 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -120,6 +120,7 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic, TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec, TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec, + TlsModel::Emulated => llvm::ThreadLocalMode::GeneralDynamic, } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 51df14df644..33bfde03a31 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -58,6 +58,11 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { return; } + // The entries of the map are only used to get a list of all files with + // coverage info. In the end the list of files is passed into + // `GlobalFileTable::new()` which internally do `.sort_unstable_by()`, so + // the iteration order here does not matter. + #[allow(rustc::potential_query_instability)] let function_coverage_entries = function_coverage_map .into_iter() .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 7d69756181a..8386f067baf 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -100,6 +100,9 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let Coverage { kind } = coverage; match *kind { + // Span markers are only meaningful during MIR instrumentation, + // and have no effect during codegen. + CoverageKind::SpanMarker => {} CoverageKind::CounterIncrement { id } => { func_coverage.mark_counter_id_seen(id); // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index cf78fc56b49..acd5a1ff5c6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -17,6 +17,7 @@ use crate::debuginfo::utils::FatPtrKind; use crate::llvm; use crate::llvm::debuginfo::{ DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, + DebugNameTableKind, }; use crate::value::Value; @@ -35,9 +36,10 @@ use rustc_middle::ty::{ use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::FileName; -use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_span::{FileNameDisplayPreference, SourceFile}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; +use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; use libc::{c_char, c_longlong, c_uint}; @@ -878,6 +880,17 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let split_name = split_name.to_str().unwrap(); let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); + let dwarf_version = + tcx.sess.opts.unstable_opts.dwarf_version.unwrap_or(tcx.sess.target.default_dwarf_version); + let is_dwarf_kind = + matches!(tcx.sess.target.debuginfo_kind, DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym); + // Don't emit `.debug_pubnames` and `.debug_pubtypes` on DWARFv4 or lower. + let debug_name_table_kind = if is_dwarf_kind && dwarf_version <= 4 { + DebugNameTableKind::None + } else { + DebugNameTableKind::Default + }; + unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( debug_context.builder, @@ -907,6 +920,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( kind, 0, tcx.sess.opts.unstable_opts.split_dwarf_inlining, + debug_name_table_kind, ); if tcx.sess.opts.unstable_opts.profile { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 4832b147a54..a0ae1e9bf5d 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -32,7 +32,7 @@ use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; +use rustc_span::{BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; use rustc_target::abi::Size; use libc::c_uint; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index cc7e78b9c62..58e68a64907 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -10,7 +10,7 @@ use crate::value::Value; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; -use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; @@ -946,6 +946,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); let arg_tys = sig.inputs(); + // Vectors must be immediates (non-power-of-2 #[repr(packed)] are not) + for (ty, arg) in arg_tys.iter().zip(args) { + if ty.is_simd() && !matches!(arg.val, OperandValue::Immediate(_)) { + return_error!(InvalidMonomorphization::SimdArgument { span, name, ty: *ty }); + } + } + if name == sym::simd_select_bitmask { let (len, _) = require_simd!(arg_tys[1], SimdArgument); @@ -1492,6 +1499,198 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(v); } + if name == sym::simd_masked_load { + // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T> + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + // Loads contiguous elements from memory behind `pointer`, but only for + // those lanes whose `mask` bit is enabled. + // The memory addresses corresponding to the “off” lanes are not accessed. + + // The element type of the "mask" argument must be a signed integer type of any width + let mask_ty = in_ty; + let (mask_len, mask_elem) = (in_len, in_elem); + + // The second argument must be a pointer matching the element type + let pointer_ty = arg_tys[1]; + + // The last argument is a passthrough vector providing values for disabled lanes + let values_ty = arg_tys[2]; + let (values_len, values_elem) = require_simd!(values_ty, SimdThird); + + require_simd!(ret_ty, SimdReturn); + + // Of the same length: + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); + + // The return type must match the last argument type + require!( + ret_ty == values_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty } + ); + + require!( + matches!( + pointer_ty.kind(), + ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: values_elem, + second_arg: pointer_ty, + in_elem: values_elem, + in_ty: values_ty, + mutability: ExpectedPointerMutability::Not, + } + ); + + require!( + matches!(mask_elem.kind(), ty::Int(_)), + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: values_elem, + third_arg: mask_ty, + } + ); + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, mask_len); + (bx.trunc(args[0].immediate(), i1xn), i1xn) + }; + + let llvm_pointer = bx.type_ptr(); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); + let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); + + let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0"); + let fn_ty = bx + .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[1].immediate(), alignment, mask, args[2].immediate()], + None, + ); + return Ok(v); + } + + if name == sym::simd_masked_store { + // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> () + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + // Stores contiguous elements to memory behind `pointer`, but only for + // those lanes whose `mask` bit is enabled. + // The memory addresses corresponding to the “off” lanes are not accessed. + + // The element type of the "mask" argument must be a signed integer type of any width + let mask_ty = in_ty; + let (mask_len, mask_elem) = (in_len, in_elem); + + // The second argument must be a pointer matching the element type + let pointer_ty = arg_tys[1]; + + // The last argument specifies the values to store to memory + let values_ty = arg_tys[2]; + let (values_len, values_elem) = require_simd!(values_ty, SimdThird); + + // Of the same length: + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); + + // The second argument must be a mutable pointer type matching the element type + require!( + matches!( + pointer_ty.kind(), + ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: values_elem, + second_arg: pointer_ty, + in_elem: values_elem, + in_ty: values_ty, + mutability: ExpectedPointerMutability::Mut, + } + ); + + require!( + matches!(mask_elem.kind(), ty::Int(_)), + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: values_elem, + third_arg: mask_ty, + } + ); + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[0].immediate(), i1xn), i1xn) + }; + + let ret_t = bx.type_void(); + + let llvm_pointer = bx.type_ptr(); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); + let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); + + let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0"); + let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[2].immediate(), args[1].immediate(), alignment, mask], + None, + ); + return Ok(v); + } + if name == sym::simd_scatter { // simd_scatter(values: <N x T>, pointers: <N x *mut T>, // mask: <N x i{M}>) -> () diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 915cf31de08..116108ae5a9 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -17,7 +17,6 @@ #![feature(never_type)] #![feature(impl_trait_in_assoc_type)] #![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] @@ -306,7 +305,9 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:"); - for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { + for name in + &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] + { writeln!(out, " {name}"); } writeln!(out); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6c3ccc9cf0d..81702baa8c0 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -5,7 +5,7 @@ use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, - DebugEmissionKind, + DebugEmissionKind, DebugNameTableKind, }; use libc::{c_char, c_int, c_uint, size_t}; @@ -794,6 +794,15 @@ pub mod debuginfo { } } } + + /// LLVMRustDebugNameTableKind + #[derive(Clone, Copy)] + #[repr(C)] + pub enum DebugNameTableKind { + Default, + Gnu, + None, + } } use bitflags::bitflags; @@ -1812,6 +1821,7 @@ extern "C" { kind: DebugEmissionKind, DWOId: u64, SplitDebugInlining: bool, + DebugNameTableKind: DebugNameTableKind, ) -> &'a DIDescriptor; pub fn LLVMRustDIBuilderCreateFile<'a>( @@ -2159,7 +2169,7 @@ extern "C" { SplitDwarfFile: *const c_char, OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, - ForceEmulatedTls: bool, + UseEmulatedTls: bool, ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index eb69efb0d59..93cb7327a01 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -263,6 +263,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { "sve2-bitperm", TargetFeatureFoldStrength::EnableOnly("neon"), ), + // The unaligned-scalar-mem feature was renamed to fast-unaligned-access. + ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { + LLVMFeature::new("unaligned-scalar-mem") + } (_, s) => LLVMFeature::new(s), } } |