diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 77 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 48 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 37 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 217 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 41 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/va_arg.rs | 100 |
17 files changed, 399 insertions, 239 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 67bc86e4c90..ca43ac4b0e7 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -60,7 +60,7 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { } } -impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { +impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { fn add_archive( &mut self, archive: &Path, @@ -107,7 +107,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { pub struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { - fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { // FIXME use ArArchiveBuilder on most targets again once reading thin archives is // implemented if true { @@ -313,7 +313,7 @@ fn get_llvm_object_symbols( llvm::LLVMRustGetSymbols( buf.as_ptr(), buf.len(), - &mut *state as *mut &mut _ as *mut c_void, + std::ptr::addr_of_mut!(*state) as *mut c_void, callback, error_callback, ) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 42bd8687042..06a681c24e6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -21,6 +21,7 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::config::{self, CrateType, Lto}; +use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; @@ -787,7 +788,7 @@ pub unsafe fn optimize_thin_module( #[derive(Debug, Default)] pub struct ThinLTOKeysMap { // key = llvm name of importing module, value = LLVM cache key - keys: FxHashMap<String, String>, + keys: BTreeMap<String, String>, } impl ThinLTOKeysMap { @@ -797,7 +798,6 @@ impl ThinLTOKeysMap { let mut writer = io::BufWriter::new(file); // The entries are loaded back into a hash map in `load_from_file()`, so // the order in which we write them to file here does not matter. - #[allow(rustc::potential_query_instability)] for (module, key) in &self.keys { writeln!(writer, "{module} {key}")?; } @@ -806,7 +806,7 @@ impl ThinLTOKeysMap { fn load_from_file(path: &Path) -> io::Result<Self> { use std::io::BufRead; - let mut keys = FxHashMap::default(); + let mut keys = BTreeMap::default(); let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 27cb0366f17..031bbd63361 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -36,7 +36,7 @@ use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use crate::llvm::diagnostic::OptimizationDiagnosticKind; -use libc::{c_char, c_int, c_uint, c_void, size_t}; +use libc::{c_char, c_int, c_void, size_t}; use std::ffi::CString; use std::fs; use std::io::{self, Write}; @@ -406,7 +406,7 @@ fn report_inline_asm( cgcx: &CodegenContext<LlvmCodegenBackend>, msg: String, level: llvm::DiagnosticLevel, - mut cookie: c_uint, + mut cookie: u64, source: Option<(String, Vec<InnerSpan>)>, ) { // In LTO build we may get srcloc values from other crates which are invalid @@ -420,7 +420,7 @@ fn report_inline_asm( llvm::DiagnosticLevel::Warning => Level::Warning, llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; - cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); + cgcx.diag_emitter.inline_asm_error(cookie.try_into().unwrap(), msg, level, source); } unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { @@ -519,12 +519,22 @@ pub(crate) unsafe fn llvm_optimize( let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; let instr_profile_output_path = get_instr_profile_output_path(config); + let sanitize_dataflow_abilist: Vec<_> = config + .sanitizer_dataflow_abilist + .iter() + .map(|file| CString::new(file.as_str()).unwrap()) + .collect(); + let sanitize_dataflow_abilist_ptrs: Vec<_> = + sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect(); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI), + sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW), + sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(), + sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(), sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI), sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7ed27b33dce..eaedaec635f 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + fn checked_binop( &mut self, oop: OverflowOp, @@ -528,7 +568,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } } - abi::F32 | abi::F64 => {} + abi::F16 | abi::F32 | abi::F64 | abi::F128 => {} } } @@ -563,11 +603,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let llptr = if i == 0 { place.llval } else { - self.inbounds_gep( - self.type_i8(), - place.llval, - &[self.const_usize(b_offset.bytes())], - ) + self.inbounds_ptradd(place.llval, self.const_usize(b_offset.bytes())) }; let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); let load = self.load(llty, llptr, align); @@ -738,11 +774,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn struct_gep(&mut self, ty: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value { - assert_eq!(idx as c_uint as u64, idx); - unsafe { llvm::LLVMBuildStructGEP2(self.llbuilder, ty, ptr, idx as c_uint, UNNAMED) } - } - /* Casts */ fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } @@ -1327,17 +1358,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } @@ -1366,22 +1397,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 7dfcf1ab50e..16122e5557e 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -685,8 +685,10 @@ impl<'ll> CodegenCx<'ll, '_> { let t_i64 = self.type_i64(); let t_i128 = self.type_i128(); let t_isize = self.type_isize(); + let t_f16 = self.type_f16(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_f128 = self.type_f128(); let t_metadata = self.type_metadata(); let t_token = self.type_token(); @@ -728,69 +730,115 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.debugtrap", fn() -> void); ifn!("llvm.frameaddress", fn(t_i32) -> ptr); + ifn!("llvm.powi.f16", fn(t_f16, t_i32) -> t_f16); ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); + ifn!("llvm.powi.f128", fn(t_f128, t_i32) -> t_f128); + ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.pow.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.sqrt.f16", fn(t_f16) -> t_f16); ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); + ifn!("llvm.sqrt.f128", fn(t_f128) -> t_f128); + ifn!("llvm.sin.f16", fn(t_f16) -> t_f16); ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); + ifn!("llvm.sin.f128", fn(t_f128) -> t_f128); + ifn!("llvm.cos.f16", fn(t_f16) -> t_f16); ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); + ifn!("llvm.cos.f128", fn(t_f128) -> t_f128); + ifn!("llvm.exp.f16", fn(t_f16) -> t_f16); ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); + ifn!("llvm.exp.f128", fn(t_f128) -> t_f128); + ifn!("llvm.exp2.f16", fn(t_f16) -> t_f16); ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); + ifn!("llvm.exp2.f128", fn(t_f128) -> t_f128); + ifn!("llvm.log.f16", fn(t_f16) -> t_f16); ifn!("llvm.log.f32", fn(t_f32) -> t_f32); ifn!("llvm.log.f64", fn(t_f64) -> t_f64); + ifn!("llvm.log.f128", fn(t_f128) -> t_f128); + ifn!("llvm.log10.f16", fn(t_f16) -> t_f16); ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); + ifn!("llvm.log10.f128", fn(t_f128) -> t_f128); + ifn!("llvm.log2.f16", fn(t_f16) -> t_f16); ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); + ifn!("llvm.log2.f128", fn(t_f128) -> t_f128); + ifn!("llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16); ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16); ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); + ifn!("llvm.fabs.f128", fn(t_f128) -> t_f128); + ifn!("llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128); + + ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.floor.f16", fn(t_f16) -> t_f16); ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); + ifn!("llvm.floor.f128", fn(t_f128) -> t_f128); + ifn!("llvm.ceil.f16", fn(t_f16) -> t_f16); ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); + ifn!("llvm.ceil.f128", fn(t_f128) -> t_f128); + ifn!("llvm.trunc.f16", fn(t_f16) -> t_f16); ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); + ifn!("llvm.trunc.f128", fn(t_f128) -> t_f128); + ifn!("llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.round.f16", fn(t_f16) -> t_f16); ifn!("llvm.round.f32", fn(t_f32) -> t_f32); ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + ifn!("llvm.round.f128", fn(t_f128) -> t_f128); + ifn!("llvm.roundeven.f16", fn(t_f16) -> t_f16); ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32); ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64); + ifn!("llvm.roundeven.f128", fn(t_f128) -> t_f128); + ifn!("llvm.rint.f16", fn(t_f16) -> t_f16); ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); + ifn!("llvm.rint.f128", fn(t_f128) -> t_f128); + + ifn!("llvm.nearbyint.f16", fn(t_f16) -> t_f16); ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64); + ifn!("llvm.nearbyint.f128", fn(t_f128) -> t_f128); ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8); ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index b1ceb1d4dd5..c45787f35aa 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -357,31 +357,27 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let eligible_def_ids: Vec<DefId> = tcx - .mir_keys(()) - .iter() - .filter_map(|local_def_id| { - let def_id = local_def_id.to_def_id(); - let kind = tcx.def_kind(def_id); - // `mir_keys` will give us `DefId`s for all kinds of things, not - // just "functions", like consts, statics, etc. Filter those out. - // If `ignore_unused_generics` was specified, filter out any - // generic functions from consideration as well. - if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) { - return None; - } - if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { - return None; - } - Some(local_def_id.to_def_id()) - }) - .collect(); + let eligible_def_ids = tcx.mir_keys(()).iter().filter_map(|local_def_id| { + let def_id = local_def_id.to_def_id(); + let kind = tcx.def_kind(def_id); + // `mir_keys` will give us `DefId`s for all kinds of things, not + // just "functions", like consts, statics, etc. Filter those out. + // If `ignore_unused_generics` was specified, filter out any + // generic functions from consideration as well. + if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) { + return None; + } + if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { + return None; + } + Some(local_def_id.to_def_id()) + }); let codegenned_def_ids = codegenned_and_inlined_items(tcx); // For each `DefId` that should have coverage instrumentation but wasn't // codegenned, add it to the function coverage map as an unused function. - for def_id in eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) { + for def_id in eligible_def_ids.filter(|id| !codegenned_def_ids.contains(id)) { // Skip any function that didn't have coverage data added to it by the // coverage instrumentor. let body = tcx.instance_mir(ty::InstanceDef::Item(def_id)); @@ -403,7 +399,6 @@ fn codegenned_and_inlined_items(tcx: TyCtxt<'_>) -> DefIdSet { let mut result = items.clone(); for cgu in cgus { - #[allow(rustc::potential_query_instability)] for item in cgu.items().keys() { if let mir::mono::MonoItem::Fn(ref instance) = item { let did = instance.def_id(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index f961cd2d00b..1a5f9b42947 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -695,9 +695,13 @@ impl MsvcBasicName for ty::UintTy { impl MsvcBasicName for ty::FloatTy { fn msvc_basic_name(self) -> &'static str { + // FIXME: f16 and f128 have no MSVC representation. We could improve the debuginfo. + // See: <https://github.com/rust-lang/rust/pull/114607/files#r1454683264> match self { + ty::FloatTy::F16 => "half", ty::FloatTy::F32 => "float", ty::FloatTy::F64 => "double", + ty::FloatTy::F128 => "fp128", } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 7f671d1d061..657e9ce998f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -122,8 +122,10 @@ fn tag_base_type<'ll, 'tcx>( // Niche tags are always normalized to unsized integers of the correct size. match tag.primitive() { Primitive::Int(t, _) => t, + Primitive::F16 => Integer::I16, Primitive::F32 => Integer::I32, Primitive::F64 => Integer::I64, + Primitive::F128 => Integer::I128, // FIXME(erikdesjardins): handle non-default addrspace ptr sizes Primitive::Pointer(_) => { // If the niche is the NULL value of a reference, then `discr_enum_ty` will be diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 587c5e9e8d2..5bef240340b 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -4,7 +4,7 @@ use std::path::Path; use crate::fluent_generated as fluent; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{DiagCtxt, DiagnosticBuilder, EmissionGuarantee, IntoDiagnostic, Level}; +use rustc_errors::{Diag, DiagCtxt, EmissionGuarantee, IntoDiagnostic, Level}; use rustc_macros::{Diagnostic, Subdiagnostic}; use rustc_span::Span; @@ -100,12 +100,11 @@ pub(crate) struct DynamicLinkingWithLTO; pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { - let diag: DiagnosticBuilder<'_, G> = self.0.into_diagnostic(dcx, level); + fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { + let diag: Diag<'_, G> = self.0.into_diagnostic(dcx, level); let (message, _) = diag.messages.first().expect("`LlvmError` with no message"); - let message = dcx.eagerly_translate_to_string(message.clone(), diag.args()); - - DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) + let message = dcx.eagerly_translate_to_string(message.clone(), diag.args.iter()); + Diag::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) .with_arg("error", message) } } @@ -121,17 +120,13 @@ pub(crate) struct TargetFeatureDisableOrEnable<'a> { pub(crate) struct MissingFeatures; impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { - let mut diag = DiagnosticBuilder::new( - dcx, - level, - fluent::codegen_llvm_target_feature_disable_or_enable, - ); + fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { + let mut diag = Diag::new(dcx, level, fluent::codegen_llvm_target_feature_disable_or_enable); if let Some(span) = self.span { diag.span(span); }; if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); + diag.subdiagnostic(dcx, missing_features); } diag.arg("features", self.features.join(", ")); diag @@ -185,7 +180,7 @@ pub enum LlvmError<'a> { pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String); impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { + fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { use LlvmError::*; let msg_with_llvm_err = match &self.0 { WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index e3e48ecb3aa..f33a672aff0 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -17,7 +17,7 @@ use rustc_hir as hir; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; -use rustc_span::{sym, symbol::kw, Span, Symbol}; +use rustc_span::{sym, Span, Symbol}; use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; @@ -28,51 +28,118 @@ fn get_simple_intrinsic<'ll>( name: Symbol, ) -> Option<(&'ll Type, &'ll Value)> { let llvm_name = match name { + sym::sqrtf16 => "llvm.sqrt.f16", sym::sqrtf32 => "llvm.sqrt.f32", sym::sqrtf64 => "llvm.sqrt.f64", + sym::sqrtf128 => "llvm.sqrt.f128", + + sym::powif16 => "llvm.powi.f16", sym::powif32 => "llvm.powi.f32", sym::powif64 => "llvm.powi.f64", + sym::powif128 => "llvm.powi.f128", + + sym::sinf16 => "llvm.sin.f16", sym::sinf32 => "llvm.sin.f32", sym::sinf64 => "llvm.sin.f64", + sym::sinf128 => "llvm.sin.f128", + + sym::cosf16 => "llvm.cos.f16", sym::cosf32 => "llvm.cos.f32", sym::cosf64 => "llvm.cos.f64", + sym::cosf128 => "llvm.cos.f128", + + sym::powf16 => "llvm.pow.f16", sym::powf32 => "llvm.pow.f32", sym::powf64 => "llvm.pow.f64", + sym::powf128 => "llvm.pow.f128", + + sym::expf16 => "llvm.exp.f16", sym::expf32 => "llvm.exp.f32", sym::expf64 => "llvm.exp.f64", + sym::expf128 => "llvm.exp.f128", + + sym::exp2f16 => "llvm.exp2.f16", sym::exp2f32 => "llvm.exp2.f32", sym::exp2f64 => "llvm.exp2.f64", + sym::exp2f128 => "llvm.exp2.f128", + + sym::logf16 => "llvm.log.f16", sym::logf32 => "llvm.log.f32", sym::logf64 => "llvm.log.f64", + sym::logf128 => "llvm.log.f128", + + sym::log10f16 => "llvm.log10.f16", sym::log10f32 => "llvm.log10.f32", sym::log10f64 => "llvm.log10.f64", + sym::log10f128 => "llvm.log10.f128", + + sym::log2f16 => "llvm.log2.f16", sym::log2f32 => "llvm.log2.f32", sym::log2f64 => "llvm.log2.f64", + sym::log2f128 => "llvm.log2.f128", + + sym::fmaf16 => "llvm.fma.f16", sym::fmaf32 => "llvm.fma.f32", sym::fmaf64 => "llvm.fma.f64", + sym::fmaf128 => "llvm.fma.f128", + + sym::fabsf16 => "llvm.fabs.f16", sym::fabsf32 => "llvm.fabs.f32", sym::fabsf64 => "llvm.fabs.f64", + sym::fabsf128 => "llvm.fabs.f128", + + sym::minnumf16 => "llvm.minnum.f16", sym::minnumf32 => "llvm.minnum.f32", sym::minnumf64 => "llvm.minnum.f64", + sym::minnumf128 => "llvm.minnum.f128", + + sym::maxnumf16 => "llvm.maxnum.f16", sym::maxnumf32 => "llvm.maxnum.f32", sym::maxnumf64 => "llvm.maxnum.f64", + sym::maxnumf128 => "llvm.maxnum.f128", + + sym::copysignf16 => "llvm.copysign.f16", sym::copysignf32 => "llvm.copysign.f32", sym::copysignf64 => "llvm.copysign.f64", + sym::copysignf128 => "llvm.copysign.f128", + + sym::floorf16 => "llvm.floor.f16", sym::floorf32 => "llvm.floor.f32", sym::floorf64 => "llvm.floor.f64", + sym::floorf128 => "llvm.floor.f128", + + sym::ceilf16 => "llvm.ceil.f16", sym::ceilf32 => "llvm.ceil.f32", sym::ceilf64 => "llvm.ceil.f64", + sym::ceilf128 => "llvm.ceil.f128", + + sym::truncf16 => "llvm.trunc.f16", sym::truncf32 => "llvm.trunc.f32", sym::truncf64 => "llvm.trunc.f64", + sym::truncf128 => "llvm.trunc.f128", + + sym::rintf16 => "llvm.rint.f16", sym::rintf32 => "llvm.rint.f32", sym::rintf64 => "llvm.rint.f64", + sym::rintf128 => "llvm.rint.f128", + + sym::nearbyintf16 => "llvm.nearbyint.f16", sym::nearbyintf32 => "llvm.nearbyint.f32", sym::nearbyintf64 => "llvm.nearbyint.f64", + sym::nearbyintf128 => "llvm.nearbyint.f128", + + sym::roundf16 => "llvm.round.f16", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::roundf128 => "llvm.round.f128", + sym::ptr_mask => "llvm.ptrmask", + + sym::roundevenf16 => "llvm.roundeven.f16", sym::roundevenf32 => "llvm.roundeven.f32", sym::roundevenf64 => "llvm.roundeven.f64", + sym::roundevenf128 => "llvm.roundeven.f128", + _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -86,7 +153,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, &'ll Value>], llresult: &'ll Value, span: Span, - ) { + ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); @@ -133,15 +200,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::unlikely => self .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), - kw::Try => { - try_intrinsic( + sym::catch_unwind => { + catch_unwind_intrinsic( self, args[0].immediate(), args[1].immediate(), args[2].immediate(), llresult, ); - return; + return Ok(()); } sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), sym::va_copy => { @@ -163,11 +230,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { emit_va_arg(self, args[0], ret_ty) } } + Primitive::F16 => bug!("the va_arg intrinsic does not work with `f16`"), Primitive::F64 | Primitive::Pointer(_) => { emit_va_arg(self, args[0], ret_ty) } // `va_arg` should never be used with the return type f32. Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), + Primitive::F128 => { + bug!("the va_arg intrinsic does not work with `f128`") + } } } _ => bug!("the va_arg intrinsic does not work with non-scalar types"), @@ -194,17 +265,17 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { if !result.layout.is_zst() { self.store(load, result.llval, result.align); } - return; + return Ok(()); } sym::volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.volatile_store(self, dst); - return; + return Ok(()); } sym::unaligned_volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.unaligned_volatile_store(self, dst); - return; + return Ok(()); } sym::prefetch_read_data | sym::prefetch_write_data @@ -305,7 +376,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { name, ty, }); - return; + return Ok(()); } } } @@ -387,7 +458,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); // We have copied the value to `result` already. - return; + return Ok(()); } _ if name.as_str().starts_with("simd_") => { @@ -395,11 +466,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, ) { Ok(llval) => llval, - Err(()) => return, + Err(()) => return Ok(()), } } - _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), + _ => { + debug!("unknown intrinsic '{}' -- falling back to default body", name); + // Call the fallback body instead of generating the intrinsic code + return Err(ty::Instance::new(instance.def_id(), instance.args)); + } }; if !fn_abi.ret.is_ignore() { @@ -411,6 +486,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .store(self, result); } } + Ok(()) } fn abort(&mut self) { @@ -452,7 +528,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -fn try_intrinsic<'ll>( +fn catch_unwind_intrinsic<'ll>( bx: &mut Builder<'_, 'll, '_>, try_func: &'ll Value, data: &'ll Value, @@ -1074,7 +1150,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .map(|(arg_idx, val)| { let idx = val.unwrap_leaf().try_to_i32().unwrap(); if idx >= i32::try_from(total_len).unwrap() { - bx.sess().dcx().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: arg_idx as u64, @@ -1133,24 +1209,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - bx.sess().dcx().emit_err( - InvalidMonomorphization::ShuffleIndexNotConstant { - span, - name, - arg_idx, - }, - ); - None + bug!("typeck should have already ensured that these are const") } Some(idx) if idx >= total_len => { - bx.sess().dcx().emit_err( - InvalidMonomorphization::ShuffleIndexOutOfBounds { - span, - name, - arg_idx, - total_len, - }, - ); + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx, + total_len, + }); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1179,10 +1246,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>( out_ty: arg_tys[2] } ); + let idx = bx + .const_to_opt_u128(args[1].immediate(), false) + .expect("typeck should have ensure that this is a const"); + if idx >= in_len.into() { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: 1, + total_len: in_len.into(), + }); + return Ok(bx.const_null(llret_ty)); + } return Ok(bx.insert_element( args[0].immediate(), args[2].immediate(), - args[1].immediate(), + bx.const_i32(idx as i32), )); } if name == sym::simd_extract { @@ -1190,7 +1269,19 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ret_ty == in_elem, InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); - return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); + let idx = bx + .const_to_opt_u128(args[1].immediate(), false) + .expect("typeck should have ensure that this is a const"); + if idx >= in_len.into() { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: 1, + total_len: in_len.into(), + }); + return Ok(bx.const_null(llret_ty)); + } + return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32))); } if name == sym::simd_select { @@ -1875,14 +1966,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, + vector_reduce_fadd_reassoc, false, add, 0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, + vector_reduce_fmul_reassoc, false, mul, 1.0 @@ -1915,9 +2006,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); - minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); - minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); - macro_rules! bitwise_red { ($name:ident : $red:ident, $boolean:expr) => { if name == sym::$name { @@ -2089,9 +2177,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(args[0].immediate()); } + #[derive(Copy, Clone)] + enum Sign { + Unsigned, + Signed, + } + use Sign::*; + enum Style { Float, - Int(/* is signed? */ bool), + Int(Sign), Unsupported, } @@ -2099,11 +2194,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // vectors of pointer-sized integers should've been // disallowed before here, so this unwrap is safe. ty::Int(i) => ( - Style::Int(true), + Style::Int(Signed), i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Uint(u) => ( - Style::Int(false), + Style::Int(Unsigned), u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Float(f) => (Style::Float, f.bit_width()), @@ -2111,11 +2206,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; let (out_style, out_width) = match out_elem.kind() { ty::Int(i) => ( - Style::Int(true), + Style::Int(Signed), i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Uint(u) => ( - Style::Int(false), + Style::Int(Unsigned), u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Float(f) => (Style::Float, f.bit_width()), @@ -2123,31 +2218,31 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; match (in_style, out_style) { - (Style::Int(in_is_signed), Style::Int(_)) => { + (Style::Int(sign), Style::Int(_)) => { return Ok(match in_width.cmp(&out_width) { Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), Ordering::Equal => args[0].immediate(), - Ordering::Less => { - if in_is_signed { - bx.sext(args[0].immediate(), llret_ty) - } else { - bx.zext(args[0].immediate(), llret_ty) - } - } + Ordering::Less => match sign { + Sign::Signed => bx.sext(args[0].immediate(), llret_ty), + Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty), + }, }); } - (Style::Int(in_is_signed), Style::Float) => { - return Ok(if in_is_signed { - bx.sitofp(args[0].immediate(), llret_ty) - } else { - bx.uitofp(args[0].immediate(), llret_ty) - }); + (Style::Int(Sign::Signed), Style::Float) => { + return Ok(bx.sitofp(args[0].immediate(), llret_ty)); + } + (Style::Int(Sign::Unsigned), Style::Float) => { + return Ok(bx.uitofp(args[0].immediate(), llret_ty)); } - (Style::Float, Style::Int(out_is_signed)) => { - return Ok(match (out_is_signed, name == sym::simd_as) { - (false, false) => bx.fptoui(args[0].immediate(), llret_ty), - (true, false) => bx.fptosi(args[0].immediate(), llret_ty), - (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), + (Style::Float, Style::Int(sign)) => { + return Ok(match (sign, name == sym::simd_as) { + (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty), + (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int( + matches!(sign, Sign::Signed), + args[0].immediate(), + llret_ty, + ), }); } (Style::Float, Style::Float) => { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index e688e84db61..c84461e53eb 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -169,7 +169,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_pass_timings(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintPassTimings(&mut size as *mut usize); + let cstr = llvm::LLVMRustPrintPassTimings(std::ptr::addr_of_mut!(size)); if cstr.is_null() { println!("failed to get pass timings"); } else { @@ -182,7 +182,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_statistics(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintStatistics(&mut size as *mut usize); + let cstr = llvm::LLVMRustPrintStatistics(std::ptr::addr_of_mut!(size)); if cstr.is_null() { println!("failed to get pass stats"); } else { @@ -369,7 +369,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") @@ -382,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend { }); } - Ok((codegen_results, work_products)) + (codegen_results, work_products) } fn link( diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index 06e846a2b45..f9b28178ddb 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -123,7 +123,7 @@ impl SrcMgrDiagnostic { #[derive(Clone)] pub struct InlineAsmDiagnostic { pub level: super::DiagnosticLevel, - pub cookie: c_uint, + pub cookie: u64, pub message: String, pub source: Option<(String, Vec<InnerSpan>)>, } @@ -149,7 +149,7 @@ impl InlineAsmDiagnostic { let smdiag = SrcMgrDiagnostic::unpack(super::LLVMRustGetSMDiagnostic(di, &mut cookie)); InlineAsmDiagnostic { level: smdiag.level, - cookie, + cookie: cookie.into(), message: smdiag.message, source: smdiag.source, } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d0044086c61..952f41fbd90 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -480,6 +480,9 @@ pub struct SanitizerOptions { pub sanitize_address: bool, pub sanitize_address_recover: bool, pub sanitize_cfi: bool, + pub sanitize_dataflow: bool, + pub sanitize_dataflow_abilist: *const *const c_char, + pub sanitize_dataflow_abilist_len: size_t, pub sanitize_kcfi: bool, pub sanitize_memory: bool, pub sanitize_memory_recover: bool, @@ -858,8 +861,10 @@ extern "C" { pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; // Operations on real types + pub fn LLVMHalfTypeInContext(C: &Context) -> &Type; pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + pub fn LLVMFP128TypeInContext(C: &Context) -> &Type; // Operations on function types pub fn LLVMFunctionType<'a>( @@ -1301,13 +1306,6 @@ extern "C" { NumIndices: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildStructGEP2<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Pointer: &'a Value, - Idx: c_uint, - Name: *const c_char, - ) -> &'a Value; // Casts pub fn LLVMBuildTrunc<'a>( @@ -1618,6 +1616,8 @@ extern "C" { ) -> &'a Value; pub fn LLVMRustSetFastMath(Instr: &Value); + pub fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; @@ -2254,7 +2254,7 @@ extern "C" { pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( DI: &'a DiagnosticInfo, level_out: &mut DiagnosticLevel, - cookie_out: &mut c_uint, + cookie_out: &mut u64, message_out: &mut Option<&'a Twine>, ); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index e48479c8da2..75a189aa87c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -192,14 +192,11 @@ impl<'a> IntoIterator for LLVMFeature<'a> { // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. // -// To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def -// where the * matches the architecture's name -// -// For targets not present in the above location, see llvm-project/llvm/lib/Target/{ARCH}/*.td +// To find a list of LLVM's names, see llvm-project/llvm/lib/Target/{ARCH}/*.td // where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`. // -// Beware to not use the llvm github project for this, but check the git submodule -// found in src/llvm-project +// Check the current rustc fork of LLVM in the repo at https://github.com/rust-lang/llvm-project/. +// The commit in use can be found via the `llvm-project` submodule in https://github.com/rust-lang/rust/tree/master/src // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics @@ -266,6 +263,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { LLVMFeature::new("unaligned-scalar-mem") } + // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled. + ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => { + LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")) + } (_, s) => LLVMFeature::new(s), } } @@ -431,7 +432,7 @@ pub(crate) fn print(req: &PrintRequest, mut out: &mut dyn PrintBackendInfo, sess &tm, cpu_cstring.as_ptr(), callback, - &mut out as *mut &mut dyn PrintBackendInfo as *mut c_void, + std::ptr::addr_of_mut!(out) as *mut c_void, ); } } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 447c4ed1f0c..07a4861ed73 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -107,8 +107,10 @@ impl<'ll> CodegenCx<'ll, '_> { pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { match t { + ty::FloatTy::F16 => self.type_f16(), ty::FloatTy::F32 => self.type_f32(), ty::FloatTy::F64 => self.type_f64(), + ty::FloatTy::F128 => self.type_f128(), } } @@ -156,6 +158,10 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.isize_ty } + fn type_f16(&self) -> &'ll Type { + unsafe { llvm::LLVMHalfTypeInContext(self.llcx) } + } + fn type_f32(&self) -> &'ll Type { unsafe { llvm::LLVMFloatTypeInContext(self.llcx) } } @@ -164,6 +170,10 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) } } + fn type_f128(&self) -> &'ll Type { + unsafe { llvm::LLVMFP128TypeInContext(self.llcx) } + } + fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } } @@ -195,7 +205,7 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { match self.type_kind(ty) { TypeKind::Array | TypeKind::Vector => unsafe { llvm::LLVMGetElementType(ty) }, TypeKind::Pointer => bug!("element_type is not supported for opaque pointers"), - other => bug!("element_type called on unsupported type {:?}", other), + other => bug!("element_type called on unsupported type {other:?}"), } } @@ -205,11 +215,12 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn float_width(&self, ty: &'ll Type) -> usize { match self.type_kind(ty) { + TypeKind::Half => 16, TypeKind::Float => 32, TypeKind::Double => 64, TypeKind::X86_FP80 => 80, TypeKind::FP128 | TypeKind::PPC_FP128 => 128, - _ => bug!("llvm_float_width called on a non-float type"), + other => bug!("llvm_float_width called on a non-float type {other:?}"), } } @@ -250,9 +261,6 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { layout.is_llvm_scalar_pair() } - fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64 { - layout.llvm_field_index(self, index) - } fn scalar_pair_element_backend_type( &self, layout: TyAndLayout<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 219c7025311..587adefe1d2 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -8,7 +8,7 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty, TypeVisitableExt}; use rustc_target::abi::HasDataLayout; use rustc_target::abi::{Abi, Align, FieldsShape}; -use rustc_target::abi::{Int, Pointer, F32, F64}; +use rustc_target::abi::{Int, Pointer, F128, F16, F32, F64}; use rustc_target::abi::{Scalar, Size, Variants}; use smallvec::{smallvec, SmallVec}; @@ -174,7 +174,6 @@ pub trait LayoutLlvmExt<'tcx> { index: usize, immediate: bool, ) -> &'a Type; - fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type>; } @@ -291,8 +290,10 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type { match scalar.primitive() { Int(i, _) => cx.type_from_integer(i), + F16 => cx.type_f16(), F32 => cx.type_f32(), F64 => cx.type_f64(), + F128 => cx.type_f128(), Pointer(address_space) => cx.type_ptr_ext(address_space), } } @@ -324,42 +325,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { self.scalar_llvm_type_at(cx, scalar) } - fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64 { - match self.abi { - Abi::Scalar(_) | Abi::ScalarPair(..) => { - bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) - } - _ => {} - } - match self.fields { - FieldsShape::Primitive | FieldsShape::Union(_) => { - bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) - } - - FieldsShape::Array { .. } => index as u64, - - FieldsShape::Arbitrary { .. } => { - let variant_index = match self.variants { - Variants::Single { index } => Some(index), - _ => None, - }; - - // Look up llvm field if indexes do not match memory order due to padding. If - // `field_remapping` is `None` no padding was used and the llvm field index - // matches the memory index. - match cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - Some(TypeLowering { field_remapping: Some(ref remap), .. }) => { - remap[index] as u64 - } - Some(_) => self.fields.memory_index(index) as u64, - None => { - bug!("TyAndLayout::llvm_field_index({:?}): type info not found", self) - } - } - } - } - } - fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type> { debug_assert!(self.is_sized()); diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 172c66a7af1..b406a04af74 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -44,12 +44,12 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>( let aligned_size = size.align_to(slot_size).bytes() as i32; let full_direct_size = bx.cx().const_i32(aligned_size); - let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]); + let next = bx.inbounds_ptradd(addr, full_direct_size); bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); - let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); + let adjusted = bx.inbounds_ptradd(addr, adjusted_size); (adjusted, addr_align) } else { (addr, addr_align) @@ -89,11 +89,31 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( list: OperandRef<'tcx, &'ll Value>, target_ty: Ty<'tcx>, ) -> &'ll Value { + let dl = bx.cx.data_layout(); + // Implementation of the AAPCS64 calling convention for va_args see // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + // + // typedef struct va_list { + // void * stack; // next stack param + // void * gr_top; // end of GP arg reg save area + // void * vr_top; // end of FP/SIMD arg reg save area + // int gr_offs; // offset from gr_top to next GP register arg + // int vr_offs; // offset from vr_top to next FP/SIMD register arg + // } va_list; let va_list_addr = list.immediate(); - let va_list_layout = list.deref(bx.cx).layout; - let va_list_ty = va_list_layout.llvm_type(bx); + + // There is no padding between fields since `void*` is size=8 align=8, `int` is size=4 align=4. + // See https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + // Table 1, Byte size and byte alignment of fundamental data types + // Table 3, Mapping of C & C++ built-in data types + let ptr_offset = 8; + let i32_offset = 4; + let gr_top = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(ptr_offset)); + let vr_top = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * ptr_offset)); + let gr_offs = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(3 * ptr_offset)); + let vr_offs = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(3 * ptr_offset + i32_offset)); + let layout = bx.cx.layout_of(target_ty); let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); @@ -104,16 +124,12 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( let offset_align = Align::from_bytes(4).unwrap(); let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); - let (reg_off, reg_top_index, slot_size) = if gr_type { - let gr_offs = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let (reg_off, reg_top, slot_size) = if gr_type { let nreg = (layout.size.bytes() + 7) / 8; - (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8) + (gr_offs, gr_top, nreg * 8) } else { - let vr_off = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4)); let nreg = (layout.size.bytes() + 15) / 16; - (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16) + (vr_offs, vr_top, nreg * 16) }; // if the offset >= 0 then the value will be on the stack @@ -141,15 +157,14 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); let top_type = bx.type_ptr(); - let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); - let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); + let top = bx.load(top_type, reg_top, dl.pointer_align.abi); // reg_value = *(@top + reg_off_v); - let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); + let mut reg_addr = bx.ptradd(top, reg_off_v); if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { // On big-endian systems the value is right-aligned in its slot. let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); - reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); + reg_addr = bx.ptradd(reg_addr, offset); } let reg_type = layout.llvm_type(bx); let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); @@ -173,11 +188,29 @@ fn emit_s390x_va_arg<'ll, 'tcx>( list: OperandRef<'tcx, &'ll Value>, target_ty: Ty<'tcx>, ) -> &'ll Value { + let dl = bx.cx.data_layout(); + // Implementation of the s390x ELF ABI calling convention for va_args see // https://github.com/IBM/s390x-abi (chapter 1.2.4) + // + // typedef struct __va_list_tag { + // long __gpr; + // long __fpr; + // void *__overflow_arg_area; + // void *__reg_save_area; + // } va_list[1]; let va_list_addr = list.immediate(); - let va_list_layout = list.deref(bx.cx).layout; - let va_list_ty = va_list_layout.llvm_type(bx); + + // There is no padding between fields since `long` and `void*` both have size=8 align=8. + // https://github.com/IBM/s390x-abi (Table 1.1.: Scalar types) + let i64_offset = 8; + let ptr_offset = 8; + let gpr = va_list_addr; + let fpr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(i64_offset)); + let overflow_arg_area = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * i64_offset)); + let reg_save_area = + bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * i64_offset + ptr_offset)); + let layout = bx.cx.layout_of(target_ty); let in_reg = bx.append_sibling_block("va_arg.in_reg"); @@ -192,15 +225,10 @@ fn emit_s390x_va_arg<'ll, 'tcx>( let padding = padded_size - unpadded_size; let gpr_type = indirect || !layout.is_single_fp_element(bx.cx); - let (max_regs, reg_count_field, reg_save_index, reg_padding) = - if gpr_type { (5, 0, 2, padding) } else { (4, 1, 16, 0) }; + let (max_regs, reg_count, reg_save_index, reg_padding) = + if gpr_type { (5, gpr, 2, padding) } else { (4, fpr, 16, 0) }; // Check whether the value was passed in a register or in memory. - let reg_count = bx.struct_gep( - va_list_ty, - va_list_addr, - va_list_layout.llvm_field_index(bx.cx, reg_count_field), - ); let reg_count_v = bx.load(bx.type_i64(), reg_count, Align::from_bytes(8).unwrap()); let use_regs = bx.icmp(IntPredicate::IntULT, reg_count_v, bx.const_u64(max_regs)); bx.cond_br(use_regs, in_reg, in_mem); @@ -209,12 +237,10 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); // Work out the address of the value in the register save area. - let reg_ptr = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); - let reg_ptr_v = bx.load(bx.type_ptr(), reg_ptr, bx.tcx().data_layout.pointer_align.abi); + let reg_ptr_v = bx.load(bx.type_ptr(), reg_save_area, dl.pointer_align.abi); let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8)); let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding)); - let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]); + let reg_addr = bx.ptradd(reg_ptr_v, reg_off); // Update the register count. let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1)); @@ -225,27 +251,23 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.switch_to_block(in_mem); // Work out the address of the value in the argument overflow area. - let arg_ptr = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 2)); - let arg_ptr_v = bx.load(bx.type_ptr(), arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let arg_ptr_v = + bx.load(bx.type_ptr(), overflow_arg_area, bx.tcx().data_layout.pointer_align.abi); let arg_off = bx.const_u64(padding); - let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]); + let mem_addr = bx.ptradd(arg_ptr_v, arg_off); // Update the argument overflow area pointer. let arg_size = bx.cx().const_u64(padded_size); - let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]); - bx.store(new_arg_ptr_v, arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let new_arg_ptr_v = bx.inbounds_ptradd(arg_ptr_v, arg_size); + bx.store(new_arg_ptr_v, overflow_arg_area, dl.pointer_align.abi); bx.br(end); // Return the appropriate result. bx.switch_to_block(end); let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]); let val_type = layout.llvm_type(bx); - let val_addr = if indirect { - bx.load(bx.cx.type_ptr(), val_addr, bx.tcx().data_layout.pointer_align.abi) - } else { - val_addr - }; + let val_addr = + if indirect { bx.load(bx.cx.type_ptr(), val_addr, dl.pointer_align.abi) } else { val_addr }; bx.load(val_type, val_addr, layout.align.abi) } |