diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
27 files changed, 932 insertions, 575 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 97dc401251c..e5f5146fac8 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -6,7 +6,7 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; @@ -203,57 +203,63 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { val: &'ll Value, dst: PlaceRef<'tcx, &'ll Value>, ) { - if self.is_ignore() { - return; - } - if self.is_sized_indirect() { - OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) - } else if self.is_unsized_indirect() { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); - } else if let PassMode::Cast { cast, pad_i32: _ } = &self.mode { - // FIXME(eddyb): Figure out when the simpler Store is safe, clang - // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. - let can_store_through_cast_ptr = false; - if can_store_through_cast_ptr { - bx.store(val, dst.llval, self.layout.align.abi); - } else { - // The actual return type is a struct, but the ABI - // adaptation code has cast it into some scalar type. The - // code that follows is the only reliable way I have - // found to do a transform like i64 -> {i32,i32}. - // Basically we dump the data onto the stack then memcpy it. - // - // Other approaches I tried: - // - Casting rust ret pointer to the foreign type and using Store - // is (a) unsafe if size of foreign type > size of rust type and - // (b) runs afoul of strict aliasing rules, yielding invalid - // assembly under -O (specifically, the store gets removed). - // - Truncating foreign type to correct integral type and then - // bitcasting to the struct type yields invalid cast errors. - - // We instead thus allocate some scratch space... - let scratch_size = cast.size(bx); - let scratch_align = cast.align(bx); - let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); - bx.lifetime_start(llscratch, scratch_size); - - // ... where we first store the value... - bx.store(val, llscratch, scratch_align); - - // ... and then memcpy it to the intended destination. - bx.memcpy( - dst.llval, - self.layout.align.abi, - llscratch, - scratch_align, - bx.const_usize(self.layout.size.bytes()), - MemFlags::empty(), - ); + match &self.mode { + PassMode::Ignore => {} + // Sized indirect arguments + PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { + let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); + OperandValue::Ref(val, None, align).store(bx, dst); + } + // Unsized indirect qrguments + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { + bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + } + PassMode::Cast { cast, pad_i32: _ } => { + // FIXME(eddyb): Figure out when the simpler Store is safe, clang + // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. + let can_store_through_cast_ptr = false; + if can_store_through_cast_ptr { + bx.store(val, dst.llval, self.layout.align.abi); + } else { + // The actual return type is a struct, but the ABI + // adaptation code has cast it into some scalar type. The + // code that follows is the only reliable way I have + // found to do a transform like i64 -> {i32,i32}. + // Basically we dump the data onto the stack then memcpy it. + // + // Other approaches I tried: + // - Casting rust ret pointer to the foreign type and using Store + // is (a) unsafe if size of foreign type > size of rust type and + // (b) runs afoul of strict aliasing rules, yielding invalid + // assembly under -O (specifically, the store gets removed). + // - Truncating foreign type to correct integral type and then + // bitcasting to the struct type yields invalid cast errors. + + // We instead thus allocate some scratch space... + let scratch_size = cast.size(bx); + let scratch_align = cast.align(bx); + let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); + bx.lifetime_start(llscratch, scratch_size); + + // ... where we first store the value... + bx.store(val, llscratch, scratch_align); + + // ... and then memcpy it to the intended destination. + bx.memcpy( + dst.llval, + self.layout.align.abi, + llscratch, + scratch_align, + bx.const_usize(self.layout.size.bytes()), + MemFlags::empty(), + ); - bx.lifetime_end(llscratch, scratch_size); + bx.lifetime_end(llscratch, scratch_size); + } + } + _ => { + OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst); } - } else { - OperandValue::Immediate(val).store(bx, dst); } } @@ -424,7 +430,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(attrs); - let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); + let sret = llvm::CreateStructRetAttr( + cx.llcx, + cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()), + ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); } PassMode::Cast { cast, pad_i32: _ } => { @@ -437,7 +446,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => {} PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { let i = apply(attrs); - let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); + let byval = llvm::CreateByValAttr( + cx.llcx, + cx.type_array(cx.type_i8(), arg.layout.size.bytes()), + ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } PassMode::Direct(attrs) @@ -486,7 +498,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); let i = apply(bx.cx, attrs); - let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); + let sret = llvm::CreateStructRetAttr( + bx.cx.llcx, + bx.cx.type_array(bx.cx.type_i8(), self.ret.layout.size.bytes()), + ); attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); } PassMode::Cast { cast, pad_i32: _ } => { @@ -513,7 +528,10 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Ignore => {} PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => { let i = apply(bx.cx, attrs); - let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); + let byval = llvm::CreateByValAttr( + bx.cx.llcx, + bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()), + ); attributes::apply_to_callsite( callsite, llvm::AttributePlace::Argument(i), @@ -590,7 +608,6 @@ impl From<Conv> for llvm::CallConv { Conv::Cold => llvm::ColdCallConv, Conv::PreserveMost => llvm::PreserveMost, Conv::PreserveAll => llvm::PreserveAll, - Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, Conv::ArmAapcs => llvm::ArmAapcsCallConv, diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 58b3aa438c5..ca376029735 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -134,7 +134,8 @@ fn create_wrapper_function( llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); } if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); + let uwtable = + attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 1323261ae92..74539d4d495 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option<Self::BasicBlock>, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Build the template string + let mut labels = vec![]; let mut template_str = String::new(); for piece in template { match *piece { @@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Only emit the raw symbol name template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); } + InlineAsmOperandRef::Label { label } => { + template_str.push_str(&format!("${{{}:l}}", constraints.len())); + constraints.push("!i".to_owned()); + labels.push(label); + } } } } @@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &constraints.join(","), &inputs, output_type, + &labels, volatile, alignstack, dialect, line_spans, options.contains(InlineAsmOptions::MAY_UNWIND), - dest_catch_funclet, + dest, + catch_funclet, ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); @@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` - if let Some((dest, _, _)) = dest_catch_funclet { + if let Some(dest) = dest { self.switch_to_block(dest); } @@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>( cons: &str, inputs: &[&'ll Value], output: &'ll llvm::Type, + labels: &[&'ll llvm::BasicBlock], volatile: bool, alignstack: bool, dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, - dest_catch_funclet: Option<( - &'ll llvm::BasicBlock, - &'ll llvm::BasicBlock, - Option<&Funclet<'ll>>, - )>, + dest: Option<&'ll llvm::BasicBlock>, + catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { let volatile = if volatile { llvm::True } else { llvm::False }; let alignstack = if alignstack { llvm::True } else { llvm::False }; @@ -457,8 +464,11 @@ pub(crate) fn inline_asm_call<'ll>( can_throw, ); - let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) + let call = if !labels.is_empty() { + assert!(catch_funclet.is_none()); + bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None) + } else if let Some((catch, funclet)) = catch_funclet { + bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet) } else { bx.call(fty, None, None, v, inputs, None) }; @@ -690,6 +700,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", @@ -867,7 +878,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::S390x( + S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr, + ) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 3cc33b83434..f9eaa0d94cb 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -95,11 +95,12 @@ pub fn sanitize_attrs<'ll>( /// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. #[inline] -pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { +pub fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) -> &Attribute { // NOTE: We should determine if we even need async unwind tables, as they // take have more overhead and if we can use sync unwind tables we // probably should. - llvm::CreateUWTableAttr(llcx, true) + let async_unwind = !use_sync_unwind.unwrap_or(false); + llvm::CreateUWTableAttr(llcx, async_unwind) } pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { @@ -333,7 +334,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // You can also find more info on why Windows always requires uwtables here: // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx)); + to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); } if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { @@ -355,9 +356,6 @@ pub fn from_fn_attrs<'ll, 'tcx>( if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); } - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { - to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); - } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { to_add.push(MemoryEffects::ReadOnly.create_attr(cx.llcx)); } @@ -481,7 +479,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // `+multivalue` feature because the purpose of the wasm abi is to match // the WebAssembly specification, which has this feature. This won't be // needed when LLVM enables this `multivalue` feature by default. - if !cx.tcx.is_closure(instance.def_id()) { + if !cx.tcx.is_closure_like(instance.def_id()) { let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); if abi == Abi::Wasm { function_features.push("+multivalue".to_string()); diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 67bc86e4c90..0619000364b 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -55,12 +55,13 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType { "x86_64" => LLVMMachineType::AMD64, "x86" => LLVMMachineType::I386, "aarch64" => LLVMMachineType::ARM64, + "arm64ec" => LLVMMachineType::ARM64EC, "arm" => LLVMMachineType::ARM, _ => panic!("unsupported cpu type {cpu}"), } } -impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { +impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { fn add_archive( &mut self, archive: &Path, @@ -107,7 +108,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { pub struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { - fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { // FIXME use ArArchiveBuilder on most targets again once reading thin archives is // implemented if true { @@ -313,7 +314,7 @@ fn get_llvm_object_symbols( llvm::LLVMRustGetSymbols( buf.as_ptr(), buf.len(), - &mut *state as *mut &mut _ as *mut c_void, + std::ptr::addr_of_mut!(*state) as *mut c_void, callback, error_callback, ) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index e9e8ade09b7..06a681c24e6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -21,6 +21,7 @@ use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::config::{self, CrateType, Lto}; +use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; @@ -60,7 +61,7 @@ fn prepare_lto( }; let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { - if info.level.is_below_threshold(export_threshold) || info.used || info.used_compiler { + if info.level.is_below_threshold(export_threshold) || info.used { Some(CString::new(name.as_str()).unwrap()) } else { None @@ -787,7 +788,7 @@ pub unsafe fn optimize_thin_module( #[derive(Debug, Default)] pub struct ThinLTOKeysMap { // key = llvm name of importing module, value = LLVM cache key - keys: FxHashMap<String, String>, + keys: BTreeMap<String, String>, } impl ThinLTOKeysMap { @@ -797,7 +798,6 @@ impl ThinLTOKeysMap { let mut writer = io::BufWriter::new(file); // The entries are loaded back into a hash map in `load_from_file()`, so // the order in which we write them to file here does not matter. - #[allow(rustc::potential_query_instability)] for (module, key) in &self.keys { writeln!(writer, "{module} {key}")?; } @@ -806,7 +806,7 @@ impl ThinLTOKeysMap { fn load_from_file(path: &Path) -> io::Result<Self> { use std::io::BufRead; - let mut keys = FxHashMap::default(); + let mut keys = BTreeMap::default(); let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c607533a08e..031bbd63361 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -36,7 +36,7 @@ use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use crate::llvm::diagnostic::OptimizationDiagnosticKind; -use libc::{c_char, c_int, c_uint, c_void, size_t}; +use libc::{c_char, c_int, c_void, size_t}; use std::ffi::CString; use std::fs; use std::io::{self, Write}; @@ -245,12 +245,12 @@ pub fn target_machine_factory( match sess.opts.debuginfo_compression { rustc_session::config::DebugInfoCompression::Zlib => { if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { - sess.dcx().emit_warning(UnknownCompression { algorithm: "zlib" }); + sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" }); } } rustc_session::config::DebugInfoCompression::Zstd => { if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { - sess.dcx().emit_warning(UnknownCompression { algorithm: "zstd" }); + sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" }); } } rustc_session::config::DebugInfoCompression::None => {} @@ -406,7 +406,7 @@ fn report_inline_asm( cgcx: &CodegenContext<LlvmCodegenBackend>, msg: String, level: llvm::DiagnosticLevel, - mut cookie: c_uint, + mut cookie: u64, source: Option<(String, Vec<InnerSpan>)>, ) { // In LTO build we may get srcloc values from other crates which are invalid @@ -416,11 +416,11 @@ fn report_inline_asm( cookie = 0; } let level = match level { - llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning(None), + llvm::DiagnosticLevel::Error => Level::Error, + llvm::DiagnosticLevel::Warning => Level::Warning, llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; - cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); + cgcx.diag_emitter.inline_asm_error(cookie.try_into().unwrap(), msg, level, source); } unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { @@ -457,7 +457,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - dcx.emit_warning(FromLlvmDiag { message }); + dcx.emit_warn(FromLlvmDiag { message }); } llvm::diagnostic::Unsupported(diagnostic_ref) => { let message = llvm::build_string(|s| { @@ -519,12 +519,22 @@ pub(crate) unsafe fn llvm_optimize( let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; let instr_profile_output_path = get_instr_profile_output_path(config); + let sanitize_dataflow_abilist: Vec<_> = config + .sanitizer_dataflow_abilist + .iter() + .map(|file| CString::new(file.as_str()).unwrap()) + .collect(); + let sanitize_dataflow_abilist_ptrs: Vec<_> = + sanitize_dataflow_abilist.iter().map(|file| file.as_ptr()).collect(); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), sanitize_cfi: config.sanitizer.contains(SanitizerSet::CFI), + sanitize_dataflow: config.sanitizer.contains(SanitizerSet::DATAFLOW), + sanitize_dataflow_abilist: sanitize_dataflow_abilist_ptrs.as_ptr(), + sanitize_dataflow_abilist_len: sanitize_dataflow_abilist_ptrs.len(), sanitize_kcfi: config.sanitizer.contains(SanitizerSet::KCFI), sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), @@ -569,6 +579,7 @@ pub(crate) unsafe fn llvm_optimize( unroll_loops, config.vectorize_slp, config.vectorize_loop, + config.no_builtins, config.emit_lifetime_markers, sanitizer_options.as_ref(), pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), @@ -677,6 +688,7 @@ pub(crate) unsafe fn codegen( unsafe fn with_codegen<'ll, F, R>( tm: &'ll llvm::TargetMachine, llmod: &'ll llvm::Module, + no_builtins: bool, f: F, ) -> R where @@ -684,7 +696,7 @@ pub(crate) unsafe fn codegen( { let cpm = llvm::LLVMCreatePassManager(); llvm::LLVMAddAnalysisPasses(tm, cpm); - llvm::LLVMRustAddLibraryInfo(cpm, llmod); + llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); f(cpm) } @@ -785,7 +797,7 @@ pub(crate) unsafe fn codegen( } else { llmod }; - with_codegen(tm, llmod, |cpm| { + with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file( dcx, tm, @@ -820,7 +832,7 @@ pub(crate) unsafe fn codegen( (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; - with_codegen(tm, llmod, |cpm| { + with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file( dcx, tm, diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 8f60175a603..63e59ea13fc 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -340,6 +340,46 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn fadd_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fsub_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fmul_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn fdiv_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + + fn frem_algebraic(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetAlgebraicMath(instr); + instr + } + } + fn checked_binop( &mut self, oop: OverflowOp, @@ -528,7 +568,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } } - abi::F32 | abi::F64 => {} + abi::F16 | abi::F32 | abi::F64 | abi::F128 => {} } } @@ -558,10 +598,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Immediate(self.to_immediate(llval, place.layout)) } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { let b_offset = a.size(self).align_to(b.align(self).abi); - let pair_ty = place.layout.llvm_type(self); let mut load = |i, scalar: abi::Scalar, layout, align, offset| { - let llptr = self.struct_gep(pair_ty, place.llval, i as u64); + let llptr = if i == 0 { + place.llval + } else { + self.inbounds_ptradd(place.llval, self.const_usize(b_offset.bytes())) + }; let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); let load = self.load(llty, llptr, align); scalar_load_metadata(self, load, scalar, layout, offset); @@ -731,11 +774,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } - fn struct_gep(&mut self, ty: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value { - assert_eq!(idx as c_uint as u64, idx); - unsafe { llvm::LLVMBuildStructGEP2(self.llbuilder, ty, ptr, idx as c_uint, UNNAMED) } - } - /* Casts */ fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } @@ -1072,7 +1110,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { order: rustc_codegen_ssa::common::AtomicOrdering, failure_order: rustc_codegen_ssa::common::AtomicOrdering, weak: bool, - ) -> &'ll Value { + ) -> (&'ll Value, &'ll Value) { let weak = if weak { llvm::True } else { llvm::False }; unsafe { let value = llvm::LLVMBuildAtomicCmpXchg( @@ -1085,16 +1123,24 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llvm::False, // SingleThreaded ); llvm::LLVMSetWeak(value, weak); - value + let val = self.extract_value(value, 0); + let success = self.extract_value(value, 1); + (val, success) } } fn atomic_rmw( &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, dst: &'ll Value, - src: &'ll Value, + mut src: &'ll Value, order: rustc_codegen_ssa::common::AtomicOrdering, ) -> &'ll Value { + // The only RMW operation that LLVM supports on pointers is compare-exchange. + if self.val_ty(src) == self.type_ptr() + && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg + { + src = self.ptrtoint(src, self.type_isize()); + } unsafe { llvm::LLVMBuildAtomicRMW( self.llbuilder, @@ -1318,17 +1364,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } } - pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fadd_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } - pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + pub fn vector_reduce_fmul_reassoc(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { unsafe { let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); - llvm::LLVMRustSetFastMath(instr); + llvm::LLVMRustSetAllowReassoc(instr); instr } } @@ -1357,22 +1403,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) } } - pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } - pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - let instr = - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); - llvm::LLVMRustSetFastMath(instr); - instr - } - } pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } } @@ -1514,6 +1544,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } + pub(crate) fn callbr( + &mut self, + llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + args: &[&'ll Value], + default_dest: &'ll BasicBlock, + indirect_dest: &[&'ll BasicBlock], + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("invoke {:?} with args ({:?})", llfn, args); + + let args = self.check_call("callbr", llty, llfn, args); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } + + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } + + let callbr = unsafe { + llvm::LLVMRustBuildCallBr( + self.llbuilder, + llty, + llfn, + default_dest, + indirect_dest.as_ptr(), + indirect_dest.len() as c_uint, + args.as_ptr(), + args.len() as c_uint, + bundles.as_ptr(), + bundles.len() as c_uint, + UNNAMED, + ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, callbr); + } + callbr + } + // Emits CFI pointer type membership tests. fn cfi_type_test( &mut self, diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 8173e41aff4..25cbd90460f 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -95,11 +95,13 @@ impl<'ll> BackendTypes for CodegenCx<'ll, '_> { impl<'ll> CodegenCx<'ll, '_> { pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } + let len = u64::try_from(elts.len()).expect("LLVMConstArray2 elements len overflow"); + unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - unsafe { llvm::LLVMConstVector(elts.as_ptr(), elts.len() as c_uint) } + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } } pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { @@ -108,8 +110,8 @@ impl<'ll> CodegenCx<'ll, '_> { pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { - assert_eq!(idx as c_uint as u64, idx); - let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); + let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); + let r = llvm::LLVMGetAggregateElement(v, idx).unwrap(); debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); @@ -329,7 +331,7 @@ pub fn val_ty(v: &Value) -> &Type { pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) } } @@ -338,9 +340,8 @@ pub fn struct_in_context<'ll>( elts: &[&'ll Value], packed: bool, ) -> &'ll Value { - unsafe { - llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), elts.len() as c_uint, packed as Bool) - } + let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); + unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed as Bool) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index ec2fb2c6e54..4afa230e598 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -9,6 +9,7 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_codegen_ssa::traits::*; +use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ @@ -17,7 +18,7 @@ use rustc_middle::mir::interpret::{ }; use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use rustc_session::config::Lto; use rustc_target::abi::{ @@ -114,7 +115,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< cx.const_struct(&llvals, true) } -pub fn codegen_static_initializer<'ll, 'tcx>( +fn codegen_static_initializer<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { @@ -147,11 +148,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: fn check_and_apply_linkage<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, attrs: &CodegenFnAttrs, - ty: Ty<'tcx>, + llty: &'ll Type, sym: &str, def_id: DefId, ) -> &'ll Value { - let llty = cx.layout_of(ty).llvm_type(cx); if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); @@ -226,9 +226,28 @@ impl<'ll> CodegenCx<'ll, '_> { } } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); - if let Some(&g) = self.instances.borrow().get(&instance) { + trace!(?instance); + + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out + // the llvm type from the actual evaluated initializer. + let llty = if nested { + self.type_i8() + } else { + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + trace!(?ty); + self.layout_of(ty).llvm_type(self) + }; + self.get_static_inner(def_id, llty) + } + + #[instrument(level = "debug", skip(self, llty))] + pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value { + if let Some(&g) = self.instances.borrow().get(&Instance::mono(self.tcx, def_id)) { + trace!("used cached value"); return g; } @@ -240,14 +259,12 @@ impl<'ll> CodegenCx<'ll, '_> { statics defined in the same CGU, but did not for `{def_id:?}`" ); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let sym = self.tcx.symbol_name(instance).name; + let sym = self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name; let fn_attrs = self.tcx.codegen_fn_attrs(def_id); - debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); + debug!(?sym, ?fn_attrs); let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { - let llty = self.layout_of(ty).llvm_type(self); if let Some(g) = self.get_declared_value(sym) { if self.val_ty(g) != self.type_ptr() { span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); @@ -264,7 +281,7 @@ impl<'ll> CodegenCx<'ll, '_> { g } else { - check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) + check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) }; // Thread-local statics in some other crate need to *always* be linked @@ -332,34 +349,18 @@ impl<'ll> CodegenCx<'ll, '_> { } } - self.instances.borrow_mut().insert(instance, g); + self.instances.borrow_mut().insert(Instance::mono(self.tcx, def_id), g); g } -} - -impl<'ll> StaticMethods for CodegenCx<'ll, '_> { - fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { - if let Some(&gv) = self.const_globals.borrow().get(&cv) { - unsafe { - // Upgrade the alignment in cases where the same constant is used with different - // alignment requirements - let llalign = align.bytes() as u32; - if llalign > llvm::LLVMGetAlignment(gv) { - llvm::LLVMSetAlignment(gv, llalign); - } - } - return gv; - } - let gv = self.static_addr_of_mut(cv, align, kind); - unsafe { - llvm::LLVMSetGlobalConstant(gv, True); - } - self.const_globals.borrow_mut().insert(cv, gv); - gv - } - fn codegen_static(&self, def_id: DefId, is_mutable: bool) { + fn codegen_static_item(&self, def_id: DefId) { unsafe { + assert!( + llvm::LLVMGetInitializer( + self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap() + ) + .is_none() + ); let attrs = self.tcx.codegen_fn_attrs(def_id); let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { @@ -368,13 +369,11 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { }; let alloc = alloc.inner(); - let g = self.get_static(def_id); - let val_llty = self.val_ty(v); - let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); - let llty = self.layout_of(ty).llvm_type(self); + let g = self.get_static_inner(def_id, val_llty); + let llty = self.val_ty(g); + let g = if val_llty == llty { g } else { @@ -409,16 +408,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; - set_global_alignment(self, g, self.align_of(ty)); + set_global_alignment(self, g, alloc.align); llvm::LLVMSetInitializer(g, v); if self.should_assume_dso_local(g, true) { llvm::LLVMRustSetDSOLocal(g, true); } - // As an optimization, all shared statics which do not have interior - // mutability are placed into read-only memory. - if !is_mutable && self.type_is_freeze(ty) { + // Forward the allocation's mutability (picked by the const interner) to LLVM. + if alloc.mutability.is_not() { llvm::LLVMSetGlobalConstant(g, llvm::True); } @@ -541,6 +539,32 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { } } } +} + +impl<'ll> StaticMethods for CodegenCx<'ll, '_> { + fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + + fn codegen_static(&self, def_id: DefId) { + self.codegen_static_item(def_id) + } /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr. fn add_used_global(&self, global: &'ll Value) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1d1b6e6148d..f89c8c9f836 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -34,6 +34,7 @@ use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; use libc::c_uint; +use std::borrow::Borrow; use std::cell::{Cell, RefCell}; use std::ffi::CStr; use std::str; @@ -76,8 +77,8 @@ pub struct CodegenCx<'ll, 'tcx> { /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details pub compiler_used_statics: RefCell<Vec<&'ll Value>>, - /// Mapping of non-scalar types to llvm types and field remapping if needed. - pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), TypeLowering<'ll>>>, + /// Mapping of non-scalar types to llvm types. + pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>, /// Mapping of scalar types to llvm types. pub scalar_lltypes: RefCell<FxHashMap<Ty<'tcx>, &'ll Type>>, @@ -104,15 +105,6 @@ pub struct CodegenCx<'ll, 'tcx> { pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, } -pub struct TypeLowering<'ll> { - /// Associated LLVM type - pub lltype: &'ll Type, - - /// If padding is used the slice maps fields from source order - /// to llvm order. - pub field_remapping: Option<SmallVec<[u32; 4]>>, -} - fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { match tls_model { TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic, @@ -145,10 +137,17 @@ pub unsafe fn create_module<'ll>( .replace("-Fi64", ""); } } + if llvm_version < (18, 0, 0) { + if sess.target.arch == "x86" || sess.target.arch == "x86_64" { + // LLVM 18 adjusts i128 to be 128-bit aligned on x86 variants. + // Earlier LLVMs leave this as default alignment, so remove it. + // See https://reviews.llvm.org/D86310 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. - if sess.target.is_builtin { - // tm is disposed by its drop impl + { let tm = crate::back::write::create_informational_target_machine(tcx.sess); llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); @@ -156,33 +155,13 @@ pub unsafe fn create_module<'ll>( let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) .expect("got a non-UTF8 data-layout from LLVM"); - // Unfortunately LLVM target specs change over time, and right now we - // don't have proper support to work with any more than one - // `data_layout` than the one that is in the rust-lang/rust repo. If - // this compiler is configured against a custom LLVM, we may have a - // differing data layout, even though we should update our own to use - // that one. - // - // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we - // disable this check entirely as we may be configured with something - // that has a different target layout. - // - // Unsure if this will actually cause breakage when rustc is configured - // as such. - // - // FIXME(#34960) - let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = !cfg_llvm_root.trim().is_empty(); - - if !custom_llvm_used && target_data_layout != llvm_data_layout { - bug!( - "data-layout for target `{rustc_target}`, `{rustc_layout}`, \ - differs from LLVM target's `{llvm_target}` default layout, `{llvm_layout}`", - rustc_target = sess.opts.target_triple, - rustc_layout = target_data_layout, - llvm_target = sess.target.llvm_target, - llvm_layout = llvm_data_layout - ); + if target_data_layout != llvm_data_layout { + tcx.dcx().emit_err(crate::errors::MismatchedDataLayout { + rustc_target: sess.opts.target_triple.to_string().as_str(), + rustc_layout: target_data_layout.as_str(), + llvm_target: sess.target.llvm_target.borrow(), + llvm_layout: llvm_data_layout, + }); } } @@ -272,35 +251,29 @@ pub unsafe fn create_module<'ll>( } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { - let behavior = if llvm_version >= (15, 0, 0) { - llvm::LLVMModFlagBehavior::Min - } else { - llvm::LLVMModFlagBehavior::Error - }; - if sess.target.arch == "aarch64" { llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"branch-target-enforcement".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"sign-return-address".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"sign-return-address-all".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - behavior, + llvm::LLVMModFlagBehavior::Min, c"sign-return-address-with-bkey".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); @@ -576,11 +549,12 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let llfn = match tcx.lang_items().eh_personality() { - Some(def_id) if name.is_none() => self.get_fn_addr( - ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty()) - .unwrap() - .unwrap(), - ), + Some(def_id) if name.is_none() => self.get_fn_addr(ty::Instance::expect_resolve( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + ty::List::empty(), + )), _ => { let name = name.unwrap_or("rust_eh_personality"); if let Some(llfn) = self.get_declared_value(name) { @@ -697,8 +671,10 @@ impl<'ll> CodegenCx<'ll, '_> { let t_i64 = self.type_i64(); let t_i128 = self.type_i128(); let t_isize = self.type_isize(); + let t_f16 = self.type_f16(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_f128 = self.type_f128(); let t_metadata = self.type_metadata(); let t_token = self.type_token(); @@ -740,69 +716,115 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.debugtrap", fn() -> void); ifn!("llvm.frameaddress", fn(t_i32) -> ptr); + ifn!("llvm.powi.f16", fn(t_f16, t_i32) -> t_f16); ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); + ifn!("llvm.powi.f128", fn(t_f128, t_i32) -> t_f128); + ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.pow.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.sqrt.f16", fn(t_f16) -> t_f16); ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); + ifn!("llvm.sqrt.f128", fn(t_f128) -> t_f128); + ifn!("llvm.sin.f16", fn(t_f16) -> t_f16); ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); + ifn!("llvm.sin.f128", fn(t_f128) -> t_f128); + ifn!("llvm.cos.f16", fn(t_f16) -> t_f16); ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); + ifn!("llvm.cos.f128", fn(t_f128) -> t_f128); + ifn!("llvm.exp.f16", fn(t_f16) -> t_f16); ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); + ifn!("llvm.exp.f128", fn(t_f128) -> t_f128); + ifn!("llvm.exp2.f16", fn(t_f16) -> t_f16); ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); + ifn!("llvm.exp2.f128", fn(t_f128) -> t_f128); + ifn!("llvm.log.f16", fn(t_f16) -> t_f16); ifn!("llvm.log.f32", fn(t_f32) -> t_f32); ifn!("llvm.log.f64", fn(t_f64) -> t_f64); + ifn!("llvm.log.f128", fn(t_f128) -> t_f128); + ifn!("llvm.log10.f16", fn(t_f16) -> t_f16); ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); + ifn!("llvm.log10.f128", fn(t_f128) -> t_f128); + ifn!("llvm.log2.f16", fn(t_f16) -> t_f16); ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); + ifn!("llvm.log2.f128", fn(t_f128) -> t_f128); + ifn!("llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16); ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128); + ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16); ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); + ifn!("llvm.fabs.f128", fn(t_f128) -> t_f128); + ifn!("llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128); + + ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.floor.f16", fn(t_f16) -> t_f16); ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); + ifn!("llvm.floor.f128", fn(t_f128) -> t_f128); + ifn!("llvm.ceil.f16", fn(t_f16) -> t_f16); ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); + ifn!("llvm.ceil.f128", fn(t_f128) -> t_f128); + ifn!("llvm.trunc.f16", fn(t_f16) -> t_f16); ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); + ifn!("llvm.trunc.f128", fn(t_f128) -> t_f128); + ifn!("llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128); + ifn!("llvm.round.f16", fn(t_f16) -> t_f16); ifn!("llvm.round.f32", fn(t_f32) -> t_f32); ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + ifn!("llvm.round.f128", fn(t_f128) -> t_f128); + ifn!("llvm.roundeven.f16", fn(t_f16) -> t_f16); ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32); ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64); + ifn!("llvm.roundeven.f128", fn(t_f128) -> t_f128); + ifn!("llvm.rint.f16", fn(t_f16) -> t_f16); ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); + ifn!("llvm.rint.f128", fn(t_f128) -> t_f128); + + ifn!("llvm.nearbyint.f16", fn(t_f16) -> t_f16); ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64); + ifn!("llvm.nearbyint.f128", fn(t_f128) -> t_f128); ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8); ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16); @@ -908,6 +930,21 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void); ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void); + // FIXME: This is an infinitesimally small portion of the types you can + // pass to this intrinsic, if we can ever lazily register intrinsics we + // should register these when they're used, that way any type can be + // passed. + ifn!("llvm.is.constant.i1", fn(i1) -> i1); + ifn!("llvm.is.constant.i8", fn(t_i8) -> i1); + ifn!("llvm.is.constant.i16", fn(t_i16) -> i1); + ifn!("llvm.is.constant.i32", fn(t_i32) -> i1); + ifn!("llvm.is.constant.i64", fn(t_i64) -> i1); + ifn!("llvm.is.constant.i128", fn(t_i128) -> i1); + ifn!("llvm.is.constant.isize", fn(t_isize) -> i1); + ifn!("llvm.is.constant.f32", fn(t_f32) -> i1); + ifn!("llvm.is.constant.f64", fn(t_f64) -> i1); + ifn!("llvm.is.constant.ptr", fn(ptr) -> i1); + ifn!("llvm.expect.i1", fn(i1, i1) -> i1); ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32); ifn!("llvm.localescape", fn(...) -> void); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 7ad2d03a5ed..2af28146a51 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,4 +1,4 @@ -use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId}; +use rustc_middle::mir::coverage::{CodeRegion, CounterId, CovTerm, ExpressionId, MappingKind}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] @@ -149,6 +149,33 @@ pub struct CounterMappingRegion { } impl CounterMappingRegion { + pub(crate) fn from_mapping( + mapping_kind: &MappingKind, + local_file_id: u32, + code_region: &CodeRegion, + ) -> Self { + let &CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + match *mapping_kind { + MappingKind::Code(term) => Self::code_region( + Counter::from_term(term), + local_file_id, + start_line, + start_col, + end_line, + end_col, + ), + MappingKind::Branch { true_term, false_term } => Self::branch_region( + Counter::from_term(true_term), + Counter::from_term(false_term), + local_file_id, + start_line, + start_col, + end_line, + end_col, + ), + } + } + pub(crate) fn code_region( counter: Counter, file_id: u32, @@ -170,9 +197,6 @@ impl CounterMappingRegion { } } - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] pub(crate) fn branch_region( counter: Counter, false_counter: Counter, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index cd67fafb8e4..d85d9411f03 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -4,7 +4,8 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxIndexSet; use rustc_index::bit_set::BitSet; use rustc_middle::mir::coverage::{ - CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, Op, + CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, + MappingKind, Op, }; use rustc_middle::ty::Instance; use rustc_span::Symbol; @@ -64,8 +65,8 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { // For each expression ID that is directly used by one or more mappings, // mark it as not-yet-seen. This indicates that we expect to see a // corresponding `ExpressionUsed` statement during MIR traversal. - for Mapping { term, .. } in &function_coverage_info.mappings { - if let &CovTerm::Expression(id) = term { + for term in function_coverage_info.mappings.iter().flat_map(|m| m.kind.terms()) { + if let CovTerm::Expression(id) = term { expressions_seen.remove(id); } } @@ -221,20 +222,21 @@ impl<'tcx> FunctionCoverage<'tcx> { /// that will be used by `mapgen` when preparing for FFI. pub(crate) fn counter_regions( &self, - ) -> impl Iterator<Item = (Counter, &CodeRegion)> + ExactSizeIterator { + ) -> impl Iterator<Item = (MappingKind, &CodeRegion)> + ExactSizeIterator { self.function_coverage_info.mappings.iter().map(move |mapping| { - let &Mapping { term, ref code_region } = mapping; - let counter = self.counter_for_term(term); - (counter, code_region) + let Mapping { kind, code_region } = mapping; + let kind = + kind.map_terms(|term| if self.is_zero_term(term) { CovTerm::Zero } else { term }); + (kind, code_region) }) } fn counter_for_term(&self, term: CovTerm) -> Counter { - if is_zero_term(&self.counters_seen, &self.zero_expressions, term) { - Counter::ZERO - } else { - Counter::from_term(term) - } + if self.is_zero_term(term) { Counter::ZERO } else { Counter::from_term(term) } + } + + fn is_zero_term(&self, term: CovTerm) -> bool { + is_zero_term(&self.counters_seen, &self.zero_expressions, term) } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 33bfde03a31..ee7ea342301 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -12,7 +12,6 @@ use rustc_hir::def_id::DefId; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::mir; -use rustc_middle::mir::coverage::CodeRegion; use rustc_middle::ty::{self, TyCtxt}; use rustc_span::def_id::DefIdSet; use rustc_span::Symbol; @@ -58,11 +57,6 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { return; } - // The entries of the map are only used to get a list of all files with - // coverage info. In the end the list of files is passed into - // `GlobalFileTable::new()` which internally do `.sort_unstable_by()`, so - // the iteration order here does not matter. - #[allow(rustc::potential_query_instability)] let function_coverage_entries = function_coverage_map .into_iter() .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) @@ -242,7 +236,7 @@ fn encode_mappings_for_function( // Prepare file IDs for each filename, and prepare the mapping data so that // we can pass it through FFI to LLVM. for (file_name, counter_regions_for_file) in - &counter_regions.group_by(|(_counter, region)| region.file_name) + &counter_regions.group_by(|(_, region)| region.file_name) { // Look up the global file ID for this filename. let global_file_id = global_file_table.global_file_id_for_file_name(file_name); @@ -253,17 +247,12 @@ fn encode_mappings_for_function( // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - for (counter, region) in counter_regions_for_file { - let CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = *region; - - debug!("Adding counter {counter:?} to map for {region:?}"); - mapping_regions.push(CounterMappingRegion::code_region( - counter, + for (mapping_kind, region) in counter_regions_for_file { + debug!("Adding counter {mapping_kind:?} to map for {region:?}"); + mapping_regions.push(CounterMappingRegion::from_mapping( + &mapping_kind, local_file_id.as_u32(), - start_line, - start_col, - end_line, - end_col, + region, )); } } @@ -366,33 +355,28 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { let tcx = cx.tcx; - let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - - let eligible_def_ids: Vec<DefId> = tcx - .mir_keys(()) - .iter() - .filter_map(|local_def_id| { - let def_id = local_def_id.to_def_id(); - let kind = tcx.def_kind(def_id); - // `mir_keys` will give us `DefId`s for all kinds of things, not - // just "functions", like consts, statics, etc. Filter those out. - // If `ignore_unused_generics` was specified, filter out any - // generic functions from consideration as well. - if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) { - return None; - } - if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { - return None; - } - Some(local_def_id.to_def_id()) - }) - .collect(); + let eligible_def_ids = tcx.mir_keys(()).iter().filter_map(|local_def_id| { + let def_id = local_def_id.to_def_id(); + let kind = tcx.def_kind(def_id); + // `mir_keys` will give us `DefId`s for all kinds of things, not + // just "functions", like consts, statics, etc. Filter those out. + if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) { + return None; + } + + // FIXME(79651): Consider trying to filter out dummy instantiations of + // unused generic functions from library crates, because they can produce + // "unused instantiation" in coverage reports even when they are actually + // used by some downstream crate in the same binary. + + Some(local_def_id.to_def_id()) + }); let codegenned_def_ids = codegenned_and_inlined_items(tcx); // For each `DefId` that should have coverage instrumentation but wasn't // codegenned, add it to the function coverage map as an unused function. - for def_id in eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) { + for def_id in eligible_def_ids.filter(|id| !codegenned_def_ids.contains(id)) { // Skip any function that didn't have coverage data added to it by the // coverage instrumentor. let body = tcx.instance_mir(ty::InstanceDef::Item(def_id)); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 0befbb5a39b..133084b7c12 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -10,7 +10,7 @@ use rustc_codegen_ssa::traits::{ BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, StaticMethods, }; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; @@ -30,7 +30,7 @@ const VAR_ALIGN_BYTES: usize = 8; pub struct CrateCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: - RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, + RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -44,8 +44,8 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { pub fn take_function_coverage_map( &self, - ) -> FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { - self.function_coverage_map.replace(FxHashMap::default()) + ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { + self.function_coverage_map.replace(FxIndexMap::default()) } } @@ -88,7 +88,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { match coverage.kind { // Marker statements have no effect during codegen, // so return early and don't create `func_coverage`. - CoverageKind::SpanMarker => return, + CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => return, // Match exhaustively to ensure that newly-added kinds are classified correctly. CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. } => {} } @@ -108,7 +108,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let Coverage { kind } = coverage; match *kind { - CoverageKind::SpanMarker => unreachable!( + CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( "unexpected marker statement {kind:?} should have caused an early return" ), CoverageKind::CounterIncrement { id } => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 76c9ac6614a..5782b156335 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -26,6 +26,7 @@ use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -454,13 +455,18 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) } - // Box<T, A> may have a non-1-ZST allocator A. In that case, we - // cannot treat Box<T, A> as just an owned alias of `*mut T`. - ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_1zst() => { + // Some `Box` are newtyped pointers, make debuginfo aware of that. + // Only works if the allocator argument is a 1-ZST and hence irrelevant for layout + // (or if there is no allocator argument). + ty::Adt(def, args) + if def.is_box() + && args.get(1).map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) => + { build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), @@ -694,9 +700,13 @@ impl MsvcBasicName for ty::UintTy { impl MsvcBasicName for ty::FloatTy { fn msvc_basic_name(self) -> &'static str { + // FIXME: f16 and f128 have no MSVC representation. We could improve the debuginfo. + // See: <https://github.com/rust-lang/rust/pull/114607/files#r1454683264> match self { + ty::FloatTy::F16 => "half", ty::FloatTy::F32 => "float", ty::FloatTy::F64 => "double", + ty::FloatTy::F128 => "fp128", } } } @@ -1068,6 +1078,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), + ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { bug!( "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}", @@ -1153,7 +1164,8 @@ fn build_closure_env_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let closure_env_type = unique_type_id.expect_ty(); - let &ty::Closure(def_id, _args) = closure_env_type.kind() else { + let &(ty::Closure(def_id, _) | ty::CoroutineClosure(def_id, _)) = closure_env_type.kind() + else { bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) }; let containing_scope = get_namespace_for_item(cx, def_id); @@ -1298,6 +1310,11 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo }; let is_local_to_unit = is_node_local_to_unit(cx, def_id); + + let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else { bug!() }; + if nested { + return; + } let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); let type_di_node = type_di_node(cx, variable_type); let var_name = tcx.item_name(def_id); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 7f671d1d061..657e9ce998f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -122,8 +122,10 @@ fn tag_base_type<'ll, 'tcx>( // Niche tags are always normalized to unsized integers of the correct size. match tag.primitive() { Primitive::Int(t, _) => t, + Primitive::F16 => Integer::I16, Primitive::F32 => Integer::I32, Primitive::F64 => Integer::I64, + Primitive::F128 => Integer::I128, // FIXME(erikdesjardins): handle non-default addrspace ptr sizes Primitive::Pointer(_) => { // If the niche is the NULL value of a reference, then `discr_enum_ty` will be diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 8db97d577ca..e15eda7c66c 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -4,7 +4,7 @@ use std::path::Path; use crate::fluent_generated as fluent; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{DiagCtxt, DiagnosticBuilder, EmissionGuarantee, IntoDiagnostic, Level}; +use rustc_errors::{Diag, DiagCtxt, Diagnostic, EmissionGuarantee, Level}; use rustc_macros::{Diagnostic, Subdiagnostic}; use rustc_span::Span; @@ -99,16 +99,13 @@ pub(crate) struct DynamicLinkingWithLTO; pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); -impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { - let diag: DiagnosticBuilder<'_, G> = self.0.into_diagnostic(dcx, level); - let (message, _) = diag.messages().first().expect("`LlvmError` with no message"); - let message = dcx.eagerly_translate_to_string(message.clone(), diag.args()); - - let mut diag = - DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config); - diag.set_arg("error", message); - diag +impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { + fn into_diag(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { + let diag: Diag<'_, G> = self.0.into_diag(dcx, level); + let (message, _) = diag.messages.first().expect("`LlvmError` with no message"); + let message = dcx.eagerly_translate_to_string(message.clone(), diag.args.iter()); + Diag::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) + .with_arg("error", message) } } @@ -122,20 +119,16 @@ pub(crate) struct TargetFeatureDisableOrEnable<'a> { #[help(codegen_llvm_missing_features)] pub(crate) struct MissingFeatures; -impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { - let mut diag = DiagnosticBuilder::new( - dcx, - level, - fluent::codegen_llvm_target_feature_disable_or_enable, - ); +impl<G: EmissionGuarantee> Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { + fn into_diag(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { + let mut diag = Diag::new(dcx, level, fluent::codegen_llvm_target_feature_disable_or_enable); if let Some(span) = self.span { - diag.set_span(span); + diag.span(span); }; if let Some(missing_features) = self.missing_features { - diag.subdiagnostic(missing_features); + diag.subdiagnostic(dcx, missing_features); } - diag.set_arg("features", self.features.join(", ")); + diag.arg("features", self.features.join(", ")); diag } } @@ -186,8 +179,8 @@ pub enum LlvmError<'a> { pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String); -impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { - fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { +impl<G: EmissionGuarantee> Diagnostic<'_, G> for WithLlvmError<'_> { + fn into_diag(self, dcx: &'_ DiagCtxt, level: Level) -> Diag<'_, G> { use LlvmError::*; let msg_with_llvm_err = match &self.0 { WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err, @@ -204,10 +197,10 @@ impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, }; - let mut diag = self.0.into_diagnostic(dcx, level); - diag.set_primary_message(msg_with_llvm_err); - diag.set_arg("llvm_err", self.1); - diag + self.0 + .into_diag(dcx, level) + .with_primary_message(msg_with_llvm_err) + .with_arg("llvm_err", self.1) } } @@ -246,3 +239,18 @@ pub(crate) struct CopyBitcode { pub struct UnknownCompression { pub algorithm: &'static str, } + +#[derive(Diagnostic)] +#[diag(codegen_llvm_mismatch_data_layout)] +pub struct MismatchedDataLayout<'a> { + pub rustc_target: &'a str, + pub rustc_layout: &'a str, + pub llvm_target: &'a str, + pub llvm_layout: &'a str, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_target_feature_prefix)] +pub(crate) struct InvalidTargetFeaturePrefix<'a> { + pub feature: &'a str, +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 6043a8ebded..467e02d55e3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -17,7 +17,7 @@ use rustc_hir as hir; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; -use rustc_span::{sym, symbol::kw, Span, Symbol}; +use rustc_span::{sym, Span, Symbol}; use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; @@ -28,51 +28,118 @@ fn get_simple_intrinsic<'ll>( name: Symbol, ) -> Option<(&'ll Type, &'ll Value)> { let llvm_name = match name { + sym::sqrtf16 => "llvm.sqrt.f16", sym::sqrtf32 => "llvm.sqrt.f32", sym::sqrtf64 => "llvm.sqrt.f64", + sym::sqrtf128 => "llvm.sqrt.f128", + + sym::powif16 => "llvm.powi.f16", sym::powif32 => "llvm.powi.f32", sym::powif64 => "llvm.powi.f64", + sym::powif128 => "llvm.powi.f128", + + sym::sinf16 => "llvm.sin.f16", sym::sinf32 => "llvm.sin.f32", sym::sinf64 => "llvm.sin.f64", + sym::sinf128 => "llvm.sin.f128", + + sym::cosf16 => "llvm.cos.f16", sym::cosf32 => "llvm.cos.f32", sym::cosf64 => "llvm.cos.f64", + sym::cosf128 => "llvm.cos.f128", + + sym::powf16 => "llvm.pow.f16", sym::powf32 => "llvm.pow.f32", sym::powf64 => "llvm.pow.f64", + sym::powf128 => "llvm.pow.f128", + + sym::expf16 => "llvm.exp.f16", sym::expf32 => "llvm.exp.f32", sym::expf64 => "llvm.exp.f64", + sym::expf128 => "llvm.exp.f128", + + sym::exp2f16 => "llvm.exp2.f16", sym::exp2f32 => "llvm.exp2.f32", sym::exp2f64 => "llvm.exp2.f64", + sym::exp2f128 => "llvm.exp2.f128", + + sym::logf16 => "llvm.log.f16", sym::logf32 => "llvm.log.f32", sym::logf64 => "llvm.log.f64", + sym::logf128 => "llvm.log.f128", + + sym::log10f16 => "llvm.log10.f16", sym::log10f32 => "llvm.log10.f32", sym::log10f64 => "llvm.log10.f64", + sym::log10f128 => "llvm.log10.f128", + + sym::log2f16 => "llvm.log2.f16", sym::log2f32 => "llvm.log2.f32", sym::log2f64 => "llvm.log2.f64", + sym::log2f128 => "llvm.log2.f128", + + sym::fmaf16 => "llvm.fma.f16", sym::fmaf32 => "llvm.fma.f32", sym::fmaf64 => "llvm.fma.f64", + sym::fmaf128 => "llvm.fma.f128", + + sym::fabsf16 => "llvm.fabs.f16", sym::fabsf32 => "llvm.fabs.f32", sym::fabsf64 => "llvm.fabs.f64", + sym::fabsf128 => "llvm.fabs.f128", + + sym::minnumf16 => "llvm.minnum.f16", sym::minnumf32 => "llvm.minnum.f32", sym::minnumf64 => "llvm.minnum.f64", + sym::minnumf128 => "llvm.minnum.f128", + + sym::maxnumf16 => "llvm.maxnum.f16", sym::maxnumf32 => "llvm.maxnum.f32", sym::maxnumf64 => "llvm.maxnum.f64", + sym::maxnumf128 => "llvm.maxnum.f128", + + sym::copysignf16 => "llvm.copysign.f16", sym::copysignf32 => "llvm.copysign.f32", sym::copysignf64 => "llvm.copysign.f64", + sym::copysignf128 => "llvm.copysign.f128", + + sym::floorf16 => "llvm.floor.f16", sym::floorf32 => "llvm.floor.f32", sym::floorf64 => "llvm.floor.f64", + sym::floorf128 => "llvm.floor.f128", + + sym::ceilf16 => "llvm.ceil.f16", sym::ceilf32 => "llvm.ceil.f32", sym::ceilf64 => "llvm.ceil.f64", + sym::ceilf128 => "llvm.ceil.f128", + + sym::truncf16 => "llvm.trunc.f16", sym::truncf32 => "llvm.trunc.f32", sym::truncf64 => "llvm.trunc.f64", + sym::truncf128 => "llvm.trunc.f128", + + sym::rintf16 => "llvm.rint.f16", sym::rintf32 => "llvm.rint.f32", sym::rintf64 => "llvm.rint.f64", + sym::rintf128 => "llvm.rint.f128", + + sym::nearbyintf16 => "llvm.nearbyint.f16", sym::nearbyintf32 => "llvm.nearbyint.f32", sym::nearbyintf64 => "llvm.nearbyint.f64", + sym::nearbyintf128 => "llvm.nearbyint.f128", + + sym::roundf16 => "llvm.round.f16", sym::roundf32 => "llvm.round.f32", sym::roundf64 => "llvm.round.f64", + sym::roundf128 => "llvm.round.f128", + sym::ptr_mask => "llvm.ptrmask", + + sym::roundevenf16 => "llvm.roundeven.f16", sym::roundevenf32 => "llvm.roundeven.f32", sym::roundevenf64 => "llvm.roundeven.f64", + sym::roundevenf128 => "llvm.roundeven.f128", + _ => return None, }; Some(cx.get_intrinsic(llvm_name)) @@ -86,7 +153,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, &'ll Value>], llresult: &'ll Value, span: Span, - ) { + ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); @@ -119,17 +186,29 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::likely => { self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) } + sym::is_val_statically_known => { + let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); + match self.type_kind(intrinsic_type) { + TypeKind::Pointer | TypeKind::Integer | TypeKind::Float | TypeKind::Double => { + self.call_intrinsic( + &format!("llvm.is.constant.{:?}", intrinsic_type), + &[args[0].immediate()], + ) + } + _ => self.const_bool(false), + } + } sym::unlikely => self .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), - kw::Try => { - try_intrinsic( + sym::catch_unwind => { + catch_unwind_intrinsic( self, args[0].immediate(), args[1].immediate(), args[2].immediate(), llresult, ); - return; + return Ok(()); } sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), sym::va_copy => { @@ -151,11 +230,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { emit_va_arg(self, args[0], ret_ty) } } + Primitive::F16 => bug!("the va_arg intrinsic does not work with `f16`"), Primitive::F64 | Primitive::Pointer(_) => { emit_va_arg(self, args[0], ret_ty) } // `va_arg` should never be used with the return type f32. Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), + Primitive::F128 => { + bug!("the va_arg intrinsic does not work with `f128`") + } } } _ => bug!("the va_arg intrinsic does not work with non-scalar types"), @@ -179,17 +262,20 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { unsafe { llvm::LLVMSetAlignment(load, align); } - self.to_immediate(load, self.layout_of(tp_ty)) + if !result.layout.is_zst() { + self.store(load, result.llval, result.align); + } + return Ok(()); } sym::volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.volatile_store(self, dst); - return; + return Ok(()); } sym::unaligned_volatile_store => { let dst = args[0].deref(self.cx()); args[1].val.unaligned_volatile_store(self, dst); - return; + return Ok(()); } sym::prefetch_read_data | sym::prefetch_write_data @@ -290,7 +376,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { name, ty, }); - return; + return Ok(()); } } } @@ -362,17 +448,19 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { constraint, inputs, self.type_void(), + &[], true, false, llvm::AsmDialect::Att, &[span], false, None, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); // We have copied the value to `result` already. - return; + return Ok(()); } _ if name.as_str().starts_with("simd_") => { @@ -380,11 +468,15 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, ) { Ok(llval) => llval, - Err(()) => return, + Err(()) => return Ok(()), } } - _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), + _ => { + debug!("unknown intrinsic '{}' -- falling back to default body", name); + // Call the fallback body instead of generating the intrinsic code + return Err(ty::Instance::new(instance.def_id(), instance.args)); + } }; if !fn_abi.ret.is_ignore() { @@ -396,6 +488,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { .store(self, result); } } + Ok(()) } fn abort(&mut self) { @@ -437,7 +530,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -fn try_intrinsic<'ll>( +fn catch_unwind_intrinsic<'ll>( bx: &mut Builder<'_, 'll, '_>, try_func: &'ll Value, data: &'ll Value, @@ -1059,7 +1152,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .map(|(arg_idx, val)| { let idx = val.unwrap_leaf().try_to_i32().unwrap(); if idx >= i32::try_from(total_len).unwrap() { - bx.sess().dcx().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: arg_idx as u64, @@ -1118,24 +1211,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - bx.sess().dcx().emit_err( - InvalidMonomorphization::ShuffleIndexNotConstant { - span, - name, - arg_idx, - }, - ); - None + bug!("typeck should have already ensured that these are const") } Some(idx) if idx >= total_len => { - bx.sess().dcx().emit_err( - InvalidMonomorphization::ShuffleIndexOutOfBounds { - span, - name, - arg_idx, - total_len, - }, - ); + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx, + total_len, + }); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1164,10 +1248,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>( out_ty: arg_tys[2] } ); + let idx = bx + .const_to_opt_u128(args[1].immediate(), false) + .expect("typeck should have ensure that this is a const"); + if idx >= in_len.into() { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: 1, + total_len: in_len.into(), + }); + return Ok(bx.const_null(llret_ty)); + } return Ok(bx.insert_element( args[0].immediate(), args[2].immediate(), - args[1].immediate(), + bx.const_i32(idx as i32), )); } if name == sym::simd_extract { @@ -1175,7 +1271,19 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ret_ty == in_elem, InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } ); - return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); + let idx = bx + .const_to_opt_u128(args[1].immediate(), false) + .expect("typeck should have ensure that this is a const"); + if idx >= in_len.into() { + bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: 1, + total_len: in_len.into(), + }); + return Ok(bx.const_null(llret_ty)); + } + return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32))); } if name == sym::simd_select { @@ -1860,14 +1968,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, - vector_reduce_fadd_fast, + vector_reduce_fadd_reassoc, false, add, 0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, - vector_reduce_fmul_fast, + vector_reduce_fmul_reassoc, false, mul, 1.0 @@ -1900,9 +2008,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); - minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); - minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); - macro_rules! bitwise_red { ($name:ident : $red:ident, $boolean:expr) => { if name == sym::$name { @@ -1970,10 +2075,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match in_elem.kind() { ty::RawPtr(p) => { - let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - assert!(!check_sized); // we are in codegen, so we shouldn't see these types require!( metadata.is_unit(), InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } @@ -1985,10 +2089,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } match out_elem.kind() { ty::RawPtr(p) => { - let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { + let metadata = p.ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) }); - assert!(!check_sized); // we are in codegen, so we shouldn't see these types require!( metadata.is_unit(), InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } @@ -2076,9 +2179,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(args[0].immediate()); } + #[derive(Copy, Clone)] + enum Sign { + Unsigned, + Signed, + } + use Sign::*; + enum Style { Float, - Int(/* is signed? */ bool), + Int(Sign), Unsupported, } @@ -2086,11 +2196,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // vectors of pointer-sized integers should've been // disallowed before here, so this unwrap is safe. ty::Int(i) => ( - Style::Int(true), + Style::Int(Signed), i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Uint(u) => ( - Style::Int(false), + Style::Int(Unsigned), u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Float(f) => (Style::Float, f.bit_width()), @@ -2098,11 +2208,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; let (out_style, out_width) = match out_elem.kind() { ty::Int(i) => ( - Style::Int(true), + Style::Int(Signed), i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Uint(u) => ( - Style::Int(false), + Style::Int(Unsigned), u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), ), ty::Float(f) => (Style::Float, f.bit_width()), @@ -2110,31 +2220,31 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; match (in_style, out_style) { - (Style::Int(in_is_signed), Style::Int(_)) => { + (Style::Int(sign), Style::Int(_)) => { return Ok(match in_width.cmp(&out_width) { Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), Ordering::Equal => args[0].immediate(), - Ordering::Less => { - if in_is_signed { - bx.sext(args[0].immediate(), llret_ty) - } else { - bx.zext(args[0].immediate(), llret_ty) - } - } + Ordering::Less => match sign { + Sign::Signed => bx.sext(args[0].immediate(), llret_ty), + Sign::Unsigned => bx.zext(args[0].immediate(), llret_ty), + }, }); } - (Style::Int(in_is_signed), Style::Float) => { - return Ok(if in_is_signed { - bx.sitofp(args[0].immediate(), llret_ty) - } else { - bx.uitofp(args[0].immediate(), llret_ty) - }); + (Style::Int(Sign::Signed), Style::Float) => { + return Ok(bx.sitofp(args[0].immediate(), llret_ty)); + } + (Style::Int(Sign::Unsigned), Style::Float) => { + return Ok(bx.uitofp(args[0].immediate(), llret_ty)); } - (Style::Float, Style::Int(out_is_signed)) => { - return Ok(match (out_is_signed, name == sym::simd_as) { - (false, false) => bx.fptoui(args[0].immediate(), llret_ty), - (true, false) => bx.fptosi(args[0].immediate(), llret_ty), - (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), + (Style::Float, Style::Int(sign)) => { + return Ok(match (sign, name == sym::simd_as) { + (Sign::Unsigned, false) => bx.fptoui(args[0].immediate(), llret_ty), + (Sign::Signed, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int( + matches!(sign, Sign::Signed), + args[0].immediate(), + llret_ty, + ), }); } (Style::Float, Style::Float) => { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index a81056ed3ad..c84461e53eb 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -13,12 +13,7 @@ #![feature(hash_raw_entry)] #![feature(iter_intersperse)] #![feature(let_chains)] -#![feature(min_specialization)] -#![feature(never_type)] #![feature(impl_trait_in_assoc_type)] -#![recursion_limit = "256"] -#![deny(rustc::untranslatable_diagnostic)] -#![deny(rustc::diagnostic_outside_of_impl)] #[macro_use] extern crate rustc_macros; @@ -174,7 +169,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_pass_timings(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintPassTimings(&mut size as *mut usize); + let cstr = llvm::LLVMRustPrintPassTimings(std::ptr::addr_of_mut!(size)); if cstr.is_null() { println!("failed to get pass timings"); } else { @@ -187,7 +182,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn print_statistics(&self) { unsafe { let mut size = 0; - let cstr = llvm::LLVMRustPrintStatistics(&mut size as *mut usize); + let cstr = llvm::LLVMRustPrintStatistics(std::ptr::addr_of_mut!(size)); if cstr.is_null() { println!("failed to get pass stats"); } else { @@ -374,7 +369,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") @@ -387,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend { }); } - Ok((codegen_results, work_products)) + (codegen_results, work_products) } fn link( diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index 06e846a2b45..f9b28178ddb 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -123,7 +123,7 @@ impl SrcMgrDiagnostic { #[derive(Clone)] pub struct InlineAsmDiagnostic { pub level: super::DiagnosticLevel, - pub cookie: c_uint, + pub cookie: u64, pub message: String, pub source: Option<(String, Vec<InnerSpan>)>, } @@ -149,7 +149,7 @@ impl InlineAsmDiagnostic { let smdiag = SrcMgrDiagnostic::unpack(super::LLVMRustGetSMDiagnostic(di, &mut cookie)); InlineAsmDiagnostic { level: smdiag.level, - cookie, + cookie: cookie.into(), message: smdiag.message, source: smdiag.source, } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 81702baa8c0..d34c289887e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -56,6 +56,7 @@ pub enum LLVMMachineType { AMD64 = 0x8664, I386 = 0x14c, ARM64 = 0xaa64, + ARM64EC = 0xa641, ARM = 0x01c0, } @@ -106,7 +107,6 @@ pub enum CallConv { X86_Intr = 83, AvrNonBlockingInterrupt = 84, AvrInterrupt = 85, - AmdGpuKernel = 91, } /// LLVMRustLinkage @@ -185,7 +185,6 @@ pub enum AttributeKind { SanitizeMemory = 22, NonLazyBind = 23, OptimizeNone = 24, - ReturnsTwice = 25, ReadNone = 26, SanitizeHWAddress = 28, WillReturn = 29, @@ -482,6 +481,9 @@ pub struct SanitizerOptions { pub sanitize_address: bool, pub sanitize_address_recover: bool, pub sanitize_cfi: bool, + pub sanitize_dataflow: bool, + pub sanitize_dataflow_abilist: *const *const c_char, + pub sanitize_dataflow_abilist_len: size_t, pub sanitize_kcfi: bool, pub sanitize_memory: bool, pub sanitize_memory_recover: bool, @@ -722,7 +724,7 @@ pub mod debuginfo { // These values **must** match with LLVMRustDIFlags!! bitflags! { #[repr(transparent)] - #[derive(Default)] + #[derive(Clone, Copy, Default)] pub struct DIFlags: u32 { const FlagZero = 0; const FlagPrivate = 1; @@ -751,7 +753,7 @@ pub mod debuginfo { // These values **must** match with LLVMRustDISPFlags!! bitflags! { #[repr(transparent)] - #[derive(Default)] + #[derive(Clone, Copy, Default)] pub struct DISPFlags: u32 { const SPFlagZero = 0; const SPFlagVirtual = 1; @@ -860,8 +862,10 @@ extern "C" { pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; // Operations on real types + pub fn LLVMHalfTypeInContext(C: &Context) -> &Type; pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + pub fn LLVMFP128TypeInContext(C: &Context) -> &Type; // Operations on function types pub fn LLVMFunctionType<'a>( @@ -932,10 +936,16 @@ extern "C" { pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; // Operations on composite constants - pub fn LLVMConstStringInContext( + pub fn LLVMConstArray2<'a>( + ElementTy: &'a Type, + ConstantVals: *const &'a Value, + Length: u64, + ) -> &'a Value; + pub fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; + pub fn LLVMConstStringInContext2( C: &Context, Str: *const c_char, - Length: c_uint, + Length: size_t, DontNullTerminate: Bool, ) -> &Value; pub fn LLVMConstStructInContext<'a>( @@ -944,14 +954,6 @@ extern "C" { Count: c_uint, Packed: Bool, ) -> &'a Value; - - // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 - // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc - pub fn LLVMConstArray<'a>( - ElementTy: &'a Type, - ConstantVals: *const &'a Value, - Length: c_uint, - ) -> &'a Value; pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions @@ -1303,13 +1305,6 @@ extern "C" { NumIndices: c_uint, Name: *const c_char, ) -> &'a Value; - pub fn LLVMBuildStructGEP2<'a>( - B: &Builder<'a>, - Ty: &'a Type, - Pointer: &'a Value, - Idx: c_uint, - Name: *const c_char, - ) -> &'a Value; // Casts pub fn LLVMBuildTrunc<'a>( @@ -1533,9 +1528,6 @@ extern "C" { /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on array, pointer, and vector types (sequence types) - pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; - // Operations on all values pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; @@ -1619,7 +1611,23 @@ extern "C" { Name: *const c_char, ) -> &'a Value; + pub fn LLVMRustBuildCallBr<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + DefaultDest: &'a BasicBlock, + IndirectDests: *const &'a BasicBlock, + NumIndirectDests: c_uint, + Args: *const &'a Value, + NumArgs: c_uint, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMRustSetFastMath(Instr: &Value); + pub fn LLVMRustSetAlgebraicMath(Instr: &Value); + pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; @@ -2173,8 +2181,13 @@ extern "C" { ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; + pub fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); - pub fn LLVMRustAddLibraryInfo<'a>(PM: &PassManager<'a>, M: &'a Module); + pub fn LLVMRustAddLibraryInfo<'a>( + PM: &PassManager<'a>, + M: &'a Module, + DisableSimplifyLibCalls: bool, + ); pub fn LLVMRustWriteOutputFile<'a>( T: &'a TargetMachine, PM: &PassManager<'a>, @@ -2196,6 +2209,7 @@ extern "C" { UnrollLoops: bool, SLPVectorize: bool, LoopVectorize: bool, + DisableSimplifyLibCalls: bool, EmitLifetimeMarkers: bool, SanitizerOptions: Option<&SanitizerOptions>, PGOGenPath: *const c_char, @@ -2250,7 +2264,7 @@ extern "C" { pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( DI: &'a DiagnosticInfo, level_out: &mut DiagnosticLevel, - cookie_out: &mut c_uint, + cookie_out: &mut u64, message_out: &mut Option<&'a Twine>, ); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 03b79a143cc..e32c38644aa 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,10 +1,11 @@ use crate::back::write::create_informational_target_machine; use crate::errors::{ - PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, - UnknownCTargetFeaturePrefix, UnstableCTargetFeature, + InvalidTargetFeaturePrefix, PossibleFeature, TargetFeatureDisableOrEnable, + UnknownCTargetFeature, UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; use libc::c_int; +use rustc_codegen_ssa::base::wants_wasm_eh; use rustc_codegen_ssa::traits::PrintBackendInfo; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; @@ -98,6 +99,10 @@ unsafe fn configure_llvm(sess: &Session) { } } + if wants_wasm_eh(sess) { + add("-wasm-enable-eh", false); + } + if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { add("-enable-emscripten-cxx-exceptions", false); } @@ -192,19 +197,22 @@ impl<'a> IntoIterator for LLVMFeature<'a> { // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. // -// To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def -// where the * matches the architecture's name -// -// For targets not present in the above location, see llvm-project/llvm/lib/Target/{ARCH}/*.td +// To find a list of LLVM's names, see llvm-project/llvm/lib/Target/{ARCH}/*.td // where `{ARCH}` is the architecture name. Look for instances of `SubtargetFeature`. // -// Beware to not use the llvm github project for this, but check the git submodule -// found in src/llvm-project +// Check the current rustc fork of LLVM in the repo at https://github.com/rust-lang/llvm-project/. +// The commit in use can be found via the `llvm-project` submodule in https://github.com/rust-lang/rust/tree/master/src // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { - let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; + let arch = if sess.target.arch == "x86_64" { + "x86" + } else if sess.target.arch == "arm64ec" { + "aarch64" + } else { + &*sess.target.arch + }; match (arch, s) { ("x86", "sse4.2") => { LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32")) @@ -213,6 +221,7 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("x86", "rdrand") => LLVMFeature::new("rdrnd"), ("x86", "bmi1") => LLVMFeature::new("bmi"), ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("x86", "lahfsahf") => LLVMFeature::new("sahf"), ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), ("aarch64", "dpb") => LLVMFeature::new("ccpp"), ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), @@ -265,6 +274,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { LLVMFeature::new("unaligned-scalar-mem") } + // For LLVM 18, enable the evex512 target feature if a avx512 target feature is enabled. + ("x86", s) if get_version().0 >= 18 && s.starts_with("avx512") => { + LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")) + } (_, s) => LLVMFeature::new(s), } } @@ -430,7 +443,7 @@ pub(crate) fn print(req: &PrintRequest, mut out: &mut dyn PrintBackendInfo, sess &tm, cpu_cstring.as_ptr(), callback, - &mut out as *mut &mut dyn PrintBackendInfo as *mut c_void, + std::ptr::addr_of_mut!(out) as *mut c_void, ); } } @@ -511,10 +524,14 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str sess.target .features .split(',') - .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + .filter(|v| !v.is_empty() && backend_feature_name(sess, v).is_some()) .map(String::from), ); + if wants_wasm_eh(sess) && sess.panic_strategy() == PanicStrategy::Unwind { + features.push("+exception-handling".into()); + } + // -Ctarget-features let supported_features = sess.target.supported_target_features(); let mut featsmap = FxHashMap::default(); @@ -529,13 +546,13 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str Some(c @ ('+' | '-')) => c, Some(_) => { if diagnostics { - sess.dcx().emit_warning(UnknownCTargetFeaturePrefix { feature: s }); + sess.dcx().emit_warn(UnknownCTargetFeaturePrefix { feature: s }); } return None; } }; - let feature = backend_feature_name(s)?; + let feature = backend_feature_name(sess, s)?; // Warn against use of LLVM specific feature names and unstable features on the CLI. if diagnostics { let feature_state = supported_features.iter().find(|&&(v, _)| v == feature); @@ -557,12 +574,12 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } else { UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } }; - sess.dcx().emit_warning(unknown_feature); + sess.dcx().emit_warn(unknown_feature); } else if feature_state .is_some_and(|(_name, feature_gate)| !feature_gate.is_stable()) { // An unstable feature. Warn about using it. - sess.dcx().emit_warning(UnstableCTargetFeature { feature }); + sess.dcx().emit_warn(UnstableCTargetFeature { feature }); } } @@ -611,11 +628,11 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str /// Returns a feature name for the given `+feature` or `-feature` string. /// /// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) -fn backend_feature_name(s: &str) -> Option<&str> { +fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { // features must start with a `+` or `-`. - let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { - bug!("target feature `{}` must begin with a `+` or `-`", s); - }); + let feature = s + .strip_prefix(&['+', '-'][..]) + .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. if RUSTC_SPECIFIC_FEATURES.contains(&feature) { diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f796ce0990f..29100a64171 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -5,7 +5,9 @@ use crate::errors::SymbolAlreadyDefined; use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_middle::bug; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; @@ -21,7 +23,14 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { symbol_name: &str, ) { let instance = Instance::mono(self.tcx, def_id); - let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out + // the llvm type from the actual evaluated initializer. + let ty = if nested { + self.tcx.types.unit + } else { + instance.ty(self.tcx, ty::ParamEnv::reveal_all()) + }; let llty = self.layout_of(ty).llvm_type(self); let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { @@ -123,6 +132,17 @@ impl CodegenCx<'_, '_> { return false; } + // Match clang by only supporting COFF and ELF for now. + if self.tcx.sess.target.is_like_osx { + return false; + } + + // With pie relocation model calls of functions defined in the translation + // unit can use copy relocations. + if self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration { + return true; + } + // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); @@ -130,18 +150,12 @@ impl CodegenCx<'_, '_> { return false; } - // Match clang by only supporting COFF and ELF for now. - if self.tcx.sess.target.is_like_osx { - return false; + // Respect the direct-access-external-data to override default behavior if present. + if let Some(direct) = self.tcx.sess.direct_access_external_data() { + return direct; } // Static relocation model should force copy relocations everywhere. - if self.tcx.sess.relocation_model() == RelocModel::Static { - return true; - } - - // With pie relocation model calls of functions defined in the translation - // unit can use copy relocations. - self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration + self.tcx.sess.relocation_model() == RelocModel::Static } } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 447c4ed1f0c..af1bbda4d08 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -107,8 +107,10 @@ impl<'ll> CodegenCx<'ll, '_> { pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { match t { + ty::FloatTy::F16 => self.type_f16(), ty::FloatTy::F32 => self.type_f32(), ty::FloatTy::F64 => self.type_f64(), + ty::FloatTy::F128 => self.type_f128(), } } @@ -156,6 +158,10 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.isize_ty } + fn type_f16(&self) -> &'ll Type { + unsafe { llvm::LLVMHalfTypeInContext(self.llcx) } + } + fn type_f32(&self) -> &'ll Type { unsafe { llvm::LLVMFloatTypeInContext(self.llcx) } } @@ -164,6 +170,10 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) } } + fn type_f128(&self) -> &'ll Type { + unsafe { llvm::LLVMFP128TypeInContext(self.llcx) } + } + fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } } @@ -195,7 +205,7 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { match self.type_kind(ty) { TypeKind::Array | TypeKind::Vector => unsafe { llvm::LLVMGetElementType(ty) }, TypeKind::Pointer => bug!("element_type is not supported for opaque pointers"), - other => bug!("element_type called on unsupported type {:?}", other), + other => bug!("element_type called on unsupported type {other:?}"), } } @@ -205,11 +215,12 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn float_width(&self, ty: &'ll Type) -> usize { match self.type_kind(ty) { + TypeKind::Half => 16, TypeKind::Float => 32, TypeKind::Double => 64, TypeKind::X86_FP80 => 80, TypeKind::FP128 | TypeKind::PPC_FP128 => 128, - _ => bug!("llvm_float_width called on a non-float type"), + other => bug!("llvm_float_width called on a non-float type {other:?}"), } } @@ -222,7 +233,7 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { - unsafe { llvm::LLVMRustArrayType(ty, len) } + unsafe { llvm::LLVMArrayType2(ty, len) } } } @@ -250,9 +261,6 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { layout.is_llvm_scalar_pair() } - fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64 { - layout.llvm_field_index(self, index) - } fn scalar_pair_element_backend_type( &self, layout: TyAndLayout<'tcx>, diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 57b46382c96..d10a083765b 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -1,5 +1,4 @@ use crate::common::*; -use crate::context::TypeLowering; use crate::type_::Type; use rustc_codegen_ssa::traits::*; use rustc_middle::bug; @@ -8,9 +7,8 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty, TypeVisitableExt}; use rustc_target::abi::HasDataLayout; use rustc_target::abi::{Abi, Align, FieldsShape}; -use rustc_target::abi::{Int, Pointer, F32, F64}; +use rustc_target::abi::{Int, Pointer, F128, F16, F32, F64}; use rustc_target::abi::{Scalar, Size, Variants}; -use smallvec::{smallvec, SmallVec}; use std::fmt::Write; @@ -18,7 +16,6 @@ fn uncached_llvm_type<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, layout: TyAndLayout<'tcx>, defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, - field_remapping: &mut Option<SmallVec<[u32; 4]>>, ) -> &'a Type { match layout.abi { Abi::Scalar(_) => bug!("handled elsewhere"), @@ -26,23 +23,14 @@ fn uncached_llvm_type<'a, 'tcx>( let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } - Abi::ScalarPair(..) => { - return cx.type_struct( - &[ - layout.scalar_pair_element_llvm_type(cx, 0, false), - layout.scalar_pair_element_llvm_type(cx, 1, false), - ], - false, - ); - } - Abi::Uninhabited | Abi::Aggregate { .. } => {} + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalarPair(..) => {} } let name = match layout.ty.kind() { // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str // For performance reasons we use names only when emitting LLVM IR. if !cx.sess().fewer_names() => { @@ -80,8 +68,7 @@ fn uncached_llvm_type<'a, 'tcx>( FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), FieldsShape::Arbitrary { .. } => match name { None => { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); - *field_remapping = new_field_remapping; + let (llfields, packed) = struct_llfields(cx, layout); cx.type_struct(&llfields, packed) } Some(ref name) => { @@ -96,7 +83,7 @@ fn uncached_llvm_type<'a, 'tcx>( fn struct_llfields<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, layout: TyAndLayout<'tcx>, -) -> (Vec<&'a Type>, bool, Option<SmallVec<[u32; 4]>>) { +) -> (Vec<&'a Type>, bool) { debug!("struct_llfields: {:#?}", layout); let field_count = layout.fields.count(); @@ -104,7 +91,6 @@ fn struct_llfields<'a, 'tcx>( let mut offset = Size::ZERO; let mut prev_effective_align = layout.align.abi; let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); - let mut field_remapping = smallvec![0; field_count]; for i in layout.fields.index_by_increasing_offset() { let target_offset = layout.fields.offset(i as usize); let field = layout.field(cx, i); @@ -129,12 +115,10 @@ fn struct_llfields<'a, 'tcx>( result.push(cx.type_padding_filler(padding, padding_align)); debug!(" padding before: {:?}", padding); } - field_remapping[i] = result.len() as u32; result.push(field.llvm_type(cx)); offset = target_offset + field.size; prev_effective_align = effective_field_align; } - let padding_used = result.len() > field_count; if layout.is_sized() && field_count > 0 { if offset > layout.size { bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); @@ -152,8 +136,7 @@ fn struct_llfields<'a, 'tcx>( } else { debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); } - let field_remapping = padding_used.then_some(field_remapping); - (result, packed, field_remapping) + (result, packed) } impl<'a, 'tcx> CodegenCx<'a, 'tcx> { @@ -183,7 +166,6 @@ pub trait LayoutLlvmExt<'tcx> { index: usize, immediate: bool, ) -> &'a Type; - fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type>; } @@ -234,7 +216,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { _ => None, }; if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - return llty.lltype; + return llty; } debug!("llvm_type({:#?})", self); @@ -246,7 +228,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { let normal_ty = cx.tcx.erase_regions(self.ty); let mut defer = None; - let mut field_remapping = None; let llty = if self.ty != normal_ty { let mut layout = cx.layout_of(normal_ty); if let Some(v) = variant_index { @@ -254,40 +235,49 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } layout.llvm_type(cx) } else { - uncached_llvm_type(cx, *self, &mut defer, &mut field_remapping) + uncached_llvm_type(cx, *self, &mut defer) }; debug!("--> mapped {:#?} to llty={:?}", self, llty); - cx.type_lowering - .borrow_mut() - .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); + cx.type_lowering.borrow_mut().insert((self.ty, variant_index), llty); if let Some((llty, layout)) = defer { - let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + let (llfields, packed) = struct_llfields(cx, layout); cx.set_struct_body(llty, &llfields, packed); - cx.type_lowering - .borrow_mut() - .get_mut(&(self.ty, variant_index)) - .unwrap() - .field_remapping = new_field_remapping; } llty } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(scalar) = self.abi { - if scalar.is_bool() { - return cx.type_i1(); + match self.abi { + Abi::Scalar(scalar) => { + if scalar.is_bool() { + return cx.type_i1(); + } } - } + Abi::ScalarPair(..) => { + // An immediate pair always contains just the two elements, without any padding + // filler, as it should never be stored to memory. + return cx.type_struct( + &[ + self.scalar_pair_element_llvm_type(cx, 0, true), + self.scalar_pair_element_llvm_type(cx, 1, true), + ], + false, + ); + } + _ => {} + }; self.llvm_type(cx) } fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type { match scalar.primitive() { Int(i, _) => cx.type_from_integer(i), + F16 => cx.type_f16(), F32 => cx.type_f32(), F64 => cx.type_f64(), + F128 => cx.type_f128(), Pointer(address_space) => cx.type_ptr_ext(address_space), } } @@ -319,42 +309,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { self.scalar_llvm_type_at(cx, scalar) } - fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64 { - match self.abi { - Abi::Scalar(_) | Abi::ScalarPair(..) => { - bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) - } - _ => {} - } - match self.fields { - FieldsShape::Primitive | FieldsShape::Union(_) => { - bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) - } - - FieldsShape::Array { .. } => index as u64, - - FieldsShape::Arbitrary { .. } => { - let variant_index = match self.variants { - Variants::Single { index } => Some(index), - _ => None, - }; - - // Look up llvm field if indexes do not match memory order due to padding. If - // `field_remapping` is `None` no padding was used and the llvm field index - // matches the memory index. - match cx.type_lowering.borrow().get(&(self.ty, variant_index)) { - Some(TypeLowering { field_remapping: Some(ref remap), .. }) => { - remap[index] as u64 - } - Some(_) => self.fields.memory_index(index) as u64, - None => { - bug!("TyAndLayout::llvm_field_index({:?}): type info not found", self) - } - } - } - } - } - fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type> { debug_assert!(self.is_sized()); diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 172c66a7af1..220bb77d3fd 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -44,12 +44,12 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>( let aligned_size = size.align_to(slot_size).bytes() as i32; let full_direct_size = bx.cx().const_i32(aligned_size); - let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]); + let next = bx.inbounds_ptradd(addr, full_direct_size); bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); - let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); + let adjusted = bx.inbounds_ptradd(addr, adjusted_size); (adjusted, addr_align) } else { (addr, addr_align) @@ -89,11 +89,31 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( list: OperandRef<'tcx, &'ll Value>, target_ty: Ty<'tcx>, ) -> &'ll Value { + let dl = bx.cx.data_layout(); + // Implementation of the AAPCS64 calling convention for va_args see // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + // + // typedef struct va_list { + // void * stack; // next stack param + // void * gr_top; // end of GP arg reg save area + // void * vr_top; // end of FP/SIMD arg reg save area + // int gr_offs; // offset from gr_top to next GP register arg + // int vr_offs; // offset from vr_top to next FP/SIMD register arg + // } va_list; let va_list_addr = list.immediate(); - let va_list_layout = list.deref(bx.cx).layout; - let va_list_ty = va_list_layout.llvm_type(bx); + + // There is no padding between fields since `void*` is size=8 align=8, `int` is size=4 align=4. + // See https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + // Table 1, Byte size and byte alignment of fundamental data types + // Table 3, Mapping of C & C++ built-in data types + let ptr_offset = 8; + let i32_offset = 4; + let gr_top = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(ptr_offset)); + let vr_top = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * ptr_offset)); + let gr_offs = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(3 * ptr_offset)); + let vr_offs = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(3 * ptr_offset + i32_offset)); + let layout = bx.cx.layout_of(target_ty); let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); @@ -104,16 +124,12 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( let offset_align = Align::from_bytes(4).unwrap(); let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); - let (reg_off, reg_top_index, slot_size) = if gr_type { - let gr_offs = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let (reg_off, reg_top, slot_size) = if gr_type { let nreg = (layout.size.bytes() + 7) / 8; - (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8) + (gr_offs, gr_top, nreg * 8) } else { - let vr_off = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4)); let nreg = (layout.size.bytes() + 15) / 16; - (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16) + (vr_offs, vr_top, nreg * 16) }; // if the offset >= 0 then the value will be on the stack @@ -141,15 +157,14 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); let top_type = bx.type_ptr(); - let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); - let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); + let top = bx.load(top_type, reg_top, dl.pointer_align.abi); // reg_value = *(@top + reg_off_v); - let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); + let mut reg_addr = bx.ptradd(top, reg_off_v); if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { // On big-endian systems the value is right-aligned in its slot. let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); - reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); + reg_addr = bx.ptradd(reg_addr, offset); } let reg_type = layout.llvm_type(bx); let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); @@ -173,11 +188,29 @@ fn emit_s390x_va_arg<'ll, 'tcx>( list: OperandRef<'tcx, &'ll Value>, target_ty: Ty<'tcx>, ) -> &'ll Value { + let dl = bx.cx.data_layout(); + // Implementation of the s390x ELF ABI calling convention for va_args see // https://github.com/IBM/s390x-abi (chapter 1.2.4) + // + // typedef struct __va_list_tag { + // long __gpr; + // long __fpr; + // void *__overflow_arg_area; + // void *__reg_save_area; + // } va_list[1]; let va_list_addr = list.immediate(); - let va_list_layout = list.deref(bx.cx).layout; - let va_list_ty = va_list_layout.llvm_type(bx); + + // There is no padding between fields since `long` and `void*` both have size=8 align=8. + // https://github.com/IBM/s390x-abi (Table 1.1.: Scalar types) + let i64_offset = 8; + let ptr_offset = 8; + let gpr = va_list_addr; + let fpr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(i64_offset)); + let overflow_arg_area = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * i64_offset)); + let reg_save_area = + bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * i64_offset + ptr_offset)); + let layout = bx.cx.layout_of(target_ty); let in_reg = bx.append_sibling_block("va_arg.in_reg"); @@ -192,15 +225,10 @@ fn emit_s390x_va_arg<'ll, 'tcx>( let padding = padded_size - unpadded_size; let gpr_type = indirect || !layout.is_single_fp_element(bx.cx); - let (max_regs, reg_count_field, reg_save_index, reg_padding) = - if gpr_type { (5, 0, 2, padding) } else { (4, 1, 16, 0) }; + let (max_regs, reg_count, reg_save_index, reg_padding) = + if gpr_type { (5, gpr, 2, padding) } else { (4, fpr, 16, 0) }; // Check whether the value was passed in a register or in memory. - let reg_count = bx.struct_gep( - va_list_ty, - va_list_addr, - va_list_layout.llvm_field_index(bx.cx, reg_count_field), - ); let reg_count_v = bx.load(bx.type_i64(), reg_count, Align::from_bytes(8).unwrap()); let use_regs = bx.icmp(IntPredicate::IntULT, reg_count_v, bx.const_u64(max_regs)); bx.cond_br(use_regs, in_reg, in_mem); @@ -209,12 +237,10 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.switch_to_block(in_reg); // Work out the address of the value in the register save area. - let reg_ptr = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); - let reg_ptr_v = bx.load(bx.type_ptr(), reg_ptr, bx.tcx().data_layout.pointer_align.abi); + let reg_ptr_v = bx.load(bx.type_ptr(), reg_save_area, dl.pointer_align.abi); let scaled_reg_count = bx.mul(reg_count_v, bx.const_u64(8)); let reg_off = bx.add(scaled_reg_count, bx.const_u64(reg_save_index * 8 + reg_padding)); - let reg_addr = bx.gep(bx.type_i8(), reg_ptr_v, &[reg_off]); + let reg_addr = bx.ptradd(reg_ptr_v, reg_off); // Update the register count. let new_reg_count_v = bx.add(reg_count_v, bx.const_u64(1)); @@ -225,27 +251,23 @@ fn emit_s390x_va_arg<'ll, 'tcx>( bx.switch_to_block(in_mem); // Work out the address of the value in the argument overflow area. - let arg_ptr = - bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 2)); - let arg_ptr_v = bx.load(bx.type_ptr(), arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let arg_ptr_v = + bx.load(bx.type_ptr(), overflow_arg_area, bx.tcx().data_layout.pointer_align.abi); let arg_off = bx.const_u64(padding); - let mem_addr = bx.gep(bx.type_i8(), arg_ptr_v, &[arg_off]); + let mem_addr = bx.ptradd(arg_ptr_v, arg_off); // Update the argument overflow area pointer. let arg_size = bx.cx().const_u64(padded_size); - let new_arg_ptr_v = bx.inbounds_gep(bx.type_i8(), arg_ptr_v, &[arg_size]); - bx.store(new_arg_ptr_v, arg_ptr, bx.tcx().data_layout.pointer_align.abi); + let new_arg_ptr_v = bx.inbounds_ptradd(arg_ptr_v, arg_size); + bx.store(new_arg_ptr_v, overflow_arg_area, dl.pointer_align.abi); bx.br(end); // Return the appropriate result. bx.switch_to_block(end); let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]); let val_type = layout.llvm_type(bx); - let val_addr = if indirect { - bx.load(bx.cx.type_ptr(), val_addr, bx.tcx().data_layout.pointer_align.abi) - } else { - val_addr - }; + let val_addr = + if indirect { bx.load(bx.cx.type_ptr(), val_addr, dl.pointer_align.abi) } else { val_addr }; bx.load(val_type, val_addr, layout.align.abi) } @@ -266,7 +288,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( // Generic x86 "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true), // Windows AArch64 - "aarch64" if target.is_like_windows => { + "aarch64" | "arm64ec" if target.is_like_windows => { emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false) } // macOS / iOS AArch64 |