diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
24 files changed, 374 insertions, 551 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b8f42031263..5ff580e295a 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -5,10 +5,10 @@ use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; -use rustc_middle::bug; use rustc_middle::ty::layout::LayoutOf; pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; use rustc_middle::ty::Ty; +use rustc_middle::{bug, ty}; use rustc_session::config; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int, Size}; @@ -16,6 +16,7 @@ pub use rustc_target::spec::abi::Abi; use rustc_target::spec::SanitizerSet; use smallvec::SmallVec; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; @@ -310,7 +311,16 @@ pub trait FnAbiLlvmExt<'ll, 'tcx> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn llvm_cconv(&self) -> llvm::CallConv; - fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); + + /// Apply attributes to a function declaration/definition. + fn apply_attrs_llfn( + &self, + cx: &CodegenCx<'ll, 'tcx>, + llfn: &'ll Value, + instance: Option<ty::Instance<'tcx>>, + ); + + /// Apply attributes to a function call. fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); } @@ -396,7 +406,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { self.conv.into() } - fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { + fn apply_attrs_llfn( + &self, + cx: &CodegenCx<'ll, 'tcx>, + llfn: &'ll Value, + instance: Option<ty::Instance<'tcx>>, + ) { let mut func_attrs = SmallVec::<[_; 3]>::new(); if self.ret.layout.abi.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); @@ -415,9 +430,32 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { i += 1; i - 1 }; + + let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| { + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (19, 0, 0) + && matches!(scalar.primitive(), Int(..)) + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + && !scalar.is_bool() + // LLVM also rejects full range. + && !scalar.is_always_valid(cx) + { + attributes::apply_to_llfn( + llfn, + idx, + &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))], + ); + } + }; + match &self.ret.mode { PassMode::Direct(attrs) => { attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + apply_range_attr(llvm::AttributePlace::ReturnValue, scalar); + } } PassMode::Indirect { attrs, meta_attrs: _, on_stack } => { assert!(!on_stack); @@ -456,8 +494,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { ); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); } - PassMode::Direct(attrs) - | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { + PassMode::Direct(attrs) => { + let i = apply(attrs); + if let abi::Abi::Scalar(scalar) = arg.layout.abi { + apply_range_attr(llvm::AttributePlace::Argument(i), scalar); + } + } + PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { apply(attrs); } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { @@ -466,8 +509,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { apply(meta_attrs); } PassMode::Pair(a, b) => { - apply(a); - apply(b); + let i = apply(a); + let ii = apply(b); + if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi { + apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a); + apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b); + } } PassMode::Cast { cast, pad_i32 } => { if *pad_i32 { @@ -477,6 +524,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } } + + // If the declaration has an associated instance, compute extra attributes based on that. + if let Some(instance) = instance { + llfn_attrs_from_instance(cx, llfn, instance); + } } fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { @@ -517,15 +569,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } _ => {} } - if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { - // If the value is a boolean, the range is 0..2 and that ultimately - // become 0..0 when the type becomes i1, which would be rejected - // by the LLVM verifier. - if let Int(..) = scalar.primitive() { - if !scalar.is_bool() && !scalar.is_always_valid(bx) { - bx.range_metadata(callsite, scalar.valid_range(bx)); - } - } + if bx.cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() < (19, 0, 0) + && let abi::Abi::Scalar(scalar) = self.ret.layout.abi + && matches!(scalar.primitive(), Int(..)) + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + && !scalar.is_bool() + // LLVM also rejects full range. + && !scalar.is_always_valid(bx) + { + bx.range_metadata(callsite, scalar.valid_range(bx)); } for arg in self.args.iter() { match &arg.mode { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index aea8395441a..f931698c38f 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -1,3 +1,5 @@ +use std::assert_matches::assert_matches; + use libc::{c_char, c_uint}; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::mir::operand::OperandValue; @@ -89,7 +91,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // if the target feature needed by the register class is // disabled. This is necessary otherwise LLVM will try // to actually allocate a register for the dummy output. - assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); + assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_)); clobbers.push(format!("~{}", reg_to_llvm(reg, None))); continue; } else { diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index ad38814a68b..fde95104093 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -324,9 +324,10 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") } +/// Helper for `FnAbi::apply_attrs_llfn`: /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. -pub fn from_fn_attrs<'ll, 'tcx>( +pub fn llfn_attrs_from_instance<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::Instance<'tcx>, @@ -496,7 +497,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( to_add.extend(tune_cpu_attr(cx)); let function_features = - codegen_fn_attrs.target_features.iter().map(|f| f.as_str()).collect::<Vec<&str>>(); + codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>(); if let Some(f) = llvm_util::check_tied_features( cx.tcx.sess, diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 2f5a864d70e..2120fc1815c 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,21 +1,19 @@ //! A helper class for dealing with static archives -use std::ffi::{c_char, c_void, CStr, CString, OsString}; +use std::ffi::{c_char, c_void, CStr, CString}; use std::path::{Path, PathBuf}; -use std::{env, io, mem, ptr, str}; +use std::{io, mem, ptr, str}; use rustc_codegen_ssa::back::archive::{ - try_extract_macho_fat_archive, ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, - ArchiveBuilderBuilder, ObjectReader, UnknownArchiveKind, DEFAULT_OBJECT_READER, + create_mingw_dll_import_lib, try_extract_macho_fat_archive, ArArchiveBuilder, + ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, ObjectReader, UnknownArchiveKind, + DEFAULT_OBJECT_READER, }; -use rustc_session::cstore::DllImport; +use rustc_codegen_ssa::common; use rustc_session::Session; use tracing::trace; -use crate::common; -use crate::errors::{ - DlltoolFailImportLibrary, ErrorCallingDllTool, ErrorCreatingImportLibrary, ErrorWritingDEFFile, -}; +use crate::errors::ErrorCreatingImportLibrary; use crate::llvm::archive_ro::{ArchiveRO, Child}; use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; @@ -108,9 +106,11 @@ pub struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { - // FIXME use ArArchiveBuilder on most targets again once reading thin archives is - // implemented - if true { + // Keeping LlvmArchiveBuilder around in case of a regression caused by using + // ArArchiveBuilder. + // FIXME(#128955) remove a couple of months after #128936 gets merged in case + // no regression is found. + if false { Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) } else { Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) @@ -121,116 +121,21 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { &self, sess: &Session, lib_name: &str, - dll_imports: &[DllImport], - tmpdir: &Path, - is_direct_dependency: bool, - ) -> PathBuf { - let name_suffix = if is_direct_dependency { "_imports" } else { "_imports_indirect" }; - let output_path = tmpdir.join(format!("{lib_name}{name_suffix}.lib")); - - let target = &sess.target; - let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(target); - - let import_name_and_ordinal_vector: Vec<(String, Option<u16>)> = dll_imports - .iter() - .map(|import: &DllImport| { - if sess.target.arch == "x86" { - ( - common::i686_decorated_name(import, mingw_gnu_toolchain, false), - import.ordinal(), - ) - } else { - (import.name.to_string(), import.ordinal()) - } - }) - .collect(); - - if mingw_gnu_toolchain { + import_name_and_ordinal_vector: Vec<(String, Option<u16>)>, + output_path: &Path, + ) { + if common::is_mingw_gnu_toolchain(&sess.target) { // The binutils linker used on -windows-gnu targets cannot read the import // libraries generated by LLVM: in our attempts, the linker produced an .EXE // that loaded but crashed with an AV upon calling one of the imported // functions. Therefore, use binutils to create the import library instead, // by writing a .DEF file to the temp dir and calling binutils's dlltool. - let def_file_path = tmpdir.join(format!("{lib_name}{name_suffix}.def")); - - let def_file_content = format!( - "EXPORTS\n{}", - import_name_and_ordinal_vector - .into_iter() - .map(|(name, ordinal)| { - match ordinal { - Some(n) => format!("{name} @{n} NONAME"), - None => name, - } - }) - .collect::<Vec<String>>() - .join("\n") + create_mingw_dll_import_lib( + sess, + lib_name, + import_name_and_ordinal_vector, + output_path, ); - - match std::fs::write(&def_file_path, def_file_content) { - Ok(_) => {} - Err(e) => { - sess.dcx().emit_fatal(ErrorWritingDEFFile { error: e }); - } - }; - - // --no-leading-underscore: For the `import_name_type` feature to work, we need to be - // able to control the *exact* spelling of each of the symbols that are being imported: - // hence we don't want `dlltool` adding leading underscores automatically. - let dlltool = find_binutils_dlltool(sess); - let temp_prefix = { - let mut path = PathBuf::from(&output_path); - path.pop(); - path.push(lib_name); - path - }; - // dlltool target architecture args from: - // https://github.com/llvm/llvm-project-release-prs/blob/llvmorg-15.0.6/llvm/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp#L69 - let (dlltool_target_arch, dlltool_target_bitness) = match sess.target.arch.as_ref() { - "x86_64" => ("i386:x86-64", "--64"), - "x86" => ("i386", "--32"), - "aarch64" => ("arm64", "--64"), - "arm" => ("arm", "--32"), - _ => panic!("unsupported arch {}", sess.target.arch), - }; - let mut dlltool_cmd = std::process::Command::new(&dlltool); - dlltool_cmd - .arg("-d") - .arg(def_file_path) - .arg("-D") - .arg(lib_name) - .arg("-l") - .arg(&output_path) - .arg("-m") - .arg(dlltool_target_arch) - .arg("-f") - .arg(dlltool_target_bitness) - .arg("--no-leading-underscore") - .arg("--temp-prefix") - .arg(temp_prefix); - - match dlltool_cmd.output() { - Err(e) => { - sess.dcx().emit_fatal(ErrorCallingDllTool { - dlltool_path: dlltool.to_string_lossy(), - error: e, - }); - } - // dlltool returns '0' on failure, so check for error output instead. - Ok(output) if !output.stderr.is_empty() => { - sess.dcx().emit_fatal(DlltoolFailImportLibrary { - dlltool_path: dlltool.to_string_lossy(), - dlltool_args: dlltool_cmd - .get_args() - .map(|arg| arg.to_string_lossy()) - .collect::<Vec<_>>() - .join(" "), - stdout: String::from_utf8_lossy(&output.stdout), - stderr: String::from_utf8_lossy(&output.stderr), - }) - } - _ => {} - } } else { // we've checked for \0 characters in the library name already let dll_name_z = CString::new(lib_name).unwrap(); @@ -242,9 +147,9 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { trace!(" output_path {}", output_path.display()); trace!( " import names: {}", - dll_imports + import_name_and_ordinal_vector .iter() - .map(|import| import.name.to_string()) + .map(|(name, _ordinal)| name.clone()) .collect::<Vec<_>>() .join(", "), ); @@ -281,9 +186,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { error: llvm::last_error().unwrap_or("unknown LLVM error".to_string()), }); } - }; - - output_path + } } } @@ -297,25 +200,11 @@ static LLVM_OBJECT_READER: ObjectReader = ObjectReader { get_xcoff_member_alignment: DEFAULT_OBJECT_READER.get_xcoff_member_alignment, }; -fn should_use_llvm_reader(buf: &[u8]) -> bool { - let is_bitcode = unsafe { llvm::LLVMRustIsBitcode(buf.as_ptr(), buf.len()) }; - - // COFF bigobj file, msvc LTO file or import library. See - // https://github.com/llvm/llvm-project/blob/453f27bc9/llvm/lib/BinaryFormat/Magic.cpp#L38-L51 - let is_unsupported_windows_obj_file = buf.get(0..4) == Some(b"\0\0\xFF\xFF"); - - is_bitcode || is_unsupported_windows_obj_file -} - #[deny(unsafe_op_in_unsafe_fn)] fn get_llvm_object_symbols( buf: &[u8], f: &mut dyn FnMut(&[u8]) -> io::Result<()>, ) -> io::Result<bool> { - if !should_use_llvm_reader(buf) { - return (DEFAULT_OBJECT_READER.get_symbols)(buf, f); - } - let mut state = Box::new(f); let err = unsafe { @@ -352,18 +241,10 @@ fn get_llvm_object_symbols( } fn llvm_is_64_bit_object_file(buf: &[u8]) -> bool { - if !should_use_llvm_reader(buf) { - return (DEFAULT_OBJECT_READER.is_64_bit_object_file)(buf); - } - unsafe { llvm::LLVMRustIs64BitSymbolicFile(buf.as_ptr(), buf.len()) } } fn llvm_is_ec_object_file(buf: &[u8]) -> bool { - if !should_use_llvm_reader(buf) { - return (DEFAULT_OBJECT_READER.is_ec_object_file)(buf); - } - unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } @@ -457,39 +338,3 @@ impl<'a> LlvmArchiveBuilder<'a> { fn string_to_io_error(s: String) -> io::Error { io::Error::new(io::ErrorKind::Other, format!("bad archive: {s}")) } - -fn find_binutils_dlltool(sess: &Session) -> OsString { - assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.cg.dlltool { - return dlltool_path.clone().into_os_string(); - } - - let tool_name: OsString = if sess.host.options.is_like_windows { - // If we're compiling on Windows, always use "dlltool.exe". - "dlltool.exe" - } else { - // On other platforms, use the architecture-specific name. - match sess.target.arch.as_ref() { - "x86_64" => "x86_64-w64-mingw32-dlltool", - "x86" => "i686-w64-mingw32-dlltool", - "aarch64" => "aarch64-w64-mingw32-dlltool", - - // For non-standard architectures (e.g., aarch32) fallback to "dlltool". - _ => "dlltool", - } - } - .into(); - - // NOTE: it's not clear how useful it is to explicitly search PATH. - for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { - let full_path = dir.join(&tool_name); - if full_path.is_file() { - return full_path.into_os_string(); - } - } - - // The user didn't specify the location of the dlltool binary, and we weren't able - // to find the appropriate one on the PATH. Just return the name of the tool - // and let the invocation fail with a hopefully useful error message. - tool_name -} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 5a7909d1511..a1f2433ab6f 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -95,11 +95,14 @@ pub fn write_output_file<'ll>( } } -pub fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine { +pub fn create_informational_target_machine( + sess: &Session, + only_base_features: bool, +) -> OwnedTargetMachine { let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None }; // Can't use query system here quite yet because this function is invoked before the query // system/tcx is set up. - let features = llvm_util::global_llvm_features(sess, false); + let features = llvm_util::global_llvm_features(sess, false, only_base_features); target_machine_factory(sess, config::OptLevel::No, &features)(config) .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise()) } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 77e51f94150..cc081f29e12 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -12,7 +12,8 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, + TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; @@ -531,7 +532,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { #[instrument(level = "trace", skip(self))] fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { if place.layout.is_unsized() { - let tail = self.tcx.struct_tail_with_normalize(place.layout.ty, |ty| ty, || {}); + let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.param_env()); if matches!(tail.kind(), ty::Foreign(..)) { // Unsized locals and, at least conceptually, even unsized arguments must be copied // around, which requires dynamically determining their size. Therefore, we cannot @@ -727,13 +728,32 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llvm::LLVMSetVolatile(store, llvm::True); } if flags.contains(MemFlags::NONTEMPORAL) { - // According to LLVM [1] building a nontemporal store must - // *always* point to a metadata value of the integer 1. - // - // [1]: https://llvm.org/docs/LangRef.html#store-instruction - let one = self.cx.const_i32(1); - let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); - llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); + // Make sure that the current target architectures supports "sane" non-temporal + // stores, i.e., non-temporal stores that are equivalent to regular stores except + // for performance. LLVM doesn't seem to care about this, and will happily treat + // `!nontemporal` stores as-if they were normal stores (for reordering optimizations + // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like + // regular stores but require special fences. + // So we keep a list of architectures where `!nontemporal` is known to be truly just + // a hint, and use regular stores everywhere else. + // (In the future, we could alternatively ensure that an sfence gets emitted after a sequence of movnt + // before any kind of synchronizing operation. But it's not clear how to do that with LLVM.) + // For more context, see <https://github.com/rust-lang/rust/issues/114582> and + // <https://github.com/llvm/llvm-project/issues/64521>. + const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] = + &["aarch64", "arm", "riscv32", "riscv64"]; + + let use_nontemporal = + WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch); + if use_nontemporal { + // According to LLVM [1] building a nontemporal store must + // *always* point to a metadata value of the integer 1. + // + // [1]: https://llvm.org/docs/LangRef.html#store-instruction + let one = self.cx.const_i32(1); + let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); + llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); + } } store } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index c913bdebaaa..663c5be46e5 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -4,13 +4,14 @@ //! and methods are represented as just a fn ptr and not a full //! closure. +use rustc_codegen_ssa::common; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use tracing::debug; use crate::context::CodegenCx; +use crate::llvm; use crate::value::Value; -use crate::{attributes, common, llvm}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -46,7 +47,7 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> } else { let instance_def_id = instance.def_id(); let llfn = if tcx.sess.target.arch == "x86" - && let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) + && let Some(dllimport) = crate::common::get_dllimport(tcx, instance_def_id, sym) { // Fix for https://github.com/rust-lang/rust/issues/104453 // On x86 Windows, LLVM uses 'L' as the prefix for any private @@ -77,8 +78,6 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> }; debug!("get_fn: not casting pointer!"); - attributes::from_fn_attrs(cx, llfn, instance); - // Apply an appropriate linkage/visibility value to our item that we // just declared. // diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 197bbb9ddbf..a3997900184 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -1,7 +1,5 @@ //! Code that is useful in various codegen modules. -use std::fmt::Write; - use libc::{c_char, c_uint}; use rustc_ast::Mutability; use rustc_codegen_ssa::traits::*; @@ -10,9 +8,8 @@ use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; use rustc_middle::ty::TyCtxt; -use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; +use rustc_session::cstore::DllImport; use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; -use rustc_target::spec::Target; use tracing::debug; use crate::consts::const_alloc_to_llvm; @@ -100,11 +97,6 @@ impl<'ll> CodegenCx<'ll, '_> { unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) } } - pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { - let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); - unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } - } - pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { bytes_in_context(self.llcx, bytes) } @@ -224,6 +216,11 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { struct_in_context(self.llcx, elts, packed) } + fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } + } + fn const_to_opt_uint(&self, v: &'ll Value) -> Option<u64> { try_as_const_integral(v).and_then(|v| unsafe { let mut i = 0u64; @@ -379,64 +376,3 @@ pub(crate) fn get_dllimport<'tcx>( tcx.native_library(id) .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) } - -pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { - target.vendor == "pc" && target.os == "windows" && target.env == "gnu" && target.abi.is_empty() -} - -pub(crate) fn i686_decorated_name( - dll_import: &DllImport, - mingw: bool, - disable_name_mangling: bool, -) -> String { - let name = dll_import.name.as_str(); - - let (add_prefix, add_suffix) = match dll_import.import_name_type { - Some(PeImportNameType::NoPrefix) => (false, true), - Some(PeImportNameType::Undecorated) => (false, false), - _ => (true, true), - }; - - // Worst case: +1 for disable name mangling, +1 for prefix, +4 for suffix (@@__). - let mut decorated_name = String::with_capacity(name.len() + 6); - - if disable_name_mangling { - // LLVM uses a binary 1 ('\x01') prefix to a name to indicate that mangling needs to be disabled. - decorated_name.push('\x01'); - } - - let prefix = if add_prefix && dll_import.is_fn { - match dll_import.calling_convention { - DllCallingConvention::C | DllCallingConvention::Vectorcall(_) => None, - DllCallingConvention::Stdcall(_) => (!mingw - || dll_import.import_name_type == Some(PeImportNameType::Decorated)) - .then_some('_'), - DllCallingConvention::Fastcall(_) => Some('@'), - } - } else if !dll_import.is_fn && !mingw { - // For static variables, prefix with '_' on MSVC. - Some('_') - } else { - None - }; - if let Some(prefix) = prefix { - decorated_name.push(prefix); - } - - decorated_name.push_str(name); - - if add_suffix && dll_import.is_fn { - match dll_import.calling_convention { - DllCallingConvention::C => {} - DllCallingConvention::Stdcall(arg_list_size) - | DllCallingConvention::Fastcall(arg_list_size) => { - write!(&mut decorated_name, "@{arg_list_size}").unwrap(); - } - DllCallingConvention::Vectorcall(arg_list_size) => { - write!(&mut decorated_name, "@@{arg_list_size}").unwrap(); - } - } - } - - decorated_name -} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index c3ea4a18a71..75b298f14ca 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,5 +1,6 @@ use std::ops::Range; +use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; @@ -18,7 +19,7 @@ use rustc_target::abi::{ }; use tracing::{debug, instrument, trace}; -use crate::common::{self, CodegenCx}; +use crate::common::CodegenCx; use crate::errors::{ InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, }; @@ -195,7 +196,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( g2 } } else if cx.tcx.sess.target.arch == "x86" - && let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) + && let Some(dllimport) = crate::common::get_dllimport(cx.tcx, def_id, sym) { cx.declare_global( &common::i686_decorated_name( diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ea930421b58..dd3f39ecead 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -149,7 +149,7 @@ pub unsafe fn create_module<'ll>( // Ensure the data-layout values hardcoded remain the defaults. { - let tm = crate::back::write::create_informational_target_machine(tcx.sess); + let tm = crate::back::write::create_informational_target_machine(tcx.sess, false); unsafe { llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); } @@ -775,10 +775,10 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.debugtrap", fn() -> void); ifn!("llvm.frameaddress", fn(t_i32) -> ptr); - ifn!("llvm.powi.f16", fn(t_f16, t_i32) -> t_f16); - ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); - ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); - ifn!("llvm.powi.f128", fn(t_f128, t_i32) -> t_f128); + ifn!("llvm.powi.f16.i32", fn(t_f16, t_i32) -> t_f16); + ifn!("llvm.powi.f32.i32", fn(t_f32, t_i32) -> t_f32); + ifn!("llvm.powi.f64.i32", fn(t_f64, t_i32) -> t_f64); + ifn!("llvm.powi.f128.i32", fn(t_f128, t_i32) -> t_f128); ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16); ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index efe616838bf..0a02c230cfc 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -88,7 +88,7 @@ fn make_mir_scope<'ll, 'tcx>( let loc = cx.lookup_debug_loc(scope_data.span.lo()); let file_metadata = file_metadata(cx, &loc.file); - let parent_dbg_scope = match scope_data.inlined { + let dbg_scope = match scope_data.inlined { Some((callee, _)) => { // FIXME(eddyb) this would be `self.monomorphize(&callee)` // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. @@ -102,17 +102,15 @@ fn make_mir_scope<'ll, 'tcx>( cx.dbg_scope_fn(callee, callee_fn_abi, None) }) } - None => parent_scope.dbg_scope, - }; - - let dbg_scope = unsafe { - llvm::LLVMRustDIBuilderCreateLexicalBlock( - DIB(cx), - parent_dbg_scope, - file_metadata, - loc.line, - loc.col, - ) + None => unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_scope.dbg_scope, + file_metadata, + loc.line, + loc.col, + ) + }, }; let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index ad638588612..87bea22d8dd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -456,7 +456,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D { build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } - ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), + ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 13006638bb3..181022087f3 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; -use rustc_codegen_ssa::debuginfo::wants_c_like_enum_debuginfo; +use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstMethods; use rustc_index::IndexVec; use rustc_middle::bug; @@ -12,7 +12,7 @@ use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::common::CodegenCx; -use crate::debuginfo::metadata::enums::{tag_base_type, DiscrResult}; +use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ build_field_di_node, file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, @@ -190,7 +190,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type_and_layout = cx.layout_of(enum_type); let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); - assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); type_map::build_type_with_children( cx, @@ -265,7 +265,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( let coroutine_type_and_layout = cx.layout_of(coroutine_type); let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); - assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, coroutine_type_and_layout)); type_map::build_type_with_children( cx, @@ -381,7 +381,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( tag_field: usize, untagged_variant_index: Option<VariantIdx>, ) -> SmallVec<&'ll DIType> { - let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); let variant_names_type_di_node = build_variant_names_type_di_node( cx, @@ -676,7 +676,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); - let tag_base_type = tag_base_type(cx, coroutine_type_and_layout); + let tag_base_type = tag_base_type(cx.tcx, coroutine_type_and_layout); let variant_names_type_di_node = build_variant_names_type_di_node( cx, @@ -803,7 +803,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( assert_eq!( cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), - cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) + cx.size_and_align_of(self::tag_base_type(cx.tcx, enum_type_and_layout)) ); // ... and a field for the tag. If the tag is 128 bits wide, this will actually diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index fc3adaf0681..77cbcd86cdd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -1,17 +1,15 @@ use std::borrow::Cow; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; -use rustc_codegen_ssa::debuginfo::wants_c_like_enum_debuginfo; +use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; -use rustc_middle::ty::layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; use rustc_span::Symbol; -use rustc_target::abi::{ - FieldIdx, HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants, -}; +use rustc_target::abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{size_and_align_of, SmallVec}; @@ -39,7 +37,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type_and_layout = cx.layout_of(enum_type); - if wants_c_like_enum_debuginfo(enum_type_and_layout) { + if wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout) { return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); } @@ -74,7 +72,7 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( di_node: build_enumeration_type_di_node( cx, &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), - tag_base_type(cx, enum_type_and_layout), + tag_base_type(cx.tcx, enum_type_and_layout), enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); (name, discr.val) @@ -85,48 +83,6 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( } } -/// Extract the type with which we want to describe the tag of the given enum or coroutine. -fn tag_base_type<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - enum_type_and_layout: TyAndLayout<'tcx>, -) -> Ty<'tcx> { - assert!(match enum_type_and_layout.ty.kind() { - ty::Coroutine(..) => true, - ty::Adt(adt_def, _) => adt_def.is_enum(), - _ => false, - }); - - match enum_type_and_layout.layout.variants() { - // A single-variant enum has no discriminant. - Variants::Single { .. } => { - bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout) - } - - Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => { - // Niche tags are always normalized to unsized integers of the correct size. - match tag.primitive() { - Primitive::Int(t, _) => t, - Primitive::Float(f) => Integer::from_size(f.size()).unwrap(), - // FIXME(erikdesjardins): handle non-default addrspace ptr sizes - Primitive::Pointer(_) => { - // If the niche is the NULL value of a reference, then `discr_enum_ty` will be - // a RawPtr. CodeView doesn't know what to do with enums whose base type is a - // pointer so we fix this up to just be `usize`. - // DWARF might be able to deal with this but with an integer type we are on - // the safe side there too. - cx.data_layout().ptr_sized_integer() - } - } - .to_ty(cx.tcx, false) - } - - Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { - // Direct tags preserve the sign. - tag.primitive().to_ty(cx.tcx) - } - } -} - /// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. /// This is a helper function and does not register anything in the type map by itself. /// diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index d7e3b47e0bd..238fbad4dfd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name; -use rustc_codegen_ssa::debuginfo::wants_c_like_enum_debuginfo; +use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::ConstMethods; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; @@ -11,7 +11,6 @@ use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; use crate::common::CodegenCx; -use crate::debuginfo::metadata::enums::tag_base_type; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ file_metadata, size_and_align_of, type_di_node, unknown_file_metadata, visibility_di_flags, @@ -54,7 +53,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); - assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout)); type_map::build_type_with_children( cx, @@ -131,7 +130,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( let containing_scope = get_namespace_for_item(cx, coroutine_def_id); let coroutine_type_and_layout = cx.layout_of(coroutine_type); - assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); + assert!(!wants_c_like_enum_debuginfo(cx.tcx, coroutine_type_and_layout)); let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); @@ -321,7 +320,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( &Variants::Single { .. } => None, &Variants::Multiple { tag_field, .. } => { - let tag_base_type = tag_base_type(cx, enum_or_coroutine_type_and_layout); + let tag_base_type = tag_base_type(cx.tcx, enum_or_coroutine_type_and_layout); let (size, align) = cx.size_and_align_of(tag_base_type); unsafe { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index d51e15d12e2..e542aa96e8a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -62,7 +62,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, pointee_ty: Ty<'tcx>, ) -> Option<FatPtrKind> { - let pointee_tail_ty = cx.tcx.struct_tail_erasing_lifetimes(pointee_ty, cx.param_env()); + let pointee_tail_ty = cx.tcx.struct_tail_for_codegen(pointee_ty, cx.param_env()); let layout = cx.layout_of(pointee_tail_ty); trace!( "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index c4887464e19..2aa349b2782 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -137,7 +137,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { llvm::Visibility::Default, fn_abi.llvm_type(self), ); - fn_abi.apply_attrs_llfn(self, llfn); + fn_abi.apply_attrs_llfn(self, llfn, instance); if self.tcx.sess.is_sanitizer_cfi_enabled() { if let Some(instance) = instance { diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index a3957bc52a5..7e53d32ce8c 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::ffi::CString; use std::path::Path; @@ -72,28 +71,6 @@ pub(crate) struct InvalidMinimumAlignmentTooLarge { pub(crate) struct SanitizerMemtagRequiresMte; #[derive(Diagnostic)] -#[diag(codegen_llvm_error_writing_def_file)] -pub(crate) struct ErrorWritingDEFFile { - pub error: std::io::Error, -} - -#[derive(Diagnostic)] -#[diag(codegen_llvm_error_calling_dlltool)] -pub(crate) struct ErrorCallingDllTool<'a> { - pub dlltool_path: Cow<'a, str>, - pub error: std::io::Error, -} - -#[derive(Diagnostic)] -#[diag(codegen_llvm_dlltool_fail_import_library)] -pub(crate) struct DlltoolFailImportLibrary<'a> { - pub dlltool_path: Cow<'a, str>, - pub dlltool_args: String, - pub stdout: Cow<'a, str>, - pub stderr: Cow<'a, str>, -} - -#[derive(Diagnostic)] #[diag(codegen_llvm_dynamic_linking_with_lto)] #[note] pub(crate) struct DynamicLinkingWithLTO; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 040de1c7dd7..f5558723d11 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,3 +1,4 @@ +use std::assert_matches::assert_matches; use std::cmp::Ordering; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; @@ -35,10 +36,10 @@ fn get_simple_intrinsic<'ll>( sym::sqrtf64 => "llvm.sqrt.f64", sym::sqrtf128 => "llvm.sqrt.f128", - sym::powif16 => "llvm.powi.f16", - sym::powif32 => "llvm.powi.f32", - sym::powif64 => "llvm.powi.f64", - sym::powif128 => "llvm.powi.f128", + sym::powif16 => "llvm.powi.f16.i32", + sym::powif32 => "llvm.powi.f32.i32", + sym::powif64 => "llvm.powi.f64.i32", + sym::powif128 => "llvm.powi.f128.i32", sym::sinf16 => "llvm.sin.f16", sym::sinf32 => "llvm.sin.f32", @@ -1142,7 +1143,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if cfg!(debug_assertions) { for (ty, arg) in arg_tys.iter().zip(args) { if ty.is_simd() { - assert!(matches!(arg.val, OperandValue::Immediate(_))); + assert_matches!(arg.val, OperandValue::Immediate(_)); } } } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 41e9cfd1066..43164390a1c 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -8,6 +8,7 @@ #![allow(internal_features)] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] +#![feature(assert_matches)] #![feature(exact_size_is_empty)] #![feature(extern_types)] #![feature(hash_raw_entry)] @@ -269,7 +270,7 @@ impl CodegenBackend for LlvmCodegenBackend { fn provide(&self, providers: &mut Providers) { providers.global_backend_features = - |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) + |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true, false) } fn print(&self, req: &PrintRequest, out: &mut String, sess: &Session) { @@ -434,7 +435,7 @@ impl ModuleLlvm { ModuleLlvm { llmod_raw, llcx, - tm: ManuallyDrop::new(create_informational_target_machine(tcx.sess)), + tm: ManuallyDrop::new(create_informational_target_machine(tcx.sess, false)), } } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 80b13c0e1d4..faabbcb020d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1575,6 +1575,12 @@ extern "C" { pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute; + pub fn LLVMRustCreateRangeAttribute( + C: &Context, + num_bits: c_uint, + lower_words: *const u64, + upper_words: *const u64, + ) -> &Attribute; // Operations on functions pub fn LLVMRustGetOrInsertFunction<'a>( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 72691907c0d..d0db350a149 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -8,7 +8,7 @@ use std::string::FromUtf8Error; use libc::c_uint; use rustc_data_structures::small_c_str::SmallCStr; use rustc_llvm::RustString; -use rustc_target::abi::Align; +use rustc_target::abi::{Align, Size, WrappingRange}; pub use self::AtomicRmwBinOp::*; pub use self::CallConv::*; @@ -105,6 +105,21 @@ pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribu unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } } +pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { + let lower = range.start; + let upper = range.end.wrapping_add(1); + let lower_words = [lower as u64, (lower >> 64) as u64]; + let upper_words = [upper as u64, (upper >> 64) as u64]; + unsafe { + LLVMRustCreateRangeAttribute( + llcx, + size.bits().try_into().unwrap(), + lower_words.as_ptr(), + upper_words.as_ptr(), + ) + } +} + #[derive(Copy, Clone)] pub enum AttributePlace { ReturnValue, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index af8a9be1ccb..9fd8ca43789 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -8,6 +8,7 @@ use libc::c_int; use rustc_codegen_ssa::base::wants_wasm_eh; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_data_structures::unord::UnordSet; use rustc_fs_util::path_to_c_string; use rustc_middle::bug; use rustc_session::config::{PrintKind, PrintRequest}; @@ -239,40 +240,8 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { } // In LLVM neon implicitly enables fp, but we manually enable // neon when a feature only implicitly enables fp - ("aarch64", "f32mm") => { - LLVMFeature::with_dependency("f32mm", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "f64mm") => { - LLVMFeature::with_dependency("f64mm", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "fhm") => { - LLVMFeature::with_dependency("fp16fml", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "fp16") => { - LLVMFeature::with_dependency("fullfp16", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "jsconv") => { - LLVMFeature::with_dependency("jsconv", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "sve") => { - LLVMFeature::with_dependency("sve", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "sve2") => { - LLVMFeature::with_dependency("sve2", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "sve2-aes") => { - LLVMFeature::with_dependency("sve2-aes", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "sve2-sm4") => { - LLVMFeature::with_dependency("sve2-sm4", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "sve2-sha3") => { - LLVMFeature::with_dependency("sve2-sha3", TargetFeatureFoldStrength::EnableOnly("neon")) - } - ("aarch64", "sve2-bitperm") => LLVMFeature::with_dependency( - "sve2-bitperm", - TargetFeatureFoldStrength::EnableOnly("neon"), - ), + ("aarch64", "fhm") => LLVMFeature::new("fp16fml"), + ("aarch64", "fp16") => LLVMFeature::new("fullfp16"), // In LLVM 18, `unaligned-scalar-mem` was merged with `unaligned-vector-mem` into a single feature called // `fast-unaligned-access`. In LLVM 19, it was split back out. ("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => { @@ -308,11 +277,53 @@ pub fn check_tied_features( /// Used to generate cfg variables and apply features /// Must express features in the way Rust understands them pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { - let target_machine = create_informational_target_machine(sess); + let mut features = vec![]; + + // Add base features for the target + let target_machine = create_informational_target_machine(sess, true); + features.extend( + sess.target + .supported_target_features() + .iter() + .filter(|(feature, _, _)| { + // skip checking special features, as LLVM may not understands them + if RUSTC_SPECIAL_FEATURES.contains(feature) { + return true; + } + // check that all features in a given smallvec are enabled + for llvm_feature in to_llvm_features(sess, feature) { + let cstr = SmallCStr::new(llvm_feature); + if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { + return false; + } + } + true + }) + .map(|(feature, _, _)| Symbol::intern(feature)), + ); + + // Add enabled features + for (enabled, feature) in + sess.opts.cg.target_feature.split(',').filter_map(|s| match s.chars().next() { + Some('+') => Some((true, Symbol::intern(&s[1..]))), + Some('-') => Some((false, Symbol::intern(&s[1..]))), + _ => None, + }) + { + if enabled { + features.extend(sess.target.implied_target_features(std::iter::once(feature))); + } else { + features.retain(|f| { + !sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) + }); + } + } + + // Filter enabled features based on feature gates sess.target .supported_target_features() .iter() - .filter_map(|&(feature, gate)| { + .filter_map(|&(feature, gate, _)| { if sess.is_nightly_build() || allow_unstable || gate.is_stable() { Some(feature) } else { @@ -320,18 +331,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { } }) .filter(|feature| { - // skip checking special features, as LLVM may not understands them - if RUSTC_SPECIAL_FEATURES.contains(feature) { - return true; - } - // check that all features in a given smallvec are enabled - for llvm_feature in to_llvm_features(sess, feature) { - let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { - return false; - } - } - true + RUSTC_SPECIAL_FEATURES.contains(feature) || features.contains(&Symbol::intern(feature)) }) .map(|feature| Symbol::intern(feature)) .collect() @@ -386,7 +386,7 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach .target .supported_target_features() .iter() - .map(|(feature, _gate)| { + .map(|(feature, _gate, _implied)| { // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name; let desc = @@ -440,7 +440,7 @@ fn print_target_features(out: &mut String, sess: &Session, tm: &llvm::TargetMach pub(crate) fn print(req: &PrintRequest, mut out: &mut String, sess: &Session) { require_inited(); - let tm = create_informational_target_machine(sess); + let tm = create_informational_target_machine(sess, false); match req.kind { PrintKind::TargetCPUs => { // SAFETY generate a C compatible string from a byte slice to pass @@ -488,7 +488,11 @@ pub fn target_cpu(sess: &Session) -> &str { /// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, /// `--target` and similar). -pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<String> { +pub(crate) fn global_llvm_features( + sess: &Session, + diagnostics: bool, + only_base_features: bool, +) -> Vec<String> { // Features that come earlier are overridden by conflicting features later in the string. // Typically we'll want more explicit settings to override the implicit ones, so: // @@ -548,94 +552,124 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } // -Ctarget-features - let supported_features = sess.target.supported_target_features(); - let (llvm_major, _, _) = get_version(); - let mut featsmap = FxHashMap::default(); - let feats = sess - .opts - .cg - .target_feature - .split(',') - .filter_map(|s| { - let enable_disable = match s.chars().next() { - None => return None, - Some(c @ ('+' | '-')) => c, - Some(_) => { - if diagnostics { - sess.dcx().emit_warn(UnknownCTargetFeaturePrefix { feature: s }); + if !only_base_features { + let supported_features = sess.target.supported_target_features(); + let (llvm_major, _, _) = get_version(); + let mut featsmap = FxHashMap::default(); + + // insert implied features + let mut all_rust_features = vec![]; + for feature in sess.opts.cg.target_feature.split(',') { + match feature.strip_prefix('+') { + Some(feature) => all_rust_features.extend( + UnordSet::from( + sess.target + .implied_target_features(std::iter::once(Symbol::intern(feature))), + ) + .to_sorted_stable_ord() + .iter() + .map(|s| format!("+{}", s.as_str())), + ), + _ => all_rust_features.push(feature.to_string()), + } + } + + let feats = all_rust_features + .iter() + .filter_map(|s| { + let enable_disable = match s.chars().next() { + None => return None, + Some(c @ ('+' | '-')) => c, + Some(_) => { + if diagnostics { + sess.dcx().emit_warn(UnknownCTargetFeaturePrefix { feature: s }); + } + return None; } - return None; - } - }; + }; - let feature = backend_feature_name(sess, s)?; - // Warn against use of LLVM specific feature names and unstable features on the CLI. - if diagnostics { - let feature_state = supported_features.iter().find(|&&(v, _)| v == feature); - if feature_state.is_none() { - let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { - let llvm_features = to_llvm_features(sess, rust_feature); - if llvm_features.contains(feature) && !llvm_features.contains(rust_feature) - { - Some(rust_feature) + let feature = backend_feature_name(sess, s)?; + // Warn against use of LLVM specific feature names and unstable features on the CLI. + if diagnostics { + let feature_state = supported_features.iter().find(|&&(v, _, _)| v == feature); + if feature_state.is_none() { + let rust_feature = + supported_features.iter().find_map(|&(rust_feature, _, _)| { + let llvm_features = to_llvm_features(sess, rust_feature); + if llvm_features.contains(feature) + && !llvm_features.contains(rust_feature) + { + Some(rust_feature) + } else { + None + } + }); + let unknown_feature = if let Some(rust_feature) = rust_feature { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::Some { rust_feature }, + } } else { - None - } - }); - let unknown_feature = if let Some(rust_feature) = rust_feature { - UnknownCTargetFeature { - feature, - rust_feature: PossibleFeature::Some { rust_feature }, - } - } else { - UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } - }; - sess.dcx().emit_warn(unknown_feature); - } else if feature_state - .is_some_and(|(_name, feature_gate)| !feature_gate.is_stable()) - { - // An unstable feature. Warn about using it. - sess.dcx().emit_warn(UnstableCTargetFeature { feature }); + UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } + }; + sess.dcx().emit_warn(unknown_feature); + } else if feature_state + .is_some_and(|(_name, feature_gate, _implied)| !feature_gate.is_stable()) + { + // An unstable feature. Warn about using it. + sess.dcx().emit_warn(UnstableCTargetFeature { feature }); + } } - } - if diagnostics { - // FIXME(nagisa): figure out how to not allocate a full hashset here. - featsmap.insert(feature, enable_disable == '+'); - } + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return None; - } + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } - // if the target-feature is "backchain" and LLVM version is greater than 18 - // then we also need to add "+backchain" to the target-features attribute. - // otherwise, we will only add the naked `backchain` attribute to the attribute-group. - if feature == "backchain" && llvm_major < 18 { - return None; - } - // ... otherwise though we run through `to_llvm_features` when - // passing requests down to LLVM. This means that all in-language - // features also work on the command line instead of having two - // different names when the LLVM name and the Rust name differ. - let llvm_feature = to_llvm_features(sess, feature); - - Some( - std::iter::once(format!("{}{}", enable_disable, llvm_feature.llvm_feature_name)) - .chain(llvm_feature.dependency.into_iter().filter_map(move |feat| { - match (enable_disable, feat) { + // if the target-feature is "backchain" and LLVM version is greater than 18 + // then we also need to add "+backchain" to the target-features attribute. + // otherwise, we will only add the naked `backchain` attribute to the attribute-group. + if feature == "backchain" && llvm_major < 18 { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + let llvm_feature = to_llvm_features(sess, feature); + + Some( + std::iter::once(format!( + "{}{}", + enable_disable, llvm_feature.llvm_feature_name + )) + .chain(llvm_feature.dependency.into_iter().filter_map( + move |feat| match (enable_disable, feat) { ('-' | '+', TargetFeatureFoldStrength::Both(f)) | ('+', TargetFeatureFoldStrength::EnableOnly(f)) => { Some(format!("{enable_disable}{f}")) } _ => None, - } - })), - ) - }) - .flatten(); - features.extend(feats); + }, + )), + ) + }) + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.dcx().emit_err(TargetFeatureDisableOrEnable { + features: f, + span: None, + missing_features: None, + }); + } + } // -Zfixed-x18 if sess.opts.unstable_opts.fixed_x18 { @@ -646,30 +680,6 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } } - // This is a workaround for a LLVM bug that doesn't implicitly enable - // `simd128` when `relaxed-simd` is. - // See <https://github.com/llvm/llvm-project/pull/99803>, which didn't make - // it into a released version of LLVM yet. - // - // This doesn't use the "implicit target feature" system because it is only - // used for function attributes in other targets, which fixes this bug as - // well on the function attribute level. - if sess.target.families.contains(&"wasm".into()) { - if features.iter().any(|f| f == "+relaxed-simd") - && !features.iter().any(|f| f == "+simd128") - { - features.push("+simd128".into()); - } - } - - if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.dcx().emit_err(TargetFeatureDisableOrEnable { - features: f, - span: None, - missing_features: None, - }); - } - features } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index d59eaae1ba9..f1ef359594b 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -12,7 +12,7 @@ use tracing::debug; use crate::context::CodegenCx; use crate::errors::SymbolAlreadyDefined; use crate::type_of::LayoutLlvmExt; -use crate::{attributes, base, llvm}; +use crate::{base, llvm}; impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { fn predefine_static( @@ -87,8 +87,6 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { debug!("predefine_fn: instance = {:?}", instance); - attributes::from_fn_attrs(self, lldecl, instance); - unsafe { if self.should_assume_dso_local(lldecl, false) { llvm::LLVMRustSetDSOLocal(lldecl, true); |