diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
32 files changed, 640 insertions, 1334 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 009e7e2487b..043123fcab2 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -24,6 +24,7 @@ use crate::attributes::{self, llfn_attrs_from_instance}; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Attribute, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -500,7 +501,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { - apply(attrs); + let i = apply(attrs); + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (21, 0, 0) + { + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[llvm::AttributeKind::DeadOnReturn.create_attr(cx.llcx)], + ); + } } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 2b5090ed6db..23610aa856c 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -8,11 +8,12 @@ use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; +use smallvec::SmallVec; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; use crate::llvm::{self, False, True, Type, Value}; -use crate::{SimpleCx, attributes, debuginfo}; +use crate::{SimpleCx, attributes, debuginfo, llvm_util}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, @@ -147,6 +148,20 @@ fn create_wrapper_function( llvm::Visibility::from_generic(tcx.sess.default_visibility()), ty, ); + + let mut attrs = SmallVec::<[_; 2]>::new(); + + let target_cpu = llvm_util::target_cpu(tcx.sess); + let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu); + + let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess) + .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)); + + attrs.push(target_cpu_attr); + attrs.extend(tune_cpu_attr); + + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + let no_return = if no_return { // -> ! DIFlagNoReturn let no_return = llvm::AttributeKind::NoReturn.create_attr(cx.llcx); diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c32f11b27f3..a6daacd95ef 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,6 +1,6 @@ //! Set and unset common attributes on LLVM values. -use rustc_attr_data_structures::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; use rustc_middle::ty::{self, TyCtxt}; @@ -28,22 +28,6 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } } -pub(crate) fn has_attr(llfn: &Value, idx: AttributePlace, attr: AttributeKind) -> bool { - llvm::HasAttributeAtIndex(llfn, idx, attr) -} - -pub(crate) fn has_string_attr(llfn: &Value, name: &str) -> bool { - llvm::HasStringAttribute(llfn, name) -} - -pub(crate) fn remove_from_llfn(llfn: &Value, place: AttributePlace, kind: AttributeKind) { - llvm::RemoveRustEnumAttributeAtIndex(llfn, place, kind); -} - -pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: &str) { - llvm::RemoveStringAttrFromFn(llfn, name); -} - /// Get LLVM attribute for the provided inline heuristic. #[inline] fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 0a161442933..7a340ae83f3 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,104 +1,21 @@ //! A helper class for dealing with static archives -use std::ffi::{CStr, CString, c_char, c_void}; -use std::path::{Path, PathBuf}; -use std::{io, mem, ptr, str}; +use std::ffi::{CStr, c_char, c_void}; +use std::io; use rustc_codegen_ssa::back::archive::{ - ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, - DEFAULT_OBJECT_READER, ObjectReader, UnknownArchiveKind, try_extract_macho_fat_archive, + ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, DEFAULT_OBJECT_READER, ObjectReader, }; use rustc_session::Session; -use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind, last_error}; - -/// Helper for adding many files to an archive. -#[must_use = "must call build() to finish building the archive"] -pub(crate) struct LlvmArchiveBuilder<'a> { - sess: &'a Session, - additions: Vec<Addition>, -} - -enum Addition { - File { path: PathBuf, name_in_archive: String }, - Archive { path: PathBuf, archive: ArchiveRO, skip: Box<dyn FnMut(&str) -> bool> }, -} - -impl Addition { - fn path(&self) -> &Path { - match self { - Addition::File { path, .. } | Addition::Archive { path, .. } => path, - } - } -} - -fn is_relevant_child(c: &Child<'_>) -> bool { - match c.name() { - Some(name) => !name.contains("SYMDEF"), - None => false, - } -} - -impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { - fn add_archive( - &mut self, - archive: &Path, - skip: Box<dyn FnMut(&str) -> bool + 'static>, - ) -> io::Result<()> { - let mut archive = archive.to_path_buf(); - if self.sess.target.llvm_target.contains("-apple-macosx") { - if let Some(new_archive) = try_extract_macho_fat_archive(self.sess, &archive)? { - archive = new_archive - } - } - let archive_ro = match ArchiveRO::open(&archive) { - Ok(ar) => ar, - Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), - }; - if self.additions.iter().any(|ar| ar.path() == archive) { - return Ok(()); - } - self.additions.push(Addition::Archive { - path: archive, - archive: archive_ro, - skip: Box::new(skip), - }); - Ok(()) - } - - /// Adds an arbitrary file to this archive - fn add_file(&mut self, file: &Path) { - let name = file.file_name().unwrap().to_str().unwrap(); - self.additions - .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); - } - - /// Combine the provided files, rlibs, and native libraries into a single - /// `Archive`. - fn build(mut self: Box<Self>, output: &Path) -> bool { - match self.build_with_llvm(output) { - Ok(any_members) => any_members, - Err(error) => { - self.sess.dcx().emit_fatal(ArchiveBuildFailure { path: output.to_owned(), error }) - } - } - } -} +use crate::llvm; pub(crate) struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> { - // Keeping LlvmArchiveBuilder around in case of a regression caused by using - // ArArchiveBuilder. - // FIXME(#128955) remove a couple of months after #128936 gets merged in case - // no regression is found. - if false { - Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) - } else { - Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) - } + // Use the `object` crate to build archives, with a little bit of help from LLVM. + Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) } } @@ -178,91 +95,3 @@ fn llvm_is_64_bit_object_file(buf: &[u8]) -> bool { fn llvm_is_ec_object_file(buf: &[u8]) -> bool { unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } - -impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self, output: &Path) -> io::Result<bool> { - let kind = &*self.sess.target.archive_format; - let kind = kind - .parse::<ArchiveKind>() - .map_err(|_| kind) - .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); - - let mut additions = mem::take(&mut self.additions); - // Values in the `members` list below will contain pointers to the strings allocated here. - // So they need to get dropped after all elements of `members` get freed. - let mut strings = Vec::new(); - let mut members = Vec::new(); - - let dst = CString::new(output.to_str().unwrap())?; - - unsafe { - for addition in &mut additions { - match addition { - Addition::File { path, name_in_archive } => { - let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.as_bytes())?; - members.push(llvm::LLVMRustArchiveMemberNew( - path.as_ptr(), - name.as_ptr(), - None, - )); - strings.push(path); - strings.push(name); - } - Addition::Archive { archive, skip, .. } => { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - if !is_relevant_child(&child) { - continue; - } - let child_name = child.name().unwrap(); - if skip(child_name) { - continue; - } - - // It appears that LLVM's archive writer is a little - // buggy if the name we pass down isn't just the - // filename component, so chop that off here and - // pass it in. - // - // See LLVM bug 25877 for more info. - let child_name = - Path::new(child_name).file_name().unwrap().to_str().unwrap(); - let name = CString::new(child_name)?; - let m = llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - ); - members.push(m); - strings.push(name); - } - } - } - } - - let r = llvm::LLVMRustWriteArchive( - dst.as_ptr(), - members.len() as libc::size_t, - members.as_ptr() as *const &_, - true, - kind, - self.sess.target.arch == "arm64ec", - ); - let ret = if r.into_result().is_err() { - let msg = last_error().unwrap_or_else(|| "failed to write archive".into()); - Err(io::Error::new(io::ErrorKind::Other, msg)) - } else { - Ok(!members.is_empty()) - }; - for member in members { - llvm::LLVMRustArchiveMemberFree(member); - } - ret - } - } -} - -fn string_to_io_error(s: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("bad archive: {s}")) -} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 84302009da9..853d0295238 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,80 +1,51 @@ use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::ptr::NonNull; use std::sync::Arc; use std::{io, iter, slice}; use object::read::archive::ArchiveFile; +use object::{Object, ObjectSection}; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared}; -use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; use rustc_errors::{DiagCtxtHandle, FatalError}; -use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; -use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; -use rustc_session::config::{self, CrateType, Lto}; +use rustc_session::config::{self, Lto}; use tracing::{debug, info}; use crate::back::write::{ self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode, }; -use crate::errors::{ - DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro, -}; -use crate::llvm::AttributePlace::Function; +use crate::errors::{LlvmError, LtoBitcodeFromRlib}; use crate::llvm::{self, build_string}; -use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes}; +use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx}; /// We keep track of the computed LTO cache keys from the previous /// session to determine which CGUs we can reuse. const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; -fn crate_type_allows_lto(crate_type: CrateType) -> bool { - match crate_type { - CrateType::Executable - | CrateType::Dylib - | CrateType::Staticlib - | CrateType::Cdylib - | CrateType::ProcMacro - | CrateType::Sdylib => true, - CrateType::Rlib => false, - } -} - fn prepare_lto( cgcx: &CodegenContext<LlvmCodegenBackend>, + exported_symbols_for_lto: &[String], + each_linked_rlib_for_lto: &[PathBuf], dcx: DiagCtxtHandle<'_>, ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> { - let export_threshold = match cgcx.lto { - // We're just doing LTO for our one crate - Lto::ThinLocal => SymbolExportLevel::Rust, - - // We're doing LTO for the entire crate graph - Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types), - - Lto::No => panic!("didn't request LTO but we're doing LTO"), - }; + let mut symbols_below_threshold = exported_symbols_for_lto + .iter() + .map(|symbol| CString::new(symbol.to_owned()).unwrap()) + .collect::<Vec<CString>>(); - let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { - if info.level.is_below_threshold(export_threshold) || info.used { - Some(CString::new(name.as_str()).unwrap()) - } else { - None - } - }; - let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); - let mut symbols_below_threshold = { - let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); - exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>() - }; - info!("{} symbols to preserve in this crate", symbols_below_threshold.len()); + // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to + // __llvm_profile_runtime, therefore we won't know until link time if this symbol + // should have default visibility. + symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned()); // If we're performing LTO for the entire crate graph, then for each of our // upstream dependencies, find the corresponding rlib and load the bitcode @@ -84,37 +55,7 @@ fn prepare_lto( // with either fat or thin LTO let mut upstream_modules = Vec::new(); if cgcx.lto != Lto::ThinLocal { - // Make sure we actually can run LTO - for crate_type in cgcx.crate_types.iter() { - if !crate_type_allows_lto(*crate_type) { - dcx.emit_err(LtoDisallowed); - return Err(FatalError); - } else if *crate_type == CrateType::Dylib { - if !cgcx.opts.unstable_opts.dylib_lto { - dcx.emit_err(LtoDylib); - return Err(FatalError); - } - } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto { - dcx.emit_err(LtoProcMacro); - return Err(FatalError); - } - } - - if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto { - dcx.emit_err(DynamicLinkingWithLTO); - return Err(FatalError); - } - - for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { - let exported_symbols = - cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); - { - let _timer = - cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); - symbols_below_threshold - .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); - } - + for path in each_linked_rlib_for_lto { let archive_data = unsafe { Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib")) .expect("couldn't map rlib") @@ -147,10 +88,6 @@ fn prepare_lto( } } - // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to - // __llvm_profile_runtime, therefore we won't know until link time if this symbol - // should have default visibility. - symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned()); Ok((symbols_below_threshold, upstream_modules)) } @@ -168,46 +105,32 @@ fn get_bitcode_slice_from_object_data<'a>( // name" which in the public API for sections gets treated as part of the section name, but // internally in MachOObjectFile.cpp gets treated separately. let section_name = bitcode_section_name(cgcx).to_str().unwrap().trim_start_matches("__LLVM,"); - let mut len = 0; - let data = unsafe { - llvm::LLVMRustGetSliceFromObjectDataByName( - obj.as_ptr(), - obj.len(), - section_name.as_ptr(), - section_name.len(), - &mut len, - ) - }; - if !data.is_null() { - assert!(len != 0); - let bc = unsafe { slice::from_raw_parts(data, len) }; - // `bc` must be a sub-slice of `obj`. - assert!(obj.as_ptr() <= bc.as_ptr()); - assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr()); + let obj = + object::File::parse(obj).map_err(|err| LtoBitcodeFromRlib { err: err.to_string() })?; - Ok(bc) - } else { - assert!(len == 0); - Err(LtoBitcodeFromRlib { - llvm_err: llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()), - }) - } + let section = obj + .section_by_name(section_name) + .ok_or_else(|| LtoBitcodeFromRlib { err: format!("Can't find section {section_name}") })?; + + section.data().map_err(|err| LtoBitcodeFromRlib { err: err.to_string() }) } /// Performs fat LTO by merging all modules into a single one and returning it /// for further optimization. pub(crate) fn run_fat( cgcx: &CodegenContext<LlvmCodegenBackend>, + exported_symbols_for_lto: &[String], + each_linked_rlib_for_lto: &[PathBuf], modules: Vec<FatLtoInput<LlvmCodegenBackend>>, - cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?; + let (symbols_below_threshold, upstream_modules) = + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); - fat_lto(cgcx, dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold) + fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold) } /// Performs thin LTO by performing necessary global analysis and returning two @@ -215,12 +138,15 @@ pub(crate) fn run_fat( /// can simply be copied over from the incr. comp. cache. pub(crate) fn run_thin( cgcx: &CodegenContext<LlvmCodegenBackend>, + exported_symbols_for_lto: &[String], + each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, ) -> Result<(Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?; + let (symbols_below_threshold, upstream_modules) = + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); if cgcx.opts.cg.linker_plugin_lto.enabled() { @@ -245,7 +171,6 @@ fn fat_lto( cgcx: &CodegenContext<LlvmCodegenBackend>, dcx: DiagCtxtHandle<'_>, modules: Vec<FatLtoInput<LlvmCodegenBackend>>, - cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, symbols_below_threshold: &[*const libc::c_char], ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { @@ -258,21 +183,12 @@ fn fat_lto( // modules that are serialized in-memory. // * `in_memory` contains modules which are already parsed and in-memory, // such as from multi-CGU builds. - // - // All of `cached_modules` (cached from previous incremental builds) can - // immediately go onto the `serialized_modules` modules list and then we can - // split the `modules` array into these two lists. let mut in_memory = Vec::new(); - serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { - info!("pushing cached module {:?}", wp.cgu_name); - (buffer, CString::new(wp.cgu_name).unwrap()) - })); for module in modules { match module { FatLtoInput::InMemory(m) => in_memory.push(m), FatLtoInput::Serialized { name, buffer } => { info!("pushing serialized module {:?}", name); - let buffer = SerializedModule::Local(buffer); serialized_modules.push((buffer, CString::new(name).unwrap())); } } @@ -573,10 +489,10 @@ fn thin_lto( // Save the current ThinLTO import information for the next compilation // session, overwriting the previous serialized data (if any). - if let Some(path) = key_map_path { - if let Err(err) = curr_key_map.save_to_file(&path) { - return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err })); - } + if let Some(path) = key_map_path + && let Err(err) = curr_key_map.save_to_file(&path) + { + return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err })); } Ok((opt_jobs, copy_jobs)) @@ -676,31 +592,6 @@ pub(crate) fn run_pass_manager( } if cfg!(llvm_enzyme) && enable_ad && !thin { - let cx = - SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); - - for function in cx.get_functions() { - let enzyme_marker = "enzyme_marker"; - if attributes::has_string_attr(function, enzyme_marker) { - // Sanity check: Ensure 'noinline' is present before replacing it. - assert!( - attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), - "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'" - ); - - attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline); - attributes::remove_string_attr_from_llfn(function, enzyme_marker); - - assert!( - !attributes::has_string_attr(function, enzyme_marker), - "Expected function to not have 'enzyme_marker'" - ); - - let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx); - attributes::apply_to_llfn(function, Function, &[always_inline]); - } - } - let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { diff --git a/compiler/rustc_codegen_llvm/src/back/mod.rs b/compiler/rustc_codegen_llvm/src/back/mod.rs new file mode 100644 index 00000000000..6cb89f80ab8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod archive; +pub(crate) mod lto; +pub(crate) mod owned_target_machine; +mod profiling; +pub(crate) mod write; diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 8e82013e94a..6d8178320fe 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,4 +1,5 @@ -use std::ffi::{CStr, c_char}; +use std::assert_matches::assert_matches; +use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -41,11 +42,9 @@ impl OwnedTargetMachine { args_cstr_buff: &[u8], use_wasm_eh: bool, ) -> Result<Self, LlvmError<'static>> { - assert!(args_cstr_buff.len() > 0); - assert!( - *args_cstr_buff.last().unwrap() == 0, - "The last character must be a null terminator." - ); + // The argument list is passed as the concatenation of one or more C strings. + // This implies that there must be a last byte, and it must be 0. + assert_matches!(args_cstr_buff, [.., b'\0'], "the last byte must be a NUL terminator"); // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { @@ -71,7 +70,7 @@ impl OwnedTargetMachine { output_obj_file.as_ptr(), debug_info_compression.as_ptr(), use_emulated_tls, - args_cstr_buff.as_ptr() as *const c_char, + args_cstr_buff.as_ptr(), args_cstr_buff.len(), use_wasm_eh, ) @@ -99,7 +98,7 @@ impl Drop for OwnedTargetMachine { // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut()); + llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_ptr()); } } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 6f8fba2a30d..62998003ca1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -796,29 +796,6 @@ pub(crate) fn optimize( Ok(()) } -pub(crate) fn link( - cgcx: &CodegenContext<LlvmCodegenBackend>, - dcx: DiagCtxtHandle<'_>, - mut modules: Vec<ModuleCodegen<ModuleLlvm>>, -) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { - use super::lto::{Linker, ModuleBuffer}; - // Sort the modules by name to ensure deterministic behavior. - modules.sort_by(|a, b| a.name.cmp(&b.name)); - let (first, elements) = - modules.split_first().expect("Bug! modules must contain at least one module."); - - let mut linker = Linker::new(first.module_llvm.llmod()); - for module in elements { - let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); - let buffer = ModuleBuffer::new(module.module_llvm.llmod()); - linker - .add(buffer.data()) - .map_err(|()| llvm_err(dcx, LlvmError::SerializeModule { name: &module.name }))?; - } - drop(linker); - Ok(modules.remove(0)) -} - pub(crate) fn codegen( cgcx: &CodegenContext<LlvmCodegenBackend>, module: ModuleCodegen<ModuleLlvm>, @@ -885,7 +862,7 @@ pub(crate) fn codegen( .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); let thin_bc = module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode"); - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); + embed_bitcode(cgcx, llcx, llmod, &thin_bc); } } @@ -1081,7 +1058,6 @@ fn embed_bitcode( cgcx: &CodegenContext<LlvmCodegenBackend>, llcx: &llvm::Context, llmod: &llvm::Module, - cmdline: &str, bitcode: &[u8], ) { // We're adding custom sections to the output object file, but we definitely @@ -1097,7 +1073,9 @@ fn embed_bitcode( // * Mach-O - this is for macOS. Inspecting the source code for the native // linker here shows that the `.llvmbc` and `.llvmcmd` sections are // automatically skipped by the linker. In that case there's nothing extra - // that we need to do here. + // that we need to do here. We do need to make sure that the + // `__LLVM,__cmdline` section exists even though it is empty as otherwise + // ld64 rejects the object file. // // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and // `.llvmcmd` sections, so there's nothing extra we need to do. @@ -1134,7 +1112,7 @@ fn embed_bitcode( llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); - let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llconst = common::bytes_in_context(llcx, &[]); let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); llvm::set_initializer(llglobal, llconst); let section = if cgcx.target_is_like_darwin { @@ -1151,7 +1129,7 @@ fn embed_bitcode( let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); llvm::append_module_inline_asm(llmod, &asm); - let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]); llvm::append_module_inline_asm(llmod, &asm); } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5dda836988c..9cc5d8dbc21 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -18,9 +18,10 @@ use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; use rustc_session::config::DebugInfo; use rustc_span::Symbol; diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0ade9edb0d2..427c75d40e9 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -15,6 +15,7 @@ use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, @@ -24,7 +25,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; use rustc_span::Span; -use rustc_target::callconv::FnAbi; +use rustc_target::callconv::{FnAbi, PassMode}; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -556,13 +557,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let (size, signed) = ty.int_size_and_signed(self.tcx); let width = size.bits(); - if oop == OverflowOp::Sub && !signed { - // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these - // to be the canonical form. It will attempt to reform llvm.usub.with.overflow - // in the backend if profitable. - let sub = self.sub(lhs, rhs); - let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); - return (sub, cmp); + if !signed { + match oop { + OverflowOp::Sub => { + // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these + // to be the canonical form. It will attempt to reform llvm.usub.with.overflow + // in the backend if profitable. + let sub = self.sub(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); + return (sub, cmp); + } + OverflowOp::Add => { + // Like with sub above, using icmp is the preferred form. See + // <https://rust-lang.zulipchat.com/#narrow/channel/187780-t-compiler.2Fllvm/topic/.60uadd.2Ewith.2Eoverflow.60.20.28again.29/near/533041085> + let add = self.add(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, add, lhs); + return (add, cmp); + } + OverflowOp::Mul => {} + } } let oop_str = match oop { @@ -687,10 +700,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { bx.nonnull_metadata(load); } - if let Some(pointee) = layout.pointee_info_at(bx, offset) { - if let Some(_) = pointee.safe { - bx.align_metadata(load, pointee.align); - } + if let Some(pointee) = layout.pointee_info_at(bx, offset) + && let Some(_) = pointee.safe + { + bx.align_metadata(load, pointee.align); } } abi::Primitive::Float(_) => {} @@ -1326,15 +1339,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, dst: &'ll Value, - mut src: &'ll Value, + src: &'ll Value, order: rustc_middle::ty::AtomicOrdering, + ret_ptr: bool, ) -> &'ll Value { - // The only RMW operation that LLVM supports on pointers is compare-exchange. - let requires_cast_to_int = self.val_ty(src) == self.type_ptr() - && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg; - if requires_cast_to_int { - src = self.ptrtoint(src, self.type_isize()); - } + // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the + // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not + // currently support that (https://github.com/llvm/llvm-project/issues/120837). let mut res = unsafe { llvm::LLVMBuildAtomicRMW( self.llbuilder, @@ -1345,7 +1356,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llvm::False, // SingleThreaded ) }; - if requires_cast_to_int { + if ret_ptr && self.val_ty(res) != self.type_ptr() { res = self.inttoptr(res, self.type_ptr()); } res @@ -1431,6 +1442,28 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { call } + fn tail_call( + &mut self, + llty: Self::Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: Self::Value, + args: &[Self::Value], + funclet: Option<&Self::Funclet>, + instance: Option<Instance<'tcx>>, + ) { + let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance); + llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); + + match &fn_abi.ret.mode { + PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(), + PassMode::Direct(_) | PassMode::Pair { .. } => self.ret(call), + mode @ PassMode::Cast { .. } => { + bug!("Encountered `PassMode::{mode:?}` during codegen") + } + } + } + fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } @@ -1663,7 +1696,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + if crate::llvm_util::get_version() >= (22, 0, 0) { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]); + } else { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + } } } impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> { @@ -1863,48 +1900,4 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) { self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]); } - - /// Emits a call to `llvm.instrprof.mcdc.parameters`. - /// - /// This doesn't produce any code directly, but is used as input by - /// the LLVM pass that handles coverage instrumentation. - /// - /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.) - /// - /// [`CodeGenPGO::emitMCDCParameters`]: - /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_parameters( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - bitmap_bits: &'ll Value, - ) { - self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]); - } - - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_tvbitmap_update( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - bitmap_index: &'ll Value, - mcdc_temp: &'ll Value, - ) { - let args = &[fn_name, hash, bitmap_index, mcdc_temp]; - self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args); - } - - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) { - self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); - } - - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { - let align = self.tcx.data_layout.i32_align.abi; - let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align); - let new_tv_index = self.add(current_tv_index, cond_index); - self.store(new_tv_index, mcdc_temp, align); - } } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 829b3c513c2..e2df3265f6f 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,40 +1,92 @@ use std::ptr; -use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode}; -use rustc_codegen_ssa::ModuleCodegen; +use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; use rustc_codegen_ssa::common::TypeKind; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; -use rustc_errors::FatalError; -use rustc_middle::bug; -use tracing::{debug, trace}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; +use rustc_middle::{bug, ty}; +use tracing::debug; -use crate::back::write::llvm_err; -use crate::builder::{SBuilder, UNNAMED}; +use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; -use crate::errors::{AutoDiffWithoutEnable, LlvmError}; -use crate::llvm::AttributePlace::Function; -use crate::llvm::{Metadata, True}; +use crate::llvm; +use crate::llvm::{Metadata, True, Type}; use crate::value::Value; -use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm}; -fn get_params(fnc: &Value) -> Vec<&Value> { - let param_num = llvm::LLVMCountParams(fnc) as usize; - let mut fnc_args: Vec<&Value> = vec![]; - fnc_args.reserve(param_num); - unsafe { - llvm::LLVMGetParams(fnc, fnc_args.as_mut_ptr()); - fnc_args.set_len(param_num); +pub(crate) fn adjust_activity_to_abi<'tcx>( + tcx: TyCtxt<'tcx>, + fn_ty: Ty<'tcx>, + da: &mut Vec<DiffActivity>, +) { + if !matches!(fn_ty.kind(), ty::FnDef(..)) { + bug!("expected fn def for autodiff, got {:?}", fn_ty); } - fnc_args -} -fn has_sret(fnc: &Value) -> bool { - let num_args = llvm::LLVMCountParams(fnc) as usize; - if num_args == 0 { - false - } else { - unsafe { llvm::LLVMRustHasAttributeAtIndex(fnc, 0, llvm::AttributeKind::StructRet) } + // We don't actually pass the types back into the type system. + // All we do is decide how to handle the arguments. + let sig = fn_ty.fn_sig(tcx).skip_binder(); + + let mut new_activities = vec![]; + let mut new_positions = vec![]; + for (i, ty) in sig.inputs().iter().enumerate() { + if let Some(inner_ty) = ty.builtin_deref(true) { + if inner_ty.is_slice() { + // Now we need to figure out the size of each slice element in memory to allow + // safety checks and usability improvements in the backend. + let sty = match inner_ty.builtin_index() { + Some(sty) => sty, + None => { + panic!("slice element type unknown"); + } + }; + let pci = PseudoCanonicalInput { + typing_env: TypingEnv::fully_monomorphized(), + value: sty, + }; + + let layout = tcx.layout_of(pci); + let elem_size = match layout { + Ok(layout) => layout.size, + Err(_) => { + bug!("autodiff failed to compute slice element size"); + } + }; + let elem_size: u32 = elem_size.bytes() as u32; + + // We know that the length will be passed as extra arg. + if !da.is_empty() { + // We are looking at a slice. The length of that slice will become an + // extra integer on llvm level. Integers are always const. + // However, if the slice get's duplicated, we want to know to later check the + // size. So we mark the new size argument as FakeActivitySize. + // There is one FakeActivitySize per slice, so for convenience we store the + // slice element size in bytes in it. We will use the size in the backend. + let activity = match da[i] { + DiffActivity::DualOnly + | DiffActivity::Dual + | DiffActivity::Dualv + | DiffActivity::DuplicatedOnly + | DiffActivity::Duplicated => { + DiffActivity::FakeActivitySize(Some(elem_size)) + } + DiffActivity::Const => DiffActivity::Const, + _ => bug!("unexpected activity for ptr/ref"), + }; + new_activities.push(activity); + new_positions.push(i + 1); + } + + continue; + } + } + } + // now add the extra activities coming from slices + // Reverse order to not invalidate the indices + for _ in 0..new_activities.len() { + let pos = new_positions.pop().unwrap(); + let activity = new_activities.pop().unwrap(); + da.insert(pos, activity); } } @@ -48,14 +100,13 @@ fn has_sret(fnc: &Value) -> bool { // need to match those. // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it // using iterators and peek()? -fn match_args_from_caller_to_enzyme<'ll>( +fn match_args_from_caller_to_enzyme<'ll, 'tcx>( cx: &SimpleCx<'ll>, - builder: &SBuilder<'ll, 'll>, + builder: &mut Builder<'_, 'll, 'tcx>, width: u32, args: &mut Vec<&'ll llvm::Value>, inputs: &[DiffActivity], outer_args: &[&'ll llvm::Value], - has_sret: bool, ) { debug!("matching autodiff arguments"); // We now handle the issue that Rust level arguments not always match the llvm-ir level @@ -67,14 +118,6 @@ fn match_args_from_caller_to_enzyme<'ll>( let mut outer_pos: usize = 0; let mut activity_pos = 0; - if has_sret { - // Then the first outer arg is the sret pointer. Enzyme doesn't know about sret, so the - // inner function will still return something. We increase our outer_pos by one, - // and once we're done with all other args we will take the return of the inner call and - // update the sret pointer with it - outer_pos = 1; - } - let enzyme_const = cx.create_metadata(b"enzyme_const"); let enzyme_out = cx.create_metadata(b"enzyme_out"); let enzyme_dup = cx.create_metadata(b"enzyme_dup"); @@ -193,92 +236,6 @@ fn match_args_from_caller_to_enzyme<'ll>( } } -// On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input -// arguments. We do however need to declare them with their correct return type. -// We already figured the correct return type out in our frontend, when generating the outer_fn, -// so we can now just go ahead and use that. This is not always trivial, e.g. because sret. -// Beyond sret, this article describes our challenges nicely: -// <https://yorickpeterse.com/articles/the-mess-that-is-handling-structure-arguments-and-returns-in-llvm/> -// I.e. (i32, f32) will get merged into i64, but we don't handle that yet. -fn compute_enzyme_fn_ty<'ll>( - cx: &SimpleCx<'ll>, - attrs: &AutoDiffAttrs, - fn_to_diff: &'ll Value, - outer_fn: &'ll Value, -) -> &'ll llvm::Type { - let fn_ty = cx.get_type_of_global(outer_fn); - let mut ret_ty = cx.get_return_type(fn_ty); - - let has_sret = has_sret(outer_fn); - - if has_sret { - // Now we don't just forward the return type, so we have to figure it out based on the - // primal return type, in combination with the autodiff settings. - let fn_ty = cx.get_type_of_global(fn_to_diff); - let inner_ret_ty = cx.get_return_type(fn_ty); - - let void_ty = unsafe { llvm::LLVMVoidTypeInContext(cx.llcx) }; - if inner_ret_ty == void_ty { - // This indicates that even the inner function has an sret. - // Right now I only look for an sret in the outer function. - // This *probably* needs some extra handling, but I never ran - // into such a case. So I'll wait for user reports to have a test case. - bug!("sret in inner function"); - } - - if attrs.width == 1 { - // Enzyme returns a struct of style: - // `{ original_ret(if requested), float, float, ... }` - let mut struct_elements = vec![]; - if attrs.has_primal_ret() { - struct_elements.push(inner_ret_ty); - } - // Next, we push the list of active floats, since they will be lowered to `enzyme_out`, - // and therefore part of the return struct. - let param_tys = cx.func_params_types(fn_ty); - for (act, param_ty) in attrs.input_activity.iter().zip(param_tys) { - if matches!(act, DiffActivity::Active) { - // Now find the float type at position i based on the fn_ty, - // to know what (f16/f32/f64/...) to add to the struct. - struct_elements.push(param_ty); - } - } - ret_ty = cx.type_struct(&struct_elements, false); - } else { - // First we check if we also have to deal with the primal return. - match attrs.mode { - DiffMode::Forward => match attrs.ret_activity { - DiffActivity::Dual => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64 + 1) }; - ret_ty = arr_ty; - } - DiffActivity::DualOnly => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64) }; - ret_ty = arr_ty; - } - DiffActivity::Const => { - todo!("Not sure, do we need to do something here?"); - } - _ => { - bug!("unreachable"); - } - }, - DiffMode::Reverse => { - todo!("Handle sret for reverse mode"); - } - _ => { - bug!("unreachable"); - } - } - } - } - - // LLVM can figure out the input types on it's own, so we take a shortcut here. - unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) } -} - /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another /// function with expected naming and calling conventions[^1] which will be /// discovered by the enzyme LLVM pass and its body populated with the differentiated @@ -288,11 +245,15 @@ fn compute_enzyme_fn_ty<'ll>( /// [^1]: <https://enzyme.mit.edu/getting_started/CallingConvention/> // FIXME(ZuseZ4): `outer_fn` should include upstream safety checks to // cover some assumptions of enzyme/autodiff, which could lead to UB otherwise. -fn generate_enzyme_call<'ll>( +pub(crate) fn generate_enzyme_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, cx: &SimpleCx<'ll>, fn_to_diff: &'ll Value, - outer_fn: &'ll Value, + outer_name: &str, + ret_ty: &'ll Type, + fn_args: &[&'ll Value], attrs: AutoDiffAttrs, + dest: PlaceRef<'tcx, &'ll Value>, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -302,11 +263,9 @@ fn generate_enzyme_call<'ll>( } .to_string(); - // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple + // add outer_name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. - let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::str::from_utf8(&name).unwrap(); - ad_name.push_str(outer_fn_name); + ad_name.push_str(outer_name); // Let us assume the user wrote the following function square: // @@ -316,14 +275,8 @@ fn generate_enzyme_call<'ll>( // %0 = fmul double %x, %x // ret double %0 // } - // ``` - // - // The user now applies autodiff to the function square, in which case fn_to_diff will be `square`. - // Our macro generates the following placeholder code (slightly simplified): // - // ```llvm // define double @dsquare(double %x) { - // ; placeholder code // return 0.0; // } // ``` @@ -340,175 +293,44 @@ fn generate_enzyme_call<'ll>( // ret double %0 // } // ``` - unsafe { - let enzyme_ty = compute_enzyme_fn_ty(cx, &attrs, fn_to_diff, outer_fn); - - // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and - // think a bit more about what should go here. - let cc = llvm::LLVMGetFunctionCallConv(outer_fn); - let ad_fn = declare_simple_fn( - cx, - &ad_name, - llvm::CallConv::try_from(cc).expect("invalid callconv"), - llvm::UnnamedAddr::No, - llvm::Visibility::Default, - enzyme_ty, - ); - - // Otherwise LLVM might inline our temporary code before the enzyme pass has a chance to - // do it's work. - let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); - attributes::apply_to_llfn(ad_fn, Function, &[attr]); - - // We add a made-up attribute just such that we can recognize it after AD to update - // (no)-inline attributes. We'll then also remove this attribute. - let enzyme_marker_attr = llvm::CreateAttrString(cx.llcx, "enzyme_marker"); - attributes::apply_to_llfn(outer_fn, Function, &[enzyme_marker_attr]); - - // first, remove all calls from fnc - let entry = llvm::LLVMGetFirstBasicBlock(outer_fn); - let br = llvm::LLVMRustGetTerminator(entry); - llvm::LLVMRustEraseInstFromParent(br); - - let last_inst = llvm::LLVMRustGetLastInstruction(entry).unwrap(); - let mut builder = SBuilder::build(cx, entry); - - let num_args = llvm::LLVMCountParams(&fn_to_diff); - let mut args = Vec::with_capacity(num_args as usize + 1); - args.push(fn_to_diff); - - let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); - if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { - args.push(cx.get_metadata_value(enzyme_primal_ret)); - } - if attrs.width > 1 { - let enzyme_width = cx.create_metadata(b"enzyme_width"); - args.push(cx.get_metadata_value(enzyme_width)); - args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); - } - - let has_sret = has_sret(outer_fn); - let outer_args: Vec<&llvm::Value> = get_params(outer_fn); - match_args_from_caller_to_enzyme( - &cx, - &builder, - attrs.width, - &mut args, - &attrs.input_activity, - &outer_args, - has_sret, - ); - - let call = builder.call(enzyme_ty, ad_fn, &args, None); - - // This part is a bit iffy. LLVM requires that a call to an inlineable function has some - // metadata attached to it, but we just created this code oota. Given that the - // differentiated function already has partly confusing metadata, and given that this - // affects nothing but the auttodiff IR, we take a shortcut and just steal metadata from the - // dummy code which we inserted at a higher level. - // FIXME(ZuseZ4): Work with Enzyme core devs to clarify what debug metadata issues we have, - // and how to best improve it for enzyme core and rust-enzyme. - let md_ty = cx.get_md_kind_id("dbg"); - if llvm::LLVMRustHasMetadata(last_inst, md_ty) { - let md = llvm::LLVMRustDIGetInstMetadata(last_inst) - .expect("failed to get instruction metadata"); - let md_todiff = cx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, md_ty, md_todiff); - } else { - // We don't panic, since depending on whether we are in debug or release mode, we might - // have no debug info to copy, which would then be ok. - trace!("no dbg info"); - } - - // Now that we copied the metadata, get rid of dummy code. - llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst); - - if cx.val_ty(call) == cx.type_void() || has_sret { - if has_sret { - // This is what we already have in our outer_fn (shortened): - // define void @_foo(ptr <..> sret([32 x i8]) initializes((0, 32)) %0, <...>) { - // %7 = call [4 x double] (...) @__enzyme_fwddiff_foo(ptr @square, metadata !"enzyme_width", i64 4, <...>) - // <Here we are, we want to add the following two lines> - // store [4 x double] %7, ptr %0, align 8 - // ret void - // } - - // now store the result of the enzyme call into the sret pointer. - let sret_ptr = outer_args[0]; - let call_ty = cx.val_ty(call); - if attrs.width == 1 { - assert_eq!(cx.type_kind(call_ty), TypeKind::Struct); - } else { - assert_eq!(cx.type_kind(call_ty), TypeKind::Array); - } - llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr); - } - builder.ret_void(); - } else { - builder.ret(call); - } - - // Let's crash in case that we messed something up above and generated invalid IR. - llvm::LLVMRustVerifyFunction( - outer_fn, - llvm::LLVMRustVerifierFailureAction::LLVMAbortProcessAction, - ); - } -} - -pub(crate) fn differentiate<'ll>( - module: &'ll ModuleCodegen<ModuleLlvm>, - cgcx: &CodegenContext<LlvmCodegenBackend>, - diff_items: Vec<AutoDiffItem>, -) -> Result<(), FatalError> { - for item in &diff_items { - trace!("{}", item); + let enzyme_ty = unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) }; + + // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and + // think a bit more about what should go here. + let cc = unsafe { llvm::LLVMGetFunctionCallConv(fn_to_diff) }; + let ad_fn = declare_simple_fn( + cx, + &ad_name, + llvm::CallConv::try_from(cc).expect("invalid callconv"), + llvm::UnnamedAddr::No, + llvm::Visibility::Default, + enzyme_ty, + ); + + let num_args = llvm::LLVMCountParams(&fn_to_diff); + let mut args = Vec::with_capacity(num_args as usize + 1); + args.push(fn_to_diff); + + let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); + if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { + args.push(cx.get_metadata_value(enzyme_primal_ret)); } - - let diag_handler = cgcx.create_dcx(); - - let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - - // First of all, did the user try to use autodiff without using the -Zautodiff=Enable flag? - if !diff_items.is_empty() - && !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) - { - return Err(diag_handler.handle().emit_almost_fatal(AutoDiffWithoutEnable)); - } - - // Here we replace the placeholder code with the actual autodiff code, which calls Enzyme. - for item in diff_items.iter() { - let name = item.source.clone(); - let fn_def: Option<&llvm::Value> = cx.get_function(&name); - let Some(fn_def) = fn_def else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find source function".to_owned(), - }, - )); - }; - debug!(?item.target); - let fn_target: Option<&llvm::Value> = cx.get_function(&item.target); - let Some(fn_target) = fn_target else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find target function".to_owned(), - }, - )); - }; - - generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); + if attrs.width > 1 { + let enzyme_width = cx.create_metadata(b"enzyme_width"); + args.push(cx.get_metadata_value(enzyme_width)); + args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); } - // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts + match_args_from_caller_to_enzyme( + &cx, + builder, + attrs.width, + &mut args, + &attrs.input_activity, + fn_args, + ); - trace!("done with differentiate()"); + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); - Ok(()) + builder.store_to_place(call, dest.val); } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 5a3dd90ab24..791a71d73ae 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -103,7 +103,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t // This is a monomorphization of a generic function. if !(cx.tcx.sess.opts.share_generics() || tcx.codegen_instance_attrs(instance.def).inline - == rustc_attr_data_structures::InlineAttr::Never) + == rustc_hir::attrs::InlineAttr::Never) { // When not sharing generics, all instances are in the same // crate and have hidden visibility. diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 0b96b63bc85..9ec7b0f80ae 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -4,14 +4,15 @@ use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar, read_target_uint, }; -use rustc_middle::mir::mono::{Linkage, MonoItem}; +use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; @@ -191,8 +192,8 @@ fn check_and_apply_linkage<'ll, 'tcx>( // linkage and there are no definitions), then // `extern_with_linkage_foo` will instead be initialized to // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); + let real_name = + format!("_rust_extern_with_linkage_{:016x}_{sym}", cx.tcx.stable_crate_id(LOCAL_CRATE)); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { span: cx.tcx.def_span(def_id), diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ee77774c688..4a7de7d2e69 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use std::str; use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx}; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; -use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN}; @@ -213,6 +212,12 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("p8:128:128:128:48", "p8:128:128") } } + if llvm_version < (22, 0, 0) { + if sess.target.arch == "avr" { + // LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010 + target_data_layout = target_data_layout.replace("n8:16", "n8") + } + } // Ensure the data-layout values hardcoded remain the defaults. { @@ -372,6 +377,15 @@ pub(crate) unsafe fn create_module<'ll>( } } + if let Some(regparm_count) = sess.opts.unstable_opts.regparm { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "NumRegisterParameters", + regparm_count, + ); + } + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch == "aarch64" { llvm::add_module_flag_u32( @@ -457,6 +471,15 @@ pub(crate) unsafe fn create_module<'ll>( } } + if sess.opts.unstable_opts.indirect_branch_cs_prefix { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "indirect_branch_cs_prefix", + 1, + ); + } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) { // Set up the small-data optimization limit for architectures that use @@ -654,10 +677,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } } impl<'ll> SimpleCx<'ll> { - pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type { - assert_eq!(self.type_kind(ty), TypeKind::Function); - unsafe { llvm::LLVMGetReturnType(ty) } - } pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type { unsafe { llvm::LLVMGlobalGetValueType(val) } } @@ -721,16 +740,6 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) } } - - pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { - let mut functions = vec![]; - let mut func = unsafe { llvm::LLVMGetFirstFunction(self.llmod()) }; - while let Some(f) = func { - functions.push(f); - func = unsafe { llvm::LLVMGetNextFunction(f) } - } - functions - } } impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index f6000e72840..a4b60d420f3 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -73,48 +73,6 @@ pub(crate) struct CounterExpression { pub(crate) rhs: Counter, } -pub(crate) mod mcdc { - use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; - - /// Must match the layout of `LLVMRustMCDCDecisionParameters`. - #[repr(C)] - #[derive(Clone, Copy, Debug, Default)] - pub(crate) struct DecisionParameters { - bitmap_idx: u32, - num_conditions: u16, - } - - type LLVMConditionId = i16; - - /// Must match the layout of `LLVMRustMCDCBranchParameters`. - #[repr(C)] - #[derive(Clone, Copy, Debug, Default)] - pub(crate) struct BranchParameters { - condition_id: LLVMConditionId, - condition_ids: [LLVMConditionId; 2], - } - - impl From<ConditionInfo> for BranchParameters { - fn from(value: ConditionInfo) -> Self { - let to_llvm_cond_id = |cond_id: Option<ConditionId>| { - cond_id.and_then(|id| LLVMConditionId::try_from(id.as_usize()).ok()).unwrap_or(-1) - }; - let ConditionInfo { condition_id, true_next_id, false_next_id } = value; - Self { - condition_id: to_llvm_cond_id(Some(condition_id)), - condition_ids: [to_llvm_cond_id(false_next_id), to_llvm_cond_id(true_next_id)], - } - } - } - - impl From<DecisionInfo> for DecisionParameters { - fn from(info: DecisionInfo) -> Self { - let DecisionInfo { bitmap_idx, num_conditions } = info; - Self { bitmap_idx, num_conditions } - } - } -} - /// A span of source code coordinates to be embedded in coverage metadata. /// /// Must match the layout of `LLVMRustCoverageSpan`. @@ -148,26 +106,14 @@ pub(crate) struct Regions { pub(crate) code_regions: Vec<CodeRegion>, pub(crate) expansion_regions: Vec<ExpansionRegion>, pub(crate) branch_regions: Vec<BranchRegion>, - pub(crate) mcdc_branch_regions: Vec<MCDCBranchRegion>, - pub(crate) mcdc_decision_regions: Vec<MCDCDecisionRegion>, } impl Regions { /// Returns true if none of this structure's tables contain any regions. pub(crate) fn has_no_regions(&self) -> bool { - let Self { - code_regions, - expansion_regions, - branch_regions, - mcdc_branch_regions, - mcdc_decision_regions, - } = self; - - code_regions.is_empty() - && expansion_regions.is_empty() - && branch_regions.is_empty() - && mcdc_branch_regions.is_empty() - && mcdc_decision_regions.is_empty() + let Self { code_regions, expansion_regions, branch_regions } = self; + + code_regions.is_empty() && expansion_regions.is_empty() && branch_regions.is_empty() } } @@ -195,21 +141,3 @@ pub(crate) struct BranchRegion { pub(crate) true_counter: Counter, pub(crate) false_counter: Counter, } - -/// Must match the layout of `LLVMRustCoverageMCDCBranchRegion`. -#[derive(Clone, Debug)] -#[repr(C)] -pub(crate) struct MCDCBranchRegion { - pub(crate) cov_span: CoverageSpan, - pub(crate) true_counter: Counter, - pub(crate) false_counter: Counter, - pub(crate) mcdc_branch_params: mcdc::BranchParameters, -} - -/// Must match the layout of `LLVMRustCoverageMCDCDecisionRegion`. -#[derive(Clone, Debug)] -#[repr(C)] -pub(crate) struct MCDCDecisionRegion { - pub(crate) cov_span: CoverageSpan, - pub(crate) mcdc_decision_params: mcdc::DecisionParameters, -} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index 907d6d41a1f..bc4f6bb6a82 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -63,13 +63,7 @@ pub(crate) fn write_function_mappings_to_buffer( expressions: &[ffi::CounterExpression], regions: &ffi::Regions, ) -> Vec<u8> { - let ffi::Regions { - code_regions, - expansion_regions, - branch_regions, - mcdc_branch_regions, - mcdc_decision_regions, - } = regions; + let ffi::Regions { code_regions, expansion_regions, branch_regions } = regions; // SAFETY: // - All types are FFI-compatible and have matching representations in Rust/C++. @@ -87,10 +81,6 @@ pub(crate) fn write_function_mappings_to_buffer( expansion_regions.len(), branch_regions.as_ptr(), branch_regions.len(), - mcdc_branch_regions.as_ptr(), - mcdc_branch_regions.len(), - mcdc_decision_regions.as_ptr(), - mcdc_decision_regions.len(), buffer, ) }) diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index a9be833a643..d1cb95507d9 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,3 +1,4 @@ +use std::assert_matches::assert_matches; use std::sync::Arc; use itertools::Itertools; @@ -5,6 +6,7 @@ use rustc_abi::Align; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; use rustc_data_structures::fx::FxIndexMap; use rustc_index::IndexVec; +use rustc_macros::TryFromU32; use rustc_middle::ty::TyCtxt; use rustc_session::RemapFileNameExt; use rustc_session::config::RemapPathScopeComponents; @@ -20,6 +22,23 @@ mod covfun; mod spans; mod unused; +/// Version number that will be included the `__llvm_covmap` section header. +/// Corresponds to LLVM's `llvm::coverage::CovMapVersion` (in `CoverageMapping.h`), +/// or at least the subset that we know and care about. +/// +/// Note that version `n` is encoded as `(n-1)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, TryFromU32)] +enum CovmapVersion { + /// Used by LLVM 18 onwards. + Version7 = 6, +} + +impl CovmapVersion { + fn to_u32(self) -> u32 { + self as u32 + } +} + /// Generates and exports the coverage map, which is embedded in special /// linker sections in the final binary. /// @@ -29,38 +48,28 @@ pub(crate) fn finalize(cx: &mut CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure that LLVM is using a version of the coverage mapping format that - // agrees with our Rust-side code. Expected versions (encoded as n-1) are: - // - `CovMapVersion::Version7` (6) used by LLVM 18-19 - let covmap_version = { - let llvm_covmap_version = llvm_cov::mapping_version(); - let expected_versions = 6..=6; - assert!( - expected_versions.contains(&llvm_covmap_version), - "Coverage mapping version exposed by `llvm-wrapper` is out of sync; \ - expected {expected_versions:?} but was {llvm_covmap_version}" - ); - // This is the version number that we will embed in the covmap section: - llvm_covmap_version - }; + // agrees with our Rust-side code. Expected versions are: + // - `Version7` (6) used by LLVM 18 onwards. + let covmap_version = + CovmapVersion::try_from(llvm_cov::mapping_version()).unwrap_or_else(|raw_version: u32| { + panic!("unknown coverage mapping version reported by `llvm-wrapper`: {raw_version}") + }); + assert_matches!(covmap_version, CovmapVersion::Version7); debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); // FIXME(#132395): Can this be none even when coverage is enabled? - let instances_used = match cx.coverage_cx { - Some(ref cx) => cx.instances_used.borrow(), - None => return, - }; - - let mut covfun_records = instances_used - .iter() - .copied() + let Some(ref coverage_cx) = cx.coverage_cx else { return }; + + let mut covfun_records = coverage_cx + .instances_used() + .into_iter() // Sort by symbol name, so that the global file table is built in an // order that doesn't depend on the stable-hash-based order in which // instances were visited during codegen. .sorted_by_cached_key(|&instance| tcx.symbol_name(instance).name) .filter_map(|instance| prepare_covfun_record(tcx, instance, true)) .collect::<Vec<_>>(); - drop(instances_used); // In a single designated CGU, also prepare covfun records for functions // in this crate that were instrumented for coverage, but are unused. @@ -205,7 +214,11 @@ impl VirtualFileMapping { /// Generates the contents of the covmap record for this CGU, which mostly /// consists of a header and a list of filenames. The record is then stored /// as a global variable in the `__llvm_covmap` section. -fn generate_covmap_record<'ll>(cx: &mut CodegenCx<'ll, '_>, version: u32, filenames_buffer: &[u8]) { +fn generate_covmap_record<'ll>( + cx: &mut CodegenCx<'ll, '_>, + version: CovmapVersion, + filenames_buffer: &[u8], +) { // A covmap record consists of four target-endian u32 values, followed by // the encoded filenames table. Two of the header fields are unused in // modern versions of the LLVM coverage mapping format, and are always 0. @@ -216,7 +229,7 @@ fn generate_covmap_record<'ll>(cx: &mut CodegenCx<'ll, '_>, version: u32, filena cx.const_u32(0), // (unused) cx.const_u32(filenames_buffer.len() as u32), cx.const_u32(0), // (unused) - cx.const_u32(version), + cx.const_u32(version.to_u32()), ], /* packed */ false, ); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index b704cf2b1cd..e0da8d36876 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -27,6 +27,9 @@ use crate::llvm; /// the final record that will be embedded in the `__llvm_covfun` section. #[derive(Debug)] pub(crate) struct CovfunRecord<'tcx> { + /// Not used directly, but helpful in debug messages. + _instance: Instance<'tcx>, + mangled_function_name: &'tcx str, source_hash: u64, is_used: bool, @@ -55,6 +58,7 @@ pub(crate) fn prepare_covfun_record<'tcx>( let expressions = prepare_expressions(ids_info); let mut covfun = CovfunRecord { + _instance: instance, mangled_function_name: tcx.symbol_name(instance).name, source_hash: if is_used { fn_cov_info.function_source_hash } else { 0 }, is_used, @@ -102,11 +106,21 @@ fn fill_region_tables<'tcx>( ids_info: &'tcx CoverageIdsInfo, covfun: &mut CovfunRecord<'tcx>, ) { + // If this function is unused, replace all counters with zero. + let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { + let term = if covfun.is_used { + ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term") + } else { + CovTerm::Zero + }; + ffi::Counter::from_term(term) + }; + // Currently a function's mappings must all be in the same file, so use the // first mapping's span to determine the file. let source_map = tcx.sess.source_map(); let Some(first_span) = (try { fn_cov_info.mappings.first()?.span }) else { - debug_assert!(false, "function has no mappings: {:?}", covfun.mangled_function_name); + debug_assert!(false, "function has no mappings: {covfun:?}"); return; }; let source_file = source_map.lookup_source_file(first_span.lo()); @@ -117,7 +131,7 @@ fn fill_region_tables<'tcx>( // codegen needs to handle that gracefully to avoid #133606. // It's hard for tests to trigger this organically, so instead we set // `-Zcoverage-options=discard-all-spans-in-codegen` to force it to occur. - let discard_all = tcx.sess.coverage_discard_all_spans_in_codegen(); + let discard_all = tcx.sess.coverage_options().discard_all_spans_in_codegen; let make_coords = |span: Span| { if discard_all { None } else { spans::make_coords(source_map, &source_file, span) } }; @@ -126,23 +140,11 @@ fn fill_region_tables<'tcx>( code_regions, expansion_regions: _, // FIXME(Zalathar): Fill out support for expansion regions branch_regions, - mcdc_branch_regions, - mcdc_decision_regions, } = &mut covfun.regions; // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. for &Mapping { ref kind, span } in &fn_cov_info.mappings { - // If this function is unused, replace all counters with zero. - let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { - let term = if covfun.is_used { - ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term") - } else { - CovTerm::Zero - }; - ffi::Counter::from_term(term) - }; - let Some(coords) = make_coords(span) else { continue }; let cov_span = coords.make_coverage_span(local_file_id); @@ -157,20 +159,6 @@ fn fill_region_tables<'tcx>( false_counter: counter_for_bcb(false_bcb), }); } - MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params } => { - mcdc_branch_regions.push(ffi::MCDCBranchRegion { - cov_span, - true_counter: counter_for_bcb(true_bcb), - false_counter: counter_for_bcb(false_bcb), - mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), - }); - } - MappingKind::MCDCDecision(mcdc_decision_params) => { - mcdc_decision_regions.push(ffi::MCDCDecisionRegion { - cov_span, - mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), - }); - } } } } @@ -184,6 +172,7 @@ pub(crate) fn generate_covfun_record<'tcx>( covfun: &CovfunRecord<'tcx>, ) { let &CovfunRecord { + _instance, mangled_function_name, source_hash, is_used, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs index 39a59560c9d..574463be7ff 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs @@ -39,7 +39,10 @@ impl Coords { /// or other expansions), and if it does happen then skipping a span or function is /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid. pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) -> Option<Coords> { - let span = ensure_non_empty_span(source_map, span)?; + if span.is_empty() { + debug_assert!(false, "can't make coords from empty span: {span:?}"); + return None; + } let lo = span.lo(); let hi = span.hi(); @@ -70,29 +73,6 @@ pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) }) } -fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> { - if !span.is_empty() { - return Some(span); - } - - // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'. - source_map - .span_to_source(span, |src, start, end| try { - // Adjusting span endpoints by `BytePos(1)` is normally a bug, - // but in this case we have specifically checked that the character - // we're skipping over is one of two specific ASCII characters, so - // adjusting by exactly 1 byte is correct. - if src.as_bytes().get(end).copied() == Some(b'{') { - Some(span.with_hi(span.hi() + BytePos(1))) - } else if start > 0 && src.as_bytes()[start - 1] == b'}' { - Some(span.with_lo(span.lo() - BytePos(1))) - } else { - None - } - }) - .ok()? -} - /// If `llvm-cov` sees a source region that is improperly ordered (end < start), /// it will immediately exit with a fatal error. To prevent that from happening, /// discard regions that are improperly ordered, or might be interpreted in a diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index eefbd7cf6c4..6a58f495c9d 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,11 +1,10 @@ use std::cell::{OnceCell, RefCell}; use std::ffi::{CStr, CString}; -use rustc_abi::Size; use rustc_codegen_ssa::traits::{ - BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, + ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, }; -use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; +use rustc_data_structures::fx::FxIndexMap; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use tracing::{debug, instrument}; @@ -20,38 +19,29 @@ mod mapgen; /// Extra per-CGU context/state needed for coverage instrumentation. pub(crate) struct CguCoverageContext<'ll, 'tcx> { - /// Coverage data for each instrumented function identified by DefId. - pub(crate) instances_used: RefCell<FxIndexSet<Instance<'tcx>>>, - pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, - pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, + /// Associates function instances with an LLVM global that holds the + /// function's symbol name, as needed by LLVM coverage intrinsics. + /// + /// Instances in this map are also considered "used" for the purposes of + /// emitting covfun records. Every covfun record holds a hash of its + /// symbol name, and `llvm-cov` will exit fatally if it can't resolve that + /// hash back to an entry in the binary's `__llvm_prf_names` linker section. + pub(crate) pgo_func_name_var_map: RefCell<FxIndexMap<Instance<'tcx>, &'ll llvm::Value>>, covfun_section_name: OnceCell<CString>, } impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> { pub(crate) fn new() -> Self { - Self { - instances_used: RefCell::<FxIndexSet<_>>::default(), - pgo_func_name_var_map: Default::default(), - mcdc_condition_bitmap_map: Default::default(), - covfun_section_name: Default::default(), - } + Self { pgo_func_name_var_map: Default::default(), covfun_section_name: Default::default() } } - /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is - /// called condition bitmap. In order to handle nested decisions, several condition bitmaps can - /// be allocated for a function body. These values are named `mcdc.addr.{i}` and are a 32-bit - /// integers. They respectively hold the condition bitmaps for decisions with a depth of `i`. - fn try_get_mcdc_condition_bitmap( - &self, - instance: &Instance<'tcx>, - decision_depth: u16, - ) -> Option<&'ll llvm::Value> { - self.mcdc_condition_bitmap_map - .borrow() - .get(instance) - .and_then(|bitmap_map| bitmap_map.get(decision_depth as usize)) - .copied() // Dereference Option<&&Value> to Option<&Value> + /// Returns the list of instances considered "used" in this CGU, as + /// inferred from the keys of `pgo_func_name_var_map`. + pub(crate) fn instances_used(&self) -> Vec<Instance<'tcx>> { + // Collecting into a Vec is way easier than trying to juggle RefCell + // projections, and this should only run once per CGU anyway. + self.pgo_func_name_var_map.borrow().keys().copied().collect::<Vec<_>>() } } @@ -78,7 +68,10 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// string, to hold the function name passed to LLVM intrinsic /// `instrprof.increment()`. The `Value` is only created once per instance. /// Multiple invocations with the same instance return the same `Value`. - fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { + /// + /// This has the side-effect of causing coverage codegen to consider this + /// function "used", making it eligible to emit an associated covfun record. + fn ensure_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { debug!("getting pgo_func_name_var for instance={:?}", instance); let mut pgo_func_name_var_map = self.coverage_cx().pgo_func_name_var_map.borrow_mut(); pgo_func_name_var_map.entry(instance).or_insert_with(|| { @@ -90,38 +83,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn init_coverage(&mut self, instance: Instance<'tcx>) { - let Some(function_coverage_info) = - self.tcx.instance_mir(instance.def).function_coverage_info.as_deref() - else { - return; - }; - - // If there are no MC/DC bitmaps to set up, return immediately. - if function_coverage_info.mcdc_bitmap_bits == 0 { - return; - } - - let fn_name = self.get_pgo_func_name_var(instance); - let hash = self.const_u64(function_coverage_info.function_source_hash); - let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32); - self.mcdc_parameters(fn_name, hash, bitmap_bits); - - // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps. - let mut cond_bitmaps = vec![]; - for i in 0..function_coverage_info.mcdc_num_condition_bitmaps { - // MC/DC intrinsics will perform loads/stores that use the ABI default - // alignment for i32, so our variable declaration should match. - let align = self.tcx.data_layout.i32_align.abi; - let cond_bitmap = self.alloca(Size::from_bytes(4), align); - llvm::set_value_name(cond_bitmap, format!("mcdc.addr.{i}").as_bytes()); - self.store(self.const_i32(0), cond_bitmap, align); - cond_bitmaps.push(cond_bitmap); - } - - self.coverage_cx().mcdc_condition_bitmap_map.borrow_mut().insert(instance, cond_bitmaps); - } - #[instrument(level = "debug", skip(self))] fn add_coverage(&mut self, instance: Instance<'tcx>, kind: &CoverageKind) { // Our caller should have already taken care of inlining subtleties, @@ -138,7 +99,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { // When that happens, we currently just discard those statements, so // the corresponding code will be undercounted. // FIXME(Zalathar): Find a better solution for mixed-coverage builds. - let Some(coverage_cx) = &bx.cx.coverage_cx else { return }; + let Some(_coverage_cx) = &bx.cx.coverage_cx else { return }; let Some(function_coverage_info) = bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() @@ -151,11 +112,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - // Mark the instance as used in this CGU, for coverage purposes. - // This includes functions that were not partitioned into this CGU, - // but were MIR-inlined into one of this CGU's functions. - coverage_cx.instances_used.borrow_mut().insert(instance); - match *kind { CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!( "marker statement {kind:?} should have been removed by CleanupPostBorrowck" @@ -163,7 +119,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::VirtualCounter { bcb } if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) => { - let fn_name = bx.get_pgo_func_name_var(instance); + let fn_name = bx.ensure_pgo_func_name_var(instance); let hash = bx.const_u64(function_coverage_info.function_source_hash); let num_counters = bx.const_u32(ids_info.num_counters); let index = bx.const_u32(id.as_u32()); @@ -175,30 +131,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } // If a BCB doesn't have an associated physical counter, there's nothing to codegen. CoverageKind::VirtualCounter { .. } => {} - CoverageKind::CondBitmapUpdate { index, decision_depth } => { - let cond_bitmap = coverage_cx - .try_get_mcdc_condition_bitmap(&instance, decision_depth) - .expect("mcdc cond bitmap should have been allocated for updating"); - let cond_index = bx.const_i32(index as i32); - bx.mcdc_condbitmap_update(cond_index, cond_bitmap); - } - CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { - let cond_bitmap = - coverage_cx.try_get_mcdc_condition_bitmap(&instance, decision_depth).expect( - "mcdc cond bitmap should have been allocated for merging \ - into the global bitmap", - ); - assert!( - bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits, - "bitmap index of the decision out of range" - ); - - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_coverage_info.function_source_hash); - let bitmap_index = bx.const_u32(bitmap_idx); - bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); - bx.mcdc_condbitmap_reset(cond_bitmap); - } } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 61555ac2f6f..6eb7042da61 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -1,6 +1,5 @@ // .debug_gdb_scripts binary section. -use rustc_attr_data_structures::{AttributeKind, find_attr}; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::LOCAL_CRATE; @@ -85,9 +84,6 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( } pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { - let omit_gdb_pretty_printer_section = - find_attr!(cx.tcx.hir_krate_attrs(), AttributeKind::OmitGdbPrettyPrinterSection); - // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since // each rlib could produce a different set of visualizers that would be embedded @@ -116,8 +112,7 @@ pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { } }); - !omit_gdb_pretty_printer_section - && cx.sess().opts.debuginfo != DebugInfo::None + cx.sess().opts.debuginfo != DebugInfo::None && cx.sess().target.emit_debug_gdb_scripts && embed_visualizers } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 56fb12d3c22..d1502d2b1e6 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -285,8 +285,8 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( // Item(T), // } // ``` - let is_expanding_recursive = - debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| { + let is_expanding_recursive = adt_def.is_enum() + && debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| { if def_id == *parent_def_id { args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| { if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type()) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 5ca2505cec4..2c3a84499ac 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -533,31 +533,26 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { - // If the method does *not* belong to a trait, proceed - if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( - instance.args, - cx.typing_env(), - cx.tcx.type_of(impl_def_id), - ); - - // Only "class" methods are generally understood by LLVM, - // so avoid methods on other types (e.g., `<*mut T>::null`). - if let ty::Adt(def, ..) = impl_self_ty.kind() - && !def.is_box() - { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() - { - return (type_di_node(cx, impl_self_ty), true); - } else { - return (namespace::item_namespace(cx, def.did()), false); - } + // For trait method impls we still use the "parallel namespace" + // strategy + if let Some(imp_def_id) = cx.tcx.inherent_impl_of_assoc(instance.def_id()) { + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + cx.tcx.type_of(imp_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - } else { - // For trait method impls we still use the "parallel namespace" - // strategy } } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 31d49e86319..627b0c9ff3b 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -20,11 +20,6 @@ pub(crate) struct SymbolAlreadyDefined<'a> { #[diag(codegen_llvm_sanitizer_memtag_requires_mte)] pub(crate) struct SanitizerMemtagRequiresMte; -#[derive(Diagnostic)] -#[diag(codegen_llvm_dynamic_linking_with_lto)] -#[note] -pub(crate) struct DynamicLinkingWithLTO; - pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { @@ -42,21 +37,9 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { pub(crate) struct AutoDiffWithoutEnable; #[derive(Diagnostic)] -#[diag(codegen_llvm_lto_disallowed)] -pub(crate) struct LtoDisallowed; - -#[derive(Diagnostic)] -#[diag(codegen_llvm_lto_dylib)] -pub(crate) struct LtoDylib; - -#[derive(Diagnostic)] -#[diag(codegen_llvm_lto_proc_macro)] -pub(crate) struct LtoProcMacro; - -#[derive(Diagnostic)] #[diag(codegen_llvm_lto_bitcode_from_rlib)] pub(crate) struct LtoBitcodeFromRlib { - pub llvm_err: String, + pub err: String, } #[derive(Diagnostic)] diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index fcc0d378f06..4935f8d7dff 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -3,24 +3,29 @@ use std::cmp::Ordering; use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; +use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; -use rustc_hir as hir; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_hir::{self as hir}; use rustc_middle::mir::BinOp; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; -use rustc_middle::ty::{self, GenericArgsRef, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty, TyCtxt, TypingEnv}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; -use rustc_symbol_mangling::mangle_internal_symbol; +use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; +use rustc_target::callconv::PassMode; use rustc_target::spec::PanicStrategy; use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; +use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::context::CodegenCx; +use crate::errors::AutoDiffWithoutEnable; use crate::llvm::{self, Metadata}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -189,6 +194,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &[ptr, args[1].immediate()], ) } + sym::autodiff => { + codegen_autodiff(self, tcx, instance, args, result); + return Ok(()); + } sym::is_val_statically_known => { if let OperandValue::Immediate(imm) = args[0].val { self.call_intrinsic( @@ -382,26 +391,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let width = size.bits(); let llty = self.type_ix(width); match name { - sym::ctlz | sym::cttz => { - let y = self.const_bool(false); - let ret = self.call_intrinsic( - format!("llvm.{name}"), - &[llty], - &[args[0].immediate(), y], - ); - - self.intcast(ret, result.layout.llvm_type(self), false) - } - sym::ctlz_nonzero => { - let y = self.const_bool(true); - let ret = - self.call_intrinsic("llvm.ctlz", &[llty], &[args[0].immediate(), y]); - self.intcast(ret, result.layout.llvm_type(self), false) - } - sym::cttz_nonzero => { - let y = self.const_bool(true); + sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => { + let y = + self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero); + let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero { + "llvm.ctlz" + } else { + "llvm.cttz" + }; let ret = - self.call_intrinsic("llvm.cttz", &[llty], &[args[0].immediate(), y]); + self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]); self.intcast(ret, result.layout.llvm_type(self), false) } sym::ctpop => { @@ -1123,6 +1122,143 @@ fn get_rust_try_fn<'a, 'll, 'tcx>( rust_try } +fn codegen_autodiff<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + result: PlaceRef<'tcx, &'ll Value>, +) { + if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) { + let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable); + } + + let fn_args = instance.args; + let callee_ty = instance.ty(tcx, bx.typing_env()); + + let sig = callee_ty.fn_sig(tcx).skip_binder(); + + let ret_ty = sig.output(); + let llret_ty = bx.layout_of(ret_ty).llvm_type(bx); + + // Get source, diff, and attrs + let (source_id, source_args) = match fn_args.into_type_list(tcx)[0].kind() { + ty::FnDef(def_id, source_params) => (def_id, source_params), + _ => bug!("invalid autodiff intrinsic args"), + }; + + let fn_source = match Instance::try_resolve(tcx, bx.cx.typing_env(), *source_id, source_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + source_id, + source_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let source_symbol = symbol_name_for_instance_in_crate(tcx, fn_source.clone(), LOCAL_CRATE); + let Some(fn_to_diff) = bx.cx.get_function(&source_symbol) else { + bug!("could not find source function") + }; + + let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() { + ty::FnDef(def_id, diff_args) => (def_id, diff_args), + _ => bug!("invalid args"), + }; + + let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + diff_id, + diff_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let val_arr = get_args_from_tuple(bx, args[2], fn_diff); + let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE); + + let Some(mut diff_attrs) = autodiff_attrs(tcx, fn_diff.def_id()) else { + bug!("could not find autodiff attrs") + }; + + adjust_activity_to_abi( + tcx, + fn_source.ty(tcx, TypingEnv::fully_monomorphized()), + &mut diff_attrs.input_activity, + ); + + // Build body + generate_enzyme_call( + bx, + bx.cx, + fn_to_diff, + &diff_symbol, + llret_ty, + &val_arr, + diff_attrs.clone(), + result, + ); +} + +fn get_args_from_tuple<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tuple_op: OperandRef<'tcx, &'ll Value>, + fn_instance: Instance<'tcx>, +) -> Vec<&'ll Value> { + let cx = bx.cx; + let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty()); + + match tuple_op.val { + OperandValue::Immediate(val) => vec![val], + OperandValue::Pair(v1, v2) => vec![v1, v2], + OperandValue::Ref(ptr) => { + let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout }; + + let mut result = Vec::with_capacity(fn_abi.args.len()); + let mut tuple_index = 0; + + for arg in &fn_abi.args { + match arg.mode { + PassMode::Ignore => {} + PassMode::Direct(_) | PassMode::Cast { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(val); + tuple_index += 1; + } + PassMode::Pair(_, _) => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(bx.extract_value(pair_val, 0)); + result.push(bx.extract_value(pair_val, 1)); + tuple_index += 1; + } + PassMode::Indirect { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + result.push(field.val.llval); + tuple_index += 1; + } + } + } + + result + } + + OperandValue::ZeroSized => vec![], + } +} + fn generic_simd_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index aaf21f9ada9..0fcf31d7993 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -22,6 +22,7 @@ use std::any::Any; use std::ffi::CStr; use std::mem::ManuallyDrop; +use std::path::PathBuf; use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; @@ -29,7 +30,6 @@ use context::SimpleCx; use errors::ParseTargetMachineConfig; use llvm_util::target_config; use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -46,18 +46,11 @@ use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_span::Symbol; -mod back { - pub(crate) mod archive; - pub(crate) mod lto; - pub(crate) mod owned_target_machine; - mod profiling; - pub(crate) mod write; -} - mod abi; mod allocator; mod asm; mod attributes; +mod back; mod base; mod builder; mod callee; @@ -167,24 +160,14 @@ impl WriteBackendMethods for LlvmCodegenBackend { let stats = llvm::build_string(|s| unsafe { llvm::LLVMRustPrintStatistics(s) }).unwrap(); print!("{stats}"); } - fn run_link( - cgcx: &CodegenContext<Self>, - dcx: DiagCtxtHandle<'_>, - modules: Vec<ModuleCodegen<Self::Module>>, - ) -> Result<ModuleCodegen<Self::Module>, FatalError> { - back::write::link(cgcx, dcx, modules) - } fn run_and_optimize_fat_lto( cgcx: &CodegenContext<Self>, + exported_symbols_for_lto: &[String], + each_linked_rlib_for_lto: &[PathBuf], modules: Vec<FatLtoInput<Self>>, - cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>, - diff_fncs: Vec<AutoDiffItem>, ) -> Result<ModuleCodegen<Self::Module>, FatalError> { - let mut module = back::lto::run_fat(cgcx, modules, cached_modules)?; - - if !diff_fncs.is_empty() { - builder::autodiff::differentiate(&module, cgcx, diff_fncs)?; - } + let mut module = + back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?; let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -194,10 +177,18 @@ impl WriteBackendMethods for LlvmCodegenBackend { } fn run_thin_lto( cgcx: &CodegenContext<Self>, + exported_symbols_for_lto: &[String], + each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>, ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> { - back::lto::run_thin(cgcx, modules, cached_modules) + back::lto::run_thin( + cgcx, + exported_symbols_for_lto, + each_linked_rlib_for_lto, + modules, + cached_modules, + ) } fn optimize( cgcx: &CodegenContext<Self>, diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs deleted file mode 100644 index 51bcc4d123d..00000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! A wrapper around LLVM's archive (.a) code - -use std::path::Path; -use std::{slice, str}; - -use rustc_fs_util::path_to_c_string; - -pub(crate) struct ArchiveRO { - pub raw: &'static mut super::Archive, -} - -unsafe impl Send for ArchiveRO {} - -pub(crate) struct Iter<'a> { - raw: &'a mut super::ArchiveIterator<'a>, -} - -pub(crate) struct Child<'a> { - pub raw: &'a mut super::ArchiveChild<'a>, -} - -impl ArchiveRO { - /// Opens a static archive for read-only purposes. This is more optimized - /// than the `open` method because it uses LLVM's internal `Archive` class - /// rather than shelling out to `ar` for everything. - /// - /// If this archive is used with a mutable method, then an error will be - /// raised. - pub(crate) fn open(dst: &Path) -> Result<ArchiveRO, String> { - unsafe { - let s = path_to_c_string(dst); - let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { - super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) - })?; - Ok(ArchiveRO { raw: ar }) - } - } - - pub(crate) fn iter(&self) -> Iter<'_> { - unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } - } -} - -impl Drop for ArchiveRO { - fn drop(&mut self) { - unsafe { - super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Iterator for Iter<'a> { - type Item = Result<Child<'a>, String>; - - fn next(&mut self) -> Option<Result<Child<'a>, String>> { - unsafe { - match super::LLVMRustArchiveIteratorNext(self.raw) { - Some(raw) => Some(Ok(Child { raw })), - None => super::last_error().map(Err), - } - } - } -} - -impl<'a> Drop for Iter<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Child<'a> { - pub(crate) fn name(&self) -> Option<&'a str> { - unsafe { - let mut name_len = 0; - let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); - if name_ptr.is_null() { - None - } else { - let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); - str::from_utf8(name).ok().map(|s| s.trim()) - } - } - } -} - -impl<'a> Drop for Child<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index edfb29dd1be..55d26a97e2d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -97,6 +97,17 @@ pub(crate) enum ModuleFlagMergeBehavior { // Consts for the LLVM CallConv type, pre-cast to usize. +/// Must match the layout of `LLVMTailCallKind`. +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +#[allow(dead_code)] +pub(crate) enum TailCallKind { + None = 0, + Tail = 1, + MustTail = 2, + NoTail = 3, +} + /// LLVM CallingConv::ID. Should we wrap this? /// /// See <https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/IR/CallingConv.h> @@ -239,6 +250,7 @@ pub(crate) enum AttributeKind { FnRetThunkExtern = 41, Writable = 42, DeadOnUnwind = 43, + DeadOnReturn = 44, } /// LLVMIntPredicate @@ -321,10 +333,15 @@ impl RealPredicate { } } -/// LLVMTypeKind -#[derive(Copy, Clone, PartialEq, Debug)] +/// Must match the layout of `LLVMTypeKind`. +/// +/// Use [`RawEnum<TypeKind>`] for values of `LLVMTypeKind` returned from LLVM, +/// to avoid risk of UB if LLVM adds new enum values. +/// +/// All of LLVM's variants should be declared here, even if no Rust-side code refers +/// to them, because unknown variants will cause [`RawEnum::to_rust`] to panic. +#[derive(Copy, Clone, PartialEq, Debug, TryFromU32)] #[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] pub(crate) enum TypeKind { Void = 0, Half = 1, @@ -599,17 +616,6 @@ pub(crate) enum DiagnosticLevel { Remark, } -/// LLVMRustArchiveKind -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) enum ArchiveKind { - K_GNU, - K_BSD, - K_DARWIN, - K_COFF, - K_AIXBIG, -} - unsafe extern "C" { // LLVMRustThinLTOData pub(crate) type ThinLTOData; @@ -758,19 +764,12 @@ pub(crate) struct Builder<'a>(InvariantOpaque<'a>); pub(crate) struct PassManager<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub type TargetMachine; - pub(crate) type Archive; } -#[repr(C)] -pub(crate) struct ArchiveIterator<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub(crate) struct ArchiveChild<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub(crate) type Twine; pub(crate) type DiagnosticInfo; pub(crate) type SMDiagnostic; } -#[repr(C)] -pub(crate) struct RustArchiveMember<'a>(InvariantOpaque<'a>); /// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); @@ -1035,6 +1034,8 @@ unsafe extern "C" { CanThrow: llvm::Bool, ) -> &'ll Value; + pub(crate) safe fn LLVMGetTypeKind(Ty: &Type) -> RawEnum<TypeKind>; + // Operations on integer types pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; @@ -1186,6 +1187,7 @@ unsafe extern "C" { pub(crate) safe fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub(crate) safe fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub(crate) safe fn LLVMSetTailCallKind(CallInst: &Value, kind: TailCallKind); // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( @@ -1829,9 +1831,6 @@ unsafe extern "C" { // Create and destroy contexts. pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; - /// See llvm::LLVMTypeKind::getTypeID. - pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on all values pub(crate) fn LLVMRustGlobalAddMetadata<'a>( Val: &'a Value, @@ -2045,10 +2044,6 @@ unsafe extern "C" { NumExpansionRegions: size_t, BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, NumBranchRegions: size_t, - MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, - NumMCDCBranchRegions: size_t, - MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, - NumMCDCDecisionRegions: size_t, BufferOut: &RustString, ); @@ -2430,7 +2425,7 @@ unsafe extern "C" { OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, UseEmulatedTls: bool, - ArgsCstrBuff: *const c_char, + ArgsCstrBuff: *const c_uchar, // See "PTR_LEN_STR". ArgsCstrBuffLen: usize, UseWasmEH: bool, ) -> *mut TargetMachine; @@ -2497,19 +2492,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub(crate) fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - pub(crate) fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub(crate) fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub(crate) fn LLVMRustArchiveIteratorNext<'a>( - AIR: &ArchiveIterator<'a>, - ) -> Option<&'a mut ArchiveChild<'a>>; - pub(crate) fn LLVMRustArchiveChildName( - ACR: &ArchiveChild<'_>, - size: &mut size_t, - ) -> *const c_char; - pub(crate) fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub(crate) fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub(crate) fn LLVMRustDestroyArchive(AR: &'static mut Archive); - pub(crate) fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); pub(crate) fn LLVMRustUnpackOptimizationDiagnostic<'a>( @@ -2547,21 +2529,6 @@ unsafe extern "C" { num_ranges: &mut usize, ) -> bool; - pub(crate) fn LLVMRustWriteArchive( - Dst: *const c_char, - NumMembers: size_t, - Members: *const &RustArchiveMember<'_>, - WriteSymbtab: bool, - Kind: ArchiveKind, - isEC: bool, - ) -> LLVMRustResult; - pub(crate) fn LLVMRustArchiveMemberNew<'a>( - Filename: *const c_char, - Name: *const c_char, - Child: Option<&ArchiveChild<'a>>, - ) -> &'a mut RustArchiveMember<'a>; - pub(crate) fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); pub(crate) fn LLVMRustPositionBuilderPastAllocas<'a>(B: &Builder<'a>, Fn: &'a Value); @@ -2612,13 +2579,6 @@ unsafe extern "C" { len: usize, Identifier: *const c_char, ) -> Option<&Module>; - pub(crate) fn LLVMRustGetSliceFromObjectDataByName( - data: *const u8, - len: usize, - name: *const u8, - name_len: usize, - out_len: &mut usize, - ) -> *const u8; pub(crate) fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub(crate) fn LLVMRustLinkerAdd( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 154ba4fd690..7fea7b79a8c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -3,7 +3,6 @@ use std::ffi::{CStr, CString}; use std::num::NonZero; use std::ptr; -use std::str::FromStr; use std::string::FromUtf8Error; use libc::c_uint; @@ -16,7 +15,6 @@ pub(crate) use self::MetadataType::*; pub(crate) use self::ffi::*; use crate::common::AsCCharPtr; -pub(crate) mod archive_ro; pub(crate) mod diagnostic; pub(crate) mod enzyme_ffi; mod ffi; @@ -42,32 +40,6 @@ pub(crate) fn AddFunctionAttributes<'ll>( } } -pub(crate) fn HasAttributeAtIndex<'ll>( - llfn: &'ll Value, - idx: AttributePlace, - kind: AttributeKind, -) -> bool { - unsafe { LLVMRustHasAttributeAtIndex(llfn, idx.as_uint(), kind) } -} - -pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: &str) -> bool { - unsafe { LLVMRustHasFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) { - unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveRustEnumAttributeAtIndex( - llfn: &Value, - place: AttributePlace, - kind: AttributeKind, -) { - unsafe { - LLVMRustRemoveEnumAttributeAtIndex(llfn, place.as_uint(), kind); - } -} - pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, @@ -178,21 +150,6 @@ pub(crate) enum CodeGenOptSize { CodeGenOptSizeAggressive = 2, } -impl FromStr for ArchiveKind { - type Err = (); - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "gnu" => Ok(ArchiveKind::K_GNU), - "bsd" => Ok(ArchiveKind::K_BSD), - "darwin" => Ok(ArchiveKind::K_DARWIN), - "coff" => Ok(ArchiveKind::K_COFF), - "aix_big" => Ok(ArchiveKind::K_AIXBIG), - _ => Err(()), - } - } -} - pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 0fb987bdf82..28d2100f478 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -262,6 +262,15 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea // Filter out features that are not supported by the current LLVM version ("aarch64", "fpmr") => None, // only existed in 18 ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), + // NVPTX targets added in LLVM 20 + ("nvptx64", "sm_100") if get_version().0 < 20 => None, + ("nvptx64", "sm_100a") if get_version().0 < 20 => None, + ("nvptx64", "sm_101") if get_version().0 < 20 => None, + ("nvptx64", "sm_101a") if get_version().0 < 20 => None, + ("nvptx64", "sm_120") if get_version().0 < 20 => None, + ("nvptx64", "sm_120a") if get_version().0 < 20 => None, + ("nvptx64", "ptx86") if get_version().0 < 20 => None, + ("nvptx64", "ptx87") if get_version().0 < 20 => None, // Filter out features that are not supported by the current LLVM version ("loongarch64", "div32" | "lam-bh" | "lamcas" | "ld-seq-sa" | "scq") if get_version().0 < 20 => @@ -324,15 +333,12 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea /// /// We do not have to worry about RUSTC_SPECIFIC_FEATURES here, those are handled outside codegen. pub(crate) fn target_config(sess: &Session) -> TargetConfig { - // Add base features for the target. - // We do *not* add the -Ctarget-features there, and instead duplicate the logic for that below. - // The reason is that if LLVM considers a feature implied but we do not, we don't want that to - // show up in `cfg`. That way, `cfg` is entirely under our control -- except for the handling of - // the target CPU, that is still expanded to target features (with all their implied features) - // by LLVM. let target_machine = create_informational_target_machine(sess, true); let (unstable_target_features, target_features) = cfg_target_feature(sess, |feature| { + // This closure determines whether the target CPU has the feature according to LLVM. We do + // *not* consider the `-Ctarget-feature`s here, as that will be handled later in + // `cfg_target_feature`. if let Some(feat) = to_llvm_features(sess, feature) { // All the LLVM features this expands to must be enabled. for llvm_feature in feat { @@ -371,24 +377,25 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { let target_abi = sess.target.options.abi.as_ref(); let target_pointer_width = sess.target.pointer_width; let version = get_version(); + let lt_20_1_1 = version < (20, 1, 1); + let lt_21_0_0 = version < (21, 0, 0); cfg.has_reliable_f16 = match (target_arch, target_os) { - // Selection failure <https://github.com/llvm/llvm-project/issues/50374> - ("s390x", _) => false, - // LLVM crash without neon <https://github.com/llvm/llvm-project/issues/129394> (now fixed) + // LLVM crash without neon <https://github.com/llvm/llvm-project/issues/129394> (fixed in llvm20) ("aarch64", _) - if !cfg.target_features.iter().any(|f| f.as_str() == "neon") - && version < (20, 1, 1) => + if !cfg.target_features.iter().any(|f| f.as_str() == "neon") && lt_20_1_1 => { false } // Unsupported <https://github.com/llvm/llvm-project/issues/94434> ("arm64ec", _) => false, + // Selection failure <https://github.com/llvm/llvm-project/issues/50374> (fixed in llvm21) + ("s390x", _) if lt_21_0_0 => false, // MinGW ABI bugs <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054> ("x86_64", "windows") if target_env == "gnu" && target_abi != "llvm" => false, // Infinite recursion <https://github.com/llvm/llvm-project/issues/97981> ("csky", _) => false, - ("hexagon", _) => false, + ("hexagon", _) if lt_21_0_0 => false, // (fixed in llvm21) ("powerpc" | "powerpc64", _) => false, ("sparc" | "sparc64", _) => false, ("wasm32" | "wasm64", _) => false, @@ -401,18 +408,21 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { cfg.has_reliable_f128 = match (target_arch, target_os) { // Unsupported <https://github.com/llvm/llvm-project/issues/94434> ("arm64ec", _) => false, - // Selection bug <https://github.com/llvm/llvm-project/issues/96432> - ("mips64" | "mips64r6", _) => false, - // Selection bug <https://github.com/llvm/llvm-project/issues/95471> + // Selection bug <https://github.com/llvm/llvm-project/issues/96432> (fixed in llvm20) + ("mips64" | "mips64r6", _) if lt_20_1_1 => false, + // Selection bug <https://github.com/llvm/llvm-project/issues/95471>. This issue is closed + // but basic math still does not work. ("nvptx64", _) => false, + // Unsupported https://github.com/llvm/llvm-project/issues/121122 + ("amdgpu", _) => false, // ABI bugs <https://github.com/rust-lang/rust/issues/125109> et al. (full // list at <https://github.com/rust-lang/rust/issues/116909>) ("powerpc" | "powerpc64", _) => false, // ABI unsupported <https://github.com/llvm/llvm-project/issues/41838> ("sparc", _) => false, // Stack alignment bug <https://github.com/llvm/llvm-project/issues/77401>. NB: tests may - // not fail if our compiler-builtins is linked. - ("x86", _) => false, + // not fail if our compiler-builtins is linked. (fixed in llvm21) + ("x86", _) if lt_21_0_0 => false, // MinGW ABI bugs <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054> ("x86_64", "windows") if target_env == "gnu" && target_abi != "llvm" => false, // There are no known problems on other platforms, so the only requirement is that symbols @@ -433,6 +443,9 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { // This rules out anything that doesn't have `long double` = `binary128`; <= 32 bits // (ld is `f64`), anything other than Linux (Windows and MacOS use `f64`), and `x86` // (ld is 80-bit extended precision). + // + // musl does not implement the symbols required for f128 math at all. + _ if target_env == "musl" => false, ("x86_64", _) => false, (_, "linux") if target_pointer_width == 64 => true, _ => false, diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f9edaded60d..5075befae8a 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,8 +1,9 @@ use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 89365503138..f02d16baf94 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -204,7 +204,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_kind(&self, ty: &'ll Type) -> TypeKind { - unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } + llvm::LLVMGetTypeKind(ty).to_rust().to_generic() } fn type_ptr(&self) -> &'ll Type { |