diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/callee.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 33 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 14 |
7 files changed, 75 insertions, 20 deletions
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 51c70f0868f..659cf9ea070 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -263,6 +263,10 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: attributes::emit_uwtable(llfn, true); } + if cx.sess().opts.debugging_opts.profile_sample_use.is_some() { + llvm::AddFunctionAttrString(llfn, Function, cstr!("use-sample-profile")); + } + // FIXME: none of these three functions interact with source level attributes. set_frame_pointer_type(cx, llfn); set_instrument_function(cx, llfn); diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index ebc3b704158..2fb5a0f9faf 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -163,13 +163,13 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { // All import names are Rust identifiers and therefore cannot contain \0 characters. // FIXME: when support for #[link_name] implemented, ensure that import.name values don't // have any \0 characters - let import_name_vector: Vec<CString> = dll_imports + let import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = dll_imports .iter() .map(|import: &DllImport| { if self.config.sess.target.arch == "x86" { - LlvmArchiveBuilder::i686_decorated_name(import) + (LlvmArchiveBuilder::i686_decorated_name(import), import.ordinal) } else { - CString::new(import.name.to_string()).unwrap() + (CString::new(import.name.to_string()).unwrap(), import.ordinal) } }) .collect(); @@ -184,9 +184,9 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { dll_imports.iter().map(|import| import.name.to_string()).collect::<Vec<_>>().join(", "), ); - let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_vector + let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_and_ordinal_vector .iter() - .map(|name_z| LLVMRustCOFFShortExport::from_name(name_z.as_ptr())) + .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) .collect(); let result = unsafe { crate::llvm::LLVMRustWriteImportLibrary( diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index e2b33509b40..97780de9ba4 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -596,7 +596,7 @@ pub(crate) fn run_pass_manager( // tools/lto/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { - if write::should_use_new_llvm_pass_manager(config) { + if write::should_use_new_llvm_pass_manager(cgcx, config) { let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); write::optimize_with_new_llvm_pass_manager( diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index ab48c56a626..380dfd38723 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -370,10 +370,26 @@ fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } -pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool { +fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { + config + .pgo_sample_use + .as_ref() + .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) +} + +pub(crate) fn should_use_new_llvm_pass_manager( + cgcx: &CodegenContext<LlvmCodegenBackend>, + config: &ModuleConfig, +) -> bool { // The new pass manager is enabled by default for LLVM >= 13. // This matches Clang, which also enables it since Clang 13. - config.new_llvm_pass_manager.unwrap_or_else(|| llvm_util::get_version() >= (13, 0, 0)) + + // FIXME: There are some perf issues with the new pass manager + // when targeting s390x, so it is temporarily disabled for that + // arch, see https://github.com/rust-lang/rust/issues/89609 + config + .new_llvm_pass_manager + .unwrap_or_else(|| cgcx.target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0)) } pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( @@ -389,6 +405,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); + let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { @@ -439,6 +456,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, config.instrument_gcov, + pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + config.debug_info_for_profiling, llvm_selfprofiler, selfprofile_before_pass_callback, selfprofile_after_pass_callback, @@ -472,7 +491,7 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { - if should_use_new_llvm_pass_manager(config) { + if should_use_new_llvm_pass_manager(cgcx, config) { let opt_stage = match cgcx.lto { Lto::Fat => llvm::OptStage::PreLinkFatLTO, Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, @@ -544,6 +563,9 @@ pub(crate) unsafe fn optimize( if config.instrument_coverage { llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); } + if config.debug_info_for_profiling { + llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); + } add_sanitizer_passes(config, &mut extra_passes); @@ -1001,6 +1023,7 @@ pub unsafe fn with_llvm_pmb( let inline_threshold = config.inline_threshold; let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); + let pgo_sample_use_path = get_pgo_sample_use_path(config); llvm::LLVMRustConfigurePassManagerBuilder( builder, @@ -1011,6 +1034,7 @@ pub unsafe fn with_llvm_pmb( prepare_for_thin_lto, pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), ); llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32); diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 5d68d2b77d4..1bc924d3b90 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -175,7 +175,7 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value // should use dllimport for functions. if cx.use_dll_storage_attrs && tcx.is_dllimport_foreign_item(instance_def_id) - && tcx.sess.target.env != "gnu" + && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc") { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 22dc8d101c8..c43141c7695 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -20,7 +20,7 @@ use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; use rustc_target::abi::{self, HasDataLayout, Primitive}; -use rustc_target::spec::PanicStrategy; +use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; use std::iter; @@ -1190,11 +1190,28 @@ fn generic_simd_intrinsic( // FIXME: use: // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 - fn llvm_vector_str(elem_ty: Ty<'_>, vec_len: u64, no_pointers: usize) -> String { + fn llvm_vector_str( + elem_ty: Ty<'_>, + vec_len: u64, + no_pointers: usize, + bx: &Builder<'a, 'll, 'tcx>, + ) -> String { let p0s: String = "p0".repeat(no_pointers); match *elem_ty.kind() { - ty::Int(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()), - ty::Uint(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()), + ty::Int(v) => format!( + "v{}{}i{}", + vec_len, + p0s, + // Normalize to prevent crash if v: IntTy::Isize + v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() + ), + ty::Uint(v) => format!( + "v{}{}i{}", + vec_len, + p0s, + // Normalize to prevent crash if v: UIntTy::Usize + v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() + ), ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()), _ => unreachable!(), } @@ -1330,11 +1347,11 @@ fn generic_simd_intrinsic( // Type of the vector of pointers: let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); - let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count); + let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); - let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1); + let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); let llvm_intrinsic = format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); @@ -1458,11 +1475,11 @@ fn generic_simd_intrinsic( // Type of the vector of pointers: let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); - let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count); + let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); - let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1); + let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); let llvm_intrinsic = format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 436d906827b..63eca00de2a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -34,11 +34,18 @@ pub enum LLVMRustResult { #[repr(C)] pub struct LLVMRustCOFFShortExport { pub name: *const c_char, + pub ordinal_present: bool, + // value of `ordinal` only important when `ordinal_present` is true + pub ordinal: u16, } impl LLVMRustCOFFShortExport { - pub fn from_name(name: *const c_char) -> LLVMRustCOFFShortExport { - LLVMRustCOFFShortExport { name } + pub fn new(name: *const c_char, ordinal: Option<u16>) -> LLVMRustCOFFShortExport { + LLVMRustCOFFShortExport { + name, + ordinal_present: ordinal.is_some(), + ordinal: ordinal.unwrap_or(0), + } } } @@ -2176,6 +2183,7 @@ extern "C" { PrepareForThinLTO: bool, PGOGenPath: *const c_char, PGOUsePath: *const c_char, + PGOSampleUsePath: *const c_char, ); pub fn LLVMRustAddLibraryInfo( PM: &PassManager<'a>, @@ -2210,6 +2218,8 @@ extern "C" { PGOUsePath: *const c_char, InstrumentCoverage: bool, InstrumentGCOV: bool, + PGOSampleUsePath: *const c_char, + DebugInfoForProfiling: bool, llvm_selfprofiler: *mut c_void, begin_callback: SelfProfileBeforePassCallback, end_callback: SelfProfileAfterPassCallback, |