diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 50 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 50 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 31 |
10 files changed, 169 insertions, 46 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index dca9c1f04d3..bedd3523d89 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,7 +1,6 @@ use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, AttributePlace}; -use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -53,15 +52,10 @@ pub trait ArgAttributesExt { } fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { - // LLVM prior to version 12 has known miscompiles in the presence of - // noalias attributes (see #54878). Only enable mutable noalias by - // default for versions we believe to be safe. - cx.tcx - .sess - .opts - .debugging_opts - .mutable_noalias - .unwrap_or_else(|| llvm_util::get_version() >= (12, 0, 0)) + // LLVM prior to version 12 had known miscompiles in the presence of + // noalias attributes (see #54878), but we don't support earlier + // versions at all anymore. We now enable mutable noalias by default. + cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true) } impl ArgAttributesExt for ArgAttributes { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 341a8882416..f128f769580 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::{bug, span_bug}; +use rustc_middle::{bug, span_bug, ty::Instance}; use rustc_span::{Pos, Span, Symbol}; use rustc_target::abi::*; use rustc_target::asm::*; @@ -120,6 +120,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, line_spans: &[Span], + instance: Instance<'_>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -135,7 +136,10 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { let is_target_supported = |reg_class: InlineAsmRegClass| { for &(_, feature) in reg_class.supported_types(asm_arch) { if let Some(feature) = feature { - if self.tcx.sess.target_features.contains(&Symbol::intern(feature)) + let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); + let feature_name = Symbol::intern(feature); + if self.tcx.sess.target_features.contains(&feature_name) + || codegen_fn_attrs.target_features.contains(&feature_name) { return true; } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 380dfd38723..3d05fc15b38 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -17,6 +17,7 @@ use rustc_codegen_ssa::back::write::{ }; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; @@ -53,6 +54,7 @@ pub fn write_output_file( output: &Path, dwo_output: Option<&Path>, file_type: llvm::FileType, + self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { unsafe { let output_c = path_to_c_string(output); @@ -76,6 +78,19 @@ pub fn write_output_file( file_type, ) }; + + // Record artifact sizes for self-profiling + if result == llvm::LLVMRustResult::Success { + let artifact_kind = match file_type { + llvm::FileType::ObjectFile => "object_file", + llvm::FileType::AssemblyFile => "assembly_file", + }; + record_artifact_size(self_profiler_ref, artifact_kind, output); + if let Some(dwo_file) = dwo_output { + record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); + } + } + result.into_result().map_err(|()| { let msg = format!("could not write output to {}", output.display()); llvm_err(handler, &msg) @@ -161,6 +176,7 @@ pub fn target_machine_factory( let ffunction_sections = sess.opts.debugging_opts.function_sections.unwrap_or(sess.target.function_sections); let fdata_sections = ffunction_sections; + let funique_section_names = !sess.opts.debugging_opts.no_unique_section_names; let code_model = to_llvm_code_model(sess.code_model()); @@ -205,6 +221,7 @@ pub fn target_machine_factory( use_softfp, ffunction_sections, fdata_sections, + funique_section_names, trap_unreachable, singlethread, asm_comments, @@ -284,7 +301,7 @@ fn report_inline_asm( cookie = 0; } let level = match level { - llvm::DiagnosticLevel::Error => Level::Error, + llvm::DiagnosticLevel::Error => Level::Error { lint: false }, llvm::DiagnosticLevel::Warning => Level::Warning, llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; @@ -750,6 +767,14 @@ pub(crate) unsafe fn codegen( let thin = ThinBuffer::new(llmod); let data = thin.data(); + if let Some(bitcode_filename) = bc_out.file_name() { + cgcx.prof.artifact_size( + "llvm_bitcode", + bitcode_filename.to_string_lossy(), + data.len() as u64, + ); + } + if config.emit_bc || config.emit_obj == EmitObj::Bitcode { let _timer = cgcx.prof.generic_activity_with_arg( "LLVM_module_codegen_emit_bitcode", @@ -810,6 +835,11 @@ pub(crate) unsafe fn codegen( } let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback); + + if result == llvm::LLVMRustResult::Success { + record_artifact_size(&cgcx.prof, "llvm_ir", &out); + } + result.into_result().map_err(|()| { let msg = format!("failed to write LLVM IR to {}", out.display()); llvm_err(diag_handler, &msg) @@ -840,6 +870,7 @@ pub(crate) unsafe fn codegen( &path, None, llvm::FileType::AssemblyFile, + &cgcx.prof, ) })?; } @@ -873,6 +904,7 @@ pub(crate) unsafe fn codegen( &obj_out, dwo_out, llvm::FileType::ObjectFile, + &cgcx.prof, ) })?; } @@ -1129,3 +1161,19 @@ fn create_msvc_imps( symbol_name.starts_with(b"__llvm_profile_") } } + +fn record_artifact_size( + self_profiler_ref: &SelfProfilerRef, + artifact_kind: &'static str, + path: &Path, +) { + // Don't stat the file if we are not going to record its size. + if !self_profiler_ref.enabled() { + return; + } + + if let Some(artifact_name) = path.file_name() { + let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); + } +} diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index d5deacf3811..6c74163fb49 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -604,6 +604,32 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn type_metadata(&mut self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn typeid_metadata(&mut self, typeid: String) -> Self::Value { + unsafe { + llvm::LLVMMDStringInContext( + self.cx.llcx, + typeid.as_ptr() as *const c_char, + typeid.as_bytes().len() as c_uint, + ) + } + } + fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { self.store_with_flags(val, ptr, align, MemFlags::empty()) } @@ -705,7 +731,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() { + if !self.fptoint_sat_broken_in_llvm() { let src_ty = self.cx.val_ty(val); let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); @@ -717,7 +743,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { - if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() { + if !self.fptoint_sat_broken_in_llvm() { let src_ty = self.cx.val_ty(val); let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 257a0ac89d8..1dba264a961 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -134,9 +134,6 @@ pub unsafe fn create_module( let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); let mut target_data_layout = sess.target.data_layout.clone(); - if llvm_util::get_version() < (12, 0, 0) && sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-v256:256:256-v512:512:512", ""); - } if llvm_util::get_version() < (13, 0, 0) { if sess.target.arch == "powerpc64" { target_data_layout = target_data_layout.replace("-S128", ""); @@ -221,6 +218,15 @@ pub unsafe fn create_module( llvm::LLVMRustAddModuleFlag(llmod, avoid_plt, 1); } + if sess.is_sanitizer_cfi_enabled() { + // FIXME(rcvalle): Add support for non canonical jump tables. + let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); + // FIXME(rcvalle): Add it with Override behavior flag--LLVMRustAddModuleFlag adds it with + // Warning behavior flag. Add support for specifying the behavior flag to + // LLVMRustAddModuleFlag. + llvm::LLVMRustAddModuleFlag(llmod, canonical_jump_tables, 1); + } + // Control Flow Guard is currently only supported by the MSVC linker on Windows. if sess.target.is_like_msvc { match sess.opts.cg.control_flow_guard { @@ -779,6 +785,8 @@ impl CodegenCx<'b, 'tcx> { ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); } + ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1); + if self.sess().opts.debuginfo != DebugInfo::None { ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void); ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 1f1bd73c7d0..2a6bf7d9b1a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -322,7 +322,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { type_names::push_item_name(self.tcx(), def_id, false, &mut name); // Find the enclosing function, in case this is a closure. - let enclosing_fn_def_id = self.tcx().closure_base_def_id(def_id); + let enclosing_fn_def_id = self.tcx().typeck_root_def_id(def_id); // Get_template_parameters() will append a `<...>` clause to the function // name if necessary. diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c43141c7695..e63fb22829a 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -401,6 +401,14 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } + fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { + // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time + // optimization pass replaces calls to this intrinsic with code to test type membership. + let i8p_ty = self.type_i8p(); + let bitcast = self.bitcast(pointer, i8p_ty); + self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) + } + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { self.call_intrinsic("llvm.va_start", &[va_list]) } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index f1dacc393d5..64fedb7bc1a 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -13,7 +13,6 @@ #![feature(iter_zip)] #![feature(nll)] #![recursion_limit = "256"] -#![cfg_attr(not(bootstrap), allow(rustc::potential_query_instability))] use back::write::{create_informational_target_machine, create_target_machine}; @@ -77,6 +76,27 @@ mod value; #[derive(Clone)] pub struct LlvmCodegenBackend(()); +struct TimeTraceProfiler { + enabled: bool, +} + +impl TimeTraceProfiler { + fn new(enabled: bool) -> Self { + if enabled { + unsafe { llvm::LLVMTimeTraceProfilerInitialize() } + } + TimeTraceProfiler { enabled } + } +} + +impl Drop for TimeTraceProfiler { + fn drop(&mut self) { + if self.enabled { + unsafe { llvm::LLVMTimeTraceProfilerFinishThread() } + } + } +} + impl ExtraBackendMethods for LlvmCodegenBackend { fn new_metadata(&self, tcx: TyCtxt<'_>, mod_name: &str) -> ModuleLlvm { ModuleLlvm::new_metadata(tcx, mod_name) @@ -120,6 +140,34 @@ impl ExtraBackendMethods for LlvmCodegenBackend { fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { llvm_util::tune_cpu(sess) } + + fn spawn_thread<F, T>(time_trace: bool, f: F) -> std::thread::JoinHandle<T> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } + + fn spawn_named_thread<F, T>( + time_trace: bool, + name: String, + f: F, + ) -> std::io::Result<std::thread::JoinHandle<T>> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::Builder::new().name(name).spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } } impl WriteBackendMethods for LlvmCodegenBackend { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4c9ae4faf72..749eec459ac 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -416,6 +416,7 @@ pub enum MetadataType { MD_nontemporal = 9, MD_mem_parallel_loop_access = 10, MD_nonnull = 11, + MD_type = 19, } /// LLVMRustAsmDialect @@ -1002,6 +1003,8 @@ extern "C" { pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); pub fn LLVMReplaceAllUsesWith(OldVal: &'a Value, NewVal: &'a Value); pub fn LLVMSetMetadata(Val: &'a Value, KindID: c_uint, Node: &'a Value); + pub fn LLVMGlobalSetMetadata(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMValueAsMetadata(Node: &'a Value) -> &Metadata; // Operations on constants of any type pub fn LLVMConstNull(Ty: &Type) -> &Value; @@ -1734,6 +1737,8 @@ extern "C" { pub fn LLVMTimeTraceProfilerInitialize(); + pub fn LLVMTimeTraceProfilerFinishThread(); + pub fn LLVMTimeTraceProfilerFinish(FileName: *const c_char); pub fn LLVMAddAnalysisPasses(T: &'a TargetMachine, PM: &PassManager<'a>); @@ -1770,7 +1775,7 @@ extern "C" { pub fn LLVMDisposeMessage(message: *mut c_char); - pub fn LLVMStartMultithreaded() -> Bool; + pub fn LLVMIsMultithreaded() -> Bool; /// Returns a string describing the last error caused by an LLVMRust* call. pub fn LLVMRustGetLastError() -> *const c_char; @@ -2187,6 +2192,7 @@ extern "C" { UseSoftFP: bool, FunctionSections: bool, DataSections: bool, + UniqueSectionNames: bool, TrapUnreachable: bool, Singlethread: bool, AsmComments: bool, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 4e4487ad88c..c3e7e7169a9 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -17,35 +17,25 @@ use std::path::Path; use std::ptr; use std::slice; use std::str; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Once; -static POISONED: AtomicBool = AtomicBool::new(false); static INIT: Once = Once::new(); pub(crate) fn init(sess: &Session) { unsafe { // Before we touch LLVM, make sure that multithreading is enabled. + if llvm::LLVMIsMultithreaded() != 1 { + bug!("LLVM compiled without support for threads"); + } INIT.call_once(|| { - if llvm::LLVMStartMultithreaded() != 1 { - // use an extra bool to make sure that all future usage of LLVM - // cannot proceed despite the Once not running more than once. - POISONED.store(true, Ordering::SeqCst); - } - configure_llvm(sess); }); - - if POISONED.load(Ordering::SeqCst) { - bug!("couldn't enable multi-threaded LLVM"); - } } } fn require_inited() { - INIT.call_once(|| bug!("llvm is not initialized")); - if POISONED.load(Ordering::SeqCst) { - bug!("couldn't enable multi-threaded LLVM"); + if !INIT.is_completed() { + bug!("LLVM is not initialized"); } } @@ -123,11 +113,6 @@ unsafe fn configure_llvm(sess: &Session) { } if sess.opts.debugging_opts.llvm_time_trace { - // time-trace is not thread safe and running it in parallel will cause seg faults. - if !sess.opts.debugging_opts.no_parallel_llvm { - bug!("`-Z llvm-time-trace` requires `-Z no-parallel-llvm") - } - llvm::LLVMTimeTraceProfilerInitialize(); } @@ -190,6 +175,7 @@ pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> Vec<&'a str> { ("aarch64", "dpb2") => vec!["ccdp"], ("aarch64", "frintts") => vec!["fptoint"], ("aarch64", "fcma") => vec!["complxnum"], + ("aarch64", "pmuv3") => vec!["perfmon"], (_, s) => vec![s], } } @@ -415,11 +401,6 @@ pub fn llvm_global_features(sess: &Session) -> Vec<String> { // -Ctarget-features features.extend(sess.opts.cg.target_feature.split(',').flat_map(&filter)); - // FIXME: Move outline-atomics to target definition when earliest supported LLVM is 12. - if get_version() >= (12, 0, 0) && sess.target.llvm_target.contains("aarch64-unknown-linux") { - features.push("+outline-atomics".to_string()); - } - features } |