diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 39 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 101 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 70 |
9 files changed, 194 insertions, 67 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b9baa87bac7..cc8b3a1a4e4 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -393,6 +393,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn llvm_cconv(&self) -> llvm::CallConv { match self.conv { Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index c7497bfd355..38402e04313 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -586,9 +586,21 @@ pub(crate) fn run_pass_manager( // LTO-specific optimization passes that LLVM provides. // // This code is based off the code found in llvm's LTO code generator: - // tools/lto/LTOCodeGenerator.cpp + // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { + if !llvm::LLVMRustHasModuleFlag( + module.module_llvm.llmod(), + "LTOPostLink".as_ptr().cast(), + 11, + ) { + llvm::LLVMRustAddModuleFlag( + module.module_llvm.llmod(), + llvm::LLVMModFlagBehavior::Error, + "LTOPostLink\0".as_ptr().cast(), + 1, + ); + } if llvm_util::should_use_new_llvm_pass_manager( &config.new_llvm_pass_manager, &cgcx.target_arch, diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 99e30531c22..ab8874d796d 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -56,28 +56,24 @@ pub fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, ) -> Result<(), FatalError> { + debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); unsafe { let output_c = path_to_c_string(output); - let result = if let Some(dwo_output) = dwo_output { - let dwo_output_c = path_to_c_string(dwo_output); - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - dwo_output_c.as_ptr(), - file_type, - ) + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() } else { - llvm::LLVMRustWriteOutputFile( - target, - pm, - m, - output_c.as_ptr(), - std::ptr::null(), - file_type, - ) + std::ptr::null() }; + let result = llvm::LLVMRustWriteOutputFile( + target, + pm, + m, + output_c.as_ptr(), + dwo_output_ptr, + file_type, + ); // Record artifact sizes for self-profiling if result == llvm::LLVMRustResult::Success { @@ -340,7 +336,7 @@ fn report_inline_asm( } let level = match level { llvm::DiagnosticLevel::Error => Level::Error { lint: false }, - llvm::DiagnosticLevel::Warning => Level::Warning, + llvm::DiagnosticLevel::Warning => Level::Warning(None), llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); @@ -467,7 +463,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); - let extra_passes = config.passes.join(","); + let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; let llvm_plugins = config.llvm_plugins.join(","); @@ -1040,7 +1036,8 @@ unsafe fn embed_bitcode( // reason (see issue #90326 for historical background). let is_apple = cgcx.opts.target_triple.triple().contains("-ios") || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos"); + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos"); if is_apple || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 5544f0d3f60..b5c31fcebe0 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -326,6 +326,15 @@ pub unsafe fn create_module<'ll>( ) } + if sess.opts.debugging_opts.virtual_function_elimination { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "Virtual Function Elim\0".as_ptr().cast(), + 1, + ); + } + llmod } @@ -656,6 +665,7 @@ impl<'ll> CodegenCx<'ll, '_> { let t_isize = self.type_isize(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); + let t_metadata = self.type_metadata(); ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); @@ -881,11 +891,12 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); } - ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1); + ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void); - ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void); + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } None } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index dd3adbf70a6..d5f39a45670 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -30,20 +30,21 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{self, GeneratorLayout}; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; -use rustc_middle::ty::{self, AdtKind, Instance, ParamEnv, Ty, TyCtxt}; -use rustc_session::config::{self, DebugInfo}; +use rustc_middle::ty::{ + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, +}; +use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::FileName; -use rustc_span::FileNameDisplayPreference; -use rustc_span::{self, SourceFile}; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; use tracing::debug; -use libc::{c_longlong, c_uint}; +use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; use std::fmt::{self, Write}; use std::hash::{Hash, Hasher}; @@ -1365,7 +1366,7 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo is_local_to_unit, global, None, - global_align.bytes() as u32, + global_align.bits() as u32, ); } } @@ -1468,6 +1469,84 @@ fn build_vtable_type_di_node<'ll, 'tcx>( .di_node } +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option<PolyExistentialTraitRef<'tcx>>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); + + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { + VCallVisibility::Public + } + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | ( + Lto::ThinLocal | Lto::Thin | Lto::Fat, + Visibility::Restricted(_) | Visibility::Invisible, + false, + ) => VCallVisibility::LinkageUnit, + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { + VCallVisibility::TranslationUnit + } + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } +} + /// Creates debug information for the given vtable, which is for the /// given type. /// @@ -1478,6 +1557,12 @@ pub fn create_vtable_di_node<'ll, 'tcx>( poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, vtable: &'ll Value, ) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.debugging_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + if cx.dbg_cx.is_none() { return; } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9927f5f399b..a18f5b9dd7f 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -406,6 +406,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) } + fn type_checked_load( + &mut self, + llvtable: &'ll Value, + vtable_byte_offset: u64, + typeid: &'ll Value, + ) -> Self::Value { + let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + } + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { self.call_intrinsic("llvm.va_start", &[va_list]) } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 913cf4eea13..6713a756735 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -9,7 +9,6 @@ #![feature(let_else)] #![feature(extern_types)] #![feature(once_cell)] -#![feature(nll)] #![feature(iter_intersperse)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 1d9a4655db6..b831423994f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -442,6 +442,7 @@ pub enum MetadataType { MD_nonnull = 11, MD_align = 17, MD_type = 19, + MD_vcall_visibility = 28, MD_noundef = 29, } @@ -1067,6 +1068,7 @@ extern "C" { pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1080,6 +1082,11 @@ extern "C" { Vals: *const &'a Value, Count: c_uint, ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); // Operations on scalar constants @@ -1936,6 +1943,7 @@ extern "C" { name: *const c_char, value: u32, ); + pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 7b407c94e7b..ce6c6e3215c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -218,15 +218,17 @@ pub fn check_tied_features( sess: &Session, features: &FxHashMap<&str, bool>, ) -> Option<&'static [&'static str]> { - for tied in tied_target_features(sess) { - // Tied features must be set to the same value, or not set at all - let mut tied_iter = tied.iter(); - let enabled = features.get(tied_iter.next().unwrap()); - if tied_iter.any(|f| enabled != features.get(f)) { - return Some(tied); + if !features.is_empty() { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } } } - None + return None; } // Used to generate cfg variables and apply features @@ -440,6 +442,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // -Ctarget-features let supported_features = supported_target_features(sess); + let mut featsmap = FxHashMap::default(); let feats = sess .opts .cg @@ -485,35 +488,36 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } diag.emit(); } - Some((enable_disable, feature)) + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } + + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some( + to_llvm_features(sess, feature) + .into_iter() + .map(move |f| format!("{}{}", enable_disable, f)), + ) }) - .collect::<SmallVec<[(char, &str); 8]>>(); - - if diagnostics { - // FIXME(nagisa): figure out how to not allocate a full hashset here. - let featmap = feats.iter().map(|&(flag, feat)| (feat, flag == '+')).collect(); - if let Some(f) = check_tied_features(sess, &featmap) { - sess.err(&format!( - "target features {} must all be enabled or disabled together", - f.join(", ") - )); - } + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); } - features.extend(feats.into_iter().flat_map(|(enable_disable, feature)| { - // rustc-specific features do not get passed down to LLVM… - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { - return SmallVec::<[_; 2]>::new(); - } - // ... otherwise though we run through `to_llvm_features` when - // passing requests down to LLVM. This means that all in-language - // features also work on the command line instead of having two - // different names when the LLVM name and the Rust name differ. - to_llvm_features(sess, feature) - .into_iter() - .map(|f| format!("{}{}", enable_disable, f)) - .collect() - })); features } |