diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src/context.rs')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 191 |
1 files changed, 93 insertions, 98 deletions
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 5857b83f6c9..e8a7afcc632 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use crate::llvm_util; use crate::type_::Type; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; @@ -26,6 +25,7 @@ use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; +use rustc_span::source_map::Spanned; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; @@ -142,27 +142,16 @@ pub unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (13, 0, 0) { - if sess.target.arch == "powerpc64" { - target_data_layout = target_data_layout.replace("-S128", ""); - } - if sess.target.arch == "wasm32" { - target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); - } - if sess.target.arch == "wasm64" { - target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); - } - } - if llvm_version < (14, 0, 0) { - if sess.target.llvm_target == "i686-pc-windows-msvc" - || sess.target.llvm_target == "i586-pc-windows-msvc" - { - target_data_layout = - "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" - .to_string(); - } - if sess.target.arch == "wasm32" { - target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); + if llvm_version < (16, 0, 0) { + if sess.target.arch == "s390x" { + // LLVM 16 data layout changed to always set 64-bit vector alignment, + // which is conditional in earlier LLVM versions. + // https://reviews.llvm.org/D131158 for the discussion. + target_data_layout = target_data_layout.replace("-v128:64", ""); + } else if sess.target.arch == "riscv64" { + // LLVM 16 introduced this change so as to produce more efficient code. + // See https://reviews.llvm.org/D116735 for the discussion. + target_data_layout = target_data_layout.replace("-n32:64-", "-n64-"); } } @@ -192,7 +181,7 @@ pub unsafe fn create_module<'ll>( // // FIXME(#34960) let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); - let custom_llvm_used = cfg_llvm_root.trim() != ""; + let custom_llvm_used = !cfg_llvm_root.trim().is_empty(); if !custom_llvm_used && target_data_layout != llvm_data_layout { bug!( @@ -234,18 +223,40 @@ pub unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); - llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + c"RtLibUseGOT".as_ptr().cast(), + 1, + ); } - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. - let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. + // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) + if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Warning, - canonical_jump_tables, + llvm::LLVMModFlagBehavior::Override, + c"CFI Canonical Jump Tables".as_ptr().cast(), + 1, + ); + } + + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) + if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"EnableSplitLTOUnit".as_ptr().cast(), + 1, + ); + } + + // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) + if sess.is_sanitizer_kcfi_enabled() { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"kcfi".as_ptr().cast(), 1, ); } @@ -259,7 +270,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 1, ) } @@ -268,7 +279,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 2, ) } @@ -276,34 +287,43 @@ pub unsafe fn create_module<'ll>( } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { - if sess.target.arch != "aarch64" { - sess.err("-Zbranch-protection is only supported on aarch64"); + let behavior = if llvm_version >= (15, 0, 0) { + llvm::LLVMModFlagBehavior::Min } else { + llvm::LLVMModFlagBehavior::Error + }; + + if sess.target.arch == "aarch64" { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "branch-target-enforcement\0".as_ptr().cast(), + behavior, + c"branch-target-enforcement".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address\0".as_ptr().cast(), + behavior, + c"sign-return-address".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address-all\0".as_ptr().cast(), + behavior, + c"sign-return-address-all".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Error, - "sign-return-address-with-bkey\0".as_ptr().cast(), + behavior, + c"sign-return-address-with-bkey".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); + } else { + bug!( + "branch-protection used on non-AArch64 target; \ + this should be checked in rustc_session." + ); } } @@ -312,7 +332,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-branch\0".as_ptr().cast(), + c"cf-protection-branch".as_ptr().cast(), 1, ) } @@ -320,7 +340,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-return\0".as_ptr().cast(), + c"cf-protection-return".as_ptr().cast(), 1, ) } @@ -329,7 +349,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Error, - "Virtual Function Elim\0".as_ptr().cast(), + c"Virtual Function Elim".as_ptr().cast(), 1, ); } @@ -403,12 +423,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); - let coverage_cx = if tcx.sess.instrument_coverage() { - let covctx = coverageinfo::CrateCoverageContext::new(); - Some(covctx) - } else { - None - }; + let coverage_cx = + tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new); let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); @@ -464,15 +480,14 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { self.coverage_cx.as_ref() } - fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { - let section = cstr!("llvm.metadata"); + pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { let array = self.const_array(self.type_ptr_to(self.type_i8()), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, section.as_ptr()); + llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); } } } @@ -520,14 +535,9 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let llfn = match tcx.lang_items().eh_personality() { Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( - ty::Instance::resolve( - tcx, - ty::ParamEnv::reveal_all(), - def_id, - tcx.intern_substs(&[]), - ) - .unwrap() - .unwrap(), + ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty()) + .unwrap() + .unwrap(), ), _ => { let name = if wants_msvc_seh(self.sess()) { @@ -562,14 +572,6 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.codegen_unit } - fn used_statics(&self) -> &RefCell<Vec<&'ll Value>> { - &self.used_statics - } - - fn compiler_used_statics(&self) -> &RefCell<Vec<&'ll Value>> { - &self.compiler_used_statics - } - fn set_frame_pointer_type(&self, llfn: &'ll Value) { if let Some(attr) = attributes::frame_pointer_type_attr(self) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); @@ -583,20 +585,15 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } - fn create_used_variable(&self) { - self.create_used_variable_impl(cstr!("llvm.used"), &*self.used_statics.borrow()); - } - - fn create_compiler_used_variable(&self) { - self.create_used_variable_impl( - cstr!("llvm.compiler.used"), - &*self.compiler_used_statics.borrow(), - ); - } - fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> { - if self.get_declared_value("main").is_none() { - Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) + let entry_name = self.sess().target.entry_name.as_ref(); + if self.get_declared_value(entry_name).is_none() { + Some(self.declare_entry_fn( + entry_name, + self.sess().target.entry_abi.into(), + llvm::UnnamedAddr::Global, + fn_type, + )) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} @@ -753,9 +750,13 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.round.f32", fn(t_f32) -> t_f32); ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32); + ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64); + ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); @@ -897,6 +898,9 @@ impl<'ll> CodegenCx<'ll, '_> { ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); } + + ifn!("llvm.ptrmask", fn(i8p, t_isize) -> i8p); + None } @@ -969,9 +973,9 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { if let LayoutError::SizeOverflow(_) = err { - self.sess().span_fatal(span, &err.to_string()) + self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { - span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + span_bug!(span, "failed to get layout for `{ty}`: {err:?}") } } } @@ -987,25 +991,16 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.sess().span_fatal(span, &err.to_string()) + self.sess().emit_fatal(Spanned { span, node: err }) } else { match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { - span_bug!( - span, - "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", - sig, - extra_args, - err - ); + span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } FnAbiRequest::OfInstance { instance, extra_args } => { span_bug!( span, - "`fn_abi_of_instance({}, {:?})` failed: {}", - instance, - extra_args, - err + "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } } |