diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 46 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 20 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 101 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 20 |
12 files changed, 123 insertions, 106 deletions
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 92a857c2adc..9d4497d73a8 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -411,26 +411,31 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // the string "false". Now it is disabled by absence of the attribute. to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); } - } else if llvm_util::get_version() >= (19, 0, 0) { - // For non-naked functions, set branch protection attributes on aarch64. - if let Some(BranchProtection { bti, pac_ret }) = - cx.sess().opts.unstable_opts.branch_protection - { - assert!(cx.sess().target.arch == "aarch64"); - if bti { - to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); - } - if let Some(PacRet { leaf, key }) = pac_ret { - to_add.push(llvm::CreateAttrStringValue( - cx.llcx, - "sign-return-address", - if leaf { "all" } else { "non-leaf" }, - )); - to_add.push(llvm::CreateAttrStringValue( - cx.llcx, - "sign-return-address-key", - if key == PAuthKey::A { "a_key" } else { "b_key" }, - )); + } else { + // Do not set sanitizer attributes for naked functions. + to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + + if llvm_util::get_version() >= (19, 0, 0) { + // For non-naked functions, set branch protection attributes on aarch64. + if let Some(BranchProtection { bti, pac_ret }) = + cx.sess().opts.unstable_opts.branch_protection + { + assert!(cx.sess().target.arch == "aarch64"); + if bti { + to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); + } + if let Some(PacRet { leaf, key }) = pac_ret { + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "sign-return-address", + if leaf { "all" } else { "non-leaf" }, + )); + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "sign-return-address-key", + if key == PAuthKey::A { "a_key" } else { "b_key" }, + )); + } } } } @@ -485,7 +490,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(backchain) = backchain_attr(cx) { to_add.push(backchain); } - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); // Always annotate functions with the target-cpu they are compiled for. diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index aa6842c75ce..2ebe0be53aa 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -92,11 +92,9 @@ fn prepare_lto( dcx.emit_err(LtoDylib); return Err(FatalError); } - } else if *crate_type == CrateType::ProcMacro { - if !cgcx.opts.unstable_opts.dylib_lto { - dcx.emit_err(LtoProcMacro); - return Err(FatalError); - } + } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto { + dcx.emit_err(LtoProcMacro); + return Err(FatalError); } } @@ -314,7 +312,6 @@ fn fat_lto( } } }; - let mut serialized_bitcode = Vec::new(); { let (llcx, llmod) = { let llvm = &module.module_llvm; @@ -342,9 +339,7 @@ fn fat_lto( serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1)); // For all serialized bitcode files we parse them and link them in as we did - // above, this is all mostly handled in C++. Like above, though, we don't - // know much about the memory management here so we err on the side of being - // save and persist everything with the original module. + // above, this is all mostly handled in C++. let mut linker = Linker::new(llmod); for (bc_decoded, name) in serialized_modules { let _timer = cgcx @@ -355,7 +350,6 @@ fn fat_lto( info!("linking {:?}", name); let data = bc_decoded.data(); linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?; - serialized_bitcode.push(bc_decoded); } drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); @@ -372,7 +366,7 @@ fn fat_lto( } } - Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) + Ok(LtoModuleCodegen::Fat(module)) } pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index a5c27d2282e..b1b692cc027 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -185,7 +185,13 @@ pub(crate) fn target_machine_factory( let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); - let use_softfp = sess.opts.cg.soft_float; + let use_softfp = if sess.target.arch == "arm" && sess.target.abi == "eabihf" { + sess.opts.cg.soft_float + } else { + // `validate_commandline_args_with_session_available` has already warned about this being ignored. + // Let's make sure LLVM doesn't suddenly start using this flag on more targets. + false + }; let ffunction_sections = sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index b143e28c5f9..d60122fccee 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -390,7 +390,7 @@ impl<'ll> CodegenCx<'ll, '_> { let val_llty = self.val_ty(v); let g = self.get_static_inner(def_id, val_llty); - let llty = self.val_ty(g); + let llty = llvm::LLVMGlobalGetValueType(g); let g = if val_llty == llty { g diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 01aae24ab56..73c2c15717f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -27,7 +27,7 @@ use rustc_span::source_map::Spanned; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{HasDataLayout, TargetDataLayout, VariantIdx}; -use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; +use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; @@ -387,6 +387,24 @@ pub(crate) unsafe fn create_module<'ll>( } } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) + { + // Set up the small-data optimization limit for architectures that use + // an LLVM module flag to control this. + (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => { + let flag = SmallCStr::new(flag.as_ref()); + unsafe { + llvm::LLVMRustAddModuleFlagU32( + llmod, + llvm::LLVMModFlagBehavior::Error, + flag.as_c_str().as_ptr(), + threshold as u32, + ) + } + } + _ => (), + }; + // Insert `llvm.ident` metadata. // // On the wasm targets it will get hooked up to the "producer" sections diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 069b62af5e7..b5acfabfde2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -422,7 +422,7 @@ fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> { (instance.def_id(), body) }); - // Functions whose coverage statments were found inlined into other functions. + // Functions whose coverage statements were found inlined into other functions. let mut used_via_inlining = FxHashSet::default(); // Functions that were instrumented, but had all of their coverage statements // removed by later MIR transforms (e.g. UnreachablePropagation). diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 17a9630c655..d231b103964 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -456,7 +456,7 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> if def.is_box() && args.get(1).map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) => { - build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) + build_pointer_or_reference_di_node(cx, t, t.expect_boxed_ty(), unique_type_id) } ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index f5d6fc6f080..842212ac05d 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -1,8 +1,8 @@ #![doc = include_str!("doc.md")] use std::cell::{OnceCell, RefCell}; -use std::iter; use std::ops::Range; +use std::{iter, ptr}; use libc::c_uint; use rustc_codegen_ssa::debuginfo::type_names; @@ -209,6 +209,12 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { } } + fn clear_dbg_loc(&mut self) { + unsafe { + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, ptr::null()); + } + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { gdb::insert_reference_to_gdb_debug_scripts_section_global(self) } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index a300f5f707a..15c519dfcb4 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -187,9 +187,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { Some(instance), ) } - sym::likely => { - self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) - } + sym::likely => self.expect(args[0].immediate(), true), sym::is_val_statically_known => { let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); let kind = self.type_kind(intrinsic_type); @@ -210,8 +208,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.const_bool(false) } } - sym::unlikely => self - .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), + sym::unlikely => self.expect(args[0].immediate(), false), sym::select_unpredictable => { let cond = args[0].immediate(); assert_eq!(args[1].layout, args[2].layout); @@ -576,6 +573,8 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { span, ) { Ok(llval) => llval, + // If there was an error, just skip this invocation... we'll abort compilation anyway, + // but we can keep codegen'ing to find more errors. Err(()) => return Ok(()), } } @@ -604,11 +603,17 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } fn assume(&mut self, val: Self::Value) { - self.call_intrinsic("llvm.assume", &[val]); + if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No { + self.call_intrinsic("llvm.assume", &[val]); + } } fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { - self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) + if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No { + self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) + } else { + cond + } } fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { @@ -1287,24 +1292,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle { - // Make sure this is actually an array or SIMD vector, since typeck only checks the length-suffixed - // version of this intrinsic. + // Make sure this is actually a SIMD vector. let idx_ty = args[2].layout.ty; - let n: u64 = match idx_ty.kind() { - ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( - || span_bug!(span, "could not evaluate shuffle index array length"), - ) - } - _ if idx_ty.is_simd() - && matches!( - idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), - ty::Uint(ty::UintTy::U32) - ) => - { - idx_ty.simd_size_and_type(bx.cx.tcx).0 - } - _ => return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }), + let n: u64 = if idx_ty.is_simd() + && matches!(idx_ty.simd_size_and_type(bx.cx.tcx).1.kind(), ty::Uint(ty::UintTy::U32)) + { + idx_ty.simd_size_and_type(bx.cx.tcx).0 + } else { + return_error!(InvalidMonomorphization::SimdShuffle { span, name, ty: idx_ty }) }; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); @@ -1319,38 +1314,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let total_len = u128::from(in_len) * 2; - let vector = args[2].immediate(); - - let indices: Option<Vec<_>> = (0..n) - .map(|i| { - let arg_idx = i; - let val = bx.const_get_elt(vector, i as u64); - match bx.const_to_opt_u128(val, true) { - None => { - bug!("typeck should have already ensured that these are const") - } - Some(idx) if idx >= total_len => { - bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { - span, - name, - arg_idx, - total_len, - }); - None - } - Some(idx) => Some(bx.const_i32(idx as i32)), - } - }) - .collect(); - let Some(indices) = indices else { - return Ok(bx.const_null(llret_ty)); - }; + // Check that the indices are in-bounds. + let indices = args[2].immediate(); + for i in 0..n { + let val = bx.const_get_elt(indices, i as u64); + let idx = bx + .const_to_opt_u128(val, true) + .unwrap_or_else(|| bug!("typeck should have already ensured that these are const")); + if idx >= total_len { + return_error!(InvalidMonomorphization::SimdIndexOutOfBounds { + span, + name, + arg_idx: i, + total_len, + }); + } + } - return Ok(bx.shuffle_vector( - args[0].immediate(), - args[1].immediate(), - bx.const_vector(&indices), - )); + return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), indices)); } if name == sym::simd_insert { @@ -1368,13 +1349,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .const_to_opt_u128(args[1].immediate(), false) .expect("typeck should have ensure that this is a const"); if idx >= in_len.into() { - bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + return_error!(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: 1, total_len: in_len.into(), }); - return Ok(bx.const_null(llret_ty)); } return Ok(bx.insert_element( args[0].immediate(), @@ -1391,13 +1371,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .const_to_opt_u128(args[1].immediate(), false) .expect("typeck should have ensure that this is a const"); if idx >= in_len.into() { - bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { + return_error!(InvalidMonomorphization::SimdIndexOutOfBounds { span, name, arg_idx: 1, total_len: in_len.into(), }); - return Ok(bx.const_null(llret_ty)); } return Ok(bx.extract_element(args[0].immediate(), bx.const_i32(idx as i32))); } @@ -2087,14 +2066,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; } - arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0); + arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, -0.0); arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); arith_red!( simd_reduce_add_unordered: vector_reduce_add, vector_reduce_fadd_reassoc, false, add, - 0.0 + -0.0 ); arith_red!( simd_reduce_mul_unordered: vector_reduce_mul, diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 6a303e1e602..7f26bbd7f87 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -6,7 +6,6 @@ // tidy-alphabetical-start #![allow(internal_features)] -#![cfg_attr(bootstrap, feature(unsafe_extern_blocks))] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] #![feature(assert_matches)] diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 138cc3219aa..e84ab0aa538 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -974,6 +974,7 @@ unsafe extern "C" { pub fn LLVMGetAlignment(Global: &Value) -> c_uint; pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); + pub fn LLVMGlobalGetValueType(Global: &Value) -> &Type; // Operations on global variables pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; @@ -1040,7 +1041,7 @@ unsafe extern "C" { pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata); + pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 618602ed70f..29afe6f6bfc 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -14,7 +14,7 @@ use rustc_middle::bug; use rustc_session::config::{PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_target::spec::{MergeFunctions, PanicStrategy}; +use rustc_target::spec::{MergeFunctions, PanicStrategy, SmallDataThresholdSupport}; use rustc_target::target_features::{RUSTC_SPECIAL_FEATURES, RUSTC_SPECIFIC_FEATURES}; use crate::back::write::create_informational_target_machine; @@ -125,6 +125,18 @@ unsafe fn configure_llvm(sess: &Session) { for arg in sess_args { add(&(*arg), true); } + + match ( + sess.opts.unstable_opts.small_data_threshold, + sess.target.small_data_threshold_support(), + ) { + // Set up the small-data optimization limit for architectures that use + // an LLVM argument to control this. + (Some(threshold), SmallDataThresholdSupport::LlvmArg(arg)) => { + add(&format!("--{arg}={threshold}"), false) + } + _ => (), + }; } if sess.opts.unstable_opts.llvm_time_trace { @@ -290,7 +302,7 @@ pub(crate) fn check_tied_features( } } } - return None; + None } /// Used to generate cfg variables and apply features @@ -353,9 +365,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { None } }) - .filter(|feature| { - RUSTC_SPECIAL_FEATURES.contains(feature) || features.contains(&Symbol::intern(feature)) - }) + .filter(|feature| features.contains(&Symbol::intern(feature))) .map(|feature| Symbol::intern(feature)) .collect() } |