From bcfc9b5073a92bbb4b1e4db2eab535357d8973ad Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Thu, 21 Aug 2025 14:32:10 +0100 Subject: inline at the callsite & warn when target features mismatch Co-authored-by: Jamie Cunliffe --- compiler/rustc_codegen_llvm/src/attributes.rs | 31 ++++++++++++++++++--------- compiler/rustc_codegen_llvm/src/builder.rs | 29 ++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 13 deletions(-) (limited to 'compiler/rustc_codegen_llvm') diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 5affb26483a..9f2d37d39d8 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -29,8 +29,18 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } /// Get LLVM attribute for the provided inline heuristic. -#[inline] -fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { +pub(crate) fn inline_attr<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: ty::Instance<'tcx>, +) -> Option<&'ll Attribute> { + // `optnone` requires `noinline` + let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { + (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, + (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, + (inline, _) => inline, + }; + if !cx.tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); @@ -346,14 +356,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - // `optnone` requires `noinline` - let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { - (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, - (inline, _) => inline, - }; - to_add.extend(inline_attr(cx, inline)); - if cx.sess().must_emit_unwind_tables() { to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); } @@ -488,6 +490,14 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::>(); + // Apply function attributes as per usual if there are no user defined + // target features otherwise this will get applied at the callsite. + if function_features.is_empty() { + if let Some(inline_attr) = inline_attr(cx, instance) { + to_add.push(inline_attr); + } + } + let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones @@ -517,6 +527,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( let function_features = function_features.iter().map(|s| s.as_str()); let target_features: String = global_features.chain(function_features).intersperse(",").collect(); + if !target_features.is_empty() { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 37379586d58..7d0691366e6 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1392,7 +1392,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, - fn_attrs: Option<&CodegenFnAttrs>, + fn_call_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1409,10 +1409,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } // Emit CFI pointer type membership test - self.cfi_type_test(fn_attrs, fn_abi, instance, llfn); + self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn); // Emit KCFI operand bundle - let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); + let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn); if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1429,6 +1429,29 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { c"".as_ptr(), ) }; + + if let Some(instance) = instance { + // Attributes on the function definition being called + let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id()); + if let Some(fn_call_attrs) = fn_call_attrs + && !fn_call_attrs.target_features.is_empty() + // If there is an inline attribute and a target feature that matches + // we will add the attribute to the callsite otherwise we'll omit + // this and not add the attribute to prevent soundness issues. + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance) + && self.cx.tcx.is_target_feature_call_safe( + &fn_call_attrs.target_features, + &fn_defn_attrs.target_features, + ) + { + attributes::apply_to_callsite( + call, + llvm::AttributePlace::Function, + &[inlining_rule], + ); + } + } + if let Some(fn_abi) = fn_abi { fn_abi.apply_attrs_callsite(self, call); } -- cgit 1.4.1-3-g733a5 From 5d9f8fcd3ea44bdd633ecddac2aff471ffcb797f Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 2 Sep 2025 18:43:48 +0000 Subject: llvm: nvptx: Layout update to match LLVM LLVM upstream switched layouts to support 256-bit vector load/store. --- compiler/rustc_codegen_llvm/src/context.rs | 4 ++++ compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 4fd6110ac4a..257c7b95666 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -217,6 +217,10 @@ pub(crate) unsafe fn create_module<'ll>( // LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010 target_data_layout = target_data_layout.replace("n8:16", "n8") } + if sess.target.arch == "nvptx64" { + // LLVM 22 updated the NVPTX layout to indicate 256-bit vector load/store: https://github.com/llvm/llvm-project/pull/155198 + target_data_layout = target_data_layout.replace("-i256:256", ""); + } } // Ensure the data-layout values hardcoded remain the defaults. diff --git a/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs b/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs index 598f0f19f0d..cada0dd640a 100644 --- a/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs +++ b/compiler/rustc_target/src/spec/targets/nvptx64_nvidia_cuda.rs @@ -6,7 +6,7 @@ use crate::spec::{ pub(crate) fn target() -> Target { Target { arch: "nvptx64".into(), - data_layout: "e-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64".into(), + data_layout: "e-p6:32:32-i64:64-i128:128-i256:256-v16:16-v32:32-n16:32:64".into(), llvm_target: "nvptx64-nvidia-cuda".into(), metadata: TargetMetadata { description: Some("--emit=asm generates PTX code that runs on NVIDIA GPUs".into()), -- cgit 1.4.1-3-g733a5 From 62b4347e80cc86314bd98749e95eff8cdf8ef005 Mon Sep 17 00:00:00 2001 From: sayantn Date: Thu, 21 Aug 2025 04:47:05 +0530 Subject: Add `funnel_sh{l,r}` functions and intrinsics - Add a fallback implementation for the intrinsics - Add LLVM backend support for funnel shifts Co-Authored-By: folkertdev --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 26 +++++--- compiler/rustc_hir_analysis/src/check/intrinsic.rs | 3 + compiler/rustc_span/src/symbol.rs | 2 + library/core/src/intrinsics/fallback.rs | 73 ++++++++++++++++++++++ library/core/src/intrinsics/mod.rs | 55 ++++++++++++++++ library/core/src/lib.rs | 1 + library/core/src/num/mod.rs | 24 +++++++ library/core/src/num/uint_macros.rs | 73 ++++++++++++++++++++++ library/coretests/tests/lib.rs | 1 + library/coretests/tests/num/uint_macros.rs | 36 +++++++++++ library/std/src/lib.rs | 1 + src/tools/miri/tests/fail/intrinsics/funnel_shl.rs | 7 +++ .../miri/tests/fail/intrinsics/funnel_shl.stderr | 15 +++++ src/tools/miri/tests/fail/intrinsics/funnel_shr.rs | 7 +++ .../miri/tests/fail/intrinsics/funnel_shr.stderr | 15 +++++ src/tools/miri/tests/pass/intrinsics/integer.rs | 8 ++- 16 files changed, 338 insertions(+), 9 deletions(-) create mode 100644 src/tools/miri/tests/fail/intrinsics/funnel_shl.rs create mode 100644 src/tools/miri/tests/fail/intrinsics/funnel_shl.stderr create mode 100644 src/tools/miri/tests/fail/intrinsics/funnel_shr.rs create mode 100644 src/tools/miri/tests/fail/intrinsics/funnel_shr.stderr (limited to 'compiler/rustc_codegen_llvm') diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 49d3dedbeab..85f71f331a4 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -383,7 +383,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::rotate_left | sym::rotate_right | sym::saturating_add - | sym::saturating_sub => { + | sym::saturating_sub + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { let ty = args[0].layout.ty; if !ty.is_integral() { tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { @@ -424,18 +426,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::bitreverse => { self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()]) } - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same + sym::rotate_left + | sym::rotate_right + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { + let is_left = name == sym::rotate_left || name == sym::unchecked_funnel_shl; + let lhs = args[0].immediate(); + let (rhs, raw_shift) = + if name == sym::rotate_left || name == sym::rotate_right { + // rotate = funnel shift with first two args the same + (lhs, args[1].immediate()) + } else { + (args[1].immediate(), args[2].immediate()) + }; let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' }); // llvm expects shift to be the same type as the values, but rust // always uses `u32`. - let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false); - self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift]) + self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index cfc6bc2f3a0..aa2d27ab809 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -449,6 +449,9 @@ pub(crate) fn check_intrinsic_type( } sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)), sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)), + sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => { + (1, 0, vec![param(0), param(0), tcx.types.u32], param(0)) + } sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => { (1, 0, vec![param(0), param(0)], param(0)) } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index dcb1becc957..64662d48137 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2269,6 +2269,8 @@ symbols! { unboxed_closures, unchecked_add, unchecked_div, + unchecked_funnel_shl, + unchecked_funnel_shr, unchecked_mul, unchecked_rem, unchecked_shl, diff --git a/library/core/src/intrinsics/fallback.rs b/library/core/src/intrinsics/fallback.rs index eec5c4d646d..96928325328 100644 --- a/library/core/src/intrinsics/fallback.rs +++ b/library/core/src/intrinsics/fallback.rs @@ -148,3 +148,76 @@ impl_disjoint_bitor! { u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, } + +#[const_trait] +#[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")] +pub trait FunnelShift: Copy + 'static { + /// See [`super::unchecked_funnel_shl`]; we just need the trait indirection to handle + /// different types since calling intrinsics with generics doesn't work. + unsafe fn unchecked_funnel_shl(self, rhs: Self, shift: u32) -> Self; + + /// See [`super::unchecked_funnel_shr`]; we just need the trait indirection to handle + /// different types since calling intrinsics with generics doesn't work. + unsafe fn unchecked_funnel_shr(self, rhs: Self, shift: u32) -> Self; +} + +macro_rules! impl_funnel_shifts { + ($($type:ident),*) => {$( + #[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")] + impl const FunnelShift for $type { + #[cfg_attr(miri, track_caller)] + #[inline] + unsafe fn unchecked_funnel_shl(self, rhs: Self, shift: u32) -> Self { + // This implementation is also used by Miri so we have to check the precondition. + // SAFETY: this is guaranteed by the caller + unsafe { super::assume(shift < $type::BITS) }; + if shift == 0 { + self + } else { + // SAFETY: + // - `shift < T::BITS`, which satisfies `unchecked_shl` + // - this also ensures that `T::BITS - shift < T::BITS` (shift = 0 is checked + // above), which satisfies `unchecked_shr` + // - because the types are unsigned, the combination are disjoint bits (this is + // not true if they're signed, since SHR will fill in the empty space with a + // sign bit, not zero) + unsafe { + super::disjoint_bitor( + super::unchecked_shl(self, shift), + super::unchecked_shr(rhs, $type::BITS - shift), + ) + } + } + } + + #[cfg_attr(miri, track_caller)] + #[inline] + unsafe fn unchecked_funnel_shr(self, rhs: Self, shift: u32) -> Self { + // This implementation is also used by Miri so we have to check the precondition. + // SAFETY: this is guaranteed by the caller + unsafe { super::assume(shift < $type::BITS) }; + if shift == 0 { + rhs + } else { + // SAFETY: + // - `shift < T::BITS`, which satisfies `unchecked_shr` + // - this also ensures that `T::BITS - shift < T::BITS` (shift = 0 is checked + // above), which satisfies `unchecked_shl` + // - because the types are unsigned, the combination are disjoint bits (this is + // not true if they're signed, since SHR will fill in the empty space with a + // sign bit, not zero) + unsafe { + super::disjoint_bitor( + super::unchecked_shl(self, $type::BITS - shift), + super::unchecked_shr(rhs, shift), + ) + } + } + } + } + )*}; +} + +impl_funnel_shifts! { + u8, u16, u32, u64, u128, usize +} diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 904aa52c784..bffffbc29c1 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -2102,6 +2102,61 @@ pub const fn saturating_add(a: T, b: T) -> T; #[rustc_intrinsic] pub const fn saturating_sub(a: T, b: T) -> T; +/// Funnel Shift left. +/// +/// Concatenates `a` and `b` (with `a` in the most significant half), +/// creating an integer twice as wide. Then shift this integer left +/// by `shift`), and extract the most significant half. If `a` and `b` +/// are the same, this is equivalent to a rotate left operation. +/// +/// It is undefined behavior if `shift` is greater than or equal to the +/// bit size of `T`. +/// +/// Safe versions of this intrinsic are available on the integer primitives +/// via the `funnel_shl` method. For example, [`u32::funnel_shl`]. +#[rustc_intrinsic] +#[rustc_nounwind] +#[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")] +#[unstable(feature = "funnel_shifts", issue = "145686")] +#[track_caller] +#[miri::intrinsic_fallback_is_spec] +pub const unsafe fn unchecked_funnel_shl( + a: T, + b: T, + shift: u32, +) -> T { + // SAFETY: caller ensures that `shift` is in-range + unsafe { a.unchecked_funnel_shl(b, shift) } +} + +/// Funnel Shift right. +/// +/// Concatenates `a` and `b` (with `a` in the most significant half), +/// creating an integer twice as wide. Then shift this integer right +/// by `shift` (taken modulo the bit size of `T`), and extract the +/// least significant half. If `a` and `b` are the same, this is equivalent +/// to a rotate right operation. +/// +/// It is undefined behavior if `shift` is greater than or equal to the +/// bit size of `T`. +/// +/// Safer versions of this intrinsic are available on the integer primitives +/// via the `funnel_shr` method. For example, [`u32::funnel_shr`] +#[rustc_intrinsic] +#[rustc_nounwind] +#[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")] +#[unstable(feature = "funnel_shifts", issue = "145686")] +#[track_caller] +#[miri::intrinsic_fallback_is_spec] +pub const unsafe fn unchecked_funnel_shr( + a: T, + b: T, + shift: u32, +) -> T { + // SAFETY: caller ensures that `shift` is in-range + unsafe { a.unchecked_funnel_shr(b, shift) } +} + /// This is an implementation detail of [`crate::ptr::read`] and should /// not be used anywhere else. See its comments for why this exists. /// diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 71abd707374..fb6cacb0e00 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -156,6 +156,7 @@ #![feature(f128)] #![feature(freeze_impls)] #![feature(fundamental)] +#![feature(funnel_shifts)] #![feature(if_let_guard)] #![feature(intra_doc_pointers)] #![feature(intrinsics)] diff --git a/library/core/src/num/mod.rs b/library/core/src/num/mod.rs index acfe38b7a37..e2987b0b7e5 100644 --- a/library/core/src/num/mod.rs +++ b/library/core/src/num/mod.rs @@ -454,6 +454,9 @@ impl u8 { rot = 2, rot_op = "0x82", rot_result = "0xa", + fsh_op = "0x36", + fshl_result = "0x8", + fshr_result = "0x8d", swap_op = "0x12", swapped = "0x12", reversed = "0x48", @@ -1088,6 +1091,9 @@ impl u16 { rot = 4, rot_op = "0xa003", rot_result = "0x3a", + fsh_op = "0x2de", + fshl_result = "0x30", + fshr_result = "0x302d", swap_op = "0x1234", swapped = "0x3412", reversed = "0x2c48", @@ -1135,6 +1141,9 @@ impl u32 { rot = 8, rot_op = "0x10000b3", rot_result = "0xb301", + fsh_op = "0x2fe78e45", + fshl_result = "0xb32f", + fshr_result = "0xb32fe78e", swap_op = "0x12345678", swapped = "0x78563412", reversed = "0x1e6a2c48", @@ -1158,6 +1167,9 @@ impl u64 { rot = 12, rot_op = "0xaa00000000006e1", rot_result = "0x6e10aa", + fsh_op = "0x2fe78e45983acd98", + fshl_result = "0x6e12fe", + fshr_result = "0x6e12fe78e45983ac", swap_op = "0x1234567890123456", swapped = "0x5634129078563412", reversed = "0x6a2c48091e6a2c48", @@ -1181,6 +1193,9 @@ impl u128 { rot = 16, rot_op = "0x13f40000000000000000000000004f76", rot_result = "0x4f7613f4", + fsh_op = "0x2fe78e45983acd98039000008736273", + fshl_result = "0x4f7602fe", + fshr_result = "0x4f7602fe78e45983acd9803900000873", swap_op = "0x12345678901234567890123456789012", swapped = "0x12907856341290785634129078563412", reversed = "0x48091e6a2c48091e6a2c48091e6a2c48", @@ -1207,6 +1222,9 @@ impl usize { rot = 4, rot_op = "0xa003", rot_result = "0x3a", + fsh_op = "0x2fe78e45983acd98039000008736273", + fshl_result = "0x4f7602fe", + fshr_result = "0x4f7602fe78e45983acd9803900000873", swap_op = "0x1234", swapped = "0x3412", reversed = "0x2c48", @@ -1231,6 +1249,9 @@ impl usize { rot = 8, rot_op = "0x10000b3", rot_result = "0xb301", + fsh_op = "0x2fe78e45", + fshl_result = "0xb32f", + fshr_result = "0xb32fe78e", swap_op = "0x12345678", swapped = "0x78563412", reversed = "0x1e6a2c48", @@ -1255,6 +1276,9 @@ impl usize { rot = 12, rot_op = "0xaa00000000006e1", rot_result = "0x6e10aa", + fsh_op = "0x2fe78e45983acd98", + fshl_result = "0x6e12fe", + fshr_result = "0x6e12fe78e45983ac", swap_op = "0x1234567890123456", swapped = "0x5634129078563412", reversed = "0x6a2c48091e6a2c48", diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs index 10d9498d15e..1fef3c822b7 100644 --- a/library/core/src/num/uint_macros.rs +++ b/library/core/src/num/uint_macros.rs @@ -14,6 +14,9 @@ macro_rules! uint_impl { rot = $rot:literal, rot_op = $rot_op:literal, rot_result = $rot_result:literal, + fsh_op = $fsh_op:literal, + fshl_result = $fshl_result:literal, + fshr_result = $fshr_result:literal, swap_op = $swap_op:literal, swapped = $swapped:literal, reversed = $reversed:literal, @@ -375,6 +378,76 @@ macro_rules! uint_impl { return intrinsics::rotate_right(self, n); } + /// Performs a left funnel shift (concatenates `self` with `rhs`, with `self` + /// making up the most significant half, then shifts the combined value left + /// by `n`, and most significant half is extracted to produce the result). + /// + /// Please note this isn't the same operation as the `<<` shifting operator or + /// [`rotate_left`](Self::rotate_left), although `a.funnel_shl(a, n)` is *equivalent* + /// to `a.rotate_left(n)`. + /// + /// # Panics + /// + /// If `n` is greater than or equal to the number of bits in `self` + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(funnel_shifts)] + #[doc = concat!("let a = ", $rot_op, stringify!($SelfT), ";")] + #[doc = concat!("let b = ", $fsh_op, stringify!($SelfT), ";")] + #[doc = concat!("let m = ", $fshl_result, ";")] + /// + #[doc = concat!("assert_eq!(a.funnel_shl(b, ", $rot, "), m);")] + /// ``` + #[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")] + #[unstable(feature = "funnel_shifts", issue = "145686")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn funnel_shl(self, rhs: Self, n: u32) -> Self { + assert!(n < Self::BITS, "attempt to funnel shift left with overflow"); + // SAFETY: just checked that `shift` is in-range + unsafe { intrinsics::unchecked_funnel_shl(self, rhs, n) } + } + + /// Performs a right funnel shift (concatenates `self` and `rhs`, with `self` + /// making up the most significant half, then shifts the combined value right + /// by `n`, and least significant half is extracted to produce the result). + /// + /// Please note this isn't the same operation as the `>>` shifting operator or + /// [`rotate_right`](Self::rotate_right), although `a.funnel_shr(a, n)` is *equivalent* + /// to `a.rotate_right(n)`. + /// + /// # Panics + /// + /// If `n` is greater than or equal to the number of bits in `self` + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(funnel_shifts)] + #[doc = concat!("let a = ", $rot_op, stringify!($SelfT), ";")] + #[doc = concat!("let b = ", $fsh_op, stringify!($SelfT), ";")] + #[doc = concat!("let m = ", $fshr_result, ";")] + /// + #[doc = concat!("assert_eq!(a.funnel_shr(b, ", $rot, "), m);")] + /// ``` + #[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")] + #[unstable(feature = "funnel_shifts", issue = "145686")] + #[must_use = "this returns the result of the operation, \ + without modifying the original"] + #[inline(always)] + pub const fn funnel_shr(self, rhs: Self, n: u32) -> Self { + assert!(n < Self::BITS, "attempt to funnel shift right with overflow"); + // SAFETY: just checked that `shift` is in-range + unsafe { intrinsics::unchecked_funnel_shr(self, rhs, n) } + } + /// Reverses the byte order of the integer. /// /// # Examples diff --git a/library/coretests/tests/lib.rs b/library/coretests/tests/lib.rs index d2281b1df2f..c16c344776d 100644 --- a/library/coretests/tests/lib.rs +++ b/library/coretests/tests/lib.rs @@ -50,6 +50,7 @@ #![feature(fmt_internals)] #![feature(formatting_options)] #![feature(freeze)] +#![feature(funnel_shifts)] #![feature(future_join)] #![feature(generic_assert_internals)] #![feature(hasher_prefixfree_extras)] diff --git a/library/coretests/tests/num/uint_macros.rs b/library/coretests/tests/num/uint_macros.rs index c7d10ea4d88..63be8a45b5c 100644 --- a/library/coretests/tests/num/uint_macros.rs +++ b/library/coretests/tests/num/uint_macros.rs @@ -104,6 +104,19 @@ macro_rules! uint_module { assert_eq_const_safe!($T: C.rotate_left(128), C); } + fn test_funnel_shift() { + // Shifting by 0 should have no effect + assert_eq_const_safe!($T: <$T>::funnel_shl(A, B, 0), A); + assert_eq_const_safe!($T: <$T>::funnel_shr(A, B, 0), B); + + assert_eq_const_safe!($T: <$T>::funnel_shl(_0, _1, 4), 0b1111); + assert_eq_const_safe!($T: <$T>::funnel_shr(_0, _1, 4), _1 >> 4); + assert_eq_const_safe!($T: <$T>::funnel_shl(_1, _0, 4), _1 << 4); + + assert_eq_const_safe!($T: <$T>::funnel_shl(_1, _1, 4), <$T>::rotate_left(_1, 4)); + assert_eq_const_safe!($T: <$T>::funnel_shr(_1, _1, 4), <$T>::rotate_right(_1, 4)); + } + fn test_swap_bytes() { assert_eq_const_safe!($T: A.swap_bytes().swap_bytes(), A); assert_eq_const_safe!($T: B.swap_bytes().swap_bytes(), B); @@ -150,6 +163,29 @@ macro_rules! uint_module { } } + #[test] + #[should_panic = "attempt to funnel shift left with overflow"] + fn test_funnel_shl_overflow() { + let _ = <$T>::funnel_shl(A, B, $T::BITS); + } + + #[test] + #[should_panic = "attempt to funnel shift right with overflow"] + fn test_funnel_shr_overflow() { + let _ = <$T>::funnel_shr(A, B, $T::BITS); + } + + #[test] + fn test_funnel_shifts_runtime() { + for i in 0..$T::BITS - 1 { + assert_eq!(<$T>::funnel_shl(A, 0, i), A << i); + assert_eq!(<$T>::funnel_shl(A, A, i), A.rotate_left(i)); + + assert_eq!(<$T>::funnel_shr(0, A, i), A >> i); + assert_eq!(<$T>::funnel_shr(A, A, i), A.rotate_right(i)); + } + } + #[test] fn test_isolate_highest_one() { const BITS: $T = <$T>::MAX; diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index ab417b6c72f..75018424c4e 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -296,6 +296,7 @@ #![feature(f128)] #![feature(ffi_const)] #![feature(formatting_options)] +#![feature(funnel_shifts)] #![feature(hash_map_internals)] #![feature(hash_map_macro)] #![feature(if_let_guard)] diff --git a/src/tools/miri/tests/fail/intrinsics/funnel_shl.rs b/src/tools/miri/tests/fail/intrinsics/funnel_shl.rs new file mode 100644 index 00000000000..40b61f65ebe --- /dev/null +++ b/src/tools/miri/tests/fail/intrinsics/funnel_shl.rs @@ -0,0 +1,7 @@ +#![feature(core_intrinsics, funnel_shifts)] + +fn main() { + unsafe { + std::intrinsics::unchecked_funnel_shl(1_u32, 2, 32); //~ ERROR: Undefined Behavior + } +} diff --git a/src/tools/miri/tests/fail/intrinsics/funnel_shl.stderr b/src/tools/miri/tests/fail/intrinsics/funnel_shl.stderr new file mode 100644 index 00000000000..fc828021b13 --- /dev/null +++ b/src/tools/miri/tests/fail/intrinsics/funnel_shl.stderr @@ -0,0 +1,15 @@ +error: Undefined Behavior: `assume` called with `false` + --> tests/fail/intrinsics/funnel_shl.rs:LL:CC + | +LL | std::intrinsics::unchecked_funnel_shl(1_u32, 2, 32); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + = note: BACKTRACE: + = note: inside `main` at tests/fail/intrinsics/funnel_shl.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/intrinsics/funnel_shr.rs b/src/tools/miri/tests/fail/intrinsics/funnel_shr.rs new file mode 100644 index 00000000000..95822656397 --- /dev/null +++ b/src/tools/miri/tests/fail/intrinsics/funnel_shr.rs @@ -0,0 +1,7 @@ +#![feature(core_intrinsics, funnel_shifts)] + +fn main() { + unsafe { + std::intrinsics::unchecked_funnel_shr(1_u32, 2, 32); //~ ERROR: Undefined Behavior + } +} diff --git a/src/tools/miri/tests/fail/intrinsics/funnel_shr.stderr b/src/tools/miri/tests/fail/intrinsics/funnel_shr.stderr new file mode 100644 index 00000000000..7361df1b4c7 --- /dev/null +++ b/src/tools/miri/tests/fail/intrinsics/funnel_shr.stderr @@ -0,0 +1,15 @@ +error: Undefined Behavior: `assume` called with `false` + --> tests/fail/intrinsics/funnel_shr.rs:LL:CC + | +LL | std::intrinsics::unchecked_funnel_shr(1_u32, 2, 32); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + = note: BACKTRACE: + = note: inside `main` at tests/fail/intrinsics/funnel_shr.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/pass/intrinsics/integer.rs b/src/tools/miri/tests/pass/intrinsics/integer.rs index 13e7bd8e1b9..8727b6d3c87 100644 --- a/src/tools/miri/tests/pass/intrinsics/integer.rs +++ b/src/tools/miri/tests/pass/intrinsics/integer.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 // SPDX-FileCopyrightText: The Rust Project Developers (see https://thanks.rust-lang.org) -#![feature(core_intrinsics)] +#![feature(core_intrinsics, funnel_shifts)] use std::intrinsics::*; pub fn main() { @@ -143,5 +143,11 @@ pub fn main() { assert_eq!(unchecked_mul(6u8, 7), 42); assert_eq!(unchecked_mul(13, -5), -65); + + assert_eq!(unchecked_funnel_shl(1_u32, 2, 5), 32); + assert_eq!(unchecked_funnel_shl(1_u32, 2, 31), 0x80000001); + + assert_eq!(unchecked_funnel_shr(1_u32, 2, 5), 0x08000000); + assert_eq!(unchecked_funnel_shr(1_u32, 2, 31), 2); } } -- cgit 1.4.1-3-g733a5 From 319fe230f0d960b343be31a1182dc0f10753156c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:26:29 +0000 Subject: Special case allocator module submission to avoid special casing it elsewhere A lot of places had special handling just in case they would get an allocator module even though most of these places could never get one or would have a trivial implementation for the allocator module. Moving all handling of the allocator module to a single place simplifies things a fair bit. --- compiler/rustc_codegen_cranelift/src/driver/aot.rs | 11 +- compiler/rustc_codegen_gcc/src/back/lto.rs | 2 +- compiler/rustc_codegen_llvm/src/back/lto.rs | 22 ++-- compiler/rustc_codegen_ssa/src/back/write.rs | 135 ++++++++------------- compiler/rustc_codegen_ssa/src/base.rs | 53 ++++---- compiler/rustc_codegen_ssa/src/lib.rs | 2 - 6 files changed, 79 insertions(+), 146 deletions(-) (limited to 'compiler/rustc_codegen_llvm') diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 7e77781dc2f..c3adb5e767e 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -12,9 +12,7 @@ use cranelift_object::{ObjectBuilder, ObjectModule}; use rustc_codegen_ssa::assert_module_sources::CguReuse; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::base::determine_cgu_reuse; -use rustc_codegen_ssa::{ - CodegenResults, CompiledModule, CrateInfo, ModuleKind, errors as ssa_errors, -}; +use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, errors as ssa_errors}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; @@ -363,7 +361,6 @@ fn emit_cgu( invocation_temp, prof, product.object, - ModuleKind::Regular, name.clone(), producer, )?; @@ -372,7 +369,6 @@ fn emit_cgu( module_regular, module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { name: format!("{name}.asm"), - kind: ModuleKind::Regular, object: Some(global_asm_object_file), dwarf_object: None, bytecode: None, @@ -389,7 +385,6 @@ fn emit_module( invocation_temp: Option<&str>, prof: &SelfProfilerRef, mut object: cranelift_object::object::write::Object<'_>, - kind: ModuleKind, name: String, producer_str: &str, ) -> Result { @@ -430,7 +425,6 @@ fn emit_module( Ok(CompiledModule { name, - kind, object: Some(tmp_file), dwarf_object: None, bytecode: None, @@ -485,7 +479,6 @@ fn reuse_workproduct_for_cgu( Ok(ModuleCodegenResult { module_regular: CompiledModule { name: cgu.name().to_string(), - kind: ModuleKind::Regular, object: Some(obj_out_regular), dwarf_object: None, bytecode: None, @@ -495,7 +488,6 @@ fn reuse_workproduct_for_cgu( }, module_global_asm: source_file_global_asm.map(|source_file| CompiledModule { name: cgu.name().to_string(), - kind: ModuleKind::Regular, object: Some(obj_out_global_asm), dwarf_object: None, bytecode: None, @@ -651,7 +643,6 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { tcx.sess.invocation_temp.as_deref(), &tcx.sess.prof, product.object, - ModuleKind::Allocator, "allocator_shim".to_owned(), &crate::debuginfo::producer(tcx.sess), ) { diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs index fcee6b6df62..9d8ce2383f2 100644 --- a/compiler/rustc_codegen_gcc/src/back/lto.rs +++ b/compiler/rustc_codegen_gcc/src/back/lto.rs @@ -204,7 +204,7 @@ fn fat_lto( let path = tmp_path.path().to_path_buf().join(&module.name); let path = path.to_str().expect("path"); let context = &module.module_llvm.context; - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // NOTE: we need to set the optimization level here in order for LTO to do its job. context.set_optimization_level(to_gcc_opt_level(config.opt_level)); context.add_command_line_option("-flto=auto"); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index fc38c4f3e51..326b876e7e6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -11,7 +11,7 @@ use object::{Object, ObjectSection}; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; +use rustc_codegen_ssa::{ModuleCodegen, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; use rustc_errors::DiagCtxtHandle; @@ -43,9 +43,7 @@ fn prepare_lto( .map(|symbol| CString::new(symbol.to_owned()).unwrap()) .collect::>(); - if cgcx.regular_module_config.instrument_coverage - || cgcx.regular_module_config.pgo_gen.enabled() - { + if cgcx.module_config.instrument_coverage || cgcx.module_config.pgo_gen.enabled() { // These are weak symbols that point to the profile version and the // profile name, which need to be treated as exported so LTO doesn't nix // them. @@ -55,15 +53,15 @@ fn prepare_lto( symbols_below_threshold.extend(PROFILER_WEAK_SYMBOLS.iter().map(|&sym| sym.to_owned())); } - if cgcx.regular_module_config.sanitizer.contains(SanitizerSet::MEMORY) { + if cgcx.module_config.sanitizer.contains(SanitizerSet::MEMORY) { let mut msan_weak_symbols = Vec::new(); // Similar to profiling, preserve weak msan symbol during LTO. - if cgcx.regular_module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { + if cgcx.module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { msan_weak_symbols.push(c"__msan_keep_going"); } - if cgcx.regular_module_config.sanitizer_memory_track_origins != 0 { + if cgcx.module_config.sanitizer_memory_track_origins != 0 { msan_weak_symbols.push(c"__msan_track_origins"); } @@ -227,15 +225,9 @@ fn fat_lto( // All the other modules will be serialized and reparsed into the new // context, so this hopefully avoids serializing and parsing the largest // codegen unit. - // - // Additionally use a regular module as the base here to ensure that various - // file copy operations in the backend work correctly. The only other kind - // of module here should be an allocator one, and if your crate is smaller - // than the allocator module then the size doesn't really matter anyway. let costliest_module = in_memory .iter() .enumerate() - .filter(|&(_, module)| module.kind == ModuleKind::Regular) .map(|(i, module)| { let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; (cost, i) @@ -583,7 +575,7 @@ pub(crate) fn run_pass_manager( thin: bool, ) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. @@ -745,7 +737,7 @@ pub(crate) fn optimize_thin_module( let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx); let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. - if cgcx.config(ModuleKind::Regular).embed_bitcode() { + if cgcx.module_config.embed_bitcode() { module.thin_lto_buffer = Some(thin_module.data().to_vec()); } { diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index d6f289b3eee..f637e7f58db 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -333,8 +333,7 @@ pub struct CodegenContext { pub crate_types: Vec, pub output_filenames: Arc, pub invocation_temp: Option, - pub regular_module_config: Arc, - pub allocator_module_config: Arc, + pub module_config: Arc, pub tm_factory: TargetMachineFactoryFn, pub msvc_imps_needed: bool, pub is_pe_coff: bool, @@ -372,13 +371,6 @@ impl CodegenContext { pub fn create_dcx(&self) -> DiagCtxt { DiagCtxt::new(Box::new(self.diag_emitter.clone())) } - - pub fn config(&self, kind: ModuleKind) -> &ModuleConfig { - match kind { - ModuleKind::Regular => &self.regular_module_config, - ModuleKind::Allocator => &self.allocator_module_config, - } - } } fn generate_thin_lto_work( @@ -442,6 +434,7 @@ pub(crate) fn start_async_codegen( backend: B, tcx: TyCtxt<'_>, target_cpu: String, + allocator_module: Option>, ) -> OngoingCodegen { let (coordinator_send, coordinator_receive) = channel(); @@ -465,6 +458,7 @@ pub(crate) fn start_async_codegen( coordinator_receive, Arc::new(regular_config), Arc::new(allocator_config), + allocator_module, coordinator_send.clone(), ); @@ -495,7 +489,7 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir"); - for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { + for module in &compiled_modules.modules { let mut files = Vec::new(); if let Some(object_file_path) = &module.object { files.push((OutputType::Object.extension(), object_file_path.as_path())); @@ -720,15 +714,6 @@ pub(crate) enum WorkItem { } impl WorkItem { - fn module_kind(&self) -> ModuleKind { - match *self { - WorkItem::Optimize(ref m) => m.kind, - WorkItem::CopyPostLtoArtifacts(_) | WorkItem::FatLto { .. } | WorkItem::ThinLto(_) => { - ModuleKind::Regular - } - } - } - /// Generate a short description of this work item suitable for use as a thread name. fn short_description(&self) -> String { // `pthread_setname()` on *nix ignores anything beyond the first 15 @@ -836,32 +821,22 @@ pub(crate) fn compute_per_cgu_lto_type( fn execute_optimize_work_item( cgcx: &CodegenContext, mut module: ModuleCodegen, - module_config: &ModuleConfig, ) -> WorkItemResult { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - B::optimize(cgcx, dcx, &mut module, module_config); + B::optimize(cgcx, dcx, &mut module, &cgcx.module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it // back to the coordinator thread for further LTO processing (which // has to wait for all the initial modules to be optimized). - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - if module.kind == ModuleKind::Allocator { - let module = B::codegen(cgcx, module, module_config); - return WorkItemResult::Finished(module); - } - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types); // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. - let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc { + let bitcode = if cgcx.module_config.emit_pre_lto_bc { let filename = pre_lto_bitcode_filename(&module.name); cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) } else { @@ -870,7 +845,7 @@ fn execute_optimize_work_item( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } ComputedLtoType::Thin => { @@ -901,7 +876,6 @@ fn execute_optimize_work_item( fn execute_copy_from_cache_work_item( cgcx: &CodegenContext, module: CachedModuleCodegen, - module_config: &ModuleConfig, ) -> WorkItemResult { let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); @@ -961,6 +935,7 @@ fn execute_copy_from_cache_work_item( } }; + let module_config = &cgcx.module_config; let should_emit_obj = module_config.emit_obj != EmitObj::None; let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly); let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly); @@ -973,7 +948,6 @@ fn execute_copy_from_cache_work_item( WorkItemResult::Finished(CompiledModule { links_from_incr_cache, name: module.name, - kind: ModuleKind::Regular, object, dwarf_object, bytecode, @@ -988,7 +962,6 @@ fn execute_fat_lto_work_item( each_linked_rlib_for_lto: &[PathBuf], mut needs_fat_lto: Vec>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, - module_config: &ModuleConfig, ) -> WorkItemResult { for (module, wp) in import_only_modules { needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module }) @@ -1000,17 +973,16 @@ fn execute_fat_lto_work_item( each_linked_rlib_for_lto, needs_fat_lto, ); - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } fn execute_thin_lto_work_item( cgcx: &CodegenContext, module: lto::ThinModule, - module_config: &ModuleConfig, ) -> WorkItemResult { let module = B::optimize_thin(cgcx, module); - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } @@ -1095,6 +1067,7 @@ fn start_executing_work( coordinator_receive: Receiver>, regular_config: Arc, allocator_config: Arc, + allocator_module: Option>, tx_to_llvm_workers: Sender>, ) -> thread::JoinHandle> { let coordinator_send = tx_to_llvm_workers; @@ -1159,8 +1132,7 @@ fn start_executing_work( expanded_args: tcx.sess.expanded_args.clone(), diag_emitter: shared_emitter.clone(), output_filenames: Arc::clone(tcx.output_filenames(())), - regular_module_config: regular_config, - allocator_module_config: allocator_config, + module_config: regular_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), msvc_imps_needed: msvc_imps_needed(tcx), is_pe_coff: tcx.sess.target.is_like_windows, @@ -1175,6 +1147,11 @@ fn start_executing_work( invocation_temp: sess.invocation_temp.clone(), }; + let compiled_allocator_module = allocator_module.map(|mut allocator_module| { + B::optimize(&cgcx, tcx.sess.dcx(), &mut allocator_module, &allocator_config); + B::codegen(&cgcx, allocator_module, &allocator_config) + }); + // This is the "main loop" of parallel work happening for parallel codegen. // It's here that we manage parallelism, schedule work, and work with // messages coming from clients. @@ -1314,7 +1291,6 @@ fn start_executing_work( // This is where we collect codegen units that have gone all the way // through codegen and LLVM. let mut compiled_modules = vec![]; - let mut compiled_allocator_module = None; let mut needs_fat_lto = Vec::new(); let mut needs_thin_lto = Vec::new(); let mut lto_import_only_modules = Vec::new(); @@ -1588,15 +1564,7 @@ fn start_executing_work( match result { Ok(WorkItemResult::Finished(compiled_module)) => { - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); - } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); - } - } + compiled_modules.push(compiled_module); } Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => { assert!(!started_lto); @@ -1724,45 +1692,38 @@ fn spawn_work<'a, B: ExtraBackendMethods>( let cgcx = cgcx.clone(); B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { - let result = std::panic::catch_unwind(AssertUnwindSafe(|| { - let module_config = cgcx.config(work.module_kind()); - - match work { - WorkItem::Optimize(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); - execute_optimize_work_item(&cgcx, m, module_config) - } - WorkItem::CopyPostLtoArtifacts(m) => { - let _timer = cgcx.prof.generic_activity_with_arg( - "codegen_copy_artifacts_from_incr_cache", - &*m.name, - ); - execute_copy_from_cache_work_item(&cgcx, m, module_config) - } - WorkItem::FatLto { - exported_symbols_for_lto, - each_linked_rlib_for_lto, + let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { + WorkItem::Optimize(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); + execute_optimize_work_item(&cgcx, m) + } + WorkItem::CopyPostLtoArtifacts(m) => { + let _timer = cgcx + .prof + .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name); + execute_copy_from_cache_work_item(&cgcx, m) + } + WorkItem::FatLto { + exported_symbols_for_lto, + each_linked_rlib_for_lto, + needs_fat_lto, + import_only_modules, + } => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", "everything"); + execute_fat_lto_work_item( + &cgcx, + &exported_symbols_for_lto, + &each_linked_rlib_for_lto, needs_fat_lto, import_only_modules, - } => { - let _timer = cgcx - .prof - .generic_activity_with_arg("codegen_module_perform_lto", "everything"); - execute_fat_lto_work_item( - &cgcx, - &exported_symbols_for_lto, - &each_linked_rlib_for_lto, - needs_fat_lto, - import_only_modules, - module_config, - ) - } - WorkItem::ThinLto(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); - execute_thin_lto_work_item(&cgcx, m, module_config) - } + ) + } + WorkItem::ThinLto(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); + execute_thin_lto_work_item(&cgcx, m) } })); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 071bd09249a..a9a2ae1b3db 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -662,7 +662,7 @@ pub fn codegen_crate( ) -> OngoingCodegen { // Skip crate items and just output metadata in -Z no-codegen mode. if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu); + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, None); ongoing_codegen.codegen_finished(tcx); @@ -693,7 +693,27 @@ pub fn codegen_crate( } } - let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu); + // Codegen an allocator shim, if necessary. + let allocator_module = if let Some(kind) = allocator_kind_for_codegen(tcx) { + let llmod_id = + cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); + + tcx.sess.time("write_allocator_module", || { + let module = backend.codegen_allocator( + tcx, + &llmod_id, + kind, + // If allocator_kind is Some then alloc_error_handler_kind must + // also be Some. + tcx.alloc_error_handler_kind(()).unwrap(), + ); + Some(ModuleCodegen::new_allocator(llmod_id, module)) + }) + } else { + None + }; + + let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu, allocator_module); // For better throughput during parallel processing by LLVM, we used to sort // CGUs largest to smallest. This would lead to better thread utilization @@ -810,35 +830,6 @@ pub fn codegen_crate( } } - // Codegen an allocator shim, if necessary. - // Do this last to ensure the LLVM_passes timer doesn't start while no CGUs have been codegened - // yet for the backend to optimize. - if let Some(kind) = allocator_kind_for_codegen(tcx) { - let llmod_id = - cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); - let module_llvm = tcx.sess.time("write_allocator_module", || { - backend.codegen_allocator( - tcx, - &llmod_id, - kind, - // If allocator_kind is Some then alloc_error_handler_kind must - // also be Some. - tcx.alloc_error_handler_kind(()).unwrap(), - ) - }); - - ongoing_codegen.wait_for_signal_to_codegen_item(); - ongoing_codegen.check_for_errors(tcx.sess); - - // These modules are generally cheap and won't throw off scheduling. - let cost = 0; - submit_codegened_module_to_llvm( - &ongoing_codegen.coordinator, - ModuleCodegen::new_allocator(llmod_id, module_llvm), - cost, - ); - } - ongoing_codegen.codegen_finished(tcx); // Since the main thread is sometimes blocked during codegen, we keep track diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index fe0500a5d4c..5b90ffaa056 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -120,7 +120,6 @@ impl ModuleCodegen { CompiledModule { name: self.name.clone(), - kind: self.kind, object, dwarf_object, bytecode, @@ -134,7 +133,6 @@ impl ModuleCodegen { #[derive(Debug, Encodable, Decodable)] pub struct CompiledModule { pub name: String, - pub kind: ModuleKind, pub object: Option, pub dwarf_object: Option, pub bytecode: Option, -- cgit 1.4.1-3-g733a5