diff options
22 files changed, 256 insertions, 73 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 1e2e41b3122..b92885cc1a7 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -453,11 +453,6 @@ fn codegen_regular_intrinsic_call<'tcx>( fx.bcx.ins().trap(TrapCode::user(2).unwrap()); return Ok(()); } - sym::likely | sym::unlikely => { - intrinsic_args!(fx, args => (a); intrinsic); - - ret.write_cvalue(fx, a); - } sym::breakpoint => { intrinsic_args!(fx, args => (); intrinsic); @@ -1267,6 +1262,10 @@ fn codegen_regular_intrinsic_call<'tcx>( ); } + sym::cold_path => { + // This is a no-op. The intrinsic is just a hint to the optimizer. + } + // Unimplemented intrinsics must have a fallback body. The fallback body is obtained // by converting the `InstanceKind::Intrinsic` to an `InstanceKind::Item`. _ => { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index b0298a35cb0..225f294e1e4 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -139,8 +139,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), ) } - sym::likely => self.expect(args[0].immediate(), true), - sym::unlikely => self.expect(args[0].immediate(), false), sym::is_val_statically_known => { let a = args[0].immediate(); let builtin = self.context.get_builtin_function("__builtin_constant_p"); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index e9c687d75e3..b56f464975d 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -192,7 +192,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Some(instance), ) } - sym::likely => self.expect(args[0].immediate(), true), sym::is_val_statically_known => { let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); let kind = self.type_kind(intrinsic_type); @@ -213,7 +212,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.const_bool(false) } } - sym::unlikely => self.expect(args[0].immediate(), false), sym::select_unpredictable => { let cond = args[0].immediate(); assert_eq!(args[1].layout, args[2].layout); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 027d80350e4..097d37bb70c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -377,20 +377,32 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // If there are two targets (one conditional, one fallback), emit `br` instead of // `switch`. let (test_value, target) = target_iter.next().unwrap(); - let lltrue = helper.llbb_with_cleanup(self, target); - let llfalse = helper.llbb_with_cleanup(self, targets.otherwise()); + let otherwise = targets.otherwise(); + let lltarget = helper.llbb_with_cleanup(self, target); + let llotherwise = helper.llbb_with_cleanup(self, otherwise); + let target_cold = self.cold_blocks[target]; + let otherwise_cold = self.cold_blocks[otherwise]; + // If `target_cold == otherwise_cold`, the branches have the same weight + // so there is no expectation. If they differ, the `target` branch is expected + // when the `otherwise` branch is cold. + let expect = if target_cold == otherwise_cold { None } else { Some(otherwise_cold) }; if switch_ty == bx.tcx().types.bool { // Don't generate trivial icmps when switching on bool. match test_value { - 0 => bx.cond_br(discr_value, llfalse, lltrue), - 1 => bx.cond_br(discr_value, lltrue, llfalse), + 0 => { + let expect = expect.map(|e| !e); + bx.cond_br_with_expect(discr_value, llotherwise, lltarget, expect); + } + 1 => { + bx.cond_br_with_expect(discr_value, lltarget, llotherwise, expect); + } _ => bug!(), } } else { let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty)); let llval = bx.const_uint_big(switch_llty, test_value); let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval); - bx.cond_br(cmp, lltrue, llfalse); + bx.cond_br_with_expect(cmp, lltarget, llotherwise, expect); } } else if self.cx.sess().opts.optimize == OptLevel::No && target_iter.len() == 2 diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index c9e38bb80c2..2a1b9e28c1e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -498,6 +498,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + sym::cold_path => { + // This is a no-op. The intrinsic is just a hint to the optimizer. + return Ok(()); + } + _ => { // Need to use backend-specific things in the implementation. return bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 20fd08923ec..f19e3b72141 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -91,6 +91,10 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { /// Cached terminate upon unwinding block and its reason terminate_block: Option<(Bx::BasicBlock, UnwindTerminateReason)>, + /// A bool flag for each basic block indicating whether it is a cold block. + /// A cold block is a block that is unlikely to be executed at runtime. + cold_blocks: IndexVec<mir::BasicBlock, bool>, + /// The location where each MIR arg/var/tmp/ret is stored. This is /// usually an `PlaceRef` representing an alloca, but not always: /// sometimes we can skip the alloca and just store the value @@ -207,6 +211,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cleanup_kinds, landing_pads: IndexVec::from_elem(None, &mir.basic_blocks), funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()), + cold_blocks: find_cold_blocks(cx.tcx(), mir), locals: locals::Locals::empty(), debug_context, per_local_var_debug_info: None, @@ -477,3 +482,39 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( args } + +fn find_cold_blocks<'tcx>( + tcx: TyCtxt<'tcx>, + mir: &mir::Body<'tcx>, +) -> IndexVec<mir::BasicBlock, bool> { + let local_decls = &mir.local_decls; + + let mut cold_blocks: IndexVec<mir::BasicBlock, bool> = + IndexVec::from_elem(false, &mir.basic_blocks); + + // Traverse all basic blocks from end of the function to the start. + for (bb, bb_data) in traversal::postorder(mir) { + let terminator = bb_data.terminator(); + + // If a BB ends with a call to a cold function, mark it as cold. + if let mir::TerminatorKind::Call { ref func, .. } = terminator.kind + && let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind() + && let attrs = tcx.codegen_fn_attrs(def_id) + && attrs.flags.contains(CodegenFnAttrFlags::COLD) + { + cold_blocks[bb] = true; + continue; + } + + // If all successors of a BB are cold and there's at least one of them, mark this BB as cold + let mut succ = terminator.successors(); + if let Some(first) = succ.next() + && cold_blocks[first] + && succ.all(|s| cold_blocks[s]) + { + cold_blocks[bb] = true; + } + } + + cold_blocks +} diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 74cd522a30f..b0138ac8bfe 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -84,6 +84,26 @@ pub trait BuilderMethods<'a, 'tcx>: then_llbb: Self::BasicBlock, else_llbb: Self::BasicBlock, ); + + // Conditional with expectation. + // + // This function is opt-in for back ends. + // + // The default implementation calls `self.expect()` before emiting the branch + // by calling `self.cond_br()` + fn cond_br_with_expect( + &mut self, + mut cond: Self::Value, + then_llbb: Self::BasicBlock, + else_llbb: Self::BasicBlock, + expect: Option<bool>, + ) { + if let Some(expect) = expect { + cond = self.expect(cond, expect); + } + self.cond_br(cond, then_llbb, else_llbb) + } + fn switch( &mut self, v: Self::Value, diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index c7a56a80e81..d89d73824aa 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -417,6 +417,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // These just return their argument self.copy_op(&args[0], dest)?; } + sym::cold_path => { + // This is a no-op. The intrinsic is just a hint to the optimizer. + } sym::raw_eq => { let result = self.raw_eq_intrinsic(&args[0], &args[1])?; self.write_scalar(result, dest)?; diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index cb954b0adcb..3e33120901f 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -109,9 +109,8 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) - | sym::three_way_compare | sym::discriminant_value | sym::type_id - | sym::likely - | sym::unlikely | sym::select_unpredictable + | sym::cold_path | sym::ptr_guaranteed_cmp | sym::minnumf16 | sym::minnumf32 @@ -489,9 +488,8 @@ pub fn check_intrinsic_type( sym::float_to_int_unchecked => (2, 0, vec![param(0)], param(1)), sym::assume => (0, 0, vec![tcx.types.bool], tcx.types.unit), - sym::likely => (0, 0, vec![tcx.types.bool], tcx.types.bool), - sym::unlikely => (0, 0, vec![tcx.types.bool], tcx.types.bool), sym::select_unpredictable => (1, 0, vec![tcx.types.bool, param(0), param(0)], param(0)), + sym::cold_path => (0, 0, vec![], tcx.types.unit), sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)), sym::write_via_move => { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 42c6221dc57..5d4ba4be5b8 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -589,6 +589,7 @@ symbols! { cmse_nonsecure_entry, coerce_unsized, cold, + cold_path, collapse_debuginfo, column, compare_bytes, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 6a37c60d9c9..8a986337aa1 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1465,6 +1465,22 @@ pub const unsafe fn assume(b: bool) { } } +/// Hints to the compiler that current code path is cold. +/// +/// Note that, unlike most intrinsics, this is safe to call; +/// it does not require an `unsafe` block. +/// Therefore, implementations must not require the user to uphold +/// any safety invariants. +/// +/// This intrinsic does not have a stable counterpart. +#[unstable(feature = "core_intrinsics", issue = "none")] +#[cfg_attr(not(bootstrap), rustc_intrinsic)] +#[cfg(not(bootstrap))] +#[rustc_nounwind] +#[miri::intrinsic_fallback_is_spec] +#[cold] +pub const fn cold_path() {} + /// Hints to the compiler that branch condition is likely to be true. /// Returns the value passed to it. /// @@ -1480,13 +1496,21 @@ pub const unsafe fn assume(b: bool) { bootstrap, rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION") )] -#[cfg_attr(not(bootstrap), rustc_const_stable_intrinsic)] #[unstable(feature = "core_intrinsics", issue = "none")] -#[rustc_intrinsic] #[rustc_nounwind] -#[miri::intrinsic_fallback_is_spec] +#[inline(always)] pub const fn likely(b: bool) -> bool { - b + #[cfg(bootstrap)] + { + b + } + #[cfg(not(bootstrap))] + if b { + true + } else { + cold_path(); + false + } } /// Hints to the compiler that branch condition is likely to be false. @@ -1504,13 +1528,21 @@ pub const fn likely(b: bool) -> bool { bootstrap, rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION") )] -#[cfg_attr(not(bootstrap), rustc_const_stable_intrinsic)] #[unstable(feature = "core_intrinsics", issue = "none")] -#[rustc_intrinsic] #[rustc_nounwind] -#[miri::intrinsic_fallback_is_spec] +#[inline(always)] pub const fn unlikely(b: bool) -> bool { - b + #[cfg(bootstrap)] + { + b + } + #[cfg(not(bootstrap))] + if b { + cold_path(); + true + } else { + false + } } /// Returns either `true_val` or `false_val` depending on condition `b` with a diff --git a/src/tools/miri/tests/pass/shims/time-with-isolation.stdout b/src/tools/miri/tests/pass/shims/time-with-isolation.stdout index 2d7fb5f4a61..6058735f005 100644 --- a/src/tools/miri/tests/pass/shims/time-with-isolation.stdout +++ b/src/tools/miri/tests/pass/shims/time-with-isolation.stdout @@ -1,2 +1,2 @@ -The loop took around 1250ms +The loop took around 1350ms (It's fine for this number to change when you `--bless` this test.) diff --git a/tests/codegen/checked_math.rs b/tests/codegen/checked_math.rs index 63f5c3d34f7..c612ddccdaa 100644 --- a/tests/codegen/checked_math.rs +++ b/tests/codegen/checked_math.rs @@ -90,7 +90,7 @@ pub fn checked_shr_signed(a: i32, b: u32) -> Option<i32> { #[no_mangle] pub fn checked_add_one_unwrap_unsigned(x: u32) -> u32 { // CHECK: %[[IS_MAX:.+]] = icmp eq i32 %x, -1 - // CHECK: br i1 %[[IS_MAX]], label %[[NONE_BB:.+]], label %[[SOME_BB:.+]] + // CHECK: br i1 %[[IS_MAX]], label %[[NONE_BB:.+]], label %[[SOME_BB:.+]], // CHECK: [[SOME_BB]]: // CHECK: %[[R:.+]] = add nuw i32 %x, 1 // CHECK: ret i32 %[[R]] diff --git a/tests/codegen/intrinsics/cold_path.rs b/tests/codegen/intrinsics/cold_path.rs new file mode 100644 index 00000000000..24ee84e07bf --- /dev/null +++ b/tests/codegen/intrinsics/cold_path.rs @@ -0,0 +1,13 @@ +//@ compile-flags: -O +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::cold_path; + +#[no_mangle] +pub fn test_cold_path(x: bool) { + cold_path(); +} + +// CHECK-LABEL: @test_cold_path( +// CHECK-NOT: cold_path diff --git a/tests/codegen/intrinsics/likely.rs b/tests/codegen/intrinsics/likely.rs index 9dc31d21045..e318390db20 100644 --- a/tests/codegen/intrinsics/likely.rs +++ b/tests/codegen/intrinsics/likely.rs @@ -1,22 +1,35 @@ -//@ compile-flags: -C no-prepopulate-passes -Copt-level=1 - +//@ compile-flags: -O #![crate_type = "lib"] #![feature(core_intrinsics)] -use std::intrinsics::{likely, unlikely}; +use std::intrinsics::likely; +#[inline(never)] #[no_mangle] -pub fn check_likely(x: i32, y: i32) -> Option<i32> { - unsafe { - // CHECK: call i1 @llvm.expect.i1(i1 %{{.*}}, i1 true) - if likely(x == y) { None } else { Some(x + y) } - } +pub fn path_a() { + println!("path a"); +} + +#[inline(never)] +#[no_mangle] +pub fn path_b() { + println!("path b"); } #[no_mangle] -pub fn check_unlikely(x: i32, y: i32) -> Option<i32> { - unsafe { - // CHECK: call i1 @llvm.expect.i1(i1 %{{.*}}, i1 false) - if unlikely(x == y) { None } else { Some(x + y) } +pub fn test_likely(x: bool) { + if likely(x) { + path_a(); + } else { + path_b(); } } + +// CHECK-LABEL: @test_likely( +// CHECK: br i1 %x, label %bb2, label %bb3, !prof ![[NUM:[0-9]+]] +// CHECK: bb3: +// CHECK-NOT: cold_path +// CHECK: path_b +// CHECK: bb2: +// CHECK: path_a +// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1} diff --git a/tests/codegen/intrinsics/likely_assert.rs b/tests/codegen/intrinsics/likely_assert.rs new file mode 100644 index 00000000000..0ddbd6206ae --- /dev/null +++ b/tests/codegen/intrinsics/likely_assert.rs @@ -0,0 +1,17 @@ +//@ compile-flags: -O +#![crate_type = "lib"] + +#[no_mangle] +pub fn test_assert(x: bool) { + assert!(x); +} + +// check that assert! emits branch weights + +// CHECK-LABEL: @test_assert( +// CHECK: br i1 %x, label %bb2, label %bb1, !prof ![[NUM:[0-9]+]] +// CHECK: bb1: +// CHECK: panic +// CHECK: bb2: +// CHECK: ret void +// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1} diff --git a/tests/codegen/intrinsics/unlikely.rs b/tests/codegen/intrinsics/unlikely.rs new file mode 100644 index 00000000000..2d776031a52 --- /dev/null +++ b/tests/codegen/intrinsics/unlikely.rs @@ -0,0 +1,35 @@ +//@ compile-flags: -O +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::unlikely; + +#[inline(never)] +#[no_mangle] +pub fn path_a() { + println!("path a"); +} + +#[inline(never)] +#[no_mangle] +pub fn path_b() { + println!("path b"); +} + +#[no_mangle] +pub fn test_unlikely(x: bool) { + if unlikely(x) { + path_a(); + } else { + path_b(); + } +} + +// CHECK-LABEL: @test_unlikely( +// CHECK: br i1 %x, label %bb2, label %bb4, !prof ![[NUM:[0-9]+]] +// CHECK: bb4: +// CHECK: path_b +// CHECK: bb2: +// CHECK-NOT: cold_path +// CHECK: path_a +// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000} diff --git a/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-abort.mir index 935e67fc3c0..cff5b4c7243 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-abort.mir @@ -13,7 +13,9 @@ fn step_forward(_1: u16, _2: usize) -> u16 { scope 6 (inlined core::num::<impl u16>::checked_add) { let mut _5: (u16, bool); let mut _6: bool; - let mut _7: bool; + scope 7 (inlined unlikely) { + let _7: (); + } } } scope 5 (inlined convert::num::ptr_try_from_impls::<impl TryFrom<usize> for u16>::try_from) { @@ -21,11 +23,11 @@ fn step_forward(_1: u16, _2: usize) -> u16 { let mut _4: u16; } } - scope 7 (inlined Option::<u16>::is_none) { - scope 8 (inlined Option::<u16>::is_some) { + scope 8 (inlined Option::<u16>::is_none) { + scope 9 (inlined Option::<u16>::is_some) { } } - scope 9 (inlined core::num::<impl u16>::wrapping_add) { + scope 10 (inlined core::num::<impl u16>::wrapping_add) { } } @@ -39,29 +41,26 @@ fn step_forward(_1: u16, _2: usize) -> u16 { bb1: { _4 = copy _2 as u16 (IntToInt); StorageDead(_3); - StorageLive(_7); StorageLive(_6); StorageLive(_5); _5 = AddWithOverflow(copy _1, copy _4); _6 = copy (_5.1: bool); - _7 = unlikely(move _6) -> [return: bb2, unwind unreachable]; + switchInt(copy _6) -> [0: bb2, otherwise: bb3]; } bb2: { - switchInt(move _7) -> [0: bb3, otherwise: bb4]; - } - - bb3: { StorageDead(_5); StorageDead(_6); - StorageDead(_7); goto -> bb7; } + bb3: { + _7 = cold_path() -> [return: bb4, unwind unreachable]; + } + bb4: { StorageDead(_5); StorageDead(_6); - StorageDead(_7); goto -> bb6; } diff --git a/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-unwind.mir index bf1ffd1ef32..6e0242a220d 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/checked_ops.step_forward.PreCodegen.after.panic-unwind.mir @@ -13,7 +13,9 @@ fn step_forward(_1: u16, _2: usize) -> u16 { scope 6 (inlined core::num::<impl u16>::checked_add) { let mut _5: (u16, bool); let mut _6: bool; - let mut _7: bool; + scope 7 (inlined unlikely) { + let _7: (); + } } } scope 5 (inlined convert::num::ptr_try_from_impls::<impl TryFrom<usize> for u16>::try_from) { @@ -21,11 +23,11 @@ fn step_forward(_1: u16, _2: usize) -> u16 { let mut _4: u16; } } - scope 7 (inlined Option::<u16>::is_none) { - scope 8 (inlined Option::<u16>::is_some) { + scope 8 (inlined Option::<u16>::is_none) { + scope 9 (inlined Option::<u16>::is_some) { } } - scope 9 (inlined core::num::<impl u16>::wrapping_add) { + scope 10 (inlined core::num::<impl u16>::wrapping_add) { } } @@ -39,29 +41,26 @@ fn step_forward(_1: u16, _2: usize) -> u16 { bb1: { _4 = copy _2 as u16 (IntToInt); StorageDead(_3); - StorageLive(_7); StorageLive(_6); StorageLive(_5); _5 = AddWithOverflow(copy _1, copy _4); _6 = copy (_5.1: bool); - _7 = unlikely(move _6) -> [return: bb2, unwind unreachable]; + switchInt(copy _6) -> [0: bb2, otherwise: bb3]; } bb2: { - switchInt(move _7) -> [0: bb3, otherwise: bb4]; - } - - bb3: { StorageDead(_5); StorageDead(_6); - StorageDead(_7); goto -> bb7; } + bb3: { + _7 = cold_path() -> [return: bb4, unwind unreachable]; + } + bb4: { StorageDead(_5); StorageDead(_6); - StorageDead(_7); goto -> bb6; } diff --git a/tests/ui-fulldeps/stable-mir/check_intrinsics.rs b/tests/ui-fulldeps/stable-mir/check_intrinsics.rs index d7f37f36681..3534228f73e 100644 --- a/tests/ui-fulldeps/stable-mir/check_intrinsics.rs +++ b/tests/ui-fulldeps/stable-mir/check_intrinsics.rs @@ -64,7 +64,7 @@ fn check_instance(instance: &Instance) { if instance.has_body() { let Some(body) = instance.body() else { unreachable!("Expected a body") }; assert!(!body.blocks.is_empty()); - assert_eq!(&name, "likely"); + assert_eq!(&name, "select_unpredictable"); } else { assert!(instance.body().is_none()); assert_matches!(name.as_str(), "size_of_val" | "vtable_size"); @@ -78,7 +78,7 @@ fn check_def(fn_def: FnDef) { let name = intrinsic.fn_name(); match name.as_str() { - "likely" => { + "select_unpredictable" => { assert!(!intrinsic.must_be_overridden()); assert!(fn_def.has_body()); } @@ -132,7 +132,7 @@ fn generate_input(path: &str) -> std::io::Result<()> { pub fn use_intrinsics(init: bool) -> bool {{ let vtable_sz = unsafe {{ vtable_size(0 as *const ()) }}; let sz = unsafe {{ size_of_val("hi") }}; - likely(init && sz == 2) + select_unpredictable(init && sz == 2, false, true) }} "# )?; diff --git a/tests/ui/intrinsics/reify-intrinsic.rs b/tests/ui/intrinsics/reify-intrinsic.rs index 6c52651f060..0d047ccf4a3 100644 --- a/tests/ui/intrinsics/reify-intrinsic.rs +++ b/tests/ui/intrinsics/reify-intrinsic.rs @@ -13,9 +13,9 @@ fn b() { } fn c() { - let _: [unsafe extern "rust-intrinsic" fn(bool) -> bool; 2] = [ - std::intrinsics::likely, //~ ERROR cannot coerce - std::intrinsics::unlikely, + let _: [unsafe extern "rust-intrinsic" fn(f32) -> f32; 2] = [ + std::intrinsics::floorf32, //~ ERROR cannot coerce + std::intrinsics::log2f32, ]; } diff --git a/tests/ui/intrinsics/reify-intrinsic.stderr b/tests/ui/intrinsics/reify-intrinsic.stderr index 7af17147f28..a456e81e762 100644 --- a/tests/ui/intrinsics/reify-intrinsic.stderr +++ b/tests/ui/intrinsics/reify-intrinsic.stderr @@ -18,11 +18,11 @@ LL | let _ = std::mem::transmute as unsafe extern "rust-intrinsic" fn(isize) error[E0308]: cannot coerce intrinsics to function pointers --> $DIR/reify-intrinsic.rs:17:9 | -LL | std::intrinsics::likely, - | ^^^^^^^^^^^^^^^^^^^^^^^ cannot coerce intrinsics to function pointers +LL | std::intrinsics::floorf32, + | ^^^^^^^^^^^^^^^^^^^^^^^^^ cannot coerce intrinsics to function pointers | = note: expected fn pointer `unsafe extern "rust-intrinsic" fn(_) -> _` - found fn item `fn(_) -> _ {likely}` + found fn item `unsafe extern "rust-intrinsic" fn(_) -> _ {floorf32}` error: aborting due to 3 previous errors |