diff options
19 files changed, 443 insertions, 231 deletions
diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index b23d8b9e737..00a8293966b 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -37,29 +37,11 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { /// and even the full `Inline` doesn't call `visit_body`, so there's nowhere /// to put this logic in the visitor. pub(super) fn add_function_level_costs(&mut self) { - fn is_call_like(bbd: &BasicBlockData<'_>) -> bool { - use TerminatorKind::*; - match bbd.terminator().kind { - Call { .. } | TailCall { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => { - true - } - - Goto { .. } - | SwitchInt { .. } - | UnwindResume - | UnwindTerminate(_) - | Return - | Unreachable => false, - - Yield { .. } | CoroutineDrop | FalseEdge { .. } | FalseUnwind { .. } => { - unreachable!() - } - } - } - // If the only has one Call (or similar), inlining isn't increasing the total // number of calls, so give extra encouragement to inlining that. - if self.callee_body.basic_blocks.iter().filter(|bbd| is_call_like(bbd)).count() == 1 { + if self.callee_body.basic_blocks.iter().filter(|bbd| is_call_like(bbd.terminator())).count() + == 1 + { self.bonus += CALL_PENALTY; } } @@ -193,3 +175,26 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } } + +/// A terminator that's more call-like (might do a bunch of work, might panic, etc) +/// than it is goto-/return-like (no side effects, etc). +/// +/// Used to treat multi-call functions (which could inline exponentially) +/// different from those that only do one or none of these "complex" things. +pub(super) fn is_call_like(terminator: &Terminator<'_>) -> bool { + use TerminatorKind::*; + match terminator.kind { + Call { .. } | TailCall { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => true, + + Goto { .. } + | SwitchInt { .. } + | UnwindResume + | UnwindTerminate(_) + | Return + | Unreachable => false, + + Yield { .. } | CoroutineDrop | FalseEdge { .. } | FalseUnwind { .. } => { + unreachable!() + } + } +} diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 0183ba19475..0ab24e48d44 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -1,5 +1,6 @@ //! Inlining pass for MIR functions. +use std::assert_matches::debug_assert_matches; use std::iter; use std::ops::{Range, RangeFrom}; @@ -18,7 +19,7 @@ use rustc_session::config::{DebugInfo, OptLevel}; use rustc_span::source_map::Spanned; use tracing::{debug, instrument, trace, trace_span}; -use crate::cost_checker::CostChecker; +use crate::cost_checker::{CostChecker, is_call_like}; use crate::deref_separator::deref_finder; use crate::simplify::simplify_cfg; use crate::validate::validate_types; @@ -26,6 +27,7 @@ use crate::{check_inline, util}; pub(crate) mod cycle; +const HISTORY_DEPTH_LIMIT: usize = 20; const TOP_DOWN_DEPTH_LIMIT: usize = 5; #[derive(Clone, Debug)] @@ -117,6 +119,11 @@ trait Inliner<'tcx> { /// Should inlining happen for a given callee? fn should_inline_for_callee(&self, def_id: DefId) -> bool; + fn check_codegen_attributes_extra( + &self, + callee_attrs: &CodegenFnAttrs, + ) -> Result<(), &'static str>; + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool; /// Returns inlining decision that is based on the examination of callee MIR body. @@ -128,10 +135,6 @@ trait Inliner<'tcx> { callee_attrs: &CodegenFnAttrs, ) -> Result<(), &'static str>; - // How many callsites in a body are we allowed to inline? We need to limit this in order - // to prevent super-linear growth in MIR size. - fn inline_limit_for_block(&self) -> Option<usize>; - /// Called when inlining succeeds. fn on_inline_success( &mut self, @@ -142,9 +145,6 @@ trait Inliner<'tcx> { /// Called when inlining failed or was not performed. fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str); - - /// Called when the inline limit for a body is reached. - fn on_inline_limit_reached(&self) -> bool; } struct ForceInliner<'tcx> { @@ -191,6 +191,14 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { ForceInline::should_run_pass_for_callee(self.tcx(), def_id) } + fn check_codegen_attributes_extra( + &self, + callee_attrs: &CodegenFnAttrs, + ) -> Result<(), &'static str> { + debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. }); + Ok(()) + } + fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool { true } @@ -224,10 +232,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { } } - fn inline_limit_for_block(&self) -> Option<usize> { - Some(usize::MAX) - } - fn on_inline_success( &mut self, callsite: &CallSite<'tcx>, @@ -261,10 +265,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }), }); } - - fn on_inline_limit_reached(&self) -> bool { - false - } } struct NormalInliner<'tcx> { @@ -278,6 +278,10 @@ struct NormalInliner<'tcx> { /// The number of `DefId`s is finite, so checking history is enough /// to ensure that we do not loop endlessly while inlining. history: Vec<DefId>, + /// How many (multi-call) callsites have we inlined for the top-level call? + /// + /// We need to limit this in order to prevent super-linear growth in MIR size. + top_down_counter: usize, /// Indicates that the caller body has been modified. changed: bool, /// Indicates that the caller is #[inline] and just calls another function, @@ -285,6 +289,12 @@ struct NormalInliner<'tcx> { caller_is_inline_forwarder: bool, } +impl<'tcx> NormalInliner<'tcx> { + fn past_depth_limit(&self) -> bool { + self.history.len() > HISTORY_DEPTH_LIMIT || self.top_down_counter > TOP_DOWN_DEPTH_LIMIT + } +} + impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self { let typing_env = body.typing_env(tcx); @@ -295,6 +305,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { typing_env, def_id, history: Vec::new(), + top_down_counter: 0, changed: false, caller_is_inline_forwarder: matches!( codegen_fn_attrs.inline, @@ -327,6 +338,17 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { true } + fn check_codegen_attributes_extra( + &self, + callee_attrs: &CodegenFnAttrs, + ) -> Result<(), &'static str> { + if self.past_depth_limit() && matches!(callee_attrs.inline, InlineAttr::None) { + Err("Past depth limit so not inspecting unmarked callee") + } else { + Ok(()) + } + } + fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool { // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation, // which can create a cycle, even when no attempt is made to inline the function in the other @@ -351,7 +373,11 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { return Err("body has errors"); } - let mut threshold = if self.caller_is_inline_forwarder { + if self.past_depth_limit() && callee_body.basic_blocks.len() > 1 { + return Err("Not inlining multi-block body as we're past a depth limit"); + } + + let mut threshold = if self.caller_is_inline_forwarder || self.past_depth_limit() { tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) } else if tcx.cross_crate_inlinable(callsite.callee.def_id()) { tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) @@ -431,14 +457,6 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { } } - fn inline_limit_for_block(&self) -> Option<usize> { - match self.history.len() { - 0 => Some(usize::MAX), - 1..=TOP_DOWN_DEPTH_LIMIT => Some(1), - _ => None, - } - } - fn on_inline_success( &mut self, callsite: &CallSite<'tcx>, @@ -447,13 +465,21 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { ) { self.changed = true; + let new_calls_count = new_blocks + .clone() + .filter(|&bb| is_call_like(caller_body.basic_blocks[bb].terminator())) + .count(); + if new_calls_count > 1 { + self.top_down_counter += 1; + } + self.history.push(callsite.callee.def_id()); process_blocks(self, caller_body, new_blocks); self.history.pop(); - } - fn on_inline_limit_reached(&self) -> bool { - true + if self.history.is_empty() { + self.top_down_counter = 0; + } } fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {} @@ -482,8 +508,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>( caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>, ) { - let Some(inline_limit) = inliner.inline_limit_for_block() else { return }; - let mut inlined_count = 0; for bb in blocks { let bb_data = &caller_body[bb]; if bb_data.is_cleanup { @@ -505,13 +529,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>( Ok(new_blocks) => { debug!("inlined {}", callsite.callee); inliner.on_inline_success(&callsite, caller_body, new_blocks); - - inlined_count += 1; - if inlined_count == inline_limit { - if inliner.on_inline_limit_reached() { - return; - } - } } } } @@ -584,6 +601,7 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>( let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id()); check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?; check_codegen_attributes(inliner, callsite, callee_attrs)?; + inliner.check_codegen_attributes_extra(callee_attrs)?; let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; @@ -770,6 +788,8 @@ fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>( return Err("has DoNotOptimize attribute"); } + inliner.check_codegen_attributes_extra(callee_attrs)?; + // Reachability pass defines which functions are eligible for inlining. Generally inlining // other functions is incorrect because they could reference symbols that aren't exported. let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs index caab7a6ddb5..1494d7aca15 100644 --- a/library/core/src/mem/mod.rs +++ b/library/core/src/mem/mod.rs @@ -856,8 +856,13 @@ pub const fn replace<T>(dest: &mut T, src: T) -> T { // such that the old value is not duplicated. Nothing is dropped and // nothing here can panic. unsafe { - let result = ptr::read(dest); - ptr::write(dest, src); + // Ideally we wouldn't use the intrinsics here, but going through the + // `ptr` methods introduces two unnecessary UbChecks, so until we can + // remove those for pointers that come from references, this uses the + // intrinsics instead so this stays very cheap in MIR (and debug). + + let result = crate::intrinsics::read_via_copy(dest); + crate::intrinsics::write_via_move(dest, src); result } } diff --git a/src/tools/miri/tests/fail/both_borrows/aliasing_mut4.tree.stderr b/src/tools/miri/tests/fail/both_borrows/aliasing_mut4.tree.stderr index 5162368b51f..e8babf02163 100644 --- a/src/tools/miri/tests/fail/both_borrows/aliasing_mut4.tree.stderr +++ b/src/tools/miri/tests/fail/both_borrows/aliasing_mut4.tree.stderr @@ -1,8 +1,8 @@ error: Undefined Behavior: write access through <TAG> at ALLOC[0x0] is forbidden --> RUSTLIB/core/src/mem/mod.rs:LL:CC | -LL | ptr::write(dest, src); - | ^^^^^^^^^^^^^^^^^^^^^ write access through <TAG> at ALLOC[0x0] is forbidden +LL | crate::intrinsics::write_via_move(dest, src); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ write access through <TAG> at ALLOC[0x0] is forbidden | = help: this indicates a potential bug in the program: it performed an invalid operation, but the Tree Borrows rules it violated are still experimental = help: the accessed tag <TAG> is foreign to the protected tag <TAG> (i.e., it is not a child) diff --git a/tests/codegen/range-loop.rs b/tests/codegen/range-loop.rs new file mode 100644 index 00000000000..b131beb40dd --- /dev/null +++ b/tests/codegen/range-loop.rs @@ -0,0 +1,44 @@ +//@ ignore-std-debug-assertions +//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes + +#![crate_type = "lib"] + +// Ensure that MIR optimizations have cleaned things up enough that the IR we +// emit is good even without running the LLVM optimizations. + +// CHECK-NOT: define + +// CHECK-LABEL: define{{.+}}void @call_for_zero_to_n +#[no_mangle] +pub fn call_for_zero_to_n(n: u32, f: fn(u32)) { + // CHECK: start: + // CHECK-NOT: alloca + // CHECK: %[[IND:.+]] = alloca [4 x i8] + // CHECK-NEXT: %[[ALWAYS_SOME_OPTION:.+]] = alloca + // CHECK-NOT: alloca + // CHECK: store i32 0, ptr %[[IND]], + // CHECK: br label %[[HEAD:.+]] + + // CHECK: [[HEAD]]: + // CHECK: %[[T1:.+]] = load i32, ptr %[[IND]], + // CHECK: %[[NOT_DONE:.+]] = icmp ult i32 %[[T1]], %n + // CHECK: br i1 %[[NOT_DONE]], label %[[BODY:.+]], label %[[BREAK:.+]] + + // CHECK: [[BREAK]]: + // CHECK: ret void + + // CHECK: [[BODY]]: + // CHECK: %[[T2:.+]] = load i32, ptr %[[IND]], + // CHECK: %[[T3:.+]] = add nuw i32 %[[T2]], 1 + // CHECK: store i32 %[[T3]], ptr %[[IND]], + + // CHECK: store i32 %[[T2]] + // CHECK: %[[T4:.+]] = load i32 + // CHECK: call void %f(i32{{.+}}%[[T4]]) + + for i in 0..n { + f(i); + } +} + +// CHECK-NOT: define diff --git a/tests/mir-opt/inline/exponential_runtime.rs b/tests/mir-opt/inline/exponential_runtime.rs index 1199ce4e558..62c1d8be1bf 100644 --- a/tests/mir-opt/inline/exponential_runtime.rs +++ b/tests/mir-opt/inline/exponential_runtime.rs @@ -87,10 +87,15 @@ fn main() { // CHECK-LABEL: fn main( // CHECK-NOT: inlined // CHECK: (inlined <() as G>::call) + // CHECK-NOT: inlined // CHECK: (inlined <() as F>::call) + // CHECK-NOT: inlined // CHECK: (inlined <() as E>::call) + // CHECK-NOT: inlined // CHECK: (inlined <() as D>::call) + // CHECK-NOT: inlined // CHECK: (inlined <() as C>::call) + // CHECK-NOT: inlined // CHECK: (inlined <() as B>::call) // CHECK-NOT: inlined <() as G>::call(); diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff index 75fc2ea16e3..f099d763c3d 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff @@ -6,6 +6,7 @@ let _1: (!, !); + let mut _2: fn() -> ! {sleep}; + let mut _7: (); ++ let mut _8: (); + scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) { + debug f => _2; + let mut _3: &fn() -> ! {sleep}; @@ -17,6 +18,10 @@ + scope 3 { + debug b => _6; + } ++ scope 6 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { ++ scope 7 (inlined sleep) { ++ } ++ } + } + scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { + scope 5 (inlined sleep) { diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff index 407cb24df67..c33e0810739 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff @@ -6,6 +6,7 @@ let _1: (!, !); + let mut _2: fn() -> ! {sleep}; + let mut _8: (); ++ let mut _9: (); + scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) { + debug f => _2; + let mut _3: &fn() -> ! {sleep}; @@ -18,6 +19,10 @@ + scope 3 { + debug b => _6; + } ++ scope 6 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { ++ scope 7 (inlined sleep) { ++ } ++ } + } + scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { + scope 5 (inlined sleep) { diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 1e33e222b27..eb97af1e284 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -55,10 +55,46 @@ + let _26: (); + scope 9 { + } ++ scope 12 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) { ++ } ++ scope 13 (inlined <std::future::Ready<()> as Future>::poll) { ++ let mut _42: (); ++ let mut _43: std::option::Option<()>; ++ let mut _44: &mut std::option::Option<()>; ++ let mut _45: &mut std::future::Ready<()>; ++ let mut _46: &mut std::pin::Pin<&mut std::future::Ready<()>>; ++ scope 14 (inlined <Pin<&mut std::future::Ready<()>> as DerefMut>::deref_mut) { ++ let mut _47: std::pin::Pin<&mut std::future::Ready<()>>; ++ scope 15 (inlined Pin::<&mut std::future::Ready<()>>::as_mut) { ++ let mut _48: &mut &mut std::future::Ready<()>; ++ scope 16 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) { ++ } ++ scope 18 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) { ++ } ++ } ++ scope 17 (inlined Pin::<&mut std::future::Ready<()>>::get_mut) { ++ } ++ } ++ scope 19 (inlined Option::<()>::take) { ++ let mut _49: std::option::Option<()>; ++ scope 20 (inlined std::mem::replace::<Option<()>>) { ++ scope 21 { ++ } ++ } ++ } ++ scope 22 (inlined #[track_caller] Option::<()>::expect) { ++ let mut _50: isize; ++ let mut _51: !; ++ scope 23 { ++ } ++ } ++ } + } + scope 10 (inlined ready::<()>) { + let mut _41: std::option::Option<()>; + } ++ scope 11 (inlined <std::future::Ready<()> as IntoFuture>::into_future) { ++ } + } + } } @@ -113,7 +149,7 @@ + StorageLive(_40); + _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _32 = discriminant((*_33)); -+ switchInt(move _32) -> [0: bb3, 1: bb13, 3: bb12, otherwise: bb8]; ++ switchInt(move _32) -> [0: bb3, 1: bb10, 3: bb9, otherwise: bb5]; } - bb3: { @@ -164,19 +200,16 @@ + _13 = std::future::Ready::<()>(move _41); + StorageDead(_41); + StorageDead(_14); -+ _12 = <std::future::Ready<()> as IntoFuture>::into_future(move _13) -> [return: bb4, unwind unreachable]; -+ } -+ - bb4: { -- StorageDead(_2); -- return; ++ _12 = move _13; + StorageDead(_13); + _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + (((*_36) as variant#3).1: std::future::Ready<()>) = move _12; -+ goto -> bb5; ++ goto -> bb4; + } + -+ bb5: { + bb4: { +- StorageDead(_2); +- return; + StorageLive(_17); + StorageLive(_18); + StorageLive(_19); @@ -185,10 +218,7 @@ + _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>); + _20 = &mut (*_21); -+ _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb6, unwind unreachable]; -+ } -+ -+ bb6: { ++ _19 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _20 }; + StorageDead(_20); + StorageLive(_22); + StorageLive(_23); @@ -197,21 +227,36 @@ + _23 = move _24; + _22 = &mut (*_23); + StorageDead(_24); -+ _18 = <std::future::Ready<()> as Future>::poll(move _19, move _22) -> [return: bb7, unwind unreachable]; -+ } -+ -+ bb7: { -+ StorageDead(_22); -+ StorageDead(_19); -+ _25 = discriminant(_18); -+ switchInt(move _25) -> [0: bb10, 1: bb9, otherwise: bb8]; ++ StorageLive(_45); ++ StorageLive(_46); ++ StorageLive(_49); ++ StorageLive(_51); ++ StorageLive(_42); ++ StorageLive(_43); ++ StorageLive(_44); ++ _46 = &mut _19; ++ StorageLive(_47); ++ StorageLive(_48); ++ _48 = &mut (_19.0: &mut std::future::Ready<()>); ++ _45 = copy (_19.0: &mut std::future::Ready<()>); ++ StorageDead(_48); ++ _47 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _45 }; ++ StorageDead(_47); ++ _44 = &mut ((*_45).0: std::option::Option<()>); ++ _49 = Option::<()>::None; ++ _43 = copy ((*_45).0: std::option::Option<()>); ++ ((*_45).0: std::option::Option<()>) = copy _49; ++ StorageDead(_44); ++ StorageLive(_50); ++ _50 = discriminant(_43); ++ switchInt(move _50) -> [0: bb11, 1: bb12, otherwise: bb5]; + } + -+ bb8: { ++ bb5: { + unreachable; + } + -+ bb9: { ++ bb6: { + _17 = const (); + StorageDead(_23); + StorageDead(_21); @@ -229,7 +274,7 @@ + goto -> bb2; + } + -+ bb10: { ++ bb7: { + StorageLive(_26); + _26 = copy ((_18 as Ready).0: ()); + _30 = copy _26; @@ -240,17 +285,17 @@ + StorageDead(_17); + StorageDead(_12); + _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb11, unwind unreachable]; ++ drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb8, unwind unreachable]; + } + -+ bb11: { ++ bb8: { + _7 = Poll::<()>::Ready(move _30); + _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + discriminant((*_40)) = 1; + goto -> bb2; + } + -+ bb12: { ++ bb9: { + StorageLive(_12); + StorageLive(_28); + StorageLive(_29); @@ -259,11 +304,31 @@ + _31 = move _28; + StorageDead(_28); + _16 = const (); -+ goto -> bb5; ++ goto -> bb4; + } + -+ bb13: { -+ assert(const false, "`async fn` resumed after completion") -> [success: bb13, unwind unreachable]; ++ bb10: { ++ assert(const false, "`async fn` resumed after completion") -> [success: bb10, unwind unreachable]; ++ } ++ ++ bb11: { ++ _51 = option::expect_failed(const "`Ready` polled after completion") -> unwind unreachable; ++ } ++ ++ bb12: { ++ _42 = move ((_43 as Some).0: ()); ++ StorageDead(_50); ++ StorageDead(_43); ++ _18 = Poll::<()>::Ready(move _42); ++ StorageDead(_42); ++ StorageDead(_51); ++ StorageDead(_49); ++ StorageDead(_46); ++ StorageDead(_45); ++ StorageDead(_22); ++ StorageDead(_19); ++ _25 = discriminant(_18); ++ switchInt(move _25) -> [0: bb7, 1: bb6, otherwise: bb5]; } } diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index b1840beb3ef..eb757e09114 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -57,10 +57,46 @@ + let _26: (); + scope 9 { + } ++ scope 12 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) { ++ } ++ scope 13 (inlined <std::future::Ready<()> as Future>::poll) { ++ let mut _44: (); ++ let mut _45: std::option::Option<()>; ++ let mut _46: &mut std::option::Option<()>; ++ let mut _47: &mut std::future::Ready<()>; ++ let mut _48: &mut std::pin::Pin<&mut std::future::Ready<()>>; ++ scope 14 (inlined <Pin<&mut std::future::Ready<()>> as DerefMut>::deref_mut) { ++ let mut _49: std::pin::Pin<&mut std::future::Ready<()>>; ++ scope 15 (inlined Pin::<&mut std::future::Ready<()>>::as_mut) { ++ let mut _50: &mut &mut std::future::Ready<()>; ++ scope 16 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) { ++ } ++ scope 18 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) { ++ } ++ } ++ scope 17 (inlined Pin::<&mut std::future::Ready<()>>::get_mut) { ++ } ++ } ++ scope 19 (inlined Option::<()>::take) { ++ let mut _51: std::option::Option<()>; ++ scope 20 (inlined std::mem::replace::<Option<()>>) { ++ scope 21 { ++ } ++ } ++ } ++ scope 22 (inlined #[track_caller] Option::<()>::expect) { ++ let mut _52: isize; ++ let mut _53: !; ++ scope 23 { ++ } ++ } ++ } + } + scope 10 (inlined ready::<()>) { + let mut _43: std::option::Option<()>; + } ++ scope 11 (inlined <std::future::Ready<()> as IntoFuture>::into_future) { ++ } + } + } } @@ -117,7 +153,7 @@ + StorageLive(_42); + _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _32 = discriminant((*_33)); -+ switchInt(move _32) -> [0: bb5, 1: bb22, 2: bb21, 3: bb20, otherwise: bb10]; ++ switchInt(move _32) -> [0: bb5, 1: bb15, 2: bb14, 3: bb13, otherwise: bb7]; } - bb3: { @@ -181,21 +217,16 @@ + _13 = std::future::Ready::<()>(move _43); + StorageDead(_43); + StorageDead(_14); -+ _12 = <std::future::Ready<()> as IntoFuture>::into_future(move _13) -> [return: bb6, unwind: bb17]; - } - -- bb5 (cleanup): { -- drop(_2) -> [return: bb6, unwind terminate(cleanup)]; -+ bb6: { ++ _12 = move _13; + StorageDead(_13); + _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + (((*_36) as variant#3).1: std::future::Ready<()>) = move _12; -+ goto -> bb7; ++ goto -> bb6; } -- bb6 (cleanup): { -- resume; -+ bb7: { +- bb5 (cleanup): { +- drop(_2) -> [return: bb6, unwind terminate(cleanup)]; ++ bb6: { + StorageLive(_17); + StorageLive(_18); + StorageLive(_19); @@ -204,10 +235,7 @@ + _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>); + _20 = &mut (*_21); -+ _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb8, unwind: bb15]; -+ } -+ -+ bb8: { ++ _19 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _20 }; + StorageDead(_20); + StorageLive(_22); + StorageLive(_23); @@ -216,21 +244,38 @@ + _23 = move _24; + _22 = &mut (*_23); + StorageDead(_24); -+ _18 = <std::future::Ready<()> as Future>::poll(move _19, move _22) -> [return: bb9, unwind: bb14]; -+ } -+ -+ bb9: { -+ StorageDead(_22); -+ StorageDead(_19); -+ _25 = discriminant(_18); -+ switchInt(move _25) -> [0: bb12, 1: bb11, otherwise: bb10]; -+ } -+ -+ bb10: { ++ StorageLive(_47); ++ StorageLive(_48); ++ StorageLive(_51); ++ StorageLive(_53); ++ StorageLive(_44); ++ StorageLive(_45); ++ StorageLive(_46); ++ _48 = &mut _19; ++ StorageLive(_49); ++ StorageLive(_50); ++ _50 = &mut (_19.0: &mut std::future::Ready<()>); ++ _47 = copy (_19.0: &mut std::future::Ready<()>); ++ StorageDead(_50); ++ _49 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _47 }; ++ StorageDead(_49); ++ _46 = &mut ((*_47).0: std::option::Option<()>); ++ _51 = Option::<()>::None; ++ _45 = copy ((*_47).0: std::option::Option<()>); ++ ((*_47).0: std::option::Option<()>) = copy _51; ++ StorageDead(_46); ++ StorageLive(_52); ++ _52 = discriminant(_45); ++ switchInt(move _52) -> [0: bb16, 1: bb17, otherwise: bb7]; + } + +- bb6 (cleanup): { +- resume; ++ bb7: { + unreachable; + } + -+ bb11: { ++ bb8: { + _17 = const (); + StorageDead(_23); + StorageDead(_21); @@ -248,7 +293,7 @@ + goto -> bb4; + } + -+ bb12: { ++ bb9: { + StorageLive(_26); + _26 = copy ((_18 as Ready).0: ()); + _30 = copy _26; @@ -259,54 +304,35 @@ + StorageDead(_17); + StorageDead(_12); + _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb13, unwind: bb19]; ++ drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb10, unwind: bb12]; + } + -+ bb13: { ++ bb10: { + _7 = Poll::<()>::Ready(move _30); + _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + discriminant((*_40)) = 1; + goto -> bb4; + } + -+ bb14 (cleanup): { ++ bb11 (cleanup): { + StorageDead(_22); + StorageDead(_19); + StorageDead(_23); -+ goto -> bb16; -+ } -+ -+ bb15 (cleanup): { -+ StorageDead(_20); -+ StorageDead(_19); -+ goto -> bb16; -+ } -+ -+ bb16 (cleanup): { + StorageDead(_21); + StorageDead(_18); + StorageDead(_17); -+ goto -> bb18; -+ } -+ -+ bb17 (cleanup): { -+ StorageDead(_13); -+ goto -> bb18; -+ } -+ -+ bb18 (cleanup): { + StorageDead(_12); + _41 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ drop((((*_41) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb19, unwind terminate(cleanup)]; ++ drop((((*_41) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb12, unwind terminate(cleanup)]; + } + -+ bb19 (cleanup): { ++ bb12 (cleanup): { + _42 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + discriminant((*_42)) = 2; + goto -> bb2; + } + -+ bb20: { ++ bb13: { + StorageLive(_12); + StorageLive(_28); + StorageLive(_29); @@ -315,15 +341,35 @@ + _31 = move _28; + StorageDead(_28); + _16 = const (); -+ goto -> bb7; ++ goto -> bb6; + } + -+ bb21: { -+ assert(const false, "`async fn` resumed after panicking") -> [success: bb21, unwind: bb2]; ++ bb14: { ++ assert(const false, "`async fn` resumed after panicking") -> [success: bb14, unwind: bb2]; + } + -+ bb22: { -+ assert(const false, "`async fn` resumed after completion") -> [success: bb22, unwind: bb2]; ++ bb15: { ++ assert(const false, "`async fn` resumed after completion") -> [success: bb15, unwind: bb2]; ++ } ++ ++ bb16: { ++ _53 = option::expect_failed(const "`Ready` polled after completion") -> bb11; ++ } ++ ++ bb17: { ++ _44 = move ((_45 as Some).0: ()); ++ StorageDead(_52); ++ StorageDead(_45); ++ _18 = Poll::<()>::Ready(move _44); ++ StorageDead(_44); ++ StorageDead(_53); ++ StorageDead(_51); ++ StorageDead(_48); ++ StorageDead(_47); ++ StorageDead(_22); ++ StorageDead(_19); ++ _25 = discriminant(_18); ++ switchInt(move _25) -> [0: bb9, 1: bb8, otherwise: bb7]; } } diff --git a/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir index be69bbf10e7..1f9c464d633 100644 --- a/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir @@ -26,6 +26,18 @@ fn int_range(_1: usize, _2: usize) -> () { let mut _12: usize; scope 6 { debug old => _11; + scope 8 (inlined <usize as Step>::forward_unchecked) { + debug start => _11; + debug n => const 1_usize; + scope 9 (inlined core::num::<impl usize>::unchecked_add) { + debug self => _11; + debug rhs => const 1_usize; + scope 10 (inlined core::ub_checks::check_language_ub) { + scope 11 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 7 (inlined std::cmp::impls::<impl PartialOrd for usize>::lt) { debug self => _6; @@ -50,7 +62,6 @@ fn int_range(_1: usize, _2: usize) -> () { bb1: { StorageLive(_13); _5 = &mut _4; - StorageLive(_11); StorageLive(_10); StorageLive(_6); _6 = &(_4.0: usize); @@ -70,7 +81,6 @@ fn int_range(_1: usize, _2: usize) -> () { StorageDead(_7); StorageDead(_6); StorageDead(_10); - StorageDead(_11); StorageDead(_13); StorageDead(_4); return; @@ -81,20 +91,16 @@ fn int_range(_1: usize, _2: usize) -> () { StorageDead(_6); _11 = copy (_4.0: usize); StorageLive(_12); - _12 = <usize as Step>::forward_unchecked(copy _11, const 1_usize) -> [return: bb4, unwind continue]; - } - - bb4: { + _12 = AddUnchecked(copy _11, const 1_usize); (_4.0: usize) = move _12; StorageDead(_12); _13 = Option::<usize>::Some(copy _11); StorageDead(_10); - StorageDead(_11); _14 = copy ((_13 as Some).0: usize); - _15 = opaque::<usize>(move _14) -> [return: bb5, unwind continue]; + _15 = opaque::<usize>(move _14) -> [return: bb4, unwind continue]; } - bb5: { + bb4: { StorageDead(_13); goto -> bb1; } diff --git a/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-abort.mir index ed494f6e74c..958a06bcd34 100644 --- a/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-abort.mir @@ -6,10 +6,6 @@ fn mem_replace(_1: &mut u32, _2: u32) -> u32 { let mut _0: u32; scope 1 (inlined std::mem::replace::<u32>) { scope 2 { - scope 4 (inlined std::ptr::write::<u32>) { - } - } - scope 3 (inlined std::ptr::read::<u32>) { } } diff --git a/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-unwind.mir index ed494f6e74c..958a06bcd34 100644 --- a/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/mem_replace.mem_replace.PreCodegen.after.panic-unwind.mir @@ -6,10 +6,6 @@ fn mem_replace(_1: &mut u32, _2: u32) -> u32 { let mut _0: u32; scope 1 (inlined std::mem::replace::<u32>) { scope 2 { - scope 4 (inlined std::ptr::write::<u32>) { - } - } - scope 3 (inlined std::ptr::read::<u32>) { } } diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir index 5d33c33d73f..0aa37203c1b 100644 --- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -23,6 +23,14 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { let _7: u32; let mut _8: u32; scope 6 { + scope 8 (inlined <u32 as Step>::forward_unchecked) { + scope 9 (inlined core::num::<impl u32>::unchecked_add) { + scope 10 (inlined core::ub_checks::check_language_ub) { + scope 11 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 7 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) { let mut _5: u32; @@ -41,7 +49,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb1: { StorageLive(_9); - StorageLive(_7); StorageLive(_6); StorageLive(_5); _5 = copy _4; @@ -52,7 +59,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb2: { StorageDead(_6); - StorageDead(_7); StorageDead(_9); StorageDead(_4); drop(_3) -> [return: bb3, unwind unreachable]; @@ -65,24 +71,20 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb4: { _7 = copy _4; StorageLive(_8); - _8 = <u32 as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind unreachable]; - } - - bb5: { + _8 = AddUnchecked(copy _7, const 1_u32); _4 = move _8; StorageDead(_8); _9 = Option::<u32>::Some(copy _7); StorageDead(_6); - StorageDead(_7); _10 = copy ((_9 as Some).0: u32); StorageLive(_11); _11 = &_3; StorageLive(_12); _12 = (copy _10,); - _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb6, unwind unreachable]; + _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb5, unwind unreachable]; } - bb6: { + bb5: { StorageDead(_12); StorageDead(_11); StorageDead(_9); diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir index ded30a15520..699d8bc8fea 100644 --- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -23,6 +23,14 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { let _7: u32; let mut _8: u32; scope 6 { + scope 8 (inlined <u32 as Step>::forward_unchecked) { + scope 9 (inlined core::num::<impl u32>::unchecked_add) { + scope 10 (inlined core::ub_checks::check_language_ub) { + scope 11 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 7 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) { let mut _5: u32; @@ -41,7 +49,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb1: { StorageLive(_9); - StorageLive(_7); StorageLive(_6); StorageLive(_5); _5 = copy _4; @@ -52,7 +59,6 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb2: { StorageDead(_6); - StorageDead(_7); StorageDead(_9); StorageDead(_4); drop(_3) -> [return: bb3, unwind continue]; @@ -65,35 +71,31 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb4: { _7 = copy _4; StorageLive(_8); - _8 = <u32 as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind: bb7]; - } - - bb5: { + _8 = AddUnchecked(copy _7, const 1_u32); _4 = move _8; StorageDead(_8); _9 = Option::<u32>::Some(copy _7); StorageDead(_6); - StorageDead(_7); _10 = copy ((_9 as Some).0: u32); StorageLive(_11); _11 = &_3; StorageLive(_12); _12 = (copy _10,); - _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb6, unwind: bb7]; + _13 = <impl Fn(u32) as Fn<(u32,)>>::call(move _11, move _12) -> [return: bb5, unwind: bb6]; } - bb6: { + bb5: { StorageDead(_12); StorageDead(_11); StorageDead(_9); goto -> bb1; } - bb7 (cleanup): { - drop(_3) -> [return: bb8, unwind terminate(cleanup)]; + bb6 (cleanup): { + drop(_3) -> [return: bb7, unwind terminate(cleanup)]; } - bb8 (cleanup): { + bb7 (cleanup): { resume; } } diff --git a/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-abort.mir index 2621ec67531..f3033d4a2fa 100644 --- a/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-abort.mir @@ -9,6 +9,14 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> { let _5: u32; let mut _6: u32; scope 3 { + scope 5 (inlined <u32 as Step>::forward_unchecked) { + scope 6 (inlined core::num::<impl u32>::unchecked_add) { + scope 7 (inlined core::ub_checks::check_language_ub) { + scope 8 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 4 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) { let mut _2: u32; @@ -18,7 +26,6 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> { } bb0: { - StorageLive(_5); StorageLive(_4); StorageLive(_2); _2 = copy ((*_1).0: u32); @@ -32,25 +39,21 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> { bb1: { _0 = const Option::<u32>::None; - goto -> bb4; + goto -> bb3; } bb2: { _5 = copy ((*_1).0: u32); StorageLive(_6); - _6 = <u32 as Step>::forward_unchecked(copy _5, const 1_usize) -> [return: bb3, unwind unreachable]; - } - - bb3: { + _6 = AddUnchecked(copy _5, const 1_u32); ((*_1).0: u32) = move _6; StorageDead(_6); _0 = Option::<u32>::Some(copy _5); - goto -> bb4; + goto -> bb3; } - bb4: { + bb3: { StorageDead(_4); - StorageDead(_5); return; } } diff --git a/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-unwind.mir index 338fb4b9523..f3033d4a2fa 100644 --- a/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/range_iter.range_iter_next.PreCodegen.after.panic-unwind.mir @@ -9,6 +9,14 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> { let _5: u32; let mut _6: u32; scope 3 { + scope 5 (inlined <u32 as Step>::forward_unchecked) { + scope 6 (inlined core::num::<impl u32>::unchecked_add) { + scope 7 (inlined core::ub_checks::check_language_ub) { + scope 8 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 4 (inlined std::cmp::impls::<impl PartialOrd for u32>::lt) { let mut _2: u32; @@ -18,7 +26,6 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> { } bb0: { - StorageLive(_5); StorageLive(_4); StorageLive(_2); _2 = copy ((*_1).0: u32); @@ -32,25 +39,21 @@ fn range_iter_next(_1: &mut std::ops::Range<u32>) -> Option<u32> { bb1: { _0 = const Option::<u32>::None; - goto -> bb4; + goto -> bb3; } bb2: { _5 = copy ((*_1).0: u32); StorageLive(_6); - _6 = <u32 as Step>::forward_unchecked(copy _5, const 1_usize) -> [return: bb3, unwind continue]; - } - - bb3: { + _6 = AddUnchecked(copy _5, const 1_u32); ((*_1).0: u32) = move _6; StorageDead(_6); _0 = Option::<u32>::Some(copy _5); - goto -> bb4; + goto -> bb3; } - bb4: { + bb3: { StorageDead(_4); - StorageDead(_5); return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir index 151783969dd..f8d11df5185 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir @@ -28,6 +28,14 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _7: usize; let mut _8: usize; scope 7 { + scope 9 (inlined <usize as Step>::forward_unchecked) { + scope 10 (inlined core::num::<impl usize>::unchecked_add) { + scope 11 (inlined core::ub_checks::check_language_ub) { + scope 12 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 8 (inlined std::cmp::impls::<impl PartialOrd for usize>::lt) { let mut _5: usize; @@ -47,7 +55,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb1: { StorageLive(_9); - StorageLive(_7); StorageLive(_6); StorageLive(_5); _5 = copy _4; @@ -58,7 +65,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb2: { StorageDead(_6); - StorageDead(_7); StorageDead(_9); StorageDead(_4); drop(_2) -> [return: bb3, unwind unreachable]; @@ -71,30 +77,26 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb4: { _7 = copy _4; StorageLive(_8); - _8 = <usize as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind unreachable]; - } - - bb5: { + _8 = AddUnchecked(copy _7, const 1_usize); _4 = move _8; StorageDead(_8); _9 = Option::<usize>::Some(copy _7); StorageDead(_6); - StorageDead(_7); _10 = copy ((_9 as Some).0: usize); _11 = Lt(copy _10, copy _3); - assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb6, unwind unreachable]; + assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind unreachable]; } - bb6: { + bb5: { _12 = &(*_1)[_10]; StorageLive(_13); _13 = &_2; StorageLive(_14); _14 = (copy _10, copy _12); - _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb7, unwind unreachable]; + _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb6, unwind unreachable]; } - bb7: { + bb6: { StorageDead(_14); StorageDead(_13); StorageDead(_9); diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir index 006329dc20d..2c249197894 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir @@ -28,6 +28,14 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _7: usize; let mut _8: usize; scope 7 { + scope 9 (inlined <usize as Step>::forward_unchecked) { + scope 10 (inlined core::num::<impl usize>::unchecked_add) { + scope 11 (inlined core::ub_checks::check_language_ub) { + scope 12 (inlined core::ub_checks::check_language_ub::runtime) { + } + } + } + } } scope 8 (inlined std::cmp::impls::<impl PartialOrd for usize>::lt) { let mut _5: usize; @@ -47,7 +55,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb1: { StorageLive(_9); - StorageLive(_7); StorageLive(_6); StorageLive(_5); _5 = copy _4; @@ -58,7 +65,6 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb2: { StorageDead(_6); - StorageDead(_7); StorageDead(_9); StorageDead(_4); drop(_2) -> [return: bb3, unwind continue]; @@ -71,41 +77,37 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb4: { _7 = copy _4; StorageLive(_8); - _8 = <usize as Step>::forward_unchecked(copy _7, const 1_usize) -> [return: bb5, unwind: bb8]; - } - - bb5: { + _8 = AddUnchecked(copy _7, const 1_usize); _4 = move _8; StorageDead(_8); _9 = Option::<usize>::Some(copy _7); StorageDead(_6); - StorageDead(_7); _10 = copy ((_9 as Some).0: usize); _11 = Lt(copy _10, copy _3); - assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb6, unwind: bb8]; + assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind: bb7]; } - bb6: { + bb5: { _12 = &(*_1)[_10]; StorageLive(_13); _13 = &_2; StorageLive(_14); _14 = (copy _10, copy _12); - _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb7, unwind: bb8]; + _15 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _13, move _14) -> [return: bb6, unwind: bb7]; } - bb7: { + bb6: { StorageDead(_14); StorageDead(_13); StorageDead(_9); goto -> bb1; } - bb8 (cleanup): { - drop(_2) -> [return: bb9, unwind terminate(cleanup)]; + bb7 (cleanup): { + drop(_2) -> [return: bb8, unwind terminate(cleanup)]; } - bb9 (cleanup): { + bb8 (cleanup): { resume; } } |