diff options
13 files changed, 102 insertions, 201 deletions
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 8abd549c588..fc9e18378d5 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -222,7 +222,14 @@ impl<'tcx> Inliner<'tcx> { trace!(?output_type, ?destination_ty); return Err("failed to normalize return type"); } - if callsite.fn_sig.abi() == Abi::RustCall && callee_body.spread_arg.is_none() { + if callsite.fn_sig.abi() == Abi::RustCall { + // FIXME: Don't inline user-written `extern "rust-call"` functions, + // since this is generally perf-negative on rustc, and we hope that + // LLVM will inline these functions instead. + if callee_body.spread_arg.is_some() { + return Err("do not inline user-written rust-call functions"); + } + let (self_arg, arg_tuple) = match &args[..] { [arg_tuple] => (None, arg_tuple), [self_arg, arg_tuple] => (Some(self_arg), arg_tuple), diff --git a/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff b/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff index f3f4d895ae2..8f2baf4a3b6 100644 --- a/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff @@ -5,13 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {main}; -+ let mut _5: (); + scope 1 (inlined f::<fn() {main}>) { + debug g => _2; + let mut _3: &fn() {main}; + let _4: (); -+ scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { -+ } + } bb0: { @@ -22,9 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ _4 = move (*_3)() -> [return: bb2, unwind unreachable]; ++ _4 = <fn() {main} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind unreachable]; } bb1: { @@ -36,7 +31,6 @@ + } + + bb2: { -+ StorageDead(_5); + StorageDead(_3); + drop(_2) -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff b/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff index 3ce8d9acf36..ad801fd280a 100644 --- a/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff @@ -5,13 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {main}; -+ let mut _5: (); + scope 1 (inlined f::<fn() {main}>) { + debug g => _2; + let mut _3: &fn() {main}; + let _4: (); -+ scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { -+ } + } bb0: { @@ -22,9 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ _4 = move (*_3)() -> [return: bb4, unwind: bb2]; ++ _4 = <fn() {main} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind: bb3]; } bb1: { @@ -35,18 +30,17 @@ return; + } + -+ bb2 (cleanup): { -+ drop(_2) -> [return: bb3, unwind terminate]; ++ bb2: { ++ StorageDead(_3); ++ drop(_2) -> [return: bb1, unwind continue]; + } + + bb3 (cleanup): { -+ resume; ++ drop(_2) -> [return: bb4, unwind terminate]; + } + -+ bb4: { -+ StorageDead(_5); -+ StorageDead(_3); -+ drop(_2) -> [return: bb1, unwind continue]; ++ bb4 (cleanup): { ++ resume; } } diff --git a/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff b/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff index eb007635416..fd1f698c60d 100644 --- a/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff @@ -5,21 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {g}; -+ let mut _5: (); + scope 1 (inlined f::<fn() {g}>) { + debug g => _2; + let mut _3: &fn() {g}; + let _4: (); -+ scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) { -+ scope 3 (inlined g) { -+ scope 4 (inlined f::<fn() {main}>) { -+ debug g => main; -+ let _6: (); -+ scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -30,10 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ StorageLive(_6); -+ _6 = main() -> [return: bb2, unwind unreachable]; ++ _4 = <fn() {g} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind unreachable]; } bb1: { @@ -45,8 +31,6 @@ + } + + bb2: { -+ StorageDead(_6); -+ StorageDead(_5); + StorageDead(_3); + drop(_2) -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff index 198a2322618..99dc64115a9 100644 --- a/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff @@ -5,21 +5,10 @@ let mut _0: (); let _1: (); + let mut _2: fn() {g}; -+ let mut _5: (); + scope 1 (inlined f::<fn() {g}>) { + debug g => _2; + let mut _3: &fn() {g}; + let _4: (); -+ scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) { -+ scope 3 (inlined g) { -+ scope 4 (inlined f::<fn() {main}>) { -+ debug g => main; -+ let _6: (); -+ scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -30,10 +19,7 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_5); -+ _5 = const (); -+ StorageLive(_6); -+ _6 = main() -> [return: bb4, unwind: bb2]; ++ _4 = <fn() {g} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind: bb3]; } bb1: { @@ -44,19 +30,17 @@ return; + } + -+ bb2 (cleanup): { -+ drop(_2) -> [return: bb3, unwind terminate]; ++ bb2: { ++ StorageDead(_3); ++ drop(_2) -> [return: bb1, unwind continue]; + } + + bb3 (cleanup): { -+ resume; ++ drop(_2) -> [return: bb4, unwind terminate]; + } + -+ bb4: { -+ StorageDead(_6); -+ StorageDead(_5); -+ StorageDead(_3); -+ drop(_2) -> [return: bb1, unwind continue]; ++ bb4 (cleanup): { ++ resume; } } diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.diff index c665d779e61..757617e5940 100644 --- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.diff +++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.diff @@ -7,40 +7,23 @@ let mut _0: (); let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>; let mut _4: I; -+ scope 1 (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut) { -+ debug self => _3; -+ debug args => _4; -+ let mut _5: &mut dyn std::ops::FnMut<I, Output = ()>; -+ let mut _6: std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>; -+ let mut _7: *const dyn std::ops::FnMut<I, Output = ()>; -+ } bb0: { StorageLive(_3); _3 = &mut _1; StorageLive(_4); _4 = move _2; -- _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable]; -+ StorageLive(_5); -+ _6 = deref_copy (*_3); -+ _7 = (((_6.0: std::ptr::Unique<dyn std::ops::FnMut<I, Output = ()>>).0: std::ptr::NonNull<dyn std::ops::FnMut<I, Output = ()>>).0: *const dyn std::ops::FnMut<I, Output = ()>); -+ _5 = &mut (*_7); -+ _0 = <dyn FnMut<I, Output = ()> as FnMut<I>>::call_mut(move _5, move _4) -> [return: bb2, unwind unreachable]; + _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable]; } bb1: { -- StorageDead(_4); -- StorageDead(_3); -- drop(_1) -> [return: bb2, unwind unreachable]; -+ return; + StorageDead(_4); + StorageDead(_3); + drop(_1) -> [return: bb2, unwind unreachable]; } bb2: { -- return; -+ StorageDead(_5); -+ StorageDead(_4); -+ StorageDead(_3); -+ drop(_1) -> [return: bb1, unwind unreachable]; + return; } } diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.diff index 34ac637af8a..4fa04b05e89 100644 --- a/tests/mir-opt/inline/inline_box_fn.call.Inline.diff +++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.diff @@ -7,13 +7,6 @@ let _2: (); let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>; let mut _4: (i32,); -+ scope 1 (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call) { -+ debug self => _3; -+ debug args => _4; -+ let mut _5: &dyn std::ops::Fn(i32); -+ let mut _6: std::boxed::Box<dyn std::ops::Fn(i32)>; -+ let mut _7: *const dyn std::ops::Fn(i32); -+ } bb0: { StorageLive(_2); @@ -21,30 +14,19 @@ _3 = &_1; StorageLive(_4); _4 = (const 1_i32,); -- _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable]; -+ StorageLive(_5); -+ _6 = deref_copy (*_3); -+ _7 = (((_6.0: std::ptr::Unique<dyn std::ops::Fn(i32)>).0: std::ptr::NonNull<dyn std::ops::Fn(i32)>).0: *const dyn std::ops::Fn(i32)); -+ _5 = &(*_7); -+ _2 = <dyn Fn(i32) as Fn<(i32,)>>::call(move _5, move _4) -> [return: bb2, unwind unreachable]; + _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable]; } bb1: { -+ return; -+ } -+ -+ bb2: { -+ StorageDead(_5); StorageDead(_4); StorageDead(_3); StorageDead(_2); _0 = const (); -- drop(_1) -> [return: bb2, unwind unreachable]; -- } -- -- bb2: { -- return; -+ drop(_1) -> [return: bb1, unwind unreachable]; + drop(_1) -> [return: bb2, unwind unreachable]; + } + + bb2: { + return; } } diff --git a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff index d83c8d585ea..8a6eec3352a 100644 --- a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff @@ -5,20 +5,9 @@ let mut _0: (); let _1: (); + let mut _2: fn() {f}; -+ let mut _4: (); + scope 1 (inlined call::<fn() {f}>) { + debug f => _2; + let _3: (); -+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { -+ scope 3 (inlined f) { -+ scope 4 (inlined call::<fn() {f}>) { -+ debug f => f; -+ let _5: (); -+ scope 5 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -27,15 +16,10 @@ + StorageLive(_2); + _2 = f; + StorageLive(_3); -+ StorageLive(_4); -+ _4 = const (); -+ StorageLive(_5); -+ _5 = f() -> [return: bb1, unwind unreachable]; ++ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind unreachable]; } bb1: { -+ StorageDead(_5); -+ StorageDead(_4); + StorageDead(_3); + StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff index a08662959dd..a24649c1ebd 100644 --- a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff @@ -5,20 +5,9 @@ let mut _0: (); let _1: (); + let mut _2: fn() {f}; -+ let mut _4: (); + scope 1 (inlined call::<fn() {f}>) { + debug f => _2; + let _3: (); -+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { -+ scope 3 (inlined f) { -+ scope 4 (inlined call::<fn() {f}>) { -+ debug f => f; -+ let _5: (); -+ scope 5 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { -+ } -+ } -+ } -+ } + } bb0: { @@ -27,15 +16,10 @@ + StorageLive(_2); + _2 = f; + StorageLive(_3); -+ StorageLive(_4); -+ _4 = const (); -+ StorageLive(_5); -+ _5 = f() -> [return: bb1, unwind continue]; ++ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind continue]; } bb1: { -+ StorageDead(_5); -+ StorageDead(_4); + StorageDead(_3); + StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff index 0dcd5fae88d..7d5553b2f37 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff @@ -5,7 +5,6 @@ let mut _0: (); let _1: (!, !); + let mut _2: fn() -> ! {sleep}; -+ let mut _7: (); + scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) { + debug f => _2; + let mut _3: &fn() -> ! {sleep}; @@ -18,10 +17,6 @@ + debug b => _6; + } + } -+ scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { -+ scope 5 (inlined sleep) { -+ } -+ } + } bb0: { @@ -33,13 +28,24 @@ + StorageLive(_6); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_7); -+ _7 = const (); -+ goto -> bb1; ++ _4 = <fn() -> ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind unreachable]; + } + + bb1: { -+ goto -> bb1; ++ StorageDead(_3); ++ StorageLive(_5); ++ _5 = &_2; ++ _6 = <fn() -> ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind unreachable]; ++ } ++ ++ bb2: { ++ StorageDead(_5); ++ _1 = (move _4, move _6); ++ drop(_2) -> [return: bb3, unwind unreachable]; ++ } ++ ++ bb3: { ++ unreachable; } } diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff index dfc12db12a8..073ddeff7ca 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff @@ -5,7 +5,6 @@ let mut _0: (); let _1: (!, !); + let mut _2: fn() -> ! {sleep}; -+ let mut _8: (); + scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) { + debug f => _2; + let mut _3: &fn() -> ! {sleep}; @@ -19,10 +18,6 @@ + debug b => _6; + } + } -+ scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { -+ scope 5 (inlined sleep) { -+ } -+ } + } bb0: { @@ -34,13 +29,40 @@ + StorageLive(_4); + StorageLive(_3); + _3 = &_2; -+ StorageLive(_8); -+ _8 = const (); -+ goto -> bb1; ++ _4 = <fn() -> ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind: bb5]; + } + + bb1: { -+ goto -> bb1; ++ StorageDead(_3); ++ StorageLive(_5); ++ _5 = &_2; ++ _6 = <fn() -> ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind: bb4]; ++ } ++ ++ bb2: { ++ StorageDead(_5); ++ StorageLive(_7); ++ _7 = move _4; ++ _1 = (move _7, move _6); ++ StorageDead(_7); ++ StorageDead(_4); ++ drop(_2) -> [return: bb3, unwind continue]; ++ } ++ ++ bb3: { ++ unreachable; ++ } ++ ++ bb4 (cleanup): { ++ drop(_4) -> [return: bb5, unwind terminate]; ++ } ++ ++ bb5 (cleanup): { ++ drop(_2) -> [return: bb6, unwind terminate]; ++ } ++ ++ bb6 (cleanup): { ++ resume; } } diff --git a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff index b86eb5f35c9..bee01a5f97b 100644 --- a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff @@ -8,8 +8,6 @@ let mut _3: &fn() {foo}; let _4: fn() {foo}; let mut _5: (); -+ scope 1 (inlined <fn() {foo} as Fn<()>>::call - shim(fn() {foo})) { -+ } bb0: { StorageLive(_2); @@ -22,26 +20,20 @@ _3 = &_4; StorageLive(_5); _5 = (); -- _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind unreachable]; -+ _2 = move (*_3)() -> [return: bb3, unwind unreachable]; + _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind unreachable]; } bb2: { -+ return; -+ } -+ -+ bb3: { StorageDead(_5); StorageDead(_3); StorageDead(_4); StorageDead(_2); _0 = const (); -- drop(_1) -> [return: bb3, unwind unreachable]; -- } -- -- bb3: { -- return; -+ drop(_1) -> [return: bb2, unwind unreachable]; + drop(_1) -> [return: bb3, unwind unreachable]; + } + + bb3: { + return; } } diff --git a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff index c67babba23e..b750330df92 100644 --- a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff @@ -8,55 +8,40 @@ let mut _3: &fn() {foo}; let _4: fn() {foo}; let mut _5: (); -+ scope 1 (inlined <fn() {foo} as Fn<()>>::call - shim(fn() {foo})) { -+ } bb0: { StorageLive(_2); StorageLive(_3); StorageLive(_4); -- _4 = hide_foo() -> [return: bb1, unwind: bb4]; -+ _4 = hide_foo() -> [return: bb1, unwind: bb3]; + _4 = hide_foo() -> [return: bb1, unwind: bb4]; } bb1: { _3 = &_4; StorageLive(_5); _5 = (); -- _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind: bb4]; -+ _2 = move (*_3)() -> [return: bb5, unwind: bb3]; + _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind: bb4]; } bb2: { -- StorageDead(_5); -- StorageDead(_3); -- StorageDead(_4); -- StorageDead(_2); -- _0 = const (); -- drop(_1) -> [return: bb3, unwind: bb5]; -+ return; + StorageDead(_5); + StorageDead(_3); + StorageDead(_4); + StorageDead(_2); + _0 = const (); + drop(_1) -> [return: bb3, unwind: bb5]; } -- bb3: { -- return; -+ bb3 (cleanup): { -+ drop(_1) -> [return: bb4, unwind terminate]; + bb3: { + return; } bb4 (cleanup): { -- drop(_1) -> [return: bb5, unwind terminate]; -+ resume; + drop(_1) -> [return: bb5, unwind terminate]; } -- bb5 (cleanup): { -- resume; -+ bb5: { -+ StorageDead(_5); -+ StorageDead(_3); -+ StorageDead(_4); -+ StorageDead(_2); -+ _0 = const (); -+ drop(_1) -> [return: bb2, unwind: bb4]; + bb5 (cleanup): { + resume; } } |