Auto merge of #147231 - CrooseGit:dev/reucru01/extend_rustc_force_inline, r=saethlin

Extending `#[rustc_force_inline]` to be applicable to inherent methods

`#[rustc_force_inline]` is an internal-only attribute similar to `#[inline(always)]` but which emits an error if inlining cannot occur. rustc_force_inline uses the MIR inliner to do this and has limitations on where it can be applied to ensure that an error is always emitted if inlining can't happen (e.g. it can't be applied to trait methods because calls to those can't always be resolved).

`#[rustc_force_inline]` is motivated by AArch64 pointer authentication intrinsics where it is vital for the security properties of these intrinsics that they do not exist in standalone functions that could be used as gadgets in an exploit (if they could, then you could sign whatever pointers you want, for example, which is bad, but if you force inlining, then you can't jump to a reusable function containing only these instructions).

Since its initial implementation, `#[rustc_force_inline]` could only be applied to free functions. This can be relaxed to also allow inherent methods while still preserving the desired properties. In a work-in-progress patch for manual pointer authentication intrinsics, it is useful to introduce types with inherent methods that would need to be force inlined.

r? `@saethlin`

19 files changed, 341 insertions, 24 deletions

diff --git a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff
new file mode 100644
index 00000000000..8e03432c2af
--- /dev/null
+++ b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller::{closure#0}` before ForceInline
++ // MIR for `caller::{closure#0}` after ForceInline
+  
+  fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure_inherent.rs:14:6: 14:8}) -> () {
+      let mut _0: ();
+      let _2: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_2);
+-         _2 = Foo::callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff
new file mode 100644
index 00000000000..0e41fd89dac
--- /dev/null
+++ b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller::{closure#0}` before ForceInline
++ // MIR for `caller::{closure#0}` after ForceInline
+  
+  fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure_inherent.rs:14:6: 14:8}) -> () {
+      let mut _0: ();
+      let _2: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_2);
+-         _2 = Foo::callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_closure_inherent.rs b/tests/mir-opt/inline/forced_closure_inherent.rs
new file mode 100644
index 00000000000..949c7d6ecbf
--- /dev/null
+++ b/tests/mir-opt/inline/forced_closure_inherent.rs
@@ -0,0 +1,19 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+struct Foo {}
+
+impl Foo {
+    #[rustc_force_inline]
+    pub fn callee_forced() {}
+}
+
+// EMIT_MIR forced_closure_inherent.caller-{closure#0}.ForceInline.diff
+pub fn caller() {
+    (|| {
+        Foo::callee_forced();
+        // CHECK-LABEL: fn caller::{closure#0}(
+        // CHECK: (inlined Foo::callee_forced)
+    })();
+}
diff --git a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff
new file mode 100644
index 00000000000..6ea1894af98
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 00000000000..dd91c338772
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent.rs b/tests/mir-opt/inline/forced_inherent.rs
new file mode 100644
index 00000000000..24bf8daa644
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent.rs
@@ -0,0 +1,17 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+struct Foo;
+
+impl Foo {
+    #[rustc_force_inline]
+    fn bar() {}
+}
+
+// EMIT_MIR forced_inherent.caller.ForceInline.diff
+fn caller() {
+    Foo::bar();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::bar)
+}
diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff
new file mode 100644
index 00000000000..6ea1894af98
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 00000000000..dd91c338772
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.rs b/tests/mir-opt/inline/forced_inherent_ambiguous.rs
new file mode 100644
index 00000000000..e3c5d3e4f9e
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_ambiguous.rs
@@ -0,0 +1,25 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+struct Foo;
+
+impl Foo {
+    #[rustc_force_inline]
+    fn bar() {}
+}
+
+trait Tr {
+    fn bar();
+}
+
+impl Tr for Foo {
+    fn bar() {}
+}
+
+// EMIT_MIR forced_inherent_ambiguous.caller.ForceInline.diff
+fn caller() {
+    Foo::bar();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::bar)
+}
diff --git a/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-abort.diff
new file mode 100644
index 00000000000..6495ddbafba
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,12 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> {async fn body of caller()} {
+      let mut _0: {async fn body of caller()};
+  
+      bb0: {
+          _0 = {coroutine@$DIR/forced_inherent_async.rs:14:19: 18:2 (#0)};
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 00000000000..6495ddbafba
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,12 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> {async fn body of caller()} {
+      let mut _0: {async fn body of caller()};
+  
+      bb0: {
+          _0 = {coroutine@$DIR/forced_inherent_async.rs:14:19: 18:2 (#0)};
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_async.rs b/tests/mir-opt/inline/forced_inherent_async.rs
new file mode 100644
index 00000000000..ce58a0ac48f
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_async.rs
@@ -0,0 +1,18 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+//@ edition: 2021
+#![feature(rustc_attrs)]
+
+struct Foo {}
+
+impl Foo {
+    #[rustc_force_inline]
+    pub fn callee_forced() {}
+}
+
+// EMIT_MIR forced_inherent_async.caller.ForceInline.diff
+async fn caller() {
+    Foo::callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::callee_forced)
+}
diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff
new file mode 100644
index 00000000000..edaf2820d85
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 00000000000..22f8b14a724
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.rs b/tests/mir-opt/inline/forced_inherent_dead_code.rs
new file mode 100644
index 00000000000..057a4cac528
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_dead_code.rs
@@ -0,0 +1,21 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 -Clink-dead-code
+#![feature(rustc_attrs)]
+
+struct Foo {}
+
+impl Foo {
+    #[rustc_force_inline]
+    pub fn callee_forced() {}
+}
+
+// EMIT_MIR forced_inherent_dead_code.caller.ForceInline.diff
+pub fn caller() {
+    Foo::callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::callee_forced)
+}
+
+fn main() {
+    caller();
+}
diff --git a/tests/ui/force-inlining/inherent.rs b/tests/ui/force-inlining/inherent.rs
new file mode 100644
index 00000000000..25c76eaf37a
--- /dev/null
+++ b/tests/ui/force-inlining/inherent.rs
@@ -0,0 +1,21 @@
+//@ check-fail
+#![feature(rustc_attrs)]
+
+struct Foo;
+
+impl Foo {
+    #[rustc_force_inline]
+    //~^ ERROR: `Foo::bar` is incompatible with `#[rustc_force_inline]`
+    #[rustc_no_mir_inline]
+    fn bar() {}
+}
+
+fn bar_caller() {
+    unsafe {
+        Foo::bar();
+    }
+}
+
+fn main() {
+    bar_caller();
+}
diff --git a/tests/ui/force-inlining/inherent.stderr b/tests/ui/force-inlining/inherent.stderr
new file mode 100644
index 00000000000..1ffc78848fe
--- /dev/null
+++ b/tests/ui/force-inlining/inherent.stderr
@@ -0,0 +1,13 @@
+error: `Foo::bar` is incompatible with `#[rustc_force_inline]`
+  --> $DIR/inherent.rs:7:5
+   |
+LL |     #[rustc_force_inline]
+   |     ^^^^^^^^^^^^^^^^^^^^^
+...
+LL |     fn bar() {}
+   |     -------- `Foo::bar` defined here
+   |
+   = note: incompatible due to: #[rustc_no_mir_inline]
+
+error: aborting due to 1 previous error
+
diff --git a/tests/ui/force-inlining/invalid.rs b/tests/ui/force-inlining/invalid.rs
index 6047739992f..eaacbb50209 100644
--- a/tests/ui/force-inlining/invalid.rs
+++ b/tests/ui/force-inlining/invalid.rs
@@ -114,7 +114,6 @@ trait FooQux = FooBaz;
 //~^ ERROR attribute cannot be used on
 impl<T> Bar<T> {
     #[rustc_force_inline]
-//~^ ERROR attribute cannot be used on
     fn foo() {}
 }
 
diff --git a/tests/ui/force-inlining/invalid.stderr b/tests/ui/force-inlining/invalid.stderr
index 299a3ed4a46..df3b646b6ce 100644
--- a/tests/ui/force-inlining/invalid.stderr
+++ b/tests/ui/force-inlining/invalid.stderr
@@ -1,5 +1,5 @@
 error: allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters
-  --> $DIR/invalid.rs:132:11
+  --> $DIR/invalid.rs:131:11
    |
 LL | fn barqux(#[rustc_force_inline] _x: u32) {}
    |           ^^^^^^^^^^^^^^^^^^^^^
@@ -134,7 +134,7 @@ error: `#[rustc_force_inline]` attribute cannot be used on foreign functions
 LL |     #[rustc_force_inline]
    |     ^^^^^^^^^^^^^^^^^^^^^
    |
-   = help: `#[rustc_force_inline]` can only be applied to functions
+   = help: `#[rustc_force_inline]` can be applied to functions and inherent methods
 
 error: `#[rustc_force_inline]` attribute cannot be used on type aliases
   --> $DIR/invalid.rs:66:1
@@ -222,7 +222,7 @@ error: `#[rustc_force_inline]` attribute cannot be used on provided trait method
 LL |     #[rustc_force_inline]
    |     ^^^^^^^^^^^^^^^^^^^^^
    |
-   = help: `#[rustc_force_inline]` can only be applied to functions
+   = help: `#[rustc_force_inline]` can be applied to functions and inherent methods
 
 error: `#[rustc_force_inline]` attribute cannot be used on trait aliases
   --> $DIR/invalid.rs:109:1
@@ -240,16 +240,8 @@ LL | #[rustc_force_inline]
    |
    = help: `#[rustc_force_inline]` can only be applied to functions
 
-error: `#[rustc_force_inline]` attribute cannot be used on inherent methods
-  --> $DIR/invalid.rs:116:5
-   |
-LL |     #[rustc_force_inline]
-   |     ^^^^^^^^^^^^^^^^^^^^^
-   |
-   = help: `#[rustc_force_inline]` can only be applied to functions
-
 error: `#[rustc_force_inline]` attribute cannot be used on trait impl blocks
-  --> $DIR/invalid.rs:121:1
+  --> $DIR/invalid.rs:120:1
    |
 LL | #[rustc_force_inline]
    | ^^^^^^^^^^^^^^^^^^^^^
@@ -257,7 +249,7 @@ LL | #[rustc_force_inline]
    = help: `#[rustc_force_inline]` can only be applied to functions
 
 error: `#[rustc_force_inline]` attribute cannot be used on macro defs
-  --> $DIR/invalid.rs:128:1
+  --> $DIR/invalid.rs:127:1
    |
 LL | #[rustc_force_inline]
    | ^^^^^^^^^^^^^^^^^^^^^
@@ -265,7 +257,7 @@ LL | #[rustc_force_inline]
    = help: `#[rustc_force_inline]` can only be applied to functions
 
 error: `#[rustc_force_inline]` attribute cannot be used on function params
-  --> $DIR/invalid.rs:132:11
+  --> $DIR/invalid.rs:131:11
    |
 LL | fn barqux(#[rustc_force_inline] _x: u32) {}
    |           ^^^^^^^^^^^^^^^^^^^^^
@@ -273,15 +265,15 @@ LL | fn barqux(#[rustc_force_inline] _x: u32) {}
    = help: `#[rustc_force_inline]` can only be applied to functions
 
 error: `#[rustc_force_inline]` attribute cannot be used on closures
-  --> $DIR/invalid.rs:149:14
+  --> $DIR/invalid.rs:148:14
    |
 LL |     let _x = #[rustc_force_inline] || { };
    |              ^^^^^^^^^^^^^^^^^^^^^
    |
-   = help: `#[rustc_force_inline]` can only be applied to functions
+   = help: `#[rustc_force_inline]` can be applied to functions and inherent methods
 
 error: `#[rustc_force_inline]` attribute cannot be used on expressions
-  --> $DIR/invalid.rs:151:14
+  --> $DIR/invalid.rs:150:14
    |
 LL |     let _y = #[rustc_force_inline] 3 + 4;
    |              ^^^^^^^^^^^^^^^^^^^^^
@@ -289,7 +281,7 @@ LL |     let _y = #[rustc_force_inline] 3 + 4;
    = help: `#[rustc_force_inline]` can only be applied to functions
 
 error: `#[rustc_force_inline]` attribute cannot be used on statements
-  --> $DIR/invalid.rs:153:5
+  --> $DIR/invalid.rs:152:5
    |
 LL |     #[rustc_force_inline]
    |     ^^^^^^^^^^^^^^^^^^^^^
@@ -297,7 +289,7 @@ LL |     #[rustc_force_inline]
    = help: `#[rustc_force_inline]` can only be applied to functions
 
 error: `#[rustc_force_inline]` attribute cannot be used on match arms
-  --> $DIR/invalid.rs:158:9
+  --> $DIR/invalid.rs:157:9
    |
 LL |         #[rustc_force_inline]
    |         ^^^^^^^^^^^^^^^^^^^^^
@@ -305,7 +297,7 @@ LL |         #[rustc_force_inline]
    = help: `#[rustc_force_inline]` can only be applied to functions
 
 error: attribute cannot be applied to a `async`, `gen` or `async gen` function
-  --> $DIR/invalid.rs:136:1
+  --> $DIR/invalid.rs:135:1
    |
 LL | #[rustc_force_inline]
    | ^^^^^^^^^^^^^^^^^^^^^
@@ -314,7 +306,7 @@ LL | async fn async_foo() {}
    | -------------------- `async`, `gen` or `async gen` function
 
 error: attribute cannot be applied to a `async`, `gen` or `async gen` function
-  --> $DIR/invalid.rs:140:1
+  --> $DIR/invalid.rs:139:1
    |
 LL | #[rustc_force_inline]
    | ^^^^^^^^^^^^^^^^^^^^^
@@ -323,7 +315,7 @@ LL | gen fn gen_foo() {}
    | ---------------- `async`, `gen` or `async gen` function
 
 error: attribute cannot be applied to a `async`, `gen` or `async gen` function
-  --> $DIR/invalid.rs:144:1
+  --> $DIR/invalid.rs:143:1
    |
 LL | #[rustc_force_inline]
    | ^^^^^^^^^^^^^^^^^^^^^
@@ -331,7 +323,7 @@ LL |
 LL | async gen fn async_gen_foo() {}
    | ---------------------------- `async`, `gen` or `async gen` function
 
-error: aborting due to 37 previous errors
+error: aborting due to 36 previous errors
 
 Some errors have detailed explanations: E0539, E0805.
 For more information about an error, try `rustc --explain E0539`.