Rollup merge of #60300 - mjbshaw:ffi_types, r=rkruppe

Allow null-pointer-optimized enums in FFI if their underlying representation is FFI safe

I'm not sure if this requires an RFC. I attempted to start [a discussion on internals.rust-lang.org](https://internals.rust-lang.org/t/options-ffi-safety-and-guarantees-for-abi-compatibility-with-nonnull-optimizations/9784) and when no one really objected I figured I'd go ahead and try implementing this.

This allows types like `Option<NonZeroU8>` to be used in FFI without triggering the `improper_ctypes` lint. This works by changing the `is_repr_nullable_ptr` function to consider an enum `E` to be FFI-safe if:

- `E` has no explicit `#[repr(...)]`.
- It only has two variants.
- One of those variants is empty (meaning it has no fields).
- The other variant has only one field.
- That field is one of the following:
  - `&T`
  - `&mut T`
  - `extern "C" fn`
  - `core::num::NonZero*`
  - `core::ptr::NonNull<T>`
  - `#[repr(transparent)] struct` wrapper around one of the types in this list.
- The size of `E` and its field are both known and are both the same size (implying `E` is participating in the nonnull optimization).

This logic seems consistent with [the Rust nomicon](https://doc.rust-lang.org/nomicon/repr-rust.html).

4 files changed, 76 insertions, 11 deletions

diff --git a/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.rs b/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.rs
index f7ff3eb3ac9..9f5c92349e0 100644
--- a/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.rs
+++ b/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.rs
@@ -4,5 +4,6 @@
 
 #[rustc_variance] //~ ERROR the `#[rustc_variance]` attribute is just used for rustc unit tests and will never be stable
 #[rustc_error] //~ ERROR the `#[rustc_error]` attribute is just used for rustc unit tests and will never be stable
+#[rustc_nonnull_optimization_guaranteed] //~ ERROR the `#[rustc_nonnull_optimization_guaranteed]` attribute is just used to enable niche optimizations in libcore and will never be stable
 
 fn main() {}
diff --git a/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.stderr b/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.stderr
index 882feb87f42..ed98484e13c 100644
--- a/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.stderr
+++ b/src/test/ui/feature-gates/feature-gate-rustc-attrs-1.stderr
@@ -16,6 +16,15 @@ LL | #[rustc_error]
    = note: for more information, see https://github.com/rust-lang/rust/issues/29642
    = help: add #![feature(rustc_attrs)] to the crate attributes to enable
 
-error: aborting due to 2 previous errors
+error[E0658]: the `#[rustc_nonnull_optimization_guaranteed]` attribute is just used to enable niche optimizations in libcore and will never be stable
+  --> $DIR/feature-gate-rustc-attrs-1.rs:7:1
+   |
+LL | #[rustc_nonnull_optimization_guaranteed]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: for more information, see https://github.com/rust-lang/rust/issues/29642
+   = help: add #![feature(rustc_attrs)] to the crate attributes to enable
+
+error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0658`.
diff --git a/src/test/ui/lint/lint-ctypes-enum.rs b/src/test/ui/lint/lint-ctypes-enum.rs
index f347c2761c3..d3e11d2f7ed 100644
--- a/src/test/ui/lint/lint-ctypes-enum.rs
+++ b/src/test/ui/lint/lint-ctypes-enum.rs
@@ -1,6 +1,8 @@
 #![deny(improper_ctypes)]
 #![allow(dead_code)]
 
+use std::num;
+
 enum Z { }
 enum U { A }
 enum B { C, D }
@@ -15,14 +17,39 @@ enum U8 { A, B, C }
 #[repr(isize)]
 enum Isize { A, B, C }
 
+#[repr(transparent)]
+struct Transparent<T>(T, std::marker::PhantomData<Z>);
+
+struct Rust<T>(T);
+
 extern {
    fn zf(x: Z);
    fn uf(x: U); //~ ERROR enum has no representation hint
    fn bf(x: B); //~ ERROR enum has no representation hint
    fn tf(x: T); //~ ERROR enum has no representation hint
-   fn reprc(x: ReprC);
-   fn u8(x: U8);
-   fn isize(x: Isize);
+   fn repr_c(x: ReprC);
+   fn repr_u8(x: U8);
+   fn repr_isize(x: Isize);
+   fn option_ref(x: Option<&'static u8>);
+   fn option_fn(x: Option<extern "C" fn()>);
+   fn nonnull(x: Option<std::ptr::NonNull<u8>>);
+   fn nonzero_u8(x: Option<num::NonZeroU8>);
+   fn nonzero_u16(x: Option<num::NonZeroU16>);
+   fn nonzero_u32(x: Option<num::NonZeroU32>);
+   fn nonzero_u64(x: Option<num::NonZeroU64>);
+   fn nonzero_u128(x: Option<num::NonZeroU128>);
+   //~^ ERROR 128-bit integers don't currently have a known stable ABI
+   fn nonzero_usize(x: Option<num::NonZeroUsize>);
+   fn nonzero_i8(x: Option<num::NonZeroI8>);
+   fn nonzero_i16(x: Option<num::NonZeroI16>);
+   fn nonzero_i32(x: Option<num::NonZeroI32>);
+   fn nonzero_i64(x: Option<num::NonZeroI64>);
+   fn nonzero_i128(x: Option<num::NonZeroI128>);
+   //~^ ERROR 128-bit integers don't currently have a known stable ABI
+   fn nonzero_isize(x: Option<num::NonZeroIsize>);
+   fn repr_transparent(x: Option<Transparent<num::NonZeroU8>>);
+   fn repr_rust(x: Option<Rust<num::NonZeroU8>>); //~ ERROR enum has no representation hint
+   fn no_result(x: Result<(), num::NonZeroI32>); //~ ERROR enum has no representation hint
 }
 
 pub fn main() { }
diff --git a/src/test/ui/lint/lint-ctypes-enum.stderr b/src/test/ui/lint/lint-ctypes-enum.stderr
index 92f76cfc38a..6b807f48aaa 100644
--- a/src/test/ui/lint/lint-ctypes-enum.stderr
+++ b/src/test/ui/lint/lint-ctypes-enum.stderr
@@ -1,5 +1,5 @@
 error: `extern` block uses type `U` which is not FFI-safe: enum has no representation hint
-  --> $DIR/lint-ctypes-enum.rs:20:13
+  --> $DIR/lint-ctypes-enum.rs:27:13
    |
 LL |    fn uf(x: U);
    |             ^
@@ -11,36 +11,64 @@ LL | #![deny(improper_ctypes)]
    |         ^^^^^^^^^^^^^^^
    = help: consider adding a #[repr(...)] attribute to this enum
 note: type defined here
-  --> $DIR/lint-ctypes-enum.rs:5:1
+  --> $DIR/lint-ctypes-enum.rs:7:1
    |
 LL | enum U { A }
    | ^^^^^^^^^^^^
 
 error: `extern` block uses type `B` which is not FFI-safe: enum has no representation hint
-  --> $DIR/lint-ctypes-enum.rs:21:13
+  --> $DIR/lint-ctypes-enum.rs:28:13
    |
 LL |    fn bf(x: B);
    |             ^
    |
    = help: consider adding a #[repr(...)] attribute to this enum
 note: type defined here
-  --> $DIR/lint-ctypes-enum.rs:6:1
+  --> $DIR/lint-ctypes-enum.rs:8:1
    |
 LL | enum B { C, D }
    | ^^^^^^^^^^^^^^^
 
 error: `extern` block uses type `T` which is not FFI-safe: enum has no representation hint
-  --> $DIR/lint-ctypes-enum.rs:22:13
+  --> $DIR/lint-ctypes-enum.rs:29:13
    |
 LL |    fn tf(x: T);
    |             ^
    |
    = help: consider adding a #[repr(...)] attribute to this enum
 note: type defined here
-  --> $DIR/lint-ctypes-enum.rs:7:1
+  --> $DIR/lint-ctypes-enum.rs:9:1
    |
 LL | enum T { E, F, G }
    | ^^^^^^^^^^^^^^^^^^
 
-error: aborting due to 3 previous errors
+error: `extern` block uses type `u128` which is not FFI-safe: 128-bit integers don't currently have a known stable ABI
+  --> $DIR/lint-ctypes-enum.rs:40:23
+   |
+LL |    fn nonzero_u128(x: Option<num::NonZeroU128>);
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: `extern` block uses type `i128` which is not FFI-safe: 128-bit integers don't currently have a known stable ABI
+  --> $DIR/lint-ctypes-enum.rs:47:23
+   |
+LL |    fn nonzero_i128(x: Option<num::NonZeroI128>);
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: `extern` block uses type `std::option::Option<Rust<std::num::NonZeroU8>>` which is not FFI-safe: enum has no representation hint
+  --> $DIR/lint-ctypes-enum.rs:51:20
+   |
+LL |    fn repr_rust(x: Option<Rust<num::NonZeroU8>>);
+   |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: consider adding a #[repr(...)] attribute to this enum
+
+error: `extern` block uses type `std::result::Result<(), std::num::NonZeroI32>` which is not FFI-safe: enum has no representation hint
+  --> $DIR/lint-ctypes-enum.rs:52:20
+   |
+LL |    fn no_result(x: Result<(), num::NonZeroI32>);
+   |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: consider adding a #[repr(...)] attribute to this enum
+
+error: aborting due to 7 previous errors