Rollup merge of #115631 - compiler-errors:ctypes-unsized, r=davidtwco

Don't ICE when computing ctype's `repr_nullable_ptr` for possibly-unsized ty

We may not always be able to compute the layout of a type like `&T` when `T: ?Sized`, even if we're able to estimate its size skeleton.

r? davidtwco

Fixes #115628

3 files changed, 30 insertions, 1 deletions

diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index c38c6337b7b..57c2e289f20 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -923,7 +923,12 @@ pub(crate) fn repr_nullable_ptr<'tcx>(
         }
 
         // Return the nullable type this Option-like enum can be safely represented with.
-        let field_ty_abi = &tcx.layout_of(param_env.and(field_ty)).unwrap().abi;
+        let field_ty_layout = tcx.layout_of(param_env.and(field_ty));
+        if field_ty_layout.is_err() && !field_ty.has_non_region_param() {
+            bug!("should be able to compute the layout of non-polymorphic type");
+        }
+
+        let field_ty_abi = &field_ty_layout.ok()?.abi;
         if let Abi::Scalar(field_ty_scalar) = field_ty_abi {
             match field_ty_scalar.valid_range(&tcx) {
                 WrappingRange { start: 0, end }
diff --git a/tests/ui/lint/lint-ctypes-option-nonnull-unsized.rs b/tests/ui/lint/lint-ctypes-option-nonnull-unsized.rs
new file mode 100644
index 00000000000..ca08eb23a57
--- /dev/null
+++ b/tests/ui/lint/lint-ctypes-option-nonnull-unsized.rs
@@ -0,0 +1,8 @@
+#![deny(improper_ctypes_definitions)]
+
+extern "C" fn foo<T: ?Sized + 'static>() -> Option<&'static T> {
+    //~^ ERROR `extern` fn uses type `Option<&T>`, which is not FFI-safe
+    None
+}
+
+fn main() {}
diff --git a/tests/ui/lint/lint-ctypes-option-nonnull-unsized.stderr b/tests/ui/lint/lint-ctypes-option-nonnull-unsized.stderr
new file mode 100644
index 00000000000..f59fb3cc750
--- /dev/null
+++ b/tests/ui/lint/lint-ctypes-option-nonnull-unsized.stderr
@@ -0,0 +1,16 @@
+error: `extern` fn uses type `Option<&T>`, which is not FFI-safe
+  --> $DIR/lint-ctypes-option-nonnull-unsized.rs:3:45
+   |
+LL | extern "C" fn foo<T: ?Sized + 'static>() -> Option<&'static T> {
+   |                                             ^^^^^^^^^^^^^^^^^^ not FFI-safe
+   |
+   = help: consider adding a `#[repr(C)]`, `#[repr(transparent)]`, or integer `#[repr(...)]` attribute to this enum
+   = note: enum has no representation hint
+note: the lint level is defined here
+  --> $DIR/lint-ctypes-option-nonnull-unsized.rs:1:9
+   |
+LL | #![deny(improper_ctypes_definitions)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+