safe transmute: gracefully bubble-up layout errors

Changes `.unwrap()`s to `?` to avoid ICEs. Adds ui tests.

Fixes #129327

2 files changed, 10 insertions, 21 deletions

diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs
index 7c73f74e629..2c67e7d4847 100644
--- a/compiler/rustc_transmute/src/layout/tree.rs
+++ b/compiler/rustc_transmute/src/layout/tree.rs
@@ -179,7 +179,7 @@ pub(crate) mod rustc {
     };
 
     use super::Tree;
-    use crate::layout::rustc::{Def, Ref};
+    use crate::layout::rustc::{layout_of, Def, Ref};
 
     #[derive(Debug, Copy, Clone)]
     pub(crate) enum Err {
@@ -206,7 +206,7 @@ pub(crate) mod rustc {
     impl<'tcx> Tree<Def<'tcx>, Ref<'tcx>> {
         pub fn from_ty(ty: Ty<'tcx>, cx: LayoutCx<'tcx, TyCtxt<'tcx>>) -> Result<Self, Err> {
             use rustc_target::abi::HasDataLayout;
-            let layout = ty_layout(cx, ty);
+            let layout = layout_of(cx, ty)?;
 
             if let Err(e) = ty.error_reported() {
                 return Err(Err::TypeError(e));
@@ -239,7 +239,7 @@ pub(crate) mod rustc {
                     let FieldsShape::Array { stride, count } = &layout.fields else {
                         return Err(Err::NotYetSupported);
                     };
-                    let inner_layout = ty_layout(cx, *inner_ty);
+                    let inner_layout = layout_of(cx, *inner_ty)?;
                     assert_eq!(*stride, inner_layout.size);
                     let elt = Tree::from_ty(*inner_ty, cx)?;
                     Ok(std::iter::repeat(elt)
@@ -254,7 +254,7 @@ pub(crate) mod rustc {
                 },
 
                 ty::Ref(lifetime, ty, mutability) => {
-                    let layout = ty_layout(cx, *ty);
+                    let layout = layout_of(cx, *ty)?;
                     let align = layout.align.abi.bytes_usize();
                     let size = layout.size.bytes_usize();
                     Ok(Tree::Ref(Ref {
@@ -280,7 +280,7 @@ pub(crate) mod rustc {
                 FieldsShape::Primitive => {
                     assert_eq!(members.len(), 1);
                     let inner_ty = members[0];
-                    let inner_layout = ty_layout(cx, inner_ty);
+                    let inner_layout = layout_of(cx, inner_ty)?;
                     Self::from_ty(inner_ty, cx)
                 }
                 FieldsShape::Arbitrary { offsets, .. } => {
@@ -413,7 +413,7 @@ pub(crate) mod rustc {
                 let padding = Self::padding(padding_needed.bytes_usize());
 
                 let field_ty = ty_field(cx, (ty, layout), field_idx);
-                let field_layout = ty_layout(cx, field_ty);
+                let field_layout = layout_of(cx, field_ty)?;
                 let field_tree = Self::from_ty(field_ty, cx)?;
 
                 struct_tree = struct_tree.then(padding).then(field_tree);
@@ -471,7 +471,7 @@ pub(crate) mod rustc {
                 |fields, (idx, field_def)| {
                     let field_def = Def::Field(field_def);
                     let field_ty = ty_field(cx, (ty, layout), idx);
-                    let field_layout = ty_layout(cx, field_ty);
+                    let field_layout = layout_of(cx, field_ty)?;
                     let field = Self::from_ty(field_ty, cx)?;
                     let trailing_padding_needed = layout.size - field_layout.size;
                     let trailing_padding = Self::padding(trailing_padding_needed.bytes_usize());
@@ -484,10 +484,6 @@ pub(crate) mod rustc {
         }
     }
 
-    pub(crate) fn ty_layout<'tcx>(cx: LayoutCx<'tcx, TyCtxt<'tcx>>, ty: Ty<'tcx>) -> Layout<'tcx> {
-        crate::layout::rustc::layout_of(cx, ty).unwrap()
-    }
-
     fn ty_field<'tcx>(
         cx: LayoutCx<'tcx, TyCtxt<'tcx>>,
         (ty, layout): (Ty<'tcx>, Layout<'tcx>),
diff --git a/compiler/rustc_transmute/src/maybe_transmutable/mod.rs b/compiler/rustc_transmute/src/maybe_transmutable/mod.rs
index 1f3c4e3c817..2762b4e6384 100644
--- a/compiler/rustc_transmute/src/maybe_transmutable/mod.rs
+++ b/compiler/rustc_transmute/src/maybe_transmutable/mod.rs
@@ -44,18 +44,11 @@ mod rustc {
             let Self { src, dst, assume, context } = self;
 
             let layout_cx = LayoutCx { tcx: context, param_env: ParamEnv::reveal_all() };
-            let layout_of = |ty| {
-                crate::layout::rustc::layout_of(layout_cx, ty)
-                    .map_err(|_| Err::NotYetSupported)
-                    .and_then(|_| Tree::from_ty(ty, layout_cx))
-            };
 
             // Convert `src` and `dst` from their rustc representations, to `Tree`-based
-            // representations. If these conversions fail, conclude that the transmutation is
-            // unacceptable; the layouts of both the source and destination types must be
-            // well-defined.
-            let src = layout_of(src);
-            let dst = layout_of(dst);
+            // representations.
+            let src = Tree::from_ty(src, layout_cx);
+            let dst = Tree::from_ty(dst, layout_cx);
 
             match (src, dst) {
                 (Err(Err::TypeError(_)), _) | (_, Err(Err::TypeError(_))) => {