Rollup merge of #98625 - RalfJung:retag, r=oli-obk

emit Retag for compound types with reference fields

I want to add an option to Miri to do retagging inside reference fields. But that means we first have to even emit `Retag` for types that *contain* references (rather than being of reference types). :)

Stacked Borrows originally did that, but we stopped doing it when hitting bunch of issues in the standard library. However I have since realized that we actually do emit `noalias` for newtypes references, which means for soundness we should recurse into fields. Also it'd probably be bad news if newtypes lose out on optimizations (and they don't, for anything else). I want to add an option for that to Miri so that we can start experimenting with those semantics.

r? ``@oli-obk``

2 files changed, 20 insertions, 5 deletions

diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs
index 0495439385b..0f87e638d26 100644
--- a/compiler/rustc_mir_transform/src/add_retag.rs
+++ b/compiler/rustc_mir_transform/src/add_retag.rs
@@ -33,8 +33,9 @@ fn is_stable(place: PlaceRef<'_>) -> bool {
     })
 }
 
-/// Determine whether this type may be a reference (or box), and thus needs retagging.
-fn may_be_reference(ty: Ty<'_>) -> bool {
+/// Determine whether this type may contain a reference (or box), and thus needs retagging.
+/// We will only recurse `depth` times into Tuples/ADTs to bound the cost of this.
+fn may_contain_reference<'tcx>(ty: Ty<'tcx>, depth: u32, tcx: TyCtxt<'tcx>) -> bool {
     match ty.kind() {
         // Primitive types that are not references
         ty::Bool
@@ -50,8 +51,20 @@ fn may_be_reference(ty: Ty<'_>) -> bool {
         // References
         ty::Ref(..) => true,
         ty::Adt(..) if ty.is_box() => true,
-        // Compound types are not references
-        ty::Array(..) | ty::Slice(..) | ty::Tuple(..) | ty::Adt(..) => false,
+        // Compound types: recurse
+        ty::Array(ty, _) | ty::Slice(ty) => {
+            // This does not branch so we keep the depth the same.
+            may_contain_reference(*ty, depth, tcx)
+        }
+        ty::Tuple(tys) => {
+            depth == 0 || tys.iter().any(|ty| may_contain_reference(ty, depth - 1, tcx))
+        }
+        ty::Adt(adt, subst) => {
+            depth == 0
+                || adt.variants().iter().any(|v| {
+                    v.fields.iter().any(|f| may_contain_reference(f.ty(tcx, subst), depth - 1, tcx))
+                })
+        }
         // Conservative fallback
         _ => true,
     }
@@ -83,7 +96,7 @@ impl<'tcx> MirPass<'tcx> for AddRetag {
             // FIXME: Instead of giving up for unstable places, we should introduce
             // a temporary and retag on that.
             is_stable(place.as_ref())
-                && may_be_reference(place.ty(&*local_decls, tcx).ty)
+                && may_contain_reference(place.ty(&*local_decls, tcx).ty, /*depth*/ 3, tcx)
                 && is_not_temp(&local_decls[place.local])
         };
         let place_base_raw = |place: &Place<'tcx>| {
diff --git a/src/test/mir-opt/retag.array_casts.SimplifyCfg-elaborate-drops.after.mir b/src/test/mir-opt/retag.array_casts.SimplifyCfg-elaborate-drops.after.mir
index 69742d6bc3b..c8c45da1913 100644
--- a/src/test/mir-opt/retag.array_casts.SimplifyCfg-elaborate-drops.after.mir
+++ b/src/test/mir-opt/retag.array_casts.SimplifyCfg-elaborate-drops.after.mir
@@ -129,6 +129,7 @@ fn array_casts() -> () {
         _18 = &(*_35);                   // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         Retag(_18);                      // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         _13 = (move _14, move _18);      // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
+        Retag(_13);                      // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         StorageDead(_18);                // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         StorageDead(_14);                // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         StorageLive(_20);                // scope 5 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
@@ -171,6 +172,7 @@ fn array_casts() -> () {
         Retag(_32);                      // scope 8 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         StorageLive(_34);                // scope 8 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         _34 = Option::<Arguments>::None; // scope 8 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
+        Retag(_34);                      // scope 8 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
         _28 = core::panicking::assert_failed::<usize, usize>(move _29, move _30, move _32, move _34); // scope 8 at $SRC_DIR/core/src/macros/mod.rs:LL:COL
                                          // mir::Constant
                                          // + span: $SRC_DIR/core/src/macros/mod.rs:LL:COL