safe transmute: revise safety analysis

Migrate to a simplified safety analysis that does not use visibility.

Closes https://github.com/rust-lang/project-safe-transmute/issues/15

3 files changed, 67 insertions, 27 deletions

diff --git a/compiler/rustc_transmute/src/layout/mod.rs b/compiler/rustc_transmute/src/layout/mod.rs
index 76d97e0e6e7..0441b49cb14 100644
--- a/compiler/rustc_transmute/src/layout/mod.rs
+++ b/compiler/rustc_transmute/src/layout/mod.rs
@@ -29,14 +29,21 @@ impl fmt::Debug for Byte {
     }
 }
 
-pub(crate) trait Def: Debug + Hash + Eq + PartialEq + Copy + Clone {}
+pub(crate) trait Def: Debug + Hash + Eq + PartialEq + Copy + Clone {
+    fn has_safety_invariants(&self) -> bool;
+}
 pub trait Ref: Debug + Hash + Eq + PartialEq + Copy + Clone {
     fn min_align(&self) -> usize;
 
     fn is_mutable(&self) -> bool;
 }
 
-impl Def for ! {}
+impl Def for ! {
+    fn has_safety_invariants(&self) -> bool {
+        unreachable!()
+    }
+}
+
 impl Ref for ! {
     fn min_align(&self) -> usize {
         unreachable!()
@@ -83,5 +90,12 @@ pub mod rustc {
         Primitive,
     }
 
-    impl<'tcx> super::Def for Def<'tcx> {}
+    impl<'tcx> super::Def for Def<'tcx> {
+        fn has_safety_invariants(&self) -> bool {
+            // Rust presently has no notion of 'unsafe fields', so for now we
+            // make the conservative assumption that everything besides
+            // primitive types carry safety invariants.
+            self != &Self::Primitive
+        }
+    }
 }
diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs
index 86a077ee808..71b72828e4c 100644
--- a/compiler/rustc_transmute/src/layout/tree.rs
+++ b/compiler/rustc_transmute/src/layout/tree.rs
@@ -81,7 +81,8 @@ where
         Self::Seq(vec![Self::uninit(); width_in_bytes])
     }
 
-    /// Remove all `Def` nodes, and all branches of the layout for which `f` produces false.
+    /// Remove all `Def` nodes, and all branches of the layout for which `f`
+    /// produces `true`.
     pub(crate) fn prune<F>(self, f: &F) -> Tree<!, R>
     where
         F: Fn(D) -> bool,
@@ -106,7 +107,7 @@ where
             Self::Byte(b) => Tree::Byte(b),
             Self::Ref(r) => Tree::Ref(r),
             Self::Def(d) => {
-                if !f(d) {
+                if f(d) {
                     Tree::uninhabited()
                 } else {
                     Tree::unit()
diff --git a/compiler/rustc_transmute/src/layout/tree/tests.rs b/compiler/rustc_transmute/src/layout/tree/tests.rs
index 90515e92f7a..3cb47517c21 100644
--- a/compiler/rustc_transmute/src/layout/tree/tests.rs
+++ b/compiler/rustc_transmute/src/layout/tree/tests.rs
@@ -2,11 +2,15 @@ use super::Tree;
 
 #[derive(Debug, Hash, Eq, PartialEq, Clone, Copy)]
 pub enum Def {
-    Visible,
-    Invisible,
+    NoSafetyInvariants,
+    HasSafetyInvariants,
 }
 
-impl super::Def for Def {}
+impl super::Def for Def {
+    fn has_safety_invariants(&self) -> bool {
+        self == &Self::HasSafetyInvariants
+    }
+}
 
 mod prune {
     use super::*;
@@ -16,17 +20,22 @@ mod prune {
 
         #[test]
         fn seq_1() {
-            let layout: Tree<Def, !> = Tree::def(Def::Visible).then(Tree::from_bits(0x00));
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::from_bits(0x00));
+            let layout: Tree<Def, !> =
+                Tree::def(Def::NoSafetyInvariants).then(Tree::from_bits(0x00));
+            assert_eq!(
+                layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)),
+                Tree::from_bits(0x00)
+            );
         }
 
         #[test]
         fn seq_2() {
-            let layout: Tree<Def, !> =
-                Tree::from_bits(0x00).then(Tree::def(Def::Visible)).then(Tree::from_bits(0x01));
+            let layout: Tree<Def, !> = Tree::from_bits(0x00)
+                .then(Tree::def(Def::NoSafetyInvariants))
+                .then(Tree::from_bits(0x01));
 
             assert_eq!(
-                layout.prune(&|d| matches!(d, Def::Visible)),
+                layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)),
                 Tree::from_bits(0x00).then(Tree::from_bits(0x01))
             );
         }
@@ -37,21 +46,32 @@ mod prune {
 
         #[test]
         fn invisible_def() {
-            let layout: Tree<Def, !> = Tree::def(Def::Invisible);
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::uninhabited());
+            let layout: Tree<Def, !> = Tree::def(Def::HasSafetyInvariants);
+            assert_eq!(
+                layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)),
+                Tree::uninhabited()
+            );
         }
 
         #[test]
         fn invisible_def_in_seq_len_2() {
-            let layout: Tree<Def, !> = Tree::def(Def::Visible).then(Tree::def(Def::Invisible));
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::uninhabited());
+            let layout: Tree<Def, !> =
+                Tree::def(Def::NoSafetyInvariants).then(Tree::def(Def::HasSafetyInvariants));
+            assert_eq!(
+                layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)),
+                Tree::uninhabited()
+            );
         }
 
         #[test]
         fn invisible_def_in_seq_len_3() {
-            let layout: Tree<Def, !> =
-                Tree::def(Def::Visible).then(Tree::from_bits(0x00)).then(Tree::def(Def::Invisible));
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::uninhabited());
+            let layout: Tree<Def, !> = Tree::def(Def::NoSafetyInvariants)
+                .then(Tree::from_bits(0x00))
+                .then(Tree::def(Def::HasSafetyInvariants));
+            assert_eq!(
+                layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)),
+                Tree::uninhabited()
+            );
         }
     }
 
@@ -60,21 +80,26 @@ mod prune {
 
         #[test]
         fn visible_def() {
-            let layout: Tree<Def, !> = Tree::def(Def::Visible);
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::unit());
+            let layout: Tree<Def, !> = Tree::def(Def::NoSafetyInvariants);
+            assert_eq!(layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)), Tree::unit());
         }
 
         #[test]
         fn visible_def_in_seq_len_2() {
-            let layout: Tree<Def, !> = Tree::def(Def::Visible).then(Tree::def(Def::Visible));
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::unit());
+            let layout: Tree<Def, !> =
+                Tree::def(Def::NoSafetyInvariants).then(Tree::def(Def::NoSafetyInvariants));
+            assert_eq!(layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)), Tree::unit());
         }
 
         #[test]
         fn visible_def_in_seq_len_3() {
-            let layout: Tree<Def, !> =
-                Tree::def(Def::Visible).then(Tree::from_bits(0x00)).then(Tree::def(Def::Visible));
-            assert_eq!(layout.prune(&|d| matches!(d, Def::Visible)), Tree::from_bits(0x00));
+            let layout: Tree<Def, !> = Tree::def(Def::NoSafetyInvariants)
+                .then(Tree::from_bits(0x00))
+                .then(Tree::def(Def::NoSafetyInvariants));
+            assert_eq!(
+                layout.prune(&|d| matches!(d, Def::HasSafetyInvariants)),
+                Tree::from_bits(0x00)
+            );
         }
     }
 }