Prohibit casts between fat pointers to different traits

This makes them compliant with the new version of RFC 401 (i.e.
    RFC 1052).

Fixes #26391. I *hope* the tests I have are enough.

This is a [breaking-change]

4 files changed, 24 insertions, 8 deletions

diff --git a/src/librustc_typeck/check/cast.rs b/src/librustc_typeck/check/cast.rs
index 6e306047f75..2b0ed06f1ae 100644
--- a/src/librustc_typeck/check/cast.rs
+++ b/src/librustc_typeck/check/cast.rs
@@ -27,7 +27,12 @@
 //!
 //! where `&.T` and `*T` are references of either mutability,
 //! and where unsize_kind(`T`) is the kind of the unsize info
-//! in `T` - a vtable or a length (or `()` if `T: Sized`).
+//! in `T` - the vtable for a trait definition (e.g. `fmt::Display` or
+//! `Iterator`, not `Iterator<Item=u8>`) or a length (or `()` if `T: Sized`).
+//!
+//! Note that lengths are not adjusted when casting raw slices -
+//! `T: *const [u16] as *const [u8]` creates a slice that only includes
+//! half of the original memory.
 //!
 //! Casting is not transitive, that is, even if `e as U1 as U2` is a valid
 //! expression, `e as U2` is not necessarily so (in fact it will only be valid if
@@ -60,7 +65,7 @@ pub struct CastCheck<'tcx> {
 /// fat pointers if their unsize-infos have the same kind.
 #[derive(Copy, Clone, PartialEq, Eq)]
 enum UnsizeKind<'tcx> {
-    Vtable,
+    Vtable(ast::DefId),
     Length,
     /// The unsize info of this projection
     OfProjection(&'tcx ty::ProjectionTy<'tcx>),
@@ -75,7 +80,7 @@ fn unsize_kind<'a,'tcx>(fcx: &FnCtxt<'a, 'tcx>,
                         -> Option<UnsizeKind<'tcx>> {
     match t.sty {
         ty::TySlice(_) | ty::TyStr => Some(UnsizeKind::Length),
-        ty::TyTrait(_) => Some(UnsizeKind::Vtable),
+        ty::TyTrait(ref tty) => Some(UnsizeKind::Vtable(tty.principal_def_id())),
         ty::TyStruct(did, substs) => {
             match ty::struct_fields(fcx.tcx(), did, substs).pop() {
                 None => None,
diff --git a/src/test/compile-fail/cast-rfc0401.rs b/src/test/compile-fail/cast-rfc0401.rs
index f3537e54135..29ce8c15143 100644
--- a/src/test/compile-fail/cast-rfc0401.rs
+++ b/src/test/compile-fail/cast-rfc0401.rs
@@ -21,6 +21,9 @@ fn illegal_cast_2<U:?Sized>(u: *const U) -> *const str
 trait Foo { fn foo(&self) {} }
 impl<T> Foo for T {}
 
+trait Bar { fn foo(&self) {} }
+impl<T> Bar for T {}
+
 enum E {
     A, B
 }
@@ -72,4 +75,7 @@ fn main()
     // check no error cascade
     let _ = main.f as *const u32; //~ ERROR attempted access of field
 
+    let cf: *const Foo = &0;
+    let _ = cf as *const [u8]; //~ ERROR vtable kinds
+    let _ = cf as *const Bar; //~ ERROR vtable kinds
 }
diff --git a/src/test/run-pass/cast-rfc0401-vtable-kinds.rs b/src/test/run-pass/cast-rfc0401-vtable-kinds.rs
index e53d4af8e36..3a9f24ad4cc 100644
--- a/src/test/run-pass/cast-rfc0401-vtable-kinds.rs
+++ b/src/test/run-pass/cast-rfc0401-vtable-kinds.rs
@@ -23,12 +23,11 @@ impl<T> Foo<T> for () {}
 impl Foo<u32> for u32 { fn foo(&self, _: u32) -> u32 { self+43 } }
 impl Bar for () {}
 
-unsafe fn fool<'a>(t: *const (Foo<u32>+'a)) -> u32 {
-    let bar : *const Bar = t as *const Bar;
+unsafe fn round_trip_and_call<'a>(t: *const (Foo<u32>+'a)) -> u32 {
     let foo_e : *const Foo<u16> = t as *const _;
     let r_1 = foo_e as *mut Foo<u32>;
 
-    (&*r_1).foo(0)*(&*(bar as *const Foo<u32>)).foo(0)
+    (&*r_1).foo(0)
 }
 
 #[repr(C)]
@@ -43,8 +42,8 @@ fn foo_to_bar<T:?Sized>(u: *const FooS<T>) -> *const BarS<T> {
 fn main() {
     let x = 4u32;
     let y : &Foo<u32> = &x;
-    let fl = unsafe { fool(y as *const Foo<u32>) };
-    assert_eq!(fl, (43+4)*(43+4));
+    let fl = unsafe { round_trip_and_call(y as *const Foo<u32>) };
+    assert_eq!(fl, (43+4));
 
     let s = FooS([0,1,2]);
     let u: &FooS<[u32]> = &s;
diff --git a/src/test/run-pass/cast-rfc0401.rs b/src/test/run-pass/cast-rfc0401.rs
index 7c64c34fae5..efbf265bd80 100644
--- a/src/test/run-pass/cast-rfc0401.rs
+++ b/src/test/run-pass/cast-rfc0401.rs
@@ -99,6 +99,12 @@ fn main()
     let l_via_str = unsafe{&*(s as *const [u8])};
     assert_eq!(&l, l_via_str);
 
+    // ptr-ptr-cast (both vk=Length, check length is preserved)
+    let l: [[u8; 3]; 2] = [[3, 2, 6], [4, 5, 1]];
+    let p: *const [[u8; 3]] = &l;
+    let p: &[[u8; 2]] = unsafe {&*(p as *const [[u8; 2]])};
+    assert_eq!(p, [[3, 2], [6, 4]]);
+
     // enum-cast
     assert_eq!(Simple::A as u8, 0);
     assert_eq!(Simple::B as u8, 1);