Use the `enum2$` Natvis visualiser for repr128 C-style enums

2 files changed, 70 insertions, 6 deletions

diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs b/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs
index 1eaf593a6d7..0918660e6be 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/mod.rs
@@ -1,6 +1,7 @@
-use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{self};
-use rustc_target::abi::Size;
+use rustc_middle::bug;
+use rustc_middle::ty::layout::{IntegerExt, PrimitiveExt, TyAndLayout};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_target::abi::{Integer, Primitive, Size, TagEncoding, Variants};
 
 // FIXME(eddyb) find a place for this (or a way to replace it).
 pub mod type_names;
@@ -11,13 +12,25 @@ pub mod type_names;
 /// NOTE: This is somewhat inconsistent right now: For empty enums and enums with a single
 ///       fieldless variant, we generate DW_TAG_struct_type, although a
 ///       DW_TAG_enumeration_type would be a better fit.
-pub fn wants_c_like_enum_debuginfo(enum_type_and_layout: TyAndLayout<'_>) -> bool {
+pub fn wants_c_like_enum_debuginfo<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+) -> bool {
     match enum_type_and_layout.ty.kind() {
         ty::Adt(adt_def, _) => {
             if !adt_def.is_enum() {
                 return false;
             }
 
+            if type_names::cpp_like_debuginfo(tcx)
+                && tag_base_type_opt(tcx, enum_type_and_layout)
+                    .map(|ty| ty.primitive_size(tcx).bits())
+                    == Some(128)
+            {
+                // C++-like debuginfo never uses the C-like representation for 128-bit enums.
+                return false;
+            }
+
             match adt_def.variants().len() {
                 0 => false,
                 1 => {
@@ -33,3 +46,51 @@ pub fn wants_c_like_enum_debuginfo(enum_type_and_layout: TyAndLayout<'_>) -> boo
         _ => false,
     }
 }
+
+/// Extract the type with which we want to describe the tag of the given enum or coroutine.
+pub fn tag_base_type<'tcx>(tcx: TyCtxt<'tcx>, enum_type_and_layout: TyAndLayout<'tcx>) -> Ty<'tcx> {
+    tag_base_type_opt(tcx, enum_type_and_layout).unwrap_or_else(|| {
+        bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout)
+    })
+}
+
+pub fn tag_base_type_opt<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    enum_type_and_layout: TyAndLayout<'tcx>,
+) -> Option<Ty<'tcx>> {
+    assert!(match enum_type_and_layout.ty.kind() {
+        ty::Coroutine(..) => true,
+        ty::Adt(adt_def, _) => adt_def.is_enum(),
+        _ => false,
+    });
+
+    match enum_type_and_layout.layout.variants() {
+        // A single-variant enum has no discriminant.
+        Variants::Single { .. } => None,
+
+        Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => {
+            // Niche tags are always normalized to unsized integers of the correct size.
+            Some(
+                match tag.primitive() {
+                    Primitive::Int(t, _) => t,
+                    Primitive::Float(f) => Integer::from_size(f.size()).unwrap(),
+                    // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+                    Primitive::Pointer(_) => {
+                        // If the niche is the NULL value of a reference, then `discr_enum_ty` will be
+                        // a RawPtr. CodeView doesn't know what to do with enums whose base type is a
+                        // pointer so we fix this up to just be `usize`.
+                        // DWARF might be able to deal with this but with an integer type we are on
+                        // the safe side there too.
+                        tcx.data_layout.ptr_sized_integer()
+                    }
+                }
+                .to_ty(tcx, false),
+            )
+        }
+
+        Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => {
+            // Direct tags preserve the sign.
+            Some(tag.primitive().to_ty(tcx))
+        }
+    }
+}
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 27558038927..97f52ea1457 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -85,7 +85,7 @@ fn push_debuginfo_type_name<'tcx>(
             let layout_for_cpp_like_fallback = if cpp_like_debuginfo && def.is_enum() {
                 match tcx.layout_of(ParamEnv::reveal_all().and(t)) {
                     Ok(layout) => {
-                        if !wants_c_like_enum_debuginfo(layout) {
+                        if !wants_c_like_enum_debuginfo(tcx, layout) {
                             Some(layout)
                         } else {
                             // This is a C-like enum so we don't want to use the fallback encoding
@@ -106,6 +106,7 @@ fn push_debuginfo_type_name<'tcx>(
 
             if let Some(ty_and_layout) = layout_for_cpp_like_fallback {
                 msvc_enum_fallback(
+                    tcx,
                     ty_and_layout,
                     &|output, visited| {
                         push_item_name(tcx, def.did(), true, output);
@@ -421,6 +422,7 @@ fn push_debuginfo_type_name<'tcx>(
             if cpp_like_debuginfo && t.is_coroutine() {
                 let ty_and_layout = tcx.layout_of(ParamEnv::reveal_all().and(t)).unwrap();
                 msvc_enum_fallback(
+                    tcx,
                     ty_and_layout,
                     &|output, visited| {
                         push_closure_or_coroutine_name(tcx, def_id, args, true, output, visited);
@@ -455,12 +457,13 @@ fn push_debuginfo_type_name<'tcx>(
     // debugger. For more information, look in
     // rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs.
     fn msvc_enum_fallback<'tcx>(
+        tcx: TyCtxt<'tcx>,
         ty_and_layout: TyAndLayout<'tcx>,
         push_inner: &dyn Fn(/*output*/ &mut String, /*visited*/ &mut FxHashSet<Ty<'tcx>>),
         output: &mut String,
         visited: &mut FxHashSet<Ty<'tcx>>,
     ) {
-        assert!(!wants_c_like_enum_debuginfo(ty_and_layout));
+        assert!(!wants_c_like_enum_debuginfo(tcx, ty_and_layout));
         output.push_str("enum2$<");
         push_inner(output, visited);
         push_close_angle_bracket(true, output);