Auto merge of #75014 - Amanieu:fix_asm_reg, r=nagisa

Work around LLVM issues with explicit register in inline asm

Fixes #74658

7 files changed, 959 insertions, 91 deletions

diff --git a/src/librustc_codegen_llvm/asm.rs b/src/librustc_codegen_llvm/asm.rs
index 2f9e49591c3..fb2e1c43524 100644
--- a/src/librustc_codegen_llvm/asm.rs
+++ b/src/librustc_codegen_llvm/asm.rs
@@ -130,7 +130,9 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         for (idx, op) in operands.iter().enumerate() {
             match *op {
                 InlineAsmOperandRef::Out { reg, late, place } => {
-                    let ty = if let Some(place) = place {
+                    let mut layout = None;
+                    let ty = if let Some(ref place) = place {
+                        layout = Some(&place.layout);
                         llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
                     } else {
                         // If the output is discarded, we don't really care what
@@ -141,20 +143,21 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     output_types.push(ty);
                     op_idx.insert(idx, constraints.len());
                     let prefix = if late { "=" } else { "=&" };
-                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
+                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
                 }
                 InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
-                    let ty = if let Some(ref out_place) = out_place {
-                        llvm_fixup_output_type(self.cx, reg.reg_class(), &out_place.layout)
+                    let layout = if let Some(ref out_place) = out_place {
+                        &out_place.layout
                     } else {
                         // LLVM required tied operands to have the same type,
                         // so we just use the type of the input.
-                        llvm_fixup_output_type(self.cx, reg.reg_class(), &in_value.layout)
+                        &in_value.layout
                     };
+                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);
                     output_types.push(ty);
                     op_idx.insert(idx, constraints.len());
                     let prefix = if late { "=" } else { "=&" };
-                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
+                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
                 }
                 _ => {}
             }
@@ -165,11 +168,11 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         for (idx, op) in operands.iter().enumerate() {
             match *op {
                 InlineAsmOperandRef::In { reg, value } => {
-                    let value =
+                    let llval =
                         llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
-                    inputs.push(value);
+                    inputs.push(llval);
                     op_idx.insert(idx, constraints.len());
-                    constraints.push(reg_to_llvm(reg));
+                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
                 }
                 InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => {
                     let value = llvm_fixup_input(
@@ -410,10 +413,80 @@ fn inline_asm_call(
     }
 }
 
+/// If the register is an xmm/ymm/zmm register then return its index.
+fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
+    match reg {
+        InlineAsmReg::X86(reg)
+            if reg as u32 >= X86InlineAsmReg::xmm0 as u32
+                && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
+        {
+            Some(reg as u32 - X86InlineAsmReg::xmm0 as u32)
+        }
+        InlineAsmReg::X86(reg)
+            if reg as u32 >= X86InlineAsmReg::ymm0 as u32
+                && reg as u32 <= X86InlineAsmReg::ymm15 as u32 =>
+        {
+            Some(reg as u32 - X86InlineAsmReg::ymm0 as u32)
+        }
+        InlineAsmReg::X86(reg)
+            if reg as u32 >= X86InlineAsmReg::zmm0 as u32
+                && reg as u32 <= X86InlineAsmReg::zmm31 as u32 =>
+        {
+            Some(reg as u32 - X86InlineAsmReg::zmm0 as u32)
+        }
+        _ => None,
+    }
+}
+
+/// If the register is an AArch64 vector register then return its index.
+fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
+    match reg {
+        InlineAsmReg::AArch64(reg)
+            if reg as u32 >= AArch64InlineAsmReg::v0 as u32
+                && reg as u32 <= AArch64InlineAsmReg::v31 as u32 =>
+        {
+            Some(reg as u32 - AArch64InlineAsmReg::v0 as u32)
+        }
+        _ => None,
+    }
+}
+
 /// Converts a register class to an LLVM constraint code.
-fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String {
+fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) -> String {
     match reg {
-        InlineAsmRegOrRegClass::Reg(reg) => format!("{{{}}}", reg.name()),
+        // For vector registers LLVM wants the register name to match the type size.
+        InlineAsmRegOrRegClass::Reg(reg) => {
+            if let Some(idx) = xmm_reg_index(reg) {
+                let class = if let Some(layout) = layout {
+                    match layout.size.bytes() {
+                        64 => 'z',
+                        32 => 'y',
+                        _ => 'x',
+                    }
+                } else {
+                    // We use f32 as the type for discarded outputs
+                    'x'
+                };
+                format!("{{{}mm{}}}", class, idx)
+            } else if let Some(idx) = a64_vreg_index(reg) {
+                let class = if let Some(layout) = layout {
+                    match layout.size.bytes() {
+                        16 => 'q',
+                        8 => 'd',
+                        4 => 's',
+                        2 => 'h',
+                        1 => 'd', // We fixup i8 to i8x8
+                        _ => unreachable!(),
+                    }
+                } else {
+                    // We use i32 as the type for discarded outputs
+                    's'
+                };
+                format!("{{{}{}}}", class, idx)
+            } else {
+                format!("{{{}}}", reg.name())
+            }
+        }
         InlineAsmRegOrRegClass::RegClass(reg) => match reg {
             InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
@@ -601,17 +674,25 @@ fn llvm_fixup_input(
             Abi::Vector { .. },
         ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
         (
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
+            Abi::Scalar(s),
+        ) => {
+            if let Primitive::Int(Integer::I32, _) = s.value {
+                bx.bitcast(value, bx.cx.type_f32())
+            } else {
+                value
+            }
+        }
+        (
             InlineAsmRegClass::Arm(
-                ArmInlineAsmRegClass::sreg_low16
+                ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
-                | ArmInlineAsmRegClass::qreg_low4
-                | ArmInlineAsmRegClass::dreg
-                | ArmInlineAsmRegClass::qreg,
+                | ArmInlineAsmRegClass::dreg_low16,
             ),
             Abi::Scalar(s),
         ) => {
-            if let Primitive::Int(Integer::I32, _) = s.value {
-                bx.bitcast(value, bx.cx.type_f32())
+            if let Primitive::Int(Integer::I64, _) = s.value {
+                bx.bitcast(value, bx.cx.type_f64())
             } else {
                 value
             }
@@ -661,17 +742,25 @@ fn llvm_fixup_output(
             Abi::Vector { .. },
         ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
         (
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
+            Abi::Scalar(s),
+        ) => {
+            if let Primitive::Int(Integer::I32, _) = s.value {
+                bx.bitcast(value, bx.cx.type_i32())
+            } else {
+                value
+            }
+        }
+        (
             InlineAsmRegClass::Arm(
-                ArmInlineAsmRegClass::sreg_low16
+                ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
-                | ArmInlineAsmRegClass::qreg_low4
-                | ArmInlineAsmRegClass::dreg
-                | ArmInlineAsmRegClass::qreg,
+                | ArmInlineAsmRegClass::dreg_low16,
             ),
             Abi::Scalar(s),
         ) => {
-            if let Primitive::Int(Integer::I32, _) = s.value {
-                bx.bitcast(value, bx.cx.type_i32())
+            if let Primitive::Int(Integer::I64, _) = s.value {
+                bx.bitcast(value, bx.cx.type_i64())
             } else {
                 value
             }
@@ -716,17 +805,25 @@ fn llvm_fixup_output_type(
             Abi::Vector { .. },
         ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
         (
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
+            Abi::Scalar(s),
+        ) => {
+            if let Primitive::Int(Integer::I32, _) = s.value {
+                cx.type_f32()
+            } else {
+                layout.llvm_type(cx)
+            }
+        }
+        (
             InlineAsmRegClass::Arm(
-                ArmInlineAsmRegClass::sreg_low16
+                ArmInlineAsmRegClass::dreg
                 | ArmInlineAsmRegClass::dreg_low8
-                | ArmInlineAsmRegClass::qreg_low4
-                | ArmInlineAsmRegClass::dreg
-                | ArmInlineAsmRegClass::qreg,
+                | ArmInlineAsmRegClass::dreg_low16,
             ),
             Abi::Scalar(s),
         ) => {
-            if let Primitive::Int(Integer::I32, _) = s.value {
-                cx.type_f32()
+            if let Primitive::Int(Integer::I64, _) = s.value {
+                cx.type_f64()
             } else {
                 layout.llvm_type(cx)
             }
diff --git a/src/test/assembly/asm/aarch64-types.rs b/src/test/assembly/asm/aarch64-types.rs
index b78a8cbb559..a8df350ef60 100644
--- a/src/test/assembly/asm/aarch64-types.rs
+++ b/src/test/assembly/asm/aarch64-types.rs
@@ -117,6 +117,23 @@ macro_rules! check {
     };
 }
 
+macro_rules! check_reg {
+    ($func:ident $ty:ident $reg:tt $mov:literal) => {
+        #[no_mangle]
+        pub unsafe fn $func(x: $ty) -> $ty {
+            // Hack to avoid function merging
+            extern "Rust" {
+                fn dont_merge(s: &str);
+            }
+            dont_merge(stringify!($func));
+
+            let y;
+            asm!(concat!($mov, " ", $reg, ", ", $reg), lateout($reg) y, in($reg) x);
+            y
+        }
+    };
+}
+
 // CHECK-LABEL: reg_i8:
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
@@ -380,3 +397,159 @@ check!(vreg_low16_f32x4 f32x4 vreg_low16 "fmov" "s");
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f64x2 f64x2 vreg_low16 "fmov" "s");
+
+// CHECK-LABEL: x0_i8:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_i8 i8 "x0" "mov");
+
+// CHECK-LABEL: x0_i16:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_i16 i16 "x0" "mov");
+
+// CHECK-LABEL: x0_i32:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_i32 i32 "x0" "mov");
+
+// CHECK-LABEL: x0_f32:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_f32 f32 "x0" "mov");
+
+// CHECK-LABEL: x0_i64:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_i64 i64 "x0" "mov");
+
+// CHECK-LABEL: x0_f64:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_f64 f64 "x0" "mov");
+
+// CHECK-LABEL: x0_ptr:
+// CHECK: //APP
+// CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
+// CHECK: //NO_APP
+check_reg!(x0_ptr ptr "x0" "mov");
+
+// CHECK-LABEL: v0_i8:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i8 i8 "s0" "fmov");
+
+// CHECK-LABEL: v0_i16:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i16 i16 "s0" "fmov");
+
+// CHECK-LABEL: v0_i32:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i32 i32 "s0" "fmov");
+
+// CHECK-LABEL: v0_f32:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_f32 f32 "s0" "fmov");
+
+// CHECK-LABEL: v0_i64:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i64 i64 "s0" "fmov");
+
+// CHECK-LABEL: v0_f64:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_f64 f64 "s0" "fmov");
+
+// CHECK-LABEL: v0_ptr:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_ptr ptr "s0" "fmov");
+
+// CHECK-LABEL: v0_i8x8:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i8x8 i8x8 "s0" "fmov");
+
+// CHECK-LABEL: v0_i16x4:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i16x4 i16x4 "s0" "fmov");
+
+// CHECK-LABEL: v0_i32x2:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i32x2 i32x2 "s0" "fmov");
+
+// CHECK-LABEL: v0_i64x1:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i64x1 i64x1 "s0" "fmov");
+
+// CHECK-LABEL: v0_f32x2:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_f32x2 f32x2 "s0" "fmov");
+
+// CHECK-LABEL: v0_f64x1:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_f64x1 f64x1 "s0" "fmov");
+
+// CHECK-LABEL: v0_i8x16:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i8x16 i8x16 "s0" "fmov");
+
+// CHECK-LABEL: v0_i16x8:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i16x8 i16x8 "s0" "fmov");
+
+// CHECK-LABEL: v0_i32x4:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i32x4 i32x4 "s0" "fmov");
+
+// CHECK-LABEL: v0_i64x2:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_i64x2 i64x2 "s0" "fmov");
+
+// CHECK-LABEL: v0_f32x4:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_f32x4 f32x4 "s0" "fmov");
+
+// CHECK-LABEL: v0_f64x2:
+// CHECK: //APP
+// CHECK: fmov s0, s0
+// CHECK: //NO_APP
+check_reg!(v0_f64x2 f64x2 "s0" "fmov");
diff --git a/src/test/assembly/asm/arm-types.rs b/src/test/assembly/asm/arm-types.rs
index 07e25a38e45..64b7c93cbc3 100644
--- a/src/test/assembly/asm/arm-types.rs
+++ b/src/test/assembly/asm/arm-types.rs
@@ -108,6 +108,23 @@ macro_rules! check {
     };
 }
 
+macro_rules! check_reg {
+    ($func:ident $ty:ident $reg:tt $mov:literal) => {
+        #[no_mangle]
+        pub unsafe fn $func(x: $ty) -> $ty {
+            // Hack to avoid function merging
+            extern "Rust" {
+                fn dont_merge(s: &str);
+            }
+            dont_merge(stringify!($func));
+
+            let y;
+            asm!(concat!($mov, " ", $reg, ", ", $reg), lateout($reg) y, in($reg) x);
+            y
+        }
+    };
+}
+
 // CHECK-LABEL: reg_i8:
 // CHECK: @APP
 // CHECK: mov {{[a-z0-9]+}}, {{[a-z0-9]+}}
@@ -413,3 +430,123 @@ check!(qreg_low4_i64x2 i64x2 qreg_low4 "vmov");
 // CHECK: vorr q{{[0-9]+}}, q{{[0-9]+}}, q{{[0-9]+}}
 // CHECK: @NO_APP
 check!(qreg_low4_f32x4 f32x4 qreg_low4 "vmov");
+
+// CHECK-LABEL: r0_i8:
+// CHECK: @APP
+// CHECK: mov r0, r0
+// CHECK: @NO_APP
+check_reg!(r0_i8 i8 "r0" "mov");
+
+// CHECK-LABEL: r0_i16:
+// CHECK: @APP
+// CHECK: mov r0, r0
+// CHECK: @NO_APP
+check_reg!(r0_i16 i16 "r0" "mov");
+
+// CHECK-LABEL: r0_i32:
+// CHECK: @APP
+// CHECK: mov r0, r0
+// CHECK: @NO_APP
+check_reg!(r0_i32 i32 "r0" "mov");
+
+// CHECK-LABEL: r0_f32:
+// CHECK: @APP
+// CHECK: mov r0, r0
+// CHECK: @NO_APP
+check_reg!(r0_f32 f32 "r0" "mov");
+
+// CHECK-LABEL: r0_ptr:
+// CHECK: @APP
+// CHECK: mov r0, r0
+// CHECK: @NO_APP
+check_reg!(r0_ptr ptr "r0" "mov");
+
+// CHECK-LABEL: s0_i32:
+// CHECK: @APP
+// CHECK: vmov.f32 s0, s0
+// CHECK: @NO_APP
+check_reg!(s0_i32 i32 "s0" "vmov.f32");
+
+// CHECK-LABEL: s0_f32:
+// CHECK: @APP
+// CHECK: vmov.f32 s0, s0
+// CHECK: @NO_APP
+check_reg!(s0_f32 f32 "s0" "vmov.f32");
+
+// CHECK-LABEL: s0_ptr:
+// CHECK: @APP
+// CHECK: vmov.f32 s0, s0
+// CHECK: @NO_APP
+check_reg!(s0_ptr ptr "s0" "vmov.f32");
+
+// CHECK-LABEL: d0_i64:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_i64 i64 "d0" "vmov.f64");
+
+// CHECK-LABEL: d0_f64:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_f64 f64 "d0" "vmov.f64");
+
+// CHECK-LABEL: d0_i8x8:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_i8x8 i8x8 "d0" "vmov.f64");
+
+// CHECK-LABEL: d0_i16x4:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_i16x4 i16x4 "d0" "vmov.f64");
+
+// CHECK-LABEL: d0_i32x2:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_i32x2 i32x2 "d0" "vmov.f64");
+
+// CHECK-LABEL: d0_i64x1:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_i64x1 i64x1 "d0" "vmov.f64");
+
+// CHECK-LABEL: d0_f32x2:
+// CHECK: @APP
+// CHECK: vmov.f64 d0, d0
+// CHECK: @NO_APP
+check_reg!(d0_f32x2 f32x2 "d0" "vmov.f64");
+
+// CHECK-LABEL: q0_i8x16:
+// CHECK: @APP
+// CHECK: vorr q0, q0, q0
+// CHECK: @NO_APP
+check_reg!(q0_i8x16 i8x16 "q0" "vmov");
+
+// CHECK-LABEL: q0_i16x8:
+// CHECK: @APP
+// CHECK: vorr q0, q0, q0
+// CHECK: @NO_APP
+check_reg!(q0_i16x8 i16x8 "q0" "vmov");
+
+// CHECK-LABEL: q0_i32x4:
+// CHECK: @APP
+// CHECK: vorr q0, q0, q0
+// CHECK: @NO_APP
+check_reg!(q0_i32x4 i32x4 "q0" "vmov");
+
+// CHECK-LABEL: q0_i64x2:
+// CHECK: @APP
+// CHECK: vorr q0, q0, q0
+// CHECK: @NO_APP
+check_reg!(q0_i64x2 i64x2 "q0" "vmov");
+
+// CHECK-LABEL: q0_f32x4:
+// CHECK: @APP
+// CHECK: vorr q0, q0, q0
+// CHECK: @NO_APP
+check_reg!(q0_f32x4 f32x4 "q0" "vmov");
diff --git a/src/test/assembly/asm/hexagon-types.rs b/src/test/assembly/asm/hexagon-types.rs
index b6b3b54cd71..95135a3e5dd 100644
--- a/src/test/assembly/asm/hexagon-types.rs
+++ b/src/test/assembly/asm/hexagon-types.rs
@@ -13,6 +13,10 @@ macro_rules! asm {
     () => {};
 }
 #[rustc_builtin_macro]
+macro_rules! concat {
+    () => {};
+}
+#[rustc_builtin_macro]
 macro_rules! stringify {
     () => {};
 }
@@ -51,6 +55,23 @@ macro_rules! check {
     };
 }
 
+macro_rules! check_reg {
+    ($func:ident $ty:ident $reg:tt) => {
+        #[no_mangle]
+        pub unsafe fn $func(x: $ty) -> $ty {
+            // Hack to avoid function merging
+            extern "Rust" {
+                fn dont_merge(s: &str);
+            }
+            dont_merge(stringify!($func));
+
+            let y;
+            asm!(concat!($reg, " = ", $reg), lateout($reg) y, in($reg) x);
+            y
+        }
+    };
+}
+
 // CHECK-LABEL: sym_static:
 // CHECK: InlineAsm Start
 // CHECK: r0 = #extern_static
@@ -100,7 +121,7 @@ pub unsafe fn packet() {
     }}", out(reg) _, in(reg) &val);
 }
 
-// CHECK-LABEL: ptr:
+// CHECK-LABEL: reg_ptr:
 // CHECK: InlineAsm Start
 // CHECK: r{{[0-9]+}} = r{{[0-9]+}}
 // CHECK: InlineAsm End
@@ -129,3 +150,33 @@ check!(reg_i8 i8 reg);
 // CHECK: r{{[0-9]+}} = r{{[0-9]+}}
 // CHECK: InlineAsm End
 check!(reg_i16 i16 reg);
+
+// CHECK-LABEL: r0_ptr:
+// CHECK: InlineAsm Start
+// CHECK: r0 = r0
+// CHECK: InlineAsm End
+check_reg!(r0_ptr ptr "r0");
+
+// CHECK-LABEL: r0_f32:
+// CHECK: InlineAsm Start
+// CHECK: r0 = r0
+// CHECK: InlineAsm End
+check_reg!(r0_f32 f32 "r0");
+
+// CHECK-LABEL: r0_i32:
+// CHECK: InlineAsm Start
+// CHECK: r0 = r0
+// CHECK: InlineAsm End
+check_reg!(r0_i32 i32 "r0");
+
+// CHECK-LABEL: r0_i8:
+// CHECK: InlineAsm Start
+// CHECK: r0 = r0
+// CHECK: InlineAsm End
+check_reg!(r0_i8 i8 "r0");
+
+// CHECK-LABEL: r0_i16:
+// CHECK: InlineAsm Start
+// CHECK: r0 = r0
+// CHECK: InlineAsm End
+check_reg!(r0_i16 i16 "r0");
diff --git a/src/test/assembly/asm/riscv-modifiers.rs b/src/test/assembly/asm/riscv-modifiers.rs
deleted file mode 100644
index b6735153b5d..00000000000
--- a/src/test/assembly/asm/riscv-modifiers.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// no-system-llvm
-// assembly-output: emit-asm
-// compile-flags: -O
-// compile-flags: --target riscv64gc-unknown-linux-gnu
-// compile-flags: -C target-feature=+f
-// needs-llvm-components: riscv
-
-#![feature(no_core, lang_items, rustc_attrs)]
-#![crate_type = "rlib"]
-#![no_core]
-
-#[rustc_builtin_macro]
-macro_rules! asm {
-    () => {};
-}
-#[rustc_builtin_macro]
-macro_rules! concat {
-    () => {};
-}
-#[rustc_builtin_macro]
-macro_rules! stringify {
-    () => {};
-}
-
-#[lang = "sized"]
-trait Sized {}
-#[lang = "copy"]
-trait Copy {}
-
-impl Copy for f32 {}
-
-macro_rules! check {
-    ($func:ident $modifier:literal $reg:ident $mov:literal) => {
-        // -O and extern "C" guarantee that the selected register is always r0/s0/d0/q0
-        #[no_mangle]
-        pub unsafe extern "C" fn $func() -> f32 {
-            // Hack to avoid function merging
-            extern "Rust" {
-                fn dont_merge(s: &str);
-            }
-            dont_merge(stringify!($func));
-
-            let y;
-            asm!(concat!($mov, " {0:", $modifier, "}, {0:", $modifier, "}"), out($reg) y);
-            y
-        }
-    };
-}
-
-// CHECK-LABEL: reg:
-// CHECK: #APP
-// CHECK: mv a0, a0
-// CHECK: #NO_APP
-check!(reg "" reg "mv");
-
-// CHECK-LABEL: freg:
-// CHECK: #APP
-// CHECK: fmv.s fa0, fa0
-// CHECK: #NO_APP
-check!(freg "" freg "fmv.s");
diff --git a/src/test/assembly/asm/riscv-types.rs b/src/test/assembly/asm/riscv-types.rs
index 0ff0bf1f949..6b6e582442c 100644
--- a/src/test/assembly/asm/riscv-types.rs
+++ b/src/test/assembly/asm/riscv-types.rs
@@ -79,6 +79,23 @@ macro_rules! check {
     };
 }
 
+macro_rules! check_reg {
+    ($func:ident $ty:ident $reg:tt $mov:literal) => {
+        #[no_mangle]
+        pub unsafe fn $func(x: $ty) -> $ty {
+            // Hack to avoid function merging
+            extern "Rust" {
+                fn dont_merge(s: &str);
+            }
+            dont_merge(stringify!($func));
+
+            let y;
+            asm!(concat!($mov, " ", $reg, ", ", $reg), lateout($reg) y, in($reg) x);
+            y
+        }
+    };
+}
+
 // CHECK-LABEL: reg_i8:
 // CHECK: #APP
 // CHECK: mv {{[a-z0-9]+}}, {{[a-z0-9]+}}
@@ -134,3 +151,59 @@ check!(freg_f32 f32 freg "fmv.s");
 // CHECK: fmv.d f{{[a-z0-9]+}}, f{{[a-z0-9]+}}
 // CHECK: #NO_APP
 check!(freg_f64 f64 freg "fmv.d");
+
+// CHECK-LABEL: a0_i8:
+// CHECK: #APP
+// CHECK: mv a0, a0
+// CHECK: #NO_APP
+check_reg!(a0_i8 i8 "a0" "mv");
+
+// CHECK-LABEL: a0_i16:
+// CHECK: #APP
+// CHECK: mv a0, a0
+// CHECK: #NO_APP
+check_reg!(a0_i16 i16 "a0" "mv");
+
+// CHECK-LABEL: a0_i32:
+// CHECK: #APP
+// CHECK: mv a0, a0
+// CHECK: #NO_APP
+check_reg!(a0_i32 i32 "a0" "mv");
+
+// CHECK-LABEL: a0_f32:
+// CHECK: #APP
+// CHECK: mv a0, a0
+// CHECK: #NO_APP
+check_reg!(a0_f32 f32 "a0" "mv");
+
+// riscv64-LABEL: a0_i64:
+// riscv64: #APP
+// riscv64: mv a0, a0
+// riscv64: #NO_APP
+#[cfg(riscv64)]
+check_reg!(a0_i64 i64 "a0" "mv");
+
+// riscv64-LABEL: a0_f64:
+// riscv64: #APP
+// riscv64: mv a0, a0
+// riscv64: #NO_APP
+#[cfg(riscv64)]
+check_reg!(a0_f64 f64 "a0" "mv");
+
+// CHECK-LABEL: a0_ptr:
+// CHECK: #APP
+// CHECK: mv a0, a0
+// CHECK: #NO_APP
+check_reg!(a0_ptr ptr "a0" "mv");
+
+// CHECK-LABEL: fa0_f32:
+// CHECK: #APP
+// CHECK: fmv.s fa0, fa0
+// CHECK: #NO_APP
+check_reg!(fa0_f32 f32 "fa0" "fmv.s");
+
+// CHECK-LABEL: fa0_f64:
+// CHECK: #APP
+// CHECK: fmv.d fa0, fa0
+// CHECK: #NO_APP
+check_reg!(fa0_f64 f64 "fa0" "fmv.d");
diff --git a/src/test/assembly/asm/x86-types.rs b/src/test/assembly/asm/x86-types.rs
index de2e67c421f..f636f1f5296 100644
--- a/src/test/assembly/asm/x86-types.rs
+++ b/src/test/assembly/asm/x86-types.rs
@@ -261,7 +261,24 @@ macro_rules! check {
             dont_merge(stringify!($func));
 
             let y;
-            asm!(concat!($mov, " {}, {}"), out($class) y, in($class) x);
+            asm!(concat!($mov, " {}, {}"), lateout($class) y, in($class) x);
+            y
+        }
+    };
+}
+
+macro_rules! check_reg {
+    ($func:ident $ty:ident $reg:tt $mov:literal) => {
+        #[no_mangle]
+        pub unsafe fn $func(x: $ty) -> $ty {
+            // Hack to avoid function merging
+            extern "Rust" {
+                fn dont_merge(s: &str);
+            }
+            dont_merge(stringify!($func));
+
+            let y;
+            asm!(concat!($mov, " ", $reg, ", ", $reg), lateout($reg) y, in($reg) x);
             y
         }
     };
@@ -692,3 +709,383 @@ check!(kreg_i64 i64 kreg "kmovq");
 // CHECK: kmovq k{{[0-9]+}}, k{{[0-9]+}}
 // CHECK: #NO_APP
 check!(kreg_ptr ptr kreg "kmovq");
+
+// CHECK-LABEL: eax_i16:
+// CHECK: #APP
+// CHECK: mov eax, eax
+// CHECK: #NO_APP
+check_reg!(eax_i16 i16 "eax" "mov");
+
+// CHECK-LABEL: eax_i32:
+// CHECK: #APP
+// CHECK: mov eax, eax
+// CHECK: #NO_APP
+check_reg!(eax_i32 i32 "eax" "mov");
+
+// CHECK-LABEL: eax_f32:
+// CHECK: #APP
+// CHECK: mov eax, eax
+// CHECK: #NO_APP
+check_reg!(eax_f32 f32 "eax" "mov");
+
+// x86_64-LABEL: eax_i64:
+// x86_64: #APP
+// x86_64: mov eax, eax
+// x86_64: #NO_APP
+#[cfg(x86_64)]
+check_reg!(eax_i64 i64 "eax" "mov");
+
+// x86_64-LABEL: eax_f64:
+// x86_64: #APP
+// x86_64: mov eax, eax
+// x86_64: #NO_APP
+#[cfg(x86_64)]
+check_reg!(eax_f64 f64 "eax" "mov");
+
+// CHECK-LABEL: eax_ptr:
+// CHECK: #APP
+// CHECK: mov eax, eax
+// CHECK: #NO_APP
+check_reg!(eax_ptr ptr "eax" "mov");
+
+// CHECK-LABEL: ah_byte:
+// CHECK: #APP
+// CHECK: mov ah, ah
+// CHECK: #NO_APP
+check_reg!(ah_byte i8 "ah" "mov");
+
+// CHECK-LABEL: xmm0_i32:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_i32 i32 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_f32:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_f32 f32 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_i64:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_i64 i64 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_f64:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_f64 f64 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_ptr:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_ptr ptr "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_i8x16:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_i8x16 i8x16 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_i16x8:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_i16x8 i16x8 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_i32x4:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_i32x4 i32x4 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_i64x2:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_i64x2 i64x2 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_f32x4:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_f32x4 f32x4 "xmm0" "movaps");
+
+// CHECK-LABEL: xmm0_f64x2:
+// CHECK: #APP
+// CHECK: movaps xmm0, xmm0
+// CHECK: #NO_APP
+check_reg!(xmm0_f64x2 f64x2 "xmm0" "movaps");
+
+// CHECK-LABEL: ymm0_i32:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i32 i32 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_f32:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_f32 f32 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i64:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i64 i64 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_f64:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_f64 f64 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_ptr:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_ptr ptr "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i8x16:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i8x16 i8x16 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i16x8:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i16x8 i16x8 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i32x4:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i32x4 i32x4 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i64x2:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i64x2 i64x2 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_f32x4:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_f32x4 f32x4 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_f64x2:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_f64x2 f64x2 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i8x32:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i8x32 i8x32 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i16x16:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i16x16 i16x16 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i32x8:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i32x8 i32x8 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_i64x4:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_i64x4 i64x4 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_f32x8:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_f32x8 f32x8 "ymm0" "vmovaps");
+
+// CHECK-LABEL: ymm0_f64x4:
+// CHECK: #APP
+// CHECK: vmovaps ymm0, ymm0
+// CHECK: #NO_APP
+check_reg!(ymm0_f64x4 f64x4 "ymm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i32:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i32 i32 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f32:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f32 f32 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i64:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i64 i64 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f64:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f64 f64 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_ptr:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_ptr ptr "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i8x16:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i8x16 i8x16 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i16x8:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i16x8 i16x8 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i32x4:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i32x4 i32x4 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i64x2:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i64x2 i64x2 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f32x4:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f32x4 f32x4 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f64x2:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f64x2 f64x2 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i8x32:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i8x32 i8x32 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i16x16:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i16x16 i16x16 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i32x8:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i32x8 i32x8 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i64x4:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i64x4 i64x4 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f32x8:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f32x8 f32x8 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f64x4:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f64x4 f64x4 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i8x64:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i8x64 i8x64 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i16x32:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i16x32 i16x32 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i32x16:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i32x16 i32x16 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_i64x8:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_i64x8 i64x8 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f32x16:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f32x16 f32x16 "zmm0" "vmovaps");
+
+// CHECK-LABEL: zmm0_f64x8:
+// CHECK: #APP
+// CHECK: vmovaps zmm0, zmm0
+// CHECK: #NO_APP
+check_reg!(zmm0_f64x8 f64x8 "zmm0" "vmovaps");
+
+// CHECK-LABEL: k1_i8:
+// CHECK: #APP
+// CHECK: kmovb k1, k1
+// CHECK: #NO_APP
+check_reg!(k1_i8 i8 "k1" "kmovb");
+
+// CHECK-LABEL: k1_i16:
+// CHECK: #APP
+// CHECK: kmovw k1, k1
+// CHECK: #NO_APP
+check_reg!(k1_i16 i16 "k1" "kmovw");
+
+// CHECK-LABEL: k1_i32:
+// CHECK: #APP
+// CHECK: kmovd k1, k1
+// CHECK: #NO_APP
+check_reg!(k1_i32 i32 "k1" "kmovd");
+
+// CHECK-LABEL: k1_i64:
+// CHECK: #APP
+// CHECK: kmovq k1, k1
+// CHECK: #NO_APP
+check_reg!(k1_i64 i64 "k1" "kmovq");
+
+// CHECK-LABEL: k1_ptr:
+// CHECK: #APP
+// CHECK: kmovq k1, k1
+// CHECK: #NO_APP
+check_reg!(k1_ptr ptr "k1" "kmovq");