Auto merge of #123507 - dpaoliello:arm64ecasm, r=Amanieu

Add support for Arm64EC inline assembly (as unstable)

Compiler support for Arm64EC assembly mostly reuses the existing AArch64 support, except that it needs to block registers that are not permitted: <https://learn.microsoft.com/en-us/windows/arm/arm64ec-abi#register-mapping-and-blocked-registers>

For assembly authors there are several caveats and differences that need to be considered, I've provided documentation for this as part of the "Standard Library Support" PR: <https://github.com/rust-lang/rust/pull/123144/files#diff-6b08532480943c8b82f5dbda7ee1521afa74c9f626466aeb308dfa6956397edd>

r? rust-lang/compiler

5 files changed, 158 insertions, 103 deletions

diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index 500904ce188..e09869cf425 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -220,7 +220,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         constraints.append(&mut clobbers);
         if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
             match asm_arch {
-                InlineAsmArch::AArch64 | InlineAsmArch::Arm => {
+                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
                     constraints.push("~{cc}".to_string());
                 }
                 InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs
index 70528c1222c..5ae9a2e2058 100644
--- a/compiler/rustc_target/src/asm/aarch64.rs
+++ b/compiler/rustc_target/src/asm/aarch64.rs
@@ -87,6 +87,20 @@ fn reserved_x18(
     }
 }
 
+fn restricted_for_arm64ec(
+    arch: InlineAsmArch,
+    _reloc_model: RelocModel,
+    _target_features: &FxIndexSet<Symbol>,
+    _target: &Target,
+    _is_clobber: bool,
+) -> Result<(), &'static str> {
+    if arch == InlineAsmArch::Arm64EC {
+        Err("x13, x14, x23, x24, x28, v16-v31 cannot be used for Arm64EC")
+    } else {
+        Ok(())
+    }
+}
+
 def_regs! {
     AArch64 AArch64InlineAsmReg AArch64InlineAsmRegClass {
         x0: reg = ["x0", "w0"],
@@ -102,8 +116,8 @@ def_regs! {
         x10: reg = ["x10", "w10"],
         x11: reg = ["x11", "w11"],
         x12: reg = ["x12", "w12"],
-        x13: reg = ["x13", "w13"],
-        x14: reg = ["x14", "w14"],
+        x13: reg = ["x13", "w13"] % restricted_for_arm64ec,
+        x14: reg = ["x14", "w14"] % restricted_for_arm64ec,
         x15: reg = ["x15", "w15"],
         x16: reg = ["x16", "w16"],
         x17: reg = ["x17", "w17"],
@@ -111,12 +125,12 @@ def_regs! {
         x20: reg = ["x20", "w20"],
         x21: reg = ["x21", "w21"],
         x22: reg = ["x22", "w22"],
-        x23: reg = ["x23", "w23"],
-        x24: reg = ["x24", "w24"],
+        x23: reg = ["x23", "w23"] % restricted_for_arm64ec,
+        x24: reg = ["x24", "w24"] % restricted_for_arm64ec,
         x25: reg = ["x25", "w25"],
         x26: reg = ["x26", "w26"],
         x27: reg = ["x27", "w27"],
-        x28: reg = ["x28", "w28"],
+        x28: reg = ["x28", "w28"] % restricted_for_arm64ec,
         x30: reg = ["x30", "w30", "lr", "wlr"],
         v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0", "z0"],
         v1: vreg, vreg_low16 = ["v1", "b1", "h1", "s1", "d1", "q1", "z1"],
@@ -134,22 +148,22 @@ def_regs! {
         v13: vreg, vreg_low16 = ["v13", "b13", "h13", "s13", "d13", "q13", "z13"],
         v14: vreg, vreg_low16 = ["v14", "b14", "h14", "s14", "d14", "q14", "z14"],
         v15: vreg, vreg_low16 = ["v15", "b15", "h15", "s15", "d15", "q15", "z15"],
-        v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"],
-        v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"],
-        v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"],
-        v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"],
-        v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"],
-        v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"],
-        v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"],
-        v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"],
-        v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"],
-        v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"],
-        v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"],
-        v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"],
-        v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"],
-        v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"],
-        v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"],
-        v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"],
+        v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"] % restricted_for_arm64ec,
+        v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"] % restricted_for_arm64ec,
+        v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"] % restricted_for_arm64ec,
+        v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"] % restricted_for_arm64ec,
+        v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"] % restricted_for_arm64ec,
+        v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"] % restricted_for_arm64ec,
+        v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"] % restricted_for_arm64ec,
+        v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"] % restricted_for_arm64ec,
+        v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"] % restricted_for_arm64ec,
+        v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"] % restricted_for_arm64ec,
+        v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"] % restricted_for_arm64ec,
+        v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"] % restricted_for_arm64ec,
+        v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"] % restricted_for_arm64ec,
+        v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"] % restricted_for_arm64ec,
+        v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"] % restricted_for_arm64ec,
+        v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"] % restricted_for_arm64ec,
         p0: preg = ["p0"],
         p1: preg = ["p1"],
         p2: preg = ["p2"],
diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs
index 2e04dca98c5..d6bbf4f36cf 100644
--- a/compiler/rustc_target/src/asm/mod.rs
+++ b/compiler/rustc_target/src/asm/mod.rs
@@ -217,6 +217,7 @@ pub enum InlineAsmArch {
     X86_64,
     Arm,
     AArch64,
+    Arm64EC,
     RiscV32,
     RiscV64,
     Nvptx64,
@@ -246,6 +247,7 @@ impl FromStr for InlineAsmArch {
             "x86_64" => Ok(Self::X86_64),
             "arm" => Ok(Self::Arm),
             "aarch64" => Ok(Self::AArch64),
+            "arm64ec" => Ok(Self::Arm64EC),
             "riscv32" => Ok(Self::RiscV32),
             "riscv64" => Ok(Self::RiscV64),
             "nvptx64" => Ok(Self::Nvptx64),
@@ -341,7 +343,9 @@ impl InlineAsmReg {
         Ok(match arch {
             InlineAsmArch::X86 | InlineAsmArch::X86_64 => Self::X86(X86InlineAsmReg::parse(name)?),
             InlineAsmArch::Arm => Self::Arm(ArmInlineAsmReg::parse(name)?),
-            InlineAsmArch::AArch64 => Self::AArch64(AArch64InlineAsmReg::parse(name)?),
+            InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => {
+                Self::AArch64(AArch64InlineAsmReg::parse(name)?)
+            }
             InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
                 Self::RiscV(RiscVInlineAsmReg::parse(name)?)
             }
@@ -610,7 +614,9 @@ impl InlineAsmRegClass {
                 Self::X86(X86InlineAsmRegClass::parse(name)?)
             }
             InlineAsmArch::Arm => Self::Arm(ArmInlineAsmRegClass::parse(name)?),
-            InlineAsmArch::AArch64 => Self::AArch64(AArch64InlineAsmRegClass::parse(name)?),
+            InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => {
+                Self::AArch64(AArch64InlineAsmRegClass::parse(name)?)
+            }
             InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
                 Self::RiscV(RiscVInlineAsmRegClass::parse(name)?)
             }
@@ -783,7 +789,7 @@ pub fn allocatable_registers(
             arm::fill_reg_map(arch, reloc_model, target_features, target, &mut map);
             map
         }
-        InlineAsmArch::AArch64 => {
+        InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => {
             let mut map = aarch64::regclass_map();
             aarch64::fill_reg_map(arch, reloc_model, target_features, target, &mut map);
             map
@@ -909,6 +915,10 @@ impl InlineAsmClobberAbi {
                 }),
                 _ => Err(&["C", "system", "efiapi"]),
             },
+            InlineAsmArch::Arm64EC => match name {
+                "C" | "system" => Ok(InlineAsmClobberAbi::AArch64NoX18),
+                _ => Err(&["C", "system"]),
+            },
             InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => match name {
                 "C" | "system" | "efiapi" => Ok(InlineAsmClobberAbi::RiscV),
                 _ => Err(&["C", "system", "efiapi"]),
diff --git a/src/doc/unstable-book/src/language-features/asm-experimental-arch.md b/src/doc/unstable-book/src/language-features/asm-experimental-arch.md
index 59acbc73db4..43e11b6d57d 100644
--- a/src/doc/unstable-book/src/language-features/asm-experimental-arch.md
+++ b/src/doc/unstable-book/src/language-features/asm-experimental-arch.md
@@ -19,6 +19,7 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 - M68k
 - CSKY
 - s390x
+- Arm64EC
 
 ## Register classes
 
@@ -51,6 +52,9 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 | CSKY         | `freg`         | `f[0-31]`                          | `f`                  |
 | s390x        | `reg`          | `r[0-10]`, `r[12-14]`              | `r`                  |
 | s390x        | `freg`         | `f[0-15]`                          | `f`                  |
+| Arm64EC      | `reg`          | `x[0-12]`, `x[15-22]`, `x[25-27]`, `x30` | `r`            |
+| Arm64EC      | `vreg`         | `v[0-15]`                          | `w`                  |
+| Arm64EC      | `vreg_low16`   | `v[0-15]`                          | `x`                  |
 
 > **Notes**:
 > - NVPTX doesn't have a fixed register set, so named registers are not supported.
@@ -86,6 +90,8 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 | CSKY         | `freg`                          | None           | `f32`,                                  |
 | s390x        | `reg`, `reg_addr`               | None           | `i8`, `i16`, `i32`, `i64`               |
 | s390x        | `freg`                          | None           | `f32`, `f64`                            |
+| Arm64EC      | `reg`                           | None           | `i8`, `i16`, `i32`, `f32`, `i64`, `f64` |
+| Arm64EC      | `vreg`                          | None           | `i8`, `i16`, `i32`, `f32`, `i64`, `f64`, <br> `i8x8`, `i16x4`, `i32x2`, `i64x1`, `f32x2`, `f64x1`, <br> `i8x16`, `i16x8`, `i32x4`, `i64x2`, `f32x4`, `f64x2` |
 
 ## Register aliases
 
@@ -118,6 +124,12 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 | CSKY         | `r29`         | `rtb`     |
 | CSKY         | `r30`         | `svbr`    |
 | CSKY         | `r31`         | `tls`     |
+| Arm64EC      | `x[0-30]`     | `w[0-30]` |
+| Arm64EC      | `x29`         | `fp`      |
+| Arm64EC      | `x30`         | `lr`      |
+| Arm64EC      | `sp`          | `wsp`     |
+| Arm64EC      | `xzr`         | `wzr`     |
+| Arm64EC      | `v[0-15]`     | `b[0-15]`, `h[0-15]`, `s[0-15]`, `d[0-15]`, `q[0-15]` |
 
 > **Notes**:
 > - TI does not mandate a frame pointer for MSP430, but toolchains are allowed
@@ -128,8 +140,8 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 | Architecture | Unsupported register                    | Reason                                                                                                                                                                              |
 | ------------ | --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | All          | `sp`, `r15` (s390x)                     | The stack pointer must be restored to its original value at the end of an asm code block.                                                                                           |
-| All          | `fr` (Hexagon), `$fp` (MIPS), `Y` (AVR), `r4` (MSP430), `a6` (M68k), `r11` (s390x) | The frame pointer cannot be used as an input or output.                                                                                                                             |
-| All          | `r19` (Hexagon)                         | This is used internally by LLVM as a "base pointer" for functions with complex stack frames.                                                                                        |
+| All          | `fr` (Hexagon), `$fp` (MIPS), `Y` (AVR), `r4` (MSP430), `a6` (M68k), `r11` (s390x), `x29` (Arm64EC) | The frame pointer cannot be used as an input or output.                                                                                                                             |
+| All          | `r19` (Hexagon), `x19` (Arm64EC)        | This is used internally by LLVM as a "base pointer" for functions with complex stack frames.                                                                                        |
 | MIPS         | `$0` or `$zero`                         | This is a constant zero register which can't be modified.                                                                                                                           |
 | MIPS         | `$1` or `$at`                           | Reserved for assembler.                                                                                                                                                             |
 | MIPS         | `$26`/`$k0`, `$27`/`$k1`                | OS-reserved registers.                                                                                                                                                              |
@@ -145,6 +157,9 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 | CSKY         | `r15`                                   | This is the link register. |
 | CSKY         | `r[26-30]`                              | Reserved by its ABI.       |
 | CSKY         | `r31`                                   | This is the TLS register.  |
+| Arm64EC      | `xzr`                                   | This is a constant zero register which can't be modified. |
+| Arm64EC      | `x18`                                   | This is an OS-reserved register. |
+| Arm64EC      | `x13`, `x14`, `x23`, `x24`, `x28`, `v[16-31]` | These are AArch64 registers that are not supported for Arm64EC. |
 
 
 ## Template modifiers
@@ -165,6 +180,16 @@ This feature tracks `asm!` and `global_asm!` support for the following architect
 | s390x        | `freg`         | None     | `%f0`          | None          |
 | CSKY         | `reg`          | None     | `r0`           | None          |
 | CSKY         | `freg`         | None     | `f0`           | None          |
+| Arm64EC      | `reg`          | None     | `x0`           | `x`           |
+| Arm64EC      | `reg`          | `w`      | `w0`           | `w`           |
+| Arm64EC      | `reg`          | `x`      | `x0`           | `x`           |
+| Arm64EC      | `vreg`         | None     | `v0`           | None          |
+| Arm64EC      | `vreg`         | `v`      | `v0`           | None          |
+| Arm64EC      | `vreg`         | `b`      | `b0`           | `b`           |
+| Arm64EC      | `vreg`         | `h`      | `h0`           | `h`           |
+| Arm64EC      | `vreg`         | `s`      | `s0`           | `s`           |
+| Arm64EC      | `vreg`         | `d`      | `d0`           | `d`           |
+| Arm64EC      | `vreg`         | `q`      | `q0`           | `q`           |
 
 # Flags covered by `preserves_flags`
 
@@ -177,3 +202,6 @@ These flags registers must be restored upon exiting the asm block if the `preser
   - The condition code register `ccr`.
 - s390x
   - The condition code register `cc`.
+- Arm64EC
+  - Condition flags (`NZCV` register).
+  - Floating-point status (`FPSR` register).
diff --git a/tests/assembly/asm/aarch64-types.rs b/tests/assembly/asm/aarch64-types.rs
index 1b2bd4b3d81..3e2a4773703 100644
--- a/tests/assembly/asm/aarch64-types.rs
+++ b/tests/assembly/asm/aarch64-types.rs
@@ -1,8 +1,11 @@
+//@ revisions: aarch64 arm64ec
 //@ assembly-output: emit-asm
-//@ compile-flags: --target aarch64-unknown-linux-gnu
-//@ needs-llvm-components: aarch64
+//@ [aarch64] compile-flags: --target aarch64-unknown-linux-gnu
+//@ [aarch64] needs-llvm-components: aarch64
+//@ [arm64ec] compile-flags: --target arm64ec-pc-windows-msvc
+//@ [arm64ec] needs-llvm-components: aarch64
 
-#![feature(no_core, lang_items, rustc_attrs, repr_simd)]
+#![feature(no_core, lang_items, rustc_attrs, repr_simd, asm_experimental_arch)]
 #![crate_type = "rlib"]
 #![no_core]
 #![allow(asm_sub_register, non_camel_case_types)]
@@ -77,7 +80,7 @@ extern "C" {
     static extern_static: u8;
 }
 
-// CHECK-LABEL: sym_fn:
+// CHECK-LABEL: {{("#)?}}sym_fn{{"?}}
 // CHECK: //APP
 // CHECK: bl extern_func
 // CHECK: //NO_APP
@@ -86,7 +89,7 @@ pub unsafe fn sym_fn() {
     asm!("bl {}", sym extern_func);
 }
 
-// CHECK-LABEL: sym_static:
+// CHECK-LABEL: {{("#)?}}sym_static{{"?}}
 // CHECK: //APP
 // CHECK: adr x0, extern_static
 // CHECK: //NO_APP
@@ -96,7 +99,7 @@ pub unsafe fn sym_static() {
 }
 
 // Regression test for #75761
-// CHECK-LABEL: issue_75761:
+// CHECK-LABEL: {{("#)?}}issue_75761{{"?}}
 // CHECK: str {{.*}}x30
 // CHECK: //APP
 // CHECK: //NO_APP
@@ -144,421 +147,421 @@ macro_rules! check_reg {
     };
 }
 
-// CHECK-LABEL: reg_i8:
+// CHECK-LABEL: {{("#)?}}reg_i8{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_i8 i8 reg "mov" "");
 
-// CHECK-LABEL: reg_i16:
+// CHECK-LABEL: {{("#)?}}reg_i16{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_i16 i16 reg "mov" "");
 
-// CHECK-LABEL: reg_i32:
+// CHECK-LABEL: {{("#)?}}reg_i32{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_i32 i32 reg "mov" "");
 
-// CHECK-LABEL: reg_f32:
+// CHECK-LABEL: {{("#)?}}reg_f32{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_f32 f32 reg "mov" "");
 
-// CHECK-LABEL: reg_i64:
+// CHECK-LABEL: {{("#)?}}reg_i64{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_i64 i64 reg "mov" "");
 
-// CHECK-LABEL: reg_f64:
+// CHECK-LABEL: {{("#)?}}reg_f64{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_f64 f64 reg "mov" "");
 
-// CHECK-LABEL: reg_ptr:
+// CHECK-LABEL: {{("#)?}}reg_ptr{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check!(reg_ptr ptr reg "mov" "");
 
-// CHECK-LABEL: vreg_i8:
+// CHECK-LABEL: {{("#)?}}vreg_i8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i8 i8 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i16:
+// CHECK-LABEL: {{("#)?}}vreg_i16{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i16 i16 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i32:
+// CHECK-LABEL: {{("#)?}}vreg_i32{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i32 i32 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_f32:
+// CHECK-LABEL: {{("#)?}}vreg_f32{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_f32 f32 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i64:
+// CHECK-LABEL: {{("#)?}}vreg_i64{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i64 i64 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_f64:
+// CHECK-LABEL: {{("#)?}}vreg_f64{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_f64 f64 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_ptr:
+// CHECK-LABEL: {{("#)?}}vreg_ptr{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_ptr ptr vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i8x8:
+// CHECK-LABEL: {{("#)?}}vreg_i8x8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i8x8 i8x8 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i16x4:
+// CHECK-LABEL: {{("#)?}}vreg_i16x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i16x4 i16x4 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i32x2:
+// CHECK-LABEL: {{("#)?}}vreg_i32x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i32x2 i32x2 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i64x1:
+// CHECK-LABEL: {{("#)?}}vreg_i64x1{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i64x1 i64x1 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_f32x2:
+// CHECK-LABEL: {{("#)?}}vreg_f32x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_f32x2 f32x2 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_f64x1:
+// CHECK-LABEL: {{("#)?}}vreg_f64x1{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_f64x1 f64x1 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i8x16:
+// CHECK-LABEL: {{("#)?}}vreg_i8x16{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i8x16 i8x16 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i16x8:
+// CHECK-LABEL: {{("#)?}}vreg_i16x8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i16x8 i16x8 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i32x4:
+// CHECK-LABEL: {{("#)?}}vreg_i32x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i32x4 i32x4 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_i64x2:
+// CHECK-LABEL: {{("#)?}}vreg_i64x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_i64x2 i64x2 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_f32x4:
+// CHECK-LABEL: {{("#)?}}vreg_f32x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_f32x4 f32x4 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_f64x2:
+// CHECK-LABEL: {{("#)?}}vreg_f64x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_f64x2 f64x2 vreg "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i8:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i8 i8 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i16:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i16{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i16 i16 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_f32:
+// CHECK-LABEL: {{("#)?}}vreg_low16_f32{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f32 f32 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i64:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i64{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i64 i64 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_f64:
+// CHECK-LABEL: {{("#)?}}vreg_low16_f64{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f64 f64 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_ptr:
+// CHECK-LABEL: {{("#)?}}vreg_low16_ptr{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_ptr ptr vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i8x8:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i8x8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i8x8 i8x8 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i16x4:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i16x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i16x4 i16x4 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i32x2:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i32x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i32x2 i32x2 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i64x1:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i64x1{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i64x1 i64x1 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_f32x2:
+// CHECK-LABEL: {{("#)?}}vreg_low16_f32x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f32x2 f32x2 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_f64x1:
+// CHECK-LABEL: {{("#)?}}vreg_low16_f64x1{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f64x1 f64x1 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i8x16:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i8x16{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i8x16 i8x16 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i16x8:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i16x8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i16x8 i16x8 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i32x4:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i32x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i32x4 i32x4 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_i64x2:
+// CHECK-LABEL: {{("#)?}}vreg_low16_i64x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_i64x2 i64x2 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_f32x4:
+// CHECK-LABEL: {{("#)?}}vreg_low16_f32x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f32x4 f32x4 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: vreg_low16_f64x2:
+// CHECK-LABEL: {{("#)?}}vreg_low16_f64x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s{{[0-9]+}}, s{{[0-9]+}}
 // CHECK: //NO_APP
 check!(vreg_low16_f64x2 f64x2 vreg_low16 "fmov" "s");
 
-// CHECK-LABEL: x0_i8:
+// CHECK-LABEL: {{("#)?}}x0_i8{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_i8 i8 "x0" "mov");
 
-// CHECK-LABEL: x0_i16:
+// CHECK-LABEL: {{("#)?}}x0_i16{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_i16 i16 "x0" "mov");
 
-// CHECK-LABEL: x0_i32:
+// CHECK-LABEL: {{("#)?}}x0_i32{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_i32 i32 "x0" "mov");
 
-// CHECK-LABEL: x0_f32:
+// CHECK-LABEL: {{("#)?}}x0_f32{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_f32 f32 "x0" "mov");
 
-// CHECK-LABEL: x0_i64:
+// CHECK-LABEL: {{("#)?}}x0_i64{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_i64 i64 "x0" "mov");
 
-// CHECK-LABEL: x0_f64:
+// CHECK-LABEL: {{("#)?}}x0_f64{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_f64 f64 "x0" "mov");
 
-// CHECK-LABEL: x0_ptr:
+// CHECK-LABEL: {{("#)?}}x0_ptr{{"?}}
 // CHECK: //APP
 // CHECK: mov x{{[0-9]+}}, x{{[0-9]+}}
 // CHECK: //NO_APP
 check_reg!(x0_ptr ptr "x0" "mov");
 
-// CHECK-LABEL: v0_i8:
+// CHECK-LABEL: {{("#)?}}v0_i8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i8 i8 "s0" "fmov");
 
-// CHECK-LABEL: v0_i16:
+// CHECK-LABEL: {{("#)?}}v0_i16{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i16 i16 "s0" "fmov");
 
-// CHECK-LABEL: v0_i32:
+// CHECK-LABEL: {{("#)?}}v0_i32{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i32 i32 "s0" "fmov");
 
-// CHECK-LABEL: v0_f32:
+// CHECK-LABEL: {{("#)?}}v0_f32{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_f32 f32 "s0" "fmov");
 
-// CHECK-LABEL: v0_i64:
+// CHECK-LABEL: {{("#)?}}v0_i64{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i64 i64 "s0" "fmov");
 
-// CHECK-LABEL: v0_f64:
+// CHECK-LABEL: {{("#)?}}v0_f64{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_f64 f64 "s0" "fmov");
 
-// CHECK-LABEL: v0_ptr:
+// CHECK-LABEL: {{("#)?}}v0_ptr{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_ptr ptr "s0" "fmov");
 
-// CHECK-LABEL: v0_i8x8:
+// CHECK-LABEL: {{("#)?}}v0_i8x8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i8x8 i8x8 "s0" "fmov");
 
-// CHECK-LABEL: v0_i16x4:
+// CHECK-LABEL: {{("#)?}}v0_i16x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i16x4 i16x4 "s0" "fmov");
 
-// CHECK-LABEL: v0_i32x2:
+// CHECK-LABEL: {{("#)?}}v0_i32x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i32x2 i32x2 "s0" "fmov");
 
-// CHECK-LABEL: v0_i64x1:
+// CHECK-LABEL: {{("#)?}}v0_i64x1{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i64x1 i64x1 "s0" "fmov");
 
-// CHECK-LABEL: v0_f32x2:
+// CHECK-LABEL: {{("#)?}}v0_f32x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_f32x2 f32x2 "s0" "fmov");
 
-// CHECK-LABEL: v0_f64x1:
+// CHECK-LABEL: {{("#)?}}v0_f64x1{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_f64x1 f64x1 "s0" "fmov");
 
-// CHECK-LABEL: v0_i8x16:
+// CHECK-LABEL: {{("#)?}}v0_i8x16{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i8x16 i8x16 "s0" "fmov");
 
-// CHECK-LABEL: v0_i16x8:
+// CHECK-LABEL: {{("#)?}}v0_i16x8{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i16x8 i16x8 "s0" "fmov");
 
-// CHECK-LABEL: v0_i32x4:
+// CHECK-LABEL: {{("#)?}}v0_i32x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i32x4 i32x4 "s0" "fmov");
 
-// CHECK-LABEL: v0_i64x2:
+// CHECK-LABEL: {{("#)?}}v0_i64x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_i64x2 i64x2 "s0" "fmov");
 
-// CHECK-LABEL: v0_f32x4:
+// CHECK-LABEL: {{("#)?}}v0_f32x4{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP
 check_reg!(v0_f32x4 f32x4 "s0" "fmov");
 
-// CHECK-LABEL: v0_f64x2:
+// CHECK-LABEL: {{("#)?}}v0_f64x2{{"?}}
 // CHECK: //APP
 // CHECK: fmov s0, s0
 // CHECK: //NO_APP