Implement asm goto in MIR and MIR lowering

13 files changed, 53 insertions, 17 deletions

diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 94c18756083..b776643763a 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -749,7 +749,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }
diff --git a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
index 83081f83e5e..956de1dec9b 100644
--- a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
+++ b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
@@ -182,7 +182,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }
diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs
index da07b66c762..44650898de8 100644
--- a/compiler/rustc_codegen_cranelift/src/global_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@@ -78,7 +78,8 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Out { .. }
                         | InlineAsmOperand::InOut { .. }
-                        | InlineAsmOperand::SplitInOut { .. } => {
+                        | InlineAsmOperand::SplitInOut { .. }
+                        | InlineAsmOperand::Label { .. } => {
                             span_bug!(op_sp, "invalid operand type for global_asm!")
                         }
                     }
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 7793b1b7092..171ee88a11c 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -129,6 +129,9 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                 let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
                 CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
             }
+            InlineAsmOperand::Label { .. } => {
+                span_bug!(span, "asm! label operands are not yet supported");
+            }
         })
         .collect::<Vec<_>>();
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 6342802bb19..c9a1e7789c2 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -1119,6 +1119,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir::InlineAsmOperand::SymStatic { def_id } => {
                     InlineAsmOperandRef::SymStatic { def_id }
                 }
+                mir::InlineAsmOperand::Label { target_index: _ } => {
+                    todo!();
+                }
             })
             .collect();
 
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 35058458eed..e058302af31 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -830,6 +830,9 @@ impl<'tcx> TerminatorKind<'tcx> {
                         InlineAsmOperand::SymStatic { def_id } => {
                             write!(fmt, "sym_static {def_id:?}")?;
                         }
+                        InlineAsmOperand::Label { target_index } => {
+                            write!(fmt, "label {target_index}")?;
+                        }
                     }
                 }
                 write!(fmt, ", options({options:?}))")
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 6f62c7186a7..f188923f876 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -919,6 +919,10 @@ pub enum InlineAsmOperand<'tcx> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        /// This represents the index into the `targets` array in `TerminatorKind::InlineAsm`.
+        target_index: usize,
+    },
 }
 
 /// Type for MIR `Assert` terminator error messages.
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index a1418b41e05..845b1717550 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -595,7 +595,8 @@ macro_rules! make_mir_visitor {
                                     self.visit_constant(value, location);
                                 }
                                 InlineAsmOperand::Out { place: None, .. }
-                                | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                                | InlineAsmOperand::SymStatic { def_id: _ }
+                                | InlineAsmOperand::Label { target_index: _ } => {}
                             }
                         }
                     }
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index a49d864701a..09e518606e3 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -400,6 +400,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 line_spans,
             }) => {
                 use rustc_middle::{mir, thir};
+
+                let destination_block = this.cfg.start_new_block();
+                let mut targets = if options.contains(InlineAsmOptions::NORETURN) {
+                    vec![]
+                } else {
+                    vec![destination_block]
+                };
+
                 let operands = operands
                     .into_iter()
                     .map(|op| match *op {
@@ -455,17 +463,28 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         thir::InlineAsmOperand::SymStatic { def_id } => {
                             mir::InlineAsmOperand::SymStatic { def_id }
                         }
-                        thir::InlineAsmOperand::Label { .. } => {
-                            todo!()
+                        thir::InlineAsmOperand::Label { block } => {
+                            let target = this.cfg.start_new_block();
+                            let target_index = targets.len();
+                            targets.push(target);
+
+                            let tmp = this.get_unit_temp();
+                            let target = unpack!(this.ast_block(tmp, target, block, source_info));
+                            this.cfg.terminate(
+                                target,
+                                source_info,
+                                TerminatorKind::Goto { target: destination_block },
+                            );
+
+                            mir::InlineAsmOperand::Label { target_index }
                         }
                     })
                     .collect();
 
-                if !options.contains(InlineAsmOptions::NORETURN) {
+                if !expr.ty.is_never() {
                     this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
                 }
 
-                let destination_block = this.cfg.start_new_block();
                 this.cfg.terminate(
                     block,
                     source_info,
@@ -474,11 +493,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         operands,
                         options,
                         line_spans,
-                        targets: if options.contains(InlineAsmOptions::NORETURN) {
-                            Vec::new()
-                        } else {
-                            vec![destination_block]
-                        },
+                        targets,
                         unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) {
                             UnwindAction::Continue
                         } else {
diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
index 595c2ff5bf7..29169c31263 100644
--- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
@@ -271,7 +271,8 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> {
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Const { .. }
                         | InlineAsmOperand::SymFn { .. }
-                        | InlineAsmOperand::SymStatic { .. } => {}
+                        | InlineAsmOperand::SymStatic { .. }
+                        | InlineAsmOperand::Label { .. } => {}
                     }
                 }
             }
diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
index 95f12301395..db48ecd702b 100644
--- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
+++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
@@ -515,7 +515,8 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs
index 2c8201b1903..10fea09531a 100644
--- a/compiler/rustc_mir_transform/src/dest_prop.rs
+++ b/compiler/rustc_mir_transform/src/dest_prop.rs
@@ -648,7 +648,8 @@ impl WriteInfo {
                         }
                         InlineAsmOperand::Const { .. }
                         | InlineAsmOperand::SymFn { .. }
-                        | InlineAsmOperand::SymStatic { .. } => (),
+                        | InlineAsmOperand::SymStatic { .. }
+                        | InlineAsmOperand::Label { .. } => {}
                     }
                 }
             }
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
index 92e86cf0ee4..003a9a59200 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
@@ -559,7 +559,8 @@ impl<'tcx> Stable<'tcx> for mir::InlineAsmOperand<'tcx> {
             }
             InlineAsmOperand::Const { .. }
             | InlineAsmOperand::SymFn { .. }
-            | InlineAsmOperand::SymStatic { .. } => (None, None),
+            | InlineAsmOperand::SymStatic { .. }
+            | InlineAsmOperand::Label { .. } => (None, None),
         };
 
         stable_mir::mir::InlineAsmOperand { in_value, out_place, raw_rpr: format!("{self:?}") }