Merge pull request #1350 from bjorn3/inline_asm_sym

Implement const and sym operands for inline asm

3 files changed, 195 insertions, 107 deletions

diff --git a/src/allocator.rs b/src/allocator.rs
index 8508227179a..1c73957ca57 100644
--- a/src/allocator.rs
+++ b/src/allocator.rs
@@ -70,37 +70,13 @@ fn codegen_inner(
             params: arg_tys.iter().cloned().map(AbiParam::new).collect(),
             returns: output.into_iter().map(AbiParam::new).collect(),
         };
-
-        let caller_name = format!("__rust_{}", method.name);
-        let callee_name = kind.fn_name(method.name);
-
-        let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap();
-
-        let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
-
-        let mut ctx = Context::new();
-        ctx.func.signature = sig.clone();
-        {
-            let mut func_ctx = FunctionBuilderContext::new();
-            let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
-
-            let block = bcx.create_block();
-            bcx.switch_to_block(block);
-            let args = arg_tys
-                .into_iter()
-                .map(|ty| bcx.append_block_param(block, ty))
-                .collect::<Vec<Value>>();
-
-            let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
-            let call_inst = bcx.ins().call(callee_func_ref, &args);
-            let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error
-
-            bcx.ins().return_(&results);
-            bcx.seal_all_blocks();
-            bcx.finalize();
-        }
-        module.define_function(func_id, &mut ctx).unwrap();
-        unwind_context.add_function(func_id, &ctx, module.isa());
+        crate::common::create_wrapper_function(
+            module,
+            unwind_context,
+            sig,
+            &format!("__rust_{}", method.name),
+            &kind.fn_name(method.name),
+        );
     }
 
     let sig = Signature {
@@ -108,36 +84,13 @@ fn codegen_inner(
         params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)],
         returns: vec![],
     };
-
-    let callee_name = alloc_error_handler_kind.fn_name(sym::oom);
-
-    let func_id =
-        module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap();
-
-    let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
-
-    let mut ctx = Context::new();
-    ctx.func.signature = sig;
-    {
-        let mut func_ctx = FunctionBuilderContext::new();
-        let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
-
-        let block = bcx.create_block();
-        bcx.switch_to_block(block);
-        let args = (&[usize_ty, usize_ty])
-            .iter()
-            .map(|&ty| bcx.append_block_param(block, ty))
-            .collect::<Vec<Value>>();
-
-        let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
-        bcx.ins().call(callee_func_ref, &args);
-
-        bcx.ins().trap(TrapCode::UnreachableCodeReached);
-        bcx.seal_all_blocks();
-        bcx.finalize();
-    }
-    module.define_function(func_id, &mut ctx).unwrap();
-    unwind_context.add_function(func_id, &ctx, module.isa());
+    crate::common::create_wrapper_function(
+        module,
+        unwind_context,
+        sig,
+        "__rust_alloc_error_handler",
+        &alloc_error_handler_kind.fn_name(sym::oom),
+    );
 
     let data_id = module.declare_data(OomStrategy::SYMBOL, Linkage::Export, false, false).unwrap();
     let mut data_ctx = DataContext::new();
diff --git a/src/common.rs b/src/common.rs
index f41af3a9e63..a8be0d32cc8 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -254,6 +254,44 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
     }
 }
 
+pub(crate) fn create_wrapper_function(
+    module: &mut dyn Module,
+    unwind_context: &mut UnwindContext,
+    sig: Signature,
+    wrapper_name: &str,
+    callee_name: &str,
+) {
+    let wrapper_func_id = module.declare_function(wrapper_name, Linkage::Export, &sig).unwrap();
+    let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap();
+
+    let mut ctx = Context::new();
+    ctx.func.signature = sig;
+    {
+        let mut func_ctx = FunctionBuilderContext::new();
+        let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+
+        let block = bcx.create_block();
+        bcx.switch_to_block(block);
+        let func = &mut bcx.func.stencil;
+        let args = func
+            .signature
+            .params
+            .iter()
+            .map(|param| func.dfg.append_block_param(block, param.value_type))
+            .collect::<Vec<Value>>();
+
+        let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
+        let call_inst = bcx.ins().call(callee_func_ref, &args);
+        let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error
+
+        bcx.ins().return_(&results);
+        bcx.seal_all_blocks();
+        bcx.finalize();
+    }
+    module.define_function(wrapper_func_id, &mut ctx).unwrap();
+    unwind_context.add_function(wrapper_func_id, &ctx, module.isa());
+}
+
 pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) cx: &'clif mut crate::CodegenCx,
     pub(crate) module: &'m mut dyn Module,
diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index 3fcc84d3929..6206fbf7dd5 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -9,9 +9,33 @@ use rustc_middle::mir::InlineAsmOperand;
 use rustc_span::sym;
 use rustc_target::asm::*;
 
+enum CInlineAsmOperand<'tcx> {
+    In {
+        reg: InlineAsmRegOrRegClass,
+        value: CValue<'tcx>,
+    },
+    Out {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        place: Option<CPlace<'tcx>>,
+    },
+    InOut {
+        reg: InlineAsmRegOrRegClass,
+        _late: bool,
+        in_value: CValue<'tcx>,
+        out_place: Option<CPlace<'tcx>>,
+    },
+    Const {
+        value: String,
+    },
+    Symbol {
+        symbol: String,
+    },
+}
+
 pub(crate) fn codegen_inline_asm<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    _span: Span,
+    span: Span,
     template: &[InlineAsmTemplatePiece],
     operands: &[InlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
@@ -198,6 +222,81 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         }
     }
 
+    let operands = operands
+        .into_iter()
+        .map(|operand| match *operand {
+            InlineAsmOperand::In { reg, ref value } => {
+                CInlineAsmOperand::In { reg, value: crate::base::codegen_operand(fx, value) }
+            }
+            InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
+                reg,
+                late,
+                place: place.map(|place| crate::base::codegen_place(fx, place)),
+            },
+            InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
+                CInlineAsmOperand::InOut {
+                    reg,
+                    _late: late,
+                    in_value: crate::base::codegen_operand(fx, in_value),
+                    out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
+                }
+            }
+            InlineAsmOperand::Const { ref value } => {
+                let (const_value, ty) = crate::constant::eval_mir_constant(fx, &*value)
+                    .unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved"));
+                let value = rustc_codegen_ssa::common::asm_const_to_str(
+                    fx.tcx,
+                    span,
+                    const_value,
+                    fx.layout_of(ty),
+                );
+                CInlineAsmOperand::Const { value }
+            }
+            InlineAsmOperand::SymFn { ref value } => {
+                let literal = fx.monomorphize(value.literal);
+                if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
+                    let instance = ty::Instance::resolve_for_fn_ptr(
+                        fx.tcx,
+                        ty::ParamEnv::reveal_all(),
+                        def_id,
+                        substs,
+                    )
+                    .unwrap();
+                    let symbol = fx.tcx.symbol_name(instance);
+
+                    // Pass a wrapper rather than the function itself as the function itself may not
+                    // be exported from the main codegen unit and may thus be unreachable from the
+                    // object file created by an external assembler.
+                    let inline_asm_index = fx.cx.inline_asm_index.get();
+                    fx.cx.inline_asm_index.set(inline_asm_index + 1);
+                    let wrapper_name = format!(
+                        "__inline_asm_{}_wrapper_n{}",
+                        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
+                        inline_asm_index
+                    );
+                    let sig =
+                        get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
+                    create_wrapper_function(
+                        fx.module,
+                        &mut fx.cx.unwind_context,
+                        sig,
+                        &wrapper_name,
+                        symbol.name,
+                    );
+
+                    CInlineAsmOperand::Symbol { symbol: wrapper_name }
+                } else {
+                    span_bug!(span, "invalid type for asm sym (fn)");
+                }
+            }
+            InlineAsmOperand::SymStatic { def_id } => {
+                assert!(fx.tcx.is_static(def_id));
+                let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
+                CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
+            }
+        })
+        .collect::<Vec<_>>();
+
     let mut inputs = Vec::new();
     let mut outputs = Vec::new();
 
@@ -206,7 +305,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         arch: fx.tcx.sess.asm_arch.unwrap(),
         enclosing_def_id: fx.instance.def_id(),
         template,
-        operands,
+        operands: &operands,
         options,
         registers: Vec::new(),
         stack_slots_clobber: Vec::new(),
@@ -229,36 +328,22 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     fx.cx.global_asm.push_str(&generated_asm);
 
     for (i, operand) in operands.iter().enumerate() {
-        match *operand {
-            InlineAsmOperand::In { reg: _, ref value } => {
-                inputs.push((
-                    asm_gen.stack_slots_input[i].unwrap(),
-                    crate::base::codegen_operand(fx, value).load_scalar(fx),
-                ));
-            }
-            InlineAsmOperand::Out { reg: _, late: _, place } => {
+        match operand {
+            CInlineAsmOperand::In { reg: _, value } => {
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), value.load_scalar(fx)));
+            }
+            CInlineAsmOperand::Out { reg: _, late: _, place } => {
                 if let Some(place) = place {
-                    outputs.push((
-                        asm_gen.stack_slots_output[i].unwrap(),
-                        crate::base::codegen_place(fx, place),
-                    ));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), place.clone()));
                 }
             }
-            InlineAsmOperand::InOut { reg: _, late: _, ref in_value, out_place } => {
-                inputs.push((
-                    asm_gen.stack_slots_input[i].unwrap(),
-                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                ));
+            CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx)));
                 if let Some(out_place) = out_place {
-                    outputs.push((
-                        asm_gen.stack_slots_output[i].unwrap(),
-                        crate::base::codegen_place(fx, out_place),
-                    ));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), out_place.clone()));
                 }
             }
-            InlineAsmOperand::Const { value: _ } => todo!(),
-            InlineAsmOperand::SymFn { value: _ } => todo!(),
-            InlineAsmOperand::SymStatic { def_id: _ } => todo!(),
+            CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
         }
     }
 
@@ -280,7 +365,7 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
     arch: InlineAsmArch,
     enclosing_def_id: DefId,
     template: &'a [InlineAsmTemplatePiece],
-    operands: &'a [InlineAsmOperand<'tcx>],
+    operands: &'a [CInlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
     registers: Vec<Option<InlineAsmReg>>,
     stack_slots_clobber: Vec<Option<Size>>,
@@ -304,18 +389,20 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Add explicit registers to the allocated set.
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
+                CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
                     regs[i] = Some(reg);
                     allocated.entry(reg).or_default().0 = true;
                 }
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(reg), late: true, ..
+                CInlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::Reg(reg),
+                    late: true,
+                    ..
                 } => {
                     regs[i] = Some(reg);
                     allocated.entry(reg).or_default().1 = true;
                 }
-                InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
-                | InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
+                CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
+                | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
                     regs[i] = Some(reg);
                     allocated.insert(reg, (true, true));
                 }
@@ -326,12 +413,12 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate out/inout/inlateout registers first because they are more constrained.
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::Out {
+                CInlineAsmOperand::Out {
                     reg: InlineAsmRegOrRegClass::RegClass(class),
                     late: false,
                     ..
                 }
-                | InlineAsmOperand::InOut {
+                | CInlineAsmOperand::InOut {
                     reg: InlineAsmRegOrRegClass::RegClass(class), ..
                 } => {
                     let mut alloc_reg = None;
@@ -360,7 +447,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate in/lateout.
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
+                CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
                     let mut alloc_reg = None;
                     for &reg in &map[&class] {
                         let mut used = false;
@@ -380,7 +467,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
                     regs[i] = Some(reg);
                     allocated.entry(reg).or_default().0 = true;
                 }
-                InlineAsmOperand::Out {
+                CInlineAsmOperand::Out {
                     reg: InlineAsmRegOrRegClass::RegClass(class),
                     late: true,
                     ..
@@ -455,7 +542,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for inout
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
+                CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
                     let slot = new_slot(reg.reg_class());
                     slots_input[i] = Some(slot);
                     slots_output[i] = Some(slot);
@@ -470,8 +557,8 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for input
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::In { reg, .. }
-                | InlineAsmOperand::InOut { reg, out_place: None, .. } => {
+                CInlineAsmOperand::In { reg, .. }
+                | CInlineAsmOperand::InOut { reg, out_place: None, .. } => {
                     slots_input[i] = Some(new_slot(reg.reg_class()));
                 }
                 _ => (),
@@ -487,7 +574,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for output
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::Out { reg, place: Some(_), .. } => {
+                CInlineAsmOperand::Out { reg, place: Some(_), .. } => {
                     slots_output[i] = Some(new_slot(reg.reg_class()));
                 }
                 _ => (),
@@ -549,13 +636,23 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
                     generated_asm.push_str(s);
                 }
                 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
-                    if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
-                        generated_asm.push('%');
+                    match self.operands[*operand_idx] {
+                        CInlineAsmOperand::In { .. }
+                        | CInlineAsmOperand::Out { .. }
+                        | CInlineAsmOperand::InOut { .. } => {
+                            if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+                                generated_asm.push('%');
+                            }
+                            self.registers[*operand_idx]
+                                .unwrap()
+                                .emit(&mut generated_asm, self.arch, *modifier)
+                                .unwrap();
+                        }
+                        CInlineAsmOperand::Const { ref value } => {
+                            generated_asm.push_str(value);
+                        }
+                        CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol),
                     }
-                    self.registers[*operand_idx]
-                        .unwrap()
-                        .emit(&mut generated_asm, self.arch, *modifier)
-                        .unwrap();
                 }
             }
         }