Implement asm! codegen

8 files changed, 608 insertions, 17 deletions

diff --git a/src/librustc_codegen_llvm/asm.rs b/src/librustc_codegen_llvm/asm.rs
index 30bf3ce7528..21322511c99 100644
--- a/src/librustc_codegen_llvm/asm.rs
+++ b/src/librustc_codegen_llvm/asm.rs
@@ -1,14 +1,21 @@
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 use crate::llvm;
+use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 
+use rustc_ast::ast::LlvmAsmDialect;
 use rustc_codegen_ssa::mir::operand::OperandValue;
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_hir as hir;
+use rustc_middle::span_bug;
+use rustc_middle::ty::layout::TyAndLayout;
 use rustc_span::Span;
+use rustc_target::abi::*;
+use rustc_target::asm::*;
 
 use libc::{c_char, c_uint};
 use log::debug;
@@ -40,7 +47,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     indirect_outputs.push(operand.immediate());
                 }
             } else {
-                output_types.push(place.layout.llvm_type(self.cx()));
+                output_types.push(place.layout.llvm_type(self.cx));
             }
         }
         if !indirect_outputs.is_empty() {
@@ -89,6 +96,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             ia.volatile,
             ia.alignstack,
             ia.dialect,
+            span,
         );
         if r.is_none() {
             return false;
@@ -102,22 +110,210 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             OperandValue::Immediate(v).store(self, place);
         }
 
-        // Store mark in a metadata node so we can map LLVM errors
-        // back to source locations.  See #17552.
-        unsafe {
-            let key = "srcloc";
-            let kind = llvm::LLVMGetMDKindIDInContext(
-                self.llcx,
-                key.as_ptr() as *const c_char,
-                key.len() as c_uint,
-            );
+        true
+    }
+
+    fn codegen_inline_asm(
+        &mut self,
+        template: &[InlineAsmTemplatePiece],
+        operands: &[InlineAsmOperandRef<'tcx, Self>],
+        options: InlineAsmOptions,
+        span: Span,
+    ) {
+        let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
-            let val: &'ll Value = self.const_i32(span.ctxt().outer_expn().as_u32() as i32);
+        // Collect the types of output operands
+        let mut constraints = vec![];
+        let mut output_types = vec![];
+        let mut op_idx = FxHashMap::default();
+        for (idx, op) in operands.iter().enumerate() {
+            match *op {
+                InlineAsmOperandRef::Out { reg, late, place } => {
+                    let ty = if let Some(place) = place {
+                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
+                    } else {
+                        // If the output is discarded, we don't really care what
+                        // type is used. We're just using this to tell LLVM to
+                        // reserve the register.
+                        dummy_output_type(self.cx, reg.reg_class())
+                    };
+                    output_types.push(ty);
+                    op_idx.insert(idx, constraints.len());
+                    let prefix = if late { "=" } else { "=&" };
+                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
+                }
+                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
+                    let ty = if let Some(ref out_place) = out_place {
+                        llvm_fixup_output_type(self.cx, reg.reg_class(), &out_place.layout)
+                    } else {
+                        // LLVM required tied operands to have the same type,
+                        // so we just use the type of the input.
+                        llvm_fixup_output_type(self.cx, reg.reg_class(), &in_value.layout)
+                    };
+                    output_types.push(ty);
+                    op_idx.insert(idx, constraints.len());
+                    let prefix = if late { "=" } else { "=&" };
+                    constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
+                }
+                _ => {}
+            }
+        }
 
-            llvm::LLVMSetMetadata(r, kind, llvm::LLVMMDNodeInContext(self.llcx, &val, 1));
+        // Collect input operands
+        let mut inputs = vec![];
+        for (idx, op) in operands.iter().enumerate() {
+            match *op {
+                InlineAsmOperandRef::In { reg, value } => {
+                    let value =
+                        llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
+                    inputs.push(value);
+                    op_idx.insert(idx, constraints.len());
+                    constraints.push(reg_to_llvm(reg));
+                }
+                InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => {
+                    let value = llvm_fixup_input(
+                        self,
+                        in_value.immediate(),
+                        reg.reg_class(),
+                        &in_value.layout,
+                    );
+                    inputs.push(value);
+                    constraints.push(format!("{}", op_idx[&idx]));
+                }
+                InlineAsmOperandRef::SymFn { instance } => {
+                    inputs.push(self.cx.get_fn(instance));
+                    op_idx.insert(idx, constraints.len());
+                    constraints.push("s".to_string());
+                }
+                InlineAsmOperandRef::SymStatic { def_id } => {
+                    inputs.push(self.cx.get_static(def_id));
+                    op_idx.insert(idx, constraints.len());
+                    constraints.push("s".to_string());
+                }
+                _ => {}
+            }
         }
 
-        true
+        // Build the template string
+        let mut template_str = String::new();
+        for piece in template {
+            match *piece {
+                InlineAsmTemplatePiece::String(ref s) => {
+                    if s.contains('$') {
+                        for c in s.chars() {
+                            if c == '$' {
+                                template_str.push_str("$$");
+                            } else {
+                                template_str.push(c);
+                            }
+                        }
+                    } else {
+                        template_str.push_str(s)
+                    }
+                }
+                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
+                    match operands[operand_idx] {
+                        InlineAsmOperandRef::In { reg, .. }
+                        | InlineAsmOperandRef::Out { reg, .. }
+                        | InlineAsmOperandRef::InOut { reg, .. } => {
+                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
+                            if let Some(modifier) = modifier {
+                                template_str.push_str(&format!(
+                                    "${{{}:{}}}",
+                                    op_idx[&operand_idx], modifier
+                                ));
+                            } else {
+                                template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
+                            }
+                        }
+                        InlineAsmOperandRef::Const { ref string } => {
+                            // Const operands get injected directly into the template
+                            template_str.push_str(string);
+                        }
+                        InlineAsmOperandRef::SymFn { .. }
+                        | InlineAsmOperandRef::SymStatic { .. } => {
+                            // Only emit the raw symbol name
+                            template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
+                        }
+                    }
+                }
+            }
+        }
+
+        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
+            match asm_arch {
+                InlineAsmArch::AArch64 | InlineAsmArch::Arm => {
+                    constraints.push("~{cc}".to_string());
+                }
+                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
+                    constraints.extend_from_slice(&[
+                        "~{dirflag}".to_string(),
+                        "~{fpsr}".to_string(),
+                        "~{flags}".to_string(),
+                    ]);
+                }
+                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {}
+            }
+        }
+        if !options.contains(InlineAsmOptions::NOMEM) {
+            // This is actually ignored by LLVM, but it's probably best to keep
+            // it just in case. LLVM instead uses the ReadOnly/ReadNone
+            // attributes on the call instruction to optimize.
+            constraints.push("~{memory}".to_string());
+        }
+        let volatile = !options.contains(InlineAsmOptions::PURE);
+        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
+        let output_type = match &output_types[..] {
+            [] => self.type_void(),
+            [ty] => ty,
+            tys => self.type_struct(&tys, false),
+        };
+        let dialect = match asm_arch {
+            InlineAsmArch::X86 | InlineAsmArch::X86_64 => LlvmAsmDialect::Intel,
+            _ => LlvmAsmDialect::Att,
+        };
+        let result = inline_asm_call(
+            self,
+            &template_str,
+            &constraints.join(","),
+            &inputs,
+            output_type,
+            volatile,
+            alignstack,
+            dialect,
+            span,
+        )
+        .unwrap_or_else(|| span_bug!(span, "LLVM asm constraint validation failed"));
+
+        if options.contains(InlineAsmOptions::PURE) {
+            if options.contains(InlineAsmOptions::NOMEM) {
+                llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result);
+            } else if options.contains(InlineAsmOptions::READONLY) {
+                llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result);
+            }
+        } else {
+            if options.contains(InlineAsmOptions::NOMEM) {
+                llvm::Attribute::InaccessibleMemOnly
+                    .apply_callsite(llvm::AttributePlace::Function, result);
+            } else {
+                // LLVM doesn't have an attribute to represent ReadOnly + SideEffect
+            }
+        }
+
+        // Write results to outputs
+        for (idx, op) in operands.iter().enumerate() {
+            if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
+            | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
+            {
+                let value = if output_types.len() == 1 {
+                    result
+                } else {
+                    self.extract_value(result, op_idx[&idx] as u64)
+                };
+                let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout);
+                OperandValue::Immediate(value).store(self, place);
+            }
+        }
     }
 }
 
@@ -138,7 +334,8 @@ fn inline_asm_call(
     output: &'ll llvm::Type,
     volatile: bool,
     alignstack: bool,
-    dia: ::rustc_ast::ast::LlvmAsmDialect,
+    dia: LlvmAsmDialect,
+    span: Span,
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
     let alignstack = if alignstack { llvm::True } else { llvm::False };
@@ -168,10 +365,261 @@ fn inline_asm_call(
                 alignstack,
                 llvm::AsmDialect::from_generic(dia),
             );
-            Some(bx.call(v, inputs, None))
+            let call = bx.call(v, inputs, None);
+
+            // Store mark in a metadata node so we can map LLVM errors
+            // back to source locations.  See #17552.
+            let key = "srcloc";
+            let kind = llvm::LLVMGetMDKindIDInContext(
+                bx.llcx,
+                key.as_ptr() as *const c_char,
+                key.len() as c_uint,
+            );
+
+            let val: &'ll Value = bx.const_i32(span.ctxt().outer_expn().as_u32() as i32);
+            llvm::LLVMSetMetadata(call, kind, llvm::LLVMMDNodeInContext(bx.llcx, &val, 1));
+
+            Some(call)
         } else {
             // LLVM has detected an issue with our constraints, bail out
             None
         }
     }
 }
+
+/// Converts a register class to an LLVM constraint code.
+fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String {
+    match reg {
+        InlineAsmRegOrRegClass::Reg(reg) => format!("{{{}}}", reg.name()),
+        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => "l",
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
+            InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
+            | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w",
+            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
+            InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
+            InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
+            InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
+            | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
+            InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
+            InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk",
+        }
+        .to_string(),
+    }
+}
+
+/// Converts a modifier into LLVM's equivalent modifier.
+fn modifier_to_llvm(
+    arch: InlineAsmArch,
+    reg: InlineAsmRegClass,
+    modifier: Option<char>,
+) -> Option<char> {
+    match reg {
+        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
+        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
+            if modifier == Some('v') { None } else { modifier }
+        }
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => None,
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
+            if modifier.is_none() {
+                Some('q')
+            } else {
+                modifier
+            }
+        }
+        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
+        | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
+        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
+        | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
+            None if arch == InlineAsmArch::X86_64 => Some('q'),
+            None => Some('k'),
+            Some('l') => Some('b'),
+            Some('h') => Some('h'),
+            Some('x') => Some('w'),
+            Some('e') => Some('k'),
+            Some('r') => Some('q'),
+            _ => unreachable!(),
+        },
+        InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg)
+        | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg)
+        | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
+            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
+            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
+            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
+            (_, Some('x')) => Some('x'),
+            (_, Some('y')) => Some('t'),
+            (_, Some('z')) => Some('g'),
+            _ => unreachable!(),
+        },
+        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
+    }
+}
+
+/// Type to use for outputs that are discarded. It doesn't really matter what
+/// the type is, as long as it is valid for the constraint code.
+fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll Type {
+    match reg {
+        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
+        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
+        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
+            cx.type_vector(cx.type_i64(), 2)
+        }
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => cx.type_i32(),
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
+        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
+            cx.type_vector(cx.type_i64(), 2)
+        }
+        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
+        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
+        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
+        | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
+        InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
+        | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg)
+        | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
+        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
+    }
+}
+
+/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
+/// the equivalent integer type.
+fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: &Scalar) -> &'ll Type {
+    match scalar.value {
+        Primitive::Int(Integer::I8, _) => cx.type_i8(),
+        Primitive::Int(Integer::I16, _) => cx.type_i16(),
+        Primitive::Int(Integer::I32, _) => cx.type_i32(),
+        Primitive::Int(Integer::I64, _) => cx.type_i64(),
+        Primitive::F32 => cx.type_f32(),
+        Primitive::F64 => cx.type_f64(),
+        Primitive::Pointer => cx.type_isize(),
+        _ => unreachable!(),
+    }
+}
+
+/// Fix up an input value to work around LLVM bugs.
+fn llvm_fixup_input(
+    bx: &mut Builder<'a, 'll, 'tcx>,
+    mut value: &'ll Value,
+    reg: InlineAsmRegClass,
+    layout: &TyAndLayout<'tcx>,
+) -> &'ll Value {
+    match (reg, &layout.abi) {
+        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
+            if let Primitive::Int(Integer::I8, _) = s.value {
+                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
+                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
+            } else {
+                value
+            }
+        }
+        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
+            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
+            let count = 16 / layout.size.bytes();
+            let vec_ty = bx.cx.type_vector(elem_ty, count);
+            if let Primitive::Pointer = s.value {
+                value = bx.ptrtoint(value, bx.cx.type_isize());
+            }
+            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
+        }
+        (
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
+            Abi::Vector { element, count },
+        ) if layout.size.bytes() == 8 => {
+            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
+            let vec_ty = bx.cx.type_vector(elem_ty, *count);
+            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
+            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
+        }
+        _ => value,
+    }
+}
+
+/// Fix up an output value to work around LLVM bugs.
+fn llvm_fixup_output(
+    bx: &mut Builder<'a, 'll, 'tcx>,
+    mut value: &'ll Value,
+    reg: InlineAsmRegClass,
+    layout: &TyAndLayout<'tcx>,
+) -> &'ll Value {
+    match (reg, &layout.abi) {
+        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
+            if let Primitive::Int(Integer::I8, _) = s.value {
+                bx.extract_element(value, bx.const_i32(0))
+            } else {
+                value
+            }
+        }
+        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
+            value = bx.extract_element(value, bx.const_i32(0));
+            if let Primitive::Pointer = s.value {
+                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
+            }
+            value
+        }
+        (
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
+            Abi::Vector { element, count },
+        ) if layout.size.bytes() == 8 => {
+            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
+            let vec_ty = bx.cx.type_vector(elem_ty, *count * 2);
+            let indices: Vec<_> = (0..*count).map(|x| bx.const_i32(x as i32)).collect();
+            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
+        }
+        _ => value,
+    }
+}
+
+/// Output type to use for llvm_fixup_output.
+fn llvm_fixup_output_type(
+    cx: &CodegenCx<'ll, 'tcx>,
+    reg: InlineAsmRegClass,
+    layout: &TyAndLayout<'tcx>,
+) -> &'ll Type {
+    match (reg, &layout.abi) {
+        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
+            if let Primitive::Int(Integer::I8, _) = s.value {
+                cx.type_vector(cx.type_i8(), 8)
+            } else {
+                layout.llvm_type(cx)
+            }
+        }
+        (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
+            let elem_ty = llvm_asm_scalar_type(cx, s);
+            let count = 16 / layout.size.bytes();
+            cx.type_vector(elem_ty, count)
+        }
+        (
+            InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
+            Abi::Vector { element, count },
+        ) if layout.size.bytes() == 8 => {
+            let elem_ty = llvm_asm_scalar_type(cx, element);
+            cx.type_vector(elem_ty, count * 2)
+        }
+        _ => layout.llvm_type(cx),
+    }
+}
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index 21dfc113351..9cb0f0e0c2e 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -124,6 +124,8 @@ pub enum Attribute {
     NonLazyBind = 23,
     OptimizeNone = 24,
     ReturnsTwice = 25,
+    ReadNone = 26,
+    InaccessibleMemOnly = 27,
 }
 
 /// LLVMIntPredicate
diff --git a/src/librustc_codegen_ssa/mir/analyze.rs b/src/librustc_codegen_ssa/mir/analyze.rs
index 9fcaf2818e8..5e3a37e20bd 100644
--- a/src/librustc_codegen_ssa/mir/analyze.rs
+++ b/src/librustc_codegen_ssa/mir/analyze.rs
@@ -358,7 +358,8 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
                 | TerminatorKind::SwitchInt { .. }
                 | TerminatorKind::Yield { .. }
                 | TerminatorKind::FalseEdges { .. }
-                | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
+                | TerminatorKind::FalseUnwind { .. }
+                | TerminatorKind::InlineAsm { .. } => { /* nothing to do */ }
                 TerminatorKind::Call { cleanup: unwind, .. }
                 | TerminatorKind::Assert { cleanup: unwind, .. }
                 | TerminatorKind::DropAndReplace { unwind, .. }
diff --git a/src/librustc_codegen_ssa/mir/block.rs b/src/librustc_codegen_ssa/mir/block.rs
index dfb1656a6e0..f8c5caa440c 100644
--- a/src/librustc_codegen_ssa/mir/block.rs
+++ b/src/librustc_codegen_ssa/mir/block.rs
@@ -9,6 +9,7 @@ use crate::meth;
 use crate::traits::*;
 use crate::MemFlags;
 
+use rustc_ast::ast;
 use rustc_hir::lang_items;
 use rustc_index::vec::Idx;
 use rustc_middle::mir;
@@ -914,6 +915,98 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::TerminatorKind::FalseEdges { .. } | mir::TerminatorKind::FalseUnwind { .. } => {
                 bug!("borrowck false edges in codegen")
             }
+
+            mir::TerminatorKind::InlineAsm { template, ref operands, options, ref destination } => {
+                let span = terminator.source_info.span;
+
+                let operands: Vec<_> = operands
+                    .iter()
+                    .map(|op| match *op {
+                        mir::InlineAsmOperand::In { reg, ref value } => {
+                            let value = self.codegen_operand(&mut bx, value);
+                            InlineAsmOperandRef::In { reg, value }
+                        }
+                        mir::InlineAsmOperand::Out { reg, late, ref place } => {
+                            let place =
+                                place.map(|place| self.codegen_place(&mut bx, place.as_ref()));
+                            InlineAsmOperandRef::Out { reg, late, place }
+                        }
+                        mir::InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
+                            let in_value = self.codegen_operand(&mut bx, in_value);
+                            let out_place = out_place
+                                .map(|out_place| self.codegen_place(&mut bx, out_place.as_ref()));
+                            InlineAsmOperandRef::InOut { reg, late, in_value, out_place }
+                        }
+                        mir::InlineAsmOperand::Const { ref value } => {
+                            if let mir::Operand::Constant(constant) = value {
+                                let const_value =
+                                    self.eval_mir_constant(constant).unwrap_or_else(|_| {
+                                        span_bug!(span, "asm const cannot be resolved")
+                                    });
+                                let ty = constant.literal.ty;
+                                let value = const_value
+                                    .try_to_bits_for_ty(bx.tcx(), ty::ParamEnv::reveal_all(), ty)
+                                    .unwrap_or_else(|| {
+                                        span_bug!(span, "asm const has non-scalar value")
+                                    });
+                                let string = match ty.kind {
+                                    ty::Uint(_) => value.to_string(),
+                                    ty::Int(int_ty) => {
+                                        match int_ty.normalize(bx.tcx().sess.target.ptr_width) {
+                                            ast::IntTy::I8 => (value as i8).to_string(),
+                                            ast::IntTy::I16 => (value as i16).to_string(),
+                                            ast::IntTy::I32 => (value as i32).to_string(),
+                                            ast::IntTy::I64 => (value as i64).to_string(),
+                                            ast::IntTy::I128 => (value as i128).to_string(),
+                                            ast::IntTy::Isize => unreachable!(),
+                                        }
+                                    }
+                                    ty::Float(ast::FloatTy::F32) => {
+                                        f32::from_bits(value as u32).to_string()
+                                    }
+                                    ty::Float(ast::FloatTy::F64) => {
+                                        f64::from_bits(value as u64).to_string()
+                                    }
+                                    _ => span_bug!(span, "asm const has bad type {}", ty),
+                                };
+                                InlineAsmOperandRef::Const { string }
+                            } else {
+                                span_bug!(span, "asm const is not a constant");
+                            }
+                        }
+                        mir::InlineAsmOperand::SymFn { ref value } => {
+                            if let ty::FnDef(def_id, substs) = value.literal.ty.kind {
+                                let instance = ty::Instance::resolve(
+                                    bx.tcx(),
+                                    ty::ParamEnv::reveal_all(),
+                                    def_id,
+                                    substs,
+                                )
+                                .unwrap()
+                                .unwrap();
+                                InlineAsmOperandRef::SymFn { instance }
+                            } else {
+                                span_bug!(span, "invalid type for asm sym (fn)");
+                            }
+                        }
+                        mir::InlineAsmOperand::SymStatic { ref value } => {
+                            if let Some(def_id) = value.check_static_ptr(bx.tcx()) {
+                                InlineAsmOperandRef::SymStatic { def_id }
+                            } else {
+                                span_bug!(span, "invalid type for asm sym (static)");
+                            }
+                        }
+                    })
+                    .collect();
+
+                bx.codegen_inline_asm(template, &operands, options, span);
+
+                if let Some(target) = destination {
+                    helper.funclet_br(self, &mut bx, *target);
+                } else {
+                    bx.unreachable();
+                }
+            }
         }
     }
 
diff --git a/src/librustc_codegen_ssa/traits/asm.rs b/src/librustc_codegen_ssa/traits/asm.rs
index 1cdfd3ae356..cb2b3545c4c 100644
--- a/src/librustc_codegen_ssa/traits/asm.rs
+++ b/src/librustc_codegen_ssa/traits/asm.rs
@@ -1,7 +1,39 @@
 use super::BackendTypes;
+use crate::mir::operand::OperandRef;
 use crate::mir::place::PlaceRef;
+use rustc_hir::def_id::DefId;
 use rustc_hir::{GlobalAsm, LlvmInlineAsmInner};
+use rustc_middle::ty::Instance;
 use rustc_span::Span;
+use rustc_target::asm::{InlineAsmOptions, InlineAsmRegOrRegClass, InlineAsmTemplatePiece};
+
+#[derive(Debug)]
+pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
+    In {
+        reg: InlineAsmRegOrRegClass,
+        value: OperandRef<'tcx, B::Value>,
+    },
+    Out {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        place: Option<PlaceRef<'tcx, B::Value>>,
+    },
+    InOut {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        in_value: OperandRef<'tcx, B::Value>,
+        out_place: Option<PlaceRef<'tcx, B::Value>>,
+    },
+    Const {
+        string: String,
+    },
+    SymFn {
+        instance: Instance<'tcx>,
+    },
+    SymStatic {
+        def_id: DefId,
+    },
+}
 
 pub trait AsmBuilderMethods<'tcx>: BackendTypes {
     /// Take an inline assembly expression and splat it out via LLVM
@@ -12,6 +44,15 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
         inputs: Vec<Self::Value>,
         span: Span,
     ) -> bool;
+
+    /// Take an inline assembly expression and splat it out via LLVM
+    fn codegen_inline_asm(
+        &mut self,
+        template: &[InlineAsmTemplatePiece],
+        operands: &[InlineAsmOperandRef<'tcx, Self>],
+        options: InlineAsmOptions,
+        span: Span,
+    );
 }
 
 pub trait AsmMethods {
diff --git a/src/librustc_codegen_ssa/traits/mod.rs b/src/librustc_codegen_ssa/traits/mod.rs
index f4c09a33285..6b782731d53 100644
--- a/src/librustc_codegen_ssa/traits/mod.rs
+++ b/src/librustc_codegen_ssa/traits/mod.rs
@@ -28,7 +28,7 @@ mod type_;
 mod write;
 
 pub use self::abi::AbiBuilderMethods;
-pub use self::asm::{AsmBuilderMethods, AsmMethods};
+pub use self::asm::{AsmBuilderMethods, AsmMethods, InlineAsmOperandRef};
 pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods};
 pub use self::builder::{BuilderMethods, OverflowOp};
 pub use self::consts::ConstMethods;
diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp
index 28efc8bf5dd..b988d06871b 100644
--- a/src/rustllvm/RustWrapper.cpp
+++ b/src/rustllvm/RustWrapper.cpp
@@ -203,6 +203,10 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
     return Attribute::OptimizeNone;
   case ReturnsTwice:
     return Attribute::ReturnsTwice;
+  case ReadNone:
+    return Attribute::ReadNone;
+  case InaccessibleMemOnly:
+    return Attribute::InaccessibleMemOnly;
   }
   report_fatal_error("bad AttributeKind");
 }
diff --git a/src/rustllvm/rustllvm.h b/src/rustllvm/rustllvm.h
index c3f0d174d4b..da48048113b 100644
--- a/src/rustllvm/rustllvm.h
+++ b/src/rustllvm/rustllvm.h
@@ -82,6 +82,8 @@ enum LLVMRustAttribute {
   NonLazyBind = 23,
   OptimizeNone = 24,
   ReturnsTwice = 25,
+  ReadNone = 26,
+  InaccessibleMemOnly = 27,
 };
 
 typedef struct OpaqueRustString *RustStringRef;