Auto merge of #38542 - YaLTeR:fastcall-fix, r=pnkfelix

Fix fastcall not applying inreg attributes to arguments

Fixes https://github.com/rust-lang/rust/issues/18086

6 files changed, 149 insertions, 4 deletions

diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs
index d2b86ade7a2..f3dbac7ce68 100644
--- a/src/librustc_llvm/ffi.rs
+++ b/src/librustc_llvm/ffi.rs
@@ -117,6 +117,7 @@ pub enum Attribute {
     StructRet       = 16,
     UWTable         = 17,
     ZExt            = 18,
+    InReg           = 19,
 }
 
 /// LLVMIntPredicate
diff --git a/src/librustc_trans/abi.rs b/src/librustc_trans/abi.rs
index 8b4343af199..f0f25118990 100644
--- a/src/librustc_trans/abi.rs
+++ b/src/librustc_trans/abi.rs
@@ -58,7 +58,7 @@ mod attr_impl {
     // The subset of llvm::Attribute needed for arguments, packed into a bitfield.
     bitflags! {
         #[derive(Default, Debug)]
-        flags ArgAttribute : u8 {
+        flags ArgAttribute : u16 {
             const ByVal     = 1 << 0,
             const NoAlias   = 1 << 1,
             const NoCapture = 1 << 2,
@@ -67,6 +67,7 @@ mod attr_impl {
             const SExt      = 1 << 5,
             const StructRet = 1 << 6,
             const ZExt      = 1 << 7,
+            const InReg     = 1 << 8,
         }
     }
 }
@@ -80,7 +81,7 @@ macro_rules! for_each_kind {
 impl ArgAttribute {
     fn for_each_kind<F>(&self, mut f: F) where F: FnMut(llvm::Attribute) {
         for_each_kind!(self, f,
-                       ByVal, NoAlias, NoCapture, NonNull, ReadOnly, SExt, StructRet, ZExt)
+                       ByVal, NoAlias, NoCapture, NonNull, ReadOnly, SExt, StructRet, ZExt, InReg)
     }
 }
 
@@ -573,7 +574,14 @@ impl FnType {
         }
 
         match &ccx.sess().target.target.arch[..] {
-            "x86" => cabi_x86::compute_abi_info(ccx, self),
+            "x86" => {
+                let flavor = if abi == Abi::Fastcall {
+                    cabi_x86::Flavor::Fastcall
+                } else {
+                    cabi_x86::Flavor::General
+                };
+                cabi_x86::compute_abi_info(ccx, self, flavor);
+            },
             "x86_64" => if abi == Abi::SysV64 {
                 cabi_x86_64::compute_abi_info(ccx, self);
             } else if abi == Abi::Win64 || ccx.sess().target.target.options.is_like_windows {
diff --git a/src/librustc_trans/cabi_x86.rs b/src/librustc_trans/cabi_x86.rs
index ce85234f203..fea005f3d77 100644
--- a/src/librustc_trans/cabi_x86.rs
+++ b/src/librustc_trans/cabi_x86.rs
@@ -14,7 +14,13 @@ use type_::Type;
 use super::common::*;
 use super::machine::*;
 
-pub fn compute_abi_info(ccx: &CrateContext, fty: &mut FnType) {
+#[derive(PartialEq)]
+pub enum Flavor {
+    General,
+    Fastcall
+}
+
+pub fn compute_abi_info(ccx: &CrateContext, fty: &mut FnType, flavor: Flavor) {
     if !fty.ret.is_ignore() {
         if fty.ret.ty.kind() == Struct {
             // Returning a structure. Most often, this will use
@@ -51,4 +57,46 @@ pub fn compute_abi_info(ccx: &CrateContext, fty: &mut FnType) {
             arg.extend_integer_width_to(32);
         }
     }
+
+    if flavor == Flavor::Fastcall {
+        // Mark arguments as InReg like clang does it,
+        // so our fastcall is compatible with C/C++ fastcall.
+
+        // Clang reference: lib/CodeGen/TargetInfo.cpp
+        // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs()
+
+        // IsSoftFloatABI is only set to true on ARM platforms,
+        // which in turn can't be x86?
+
+        let mut free_regs = 2;
+
+        for arg in &mut fty.args {
+            if arg.is_ignore() || arg.is_indirect() { continue; }
+
+            if arg.ty.kind() == Float {
+                continue;
+            }
+
+            let size = llbitsize_of_real(ccx, arg.ty);
+            let size_in_regs = (size + 31) / 32;
+
+            if size_in_regs == 0 {
+                continue;
+            }
+
+            if size_in_regs > free_regs {
+                break;
+            }
+
+            free_regs -= size_in_regs;
+
+            if size <= 32 && (arg.ty.kind() == Pointer || arg.ty.kind() == Integer) {
+                arg.attrs.set(ArgAttribute::InReg);
+            }
+
+            if free_regs == 0 {
+                break;
+            }
+        }
+    }
 }
diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp
index f5fa66f1b0e..5d5845213e2 100644
--- a/src/rustllvm/RustWrapper.cpp
+++ b/src/rustllvm/RustWrapper.cpp
@@ -150,6 +150,8 @@ from_rust(LLVMRustAttribute kind) {
       return Attribute::UWTable;
     case ZExt:
       return Attribute::ZExt;
+    case InReg:
+      return Attribute::InReg;
     default:
       llvm_unreachable("bad AttributeKind");
   }
diff --git a/src/rustllvm/rustllvm.h b/src/rustllvm/rustllvm.h
index b8c4076f4ce..8f7e0e3d918 100644
--- a/src/rustllvm/rustllvm.h
+++ b/src/rustllvm/rustllvm.h
@@ -98,6 +98,7 @@ enum LLVMRustAttribute {
     StructRet       = 16,
     UWTable         = 17,
     ZExt            = 18,
+    InReg           = 19,
 };
 
 typedef struct OpaqueRustString *RustStringRef;
diff --git a/src/test/codegen/fastcall-inreg.rs b/src/test/codegen/fastcall-inreg.rs
new file mode 100644
index 00000000000..e1dda4d2be0
--- /dev/null
+++ b/src/test/codegen/fastcall-inreg.rs
@@ -0,0 +1,85 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Checks if the "fastcall" calling convention marks function arguments
+// as "inreg" like the C/C++ compilers for the platforms.
+// x86 only.
+
+// ignore-aarch64
+// ignore-aarch64_be
+// ignore-arm
+// ignore-armeb
+// ignore-avr
+// ignore-bpfel
+// ignore-bpfeb
+// ignore-hexagon
+// ignore-mips
+// ignore-mipsel
+// ignore-mips64
+// ignore-mips64el
+// ignore-msp430
+// ignore-powerpc64
+// ignore-powerpc64le
+// ignore-powerpc
+// ignore-r600
+// ignore-amdgcn
+// ignore-sparc
+// ignore-sparcv9
+// ignore-sparcel
+// ignore-s390x
+// ignore-tce
+// ignore-thumb
+// ignore-thumbeb
+// ignore-x86_64 no-ignore-x86
+// ignore-xcore
+// ignore-nvptx
+// ignore-nvptx64
+// ignore-le32
+// ignore-le64
+// ignore-amdil
+// ignore-amdil64
+// ignore-hsail
+// ignore-hsail64
+// ignore-spir
+// ignore-spir64
+// ignore-kalimba
+// ignore-shave
+// ignore-wasm32
+// ignore-wasm64
+
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+
+mod tests {
+    // CHECK: @f1(i32 inreg, i32 inreg, i32)
+    #[no_mangle]
+    extern "fastcall" fn f1(_: i32, _: i32, _: i32) {}
+
+    // CHECK: @f2(i32* inreg, i32* inreg, i32*)
+    #[no_mangle]
+    extern "fastcall" fn f2(_: *const i32, _: *const i32, _: *const i32) {}
+
+    // CHECK: @f3(float, i32 inreg, i32 inreg, i32)
+    #[no_mangle]
+    extern "fastcall" fn f3(_: f32, _: i32, _: i32, _: i32) {}
+
+    // CHECK: @f4(i32 inreg, float, i32 inreg, i32)
+    #[no_mangle]
+    extern "fastcall" fn f4(_: i32, _: f32, _: i32, _: i32) {}
+
+    // CHECK: @f5(i64, i32)
+    #[no_mangle]
+    extern "fastcall" fn f5(_: i64, _: i32) {}
+
+    // CHECK: @f6(i1 inreg zeroext, i32 inreg, i32)
+    #[no_mangle]
+    extern "fastcall" fn f6(_: bool, _: i32, _: i32) {}
+}