From 7d27ceb9543244867e8c75c513ecf3e5173a3ea1 Mon Sep 17 00:00:00 2001 From: Henry Jiang Date: Thu, 3 Oct 2024 11:37:41 -0400 Subject: Add AIX Calling Convention --- compiler/rustc_target/src/abi/call/powerpc64.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs index b9767bf906b..d08a30cb3d1 100644 --- a/compiler/rustc_target/src/abi/call/powerpc64.rs +++ b/compiler/rustc_target/src/abi/call/powerpc64.rs @@ -10,6 +10,7 @@ use crate::spec::HasTargetSpec; enum ABI { ELFv1, // original ABI used for powerpc64 (big-endian) ELFv2, // newer ABI used for powerpc64le and musl (both endians) + AIX, // used by AIX OS, big-endian only } use ABI::*; @@ -23,9 +24,9 @@ where C: HasDataLayout, { arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { - // ELFv1 only passes one-member aggregates transparently. + // ELFv1 and AIX only passes one-member aggregates transparently. // ELFv2 passes up to eight uniquely addressable members. - if (abi == ELFv1 && arg.layout.size > unit.size) + if ((abi == ELFv1 || abi == AIX) && arg.layout.size > unit.size) || arg.layout.size > unit.size.checked_mul(8, cx).unwrap() { return None; @@ -55,8 +56,14 @@ where return; } + // See https://github.com/llvm/llvm-project/blob/main/clang/lib/CodeGen/Targets/PPC.cpp. + if !is_ret && abi == AIX { + arg.make_indirect_byval(None); + return; + } + // The ELFv1 ABI doesn't return aggregates in registers - if is_ret && abi == ELFv1 { + if is_ret && (abi == ELFv1 || abi == AIX) { arg.make_indirect(); return; } @@ -93,6 +100,8 @@ where { let abi = if cx.target_spec().env == "musl" { ELFv2 + } else if cx.target_spec().os == "aix" { + AIX } else { match cx.data_layout().endian { Endian::Big => ELFv1, -- cgit 1.4.1-3-g733a5 From d09e27d54ace664f90583e12e4be6089a13be7e6 Mon Sep 17 00:00:00 2001 From: Henry Jiang Date: Thu, 3 Oct 2024 12:36:36 -0400 Subject: update call --- compiler/rustc_target/src/abi/call/powerpc64.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs index d08a30cb3d1..71e533b8cc5 100644 --- a/compiler/rustc_target/src/abi/call/powerpc64.rs +++ b/compiler/rustc_target/src/abi/call/powerpc64.rs @@ -56,9 +56,10 @@ where return; } + // The AIX ABI expect byval for aggregates // See https://github.com/llvm/llvm-project/blob/main/clang/lib/CodeGen/Targets/PPC.cpp. if !is_ret && abi == AIX { - arg.make_indirect_byval(None); + arg.pass_by_stack_offset(None); return; } -- cgit 1.4.1-3-g733a5 From 3743618c13d9f4afb06c9778b05b75e879b6919c Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Sun, 6 Oct 2024 08:14:44 +0900 Subject: Support clobber_abi in MSP430 inline assembly --- compiler/rustc_target/src/asm/mod.rs | 10 ++++++++++ tests/codegen/asm-msp430-clobbers.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/codegen/asm-msp430-clobbers.rs (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index df13d24cb9e..73ae1ae96ae 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -893,6 +893,7 @@ pub enum InlineAsmClobberAbi { RiscV, LoongArch, S390x, + Msp430, } impl InlineAsmClobberAbi { @@ -946,6 +947,10 @@ impl InlineAsmClobberAbi { "C" | "system" => Ok(InlineAsmClobberAbi::S390x), _ => Err(&["C", "system"]), }, + InlineAsmArch::Msp430 => match name { + "C" | "system" => Ok(InlineAsmClobberAbi::Msp430), + _ => Err(&["C", "system"]), + }, _ => Err(&[]), } } @@ -1125,6 +1130,11 @@ impl InlineAsmClobberAbi { a8, a9, a10, a11, a12, a13, a14, a15, } }, + InlineAsmClobberAbi::Msp430 => clobbered_regs! { + Msp430 Msp430InlineAsmReg { + r11, r12, r13, r14, r15, + } + }, } } } diff --git a/tests/codegen/asm-msp430-clobbers.rs b/tests/codegen/asm-msp430-clobbers.rs new file mode 100644 index 00000000000..c00c04f3088 --- /dev/null +++ b/tests/codegen/asm-msp430-clobbers.rs @@ -0,0 +1,36 @@ +//@ assembly-output: emit-asm +//@ compile-flags: --target msp430-none-elf +//@ needs-llvm-components: msp430 + +#![crate_type = "rlib"] +#![feature(no_core, rustc_attrs, lang_items, asm_experimental_arch)] +#![no_core] + +#[lang = "sized"] +trait Sized {} + +#[rustc_builtin_macro] +macro_rules! asm { + () => {}; +} + +// CHECK-LABEL: @sr_clobber +// CHECK: call void asm sideeffect "", "~{sr}"() +#[no_mangle] +pub unsafe fn sr_clobber() { + asm!("", options(nostack, nomem)); +} + +// CHECK-LABEL: @no_clobber +// CHECK: call void asm sideeffect "", ""() +#[no_mangle] +pub unsafe fn no_clobber() { + asm!("", options(nostack, nomem, preserves_flags)); +} + +// CHECK-LABEL: @clobber_abi +// CHECK: asm sideeffect "", "={r11},={r12},={r13},={r14},={r15}"() +#[no_mangle] +pub unsafe fn clobber_abi() { + asm!("", clobber_abi("C"), options(nostack, nomem, preserves_flags)); +} -- cgit 1.4.1-3-g733a5 From 8a5e03bf438ddc990ab311bc63d7c15b2f21f5a8 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Fri, 13 Sep 2024 14:30:22 +0800 Subject: Enable sanitizers for loongarch64-unknown-* --- .../rustc_target/src/spec/targets/loongarch64_unknown_linux_gnu.rs | 7 ++++++- .../src/spec/targets/loongarch64_unknown_linux_musl.rs | 7 ++++++- .../src/spec/targets/loongarch64_unknown_linux_ohos.rs | 7 ++++++- src/bootstrap/src/core/build_steps/llvm.rs | 3 +++ src/ci/docker/host-x86_64/dist-loongarch64-linux/Dockerfile | 1 + src/ci/docker/host-x86_64/dist-loongarch64-musl/Dockerfile | 1 + 6 files changed, 23 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_gnu.rs index de8d99977b4..e869314d4d8 100644 --- a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_gnu.rs +++ b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_gnu.rs @@ -1,4 +1,4 @@ -use crate::spec::{CodeModel, Target, TargetOptions, base}; +use crate::spec::{CodeModel, SanitizerSet, Target, TargetOptions, base}; pub(crate) fn target() -> Target { Target { @@ -18,6 +18,11 @@ pub(crate) fn target() -> Target { features: "+f,+d".into(), llvm_abiname: "lp64d".into(), max_atomic_width: Some(64), + supported_sanitizers: SanitizerSet::ADDRESS + | SanitizerSet::CFI + | SanitizerSet::LEAK + | SanitizerSet::MEMORY + | SanitizerSet::THREAD, direct_access_external_data: Some(false), ..base::linux_gnu::opts() }, diff --git a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs index a5088bcd5c2..70e8bf633a9 100644 --- a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs +++ b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs @@ -1,4 +1,4 @@ -use crate::spec::{CodeModel, Target, TargetOptions, base}; +use crate::spec::{CodeModel, SanitizerSet, Target, TargetOptions, base}; pub(crate) fn target() -> Target { Target { @@ -19,6 +19,11 @@ pub(crate) fn target() -> Target { llvm_abiname: "lp64d".into(), max_atomic_width: Some(64), crt_static_default: false, + supported_sanitizers: SanitizerSet::ADDRESS + | SanitizerSet::CFI + | SanitizerSet::LEAK + | SanitizerSet::MEMORY + | SanitizerSet::THREAD, ..base::linux_musl::opts() }, } diff --git a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_ohos.rs index 32a856be669..90bcd9a45cf 100644 --- a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_ohos.rs +++ b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_ohos.rs @@ -1,4 +1,4 @@ -use crate::spec::{Target, TargetOptions, base}; +use crate::spec::{SanitizerSet, Target, TargetOptions, base}; pub(crate) fn target() -> Target { Target { @@ -17,6 +17,11 @@ pub(crate) fn target() -> Target { features: "+f,+d".into(), llvm_abiname: "lp64d".into(), max_atomic_width: Some(64), + supported_sanitizers: SanitizerSet::ADDRESS + | SanitizerSet::CFI + | SanitizerSet::LEAK + | SanitizerSet::MEMORY + | SanitizerSet::THREAD, ..base::linux_ohos::opts() }, } diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 30213584157..c64ca730698 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -1215,6 +1215,9 @@ fn supported_sanitizers( "aarch64-unknown-linux-ohos" => { common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) } + "loongarch64-unknown-linux-gnu" | "loongarch64-unknown-linux-musl" => { + common_libs("linux", "loongarch64", &["asan", "lsan", "msan", "tsan"]) + } "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), diff --git a/src/ci/docker/host-x86_64/dist-loongarch64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-loongarch64-linux/Dockerfile index 865a9e32fa9..71eb72686b0 100644 --- a/src/ci/docker/host-x86_64/dist-loongarch64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-loongarch64-linux/Dockerfile @@ -47,6 +47,7 @@ ENV RUST_CONFIGURE_ARGS \ --enable-extended \ --enable-full-tools \ --enable-profiler \ + --enable-sanitizers \ --disable-docs ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-loongarch64-musl/Dockerfile b/src/ci/docker/host-x86_64/dist-loongarch64-musl/Dockerfile index 62dbfaaa673..5081f25e567 100644 --- a/src/ci/docker/host-x86_64/dist-loongarch64-musl/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-loongarch64-musl/Dockerfile @@ -29,6 +29,7 @@ ENV RUST_CONFIGURE_ARGS \ --enable-extended \ --enable-full-tools \ --enable-profiler \ + --enable-sanitizers \ --disable-docs \ --set target.loongarch64-unknown-linux-musl.crt-static=false \ --musl-root-loongarch64=/x-tools/loongarch64-unknown-linux-musl/loongarch64-unknown-linux-musl/sysroot/usr -- cgit 1.4.1-3-g733a5 From 43e198a3ae23f469e1c87b099789af1bc025fe50 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Tue, 8 Oct 2024 17:32:52 -0700 Subject: compiler: Seal off the rustc_target::abi enum glob imports --- Cargo.lock | 6 ++++++ compiler/rustc_target/src/abi/mod.rs | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/Cargo.lock b/Cargo.lock index 502920350d5..faca0619673 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3416,6 +3416,7 @@ dependencies = [ "measureme", "object 0.36.4", "rustc-demangle", + "rustc_abi", "rustc_ast", "rustc_attr", "rustc_codegen_ssa", @@ -3456,6 +3457,7 @@ dependencies = [ "object 0.36.4", "pathdiff", "regex", + "rustc_abi", "rustc_arena", "rustc_ast", "rustc_attr", @@ -3493,6 +3495,7 @@ name = "rustc_const_eval" version = "0.0.0" dependencies = [ "either", + "rustc_abi", "rustc_apfloat", "rustc_ast", "rustc_attr", @@ -3772,6 +3775,7 @@ name = "rustc_hir_typeck" version = "0.0.0" dependencies = [ "itertools", + "rustc_abi", "rustc_ast", "rustc_ast_ir", "rustc_attr", @@ -4027,6 +4031,7 @@ dependencies = [ "gsgdt", "polonius-engine", "rustc-rayon-core", + "rustc_abi", "rustc_apfloat", "rustc_arena", "rustc_ast", @@ -4522,6 +4527,7 @@ name = "rustc_ty_utils" version = "0.0.0" dependencies = [ "itertools", + "rustc_abi", "rustc_ast_ir", "rustc_data_structures", "rustc_errors", diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs index fc92e755fea..b744d5ad4ed 100644 --- a/compiler/rustc_target/src/abi/mod.rs +++ b/compiler/rustc_target/src/abi/mod.rs @@ -1,9 +1,8 @@ use std::fmt; use std::ops::Deref; -pub use Float::*; -pub use Integer::*; -pub use Primitive::*; +use Float::*; +use Primitive::*; use rustc_data_structures::intern::Interned; use rustc_macros::HashStable_Generic; -- cgit 1.4.1-3-g733a5 From 335f67b6527cbeb576151dab1dadec161eaede45 Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Tue, 3 Sep 2024 13:51:54 +0100 Subject: rustc_target: Add sme-b16b16 as an explicit aarch64 target feature LLVM 20 split out what used to be called b16b16 and correspond to aarch64 FEAT_SVE_B16B16 into sve-b16b16 and sme-b16b16. Add sme-b16b16 as an explicit feature and update the codegen accordingly. --- compiler/rustc_codegen_llvm/src/llvm_util.rs | 4 +++- compiler/rustc_target/src/target_features.rs | 4 +++- library/std/tests/run-time-detect.rs | 1 + tests/ui/check-cfg/mix.stderr | 2 +- tests/ui/check-cfg/well-known-values.stderr | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index bd847cd0068..8290e1a9544 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -247,7 +247,9 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - ("aarch64", "sve-b16b16") => Some(LLVMFeature::new("b16b16")), + // Before LLVM 20 those two features were packaged together as b16b16 + ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), + ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => { diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs index 0a98b363b1a..e92366d5c5c 100644 --- a/compiler/rustc_target/src/target_features.rs +++ b/compiler/rustc_target/src/target_features.rs @@ -191,6 +191,8 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ ("sm4", Stable, &["neon"]), // FEAT_SME ("sme", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]), + // FEAT_SME_B16B16 + ("sme-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16", "sme2", "sve-b16b16"]), // FEAT_SME_F16F16 ("sme-f16f16", Unstable(sym::aarch64_unstable_target_feature), &["sme2"]), // FEAT_SME_F64F64 @@ -227,7 +229,7 @@ const AARCH64_ALLOWED_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[ // // "For backwards compatibility, Neon and VFP are required in the latest architectures." ("sve", Stable, &["neon"]), - // FEAT_SVE_B16B16 (SVE or SME Instructions) + // FEAT_SVE_B16B16 (SVE or SME Z-targeting instructions) ("sve-b16b16", Unstable(sym::aarch64_unstable_target_feature), &["bf16"]), // FEAT_SVE2 ("sve2", Stable, &["sve"]), diff --git a/library/std/tests/run-time-detect.rs b/library/std/tests/run-time-detect.rs index dcd5cd7f6b9..dd14c0266aa 100644 --- a/library/std/tests/run-time-detect.rs +++ b/library/std/tests/run-time-detect.rs @@ -82,6 +82,7 @@ fn aarch64_linux() { println!("sha2: {}", is_aarch64_feature_detected!("sha2")); println!("sha3: {}", is_aarch64_feature_detected!("sha3")); println!("sm4: {}", is_aarch64_feature_detected!("sm4")); + println!("sme-b16b16: {}", is_aarch64_feature_detected!("sme-b16b16")); println!("sme-f16f16: {}", is_aarch64_feature_detected!("sme-f16f16")); println!("sme-f64f64: {}", is_aarch64_feature_detected!("sme-f64f64")); println!("sme-f8f16: {}", is_aarch64_feature_detected!("sme-f8f16")); diff --git a/tests/ui/check-cfg/mix.stderr b/tests/ui/check-cfg/mix.stderr index 7726c2d52f5..c21016e9290 100644 --- a/tests/ui/check-cfg/mix.stderr +++ b/tests/ui/check-cfg/mix.stderr @@ -251,7 +251,7 @@ warning: unexpected `cfg` condition value: `zebra` LL | cfg!(target_feature = "zebra"); | ^^^^^^^^^^^^^^^^^^^^^^^^ | - = note: expected values for `target_feature` are: `10e60`, `2e3`, `3e3r1`, `3e3r2`, `3e3r3`, `3e7`, `7e10`, `a`, `aclass`, `adx`, `aes`, `altivec`, `alu32`, `amx-bf16`, `amx-complex`, `amx-fp16`, `amx-int8`, `amx-tile`, `atomics`, `avx`, `avx2`, `avx512bf16`, `avx512bitalg`, `avx512bw`, `avx512cd`, `avx512dq`, `avx512f`, `avx512fp16`, `avx512ifma`, `avx512vbmi`, `avx512vbmi2`, `avx512vl`, `avx512vnni`, `avx512vp2intersect`, and `avx512vpopcntdq` and 244 more + = note: expected values for `target_feature` are: `10e60`, `2e3`, `3e3r1`, `3e3r2`, `3e3r3`, `3e7`, `7e10`, `a`, `aclass`, `adx`, `aes`, `altivec`, `alu32`, `amx-bf16`, `amx-complex`, `amx-fp16`, `amx-int8`, `amx-tile`, `atomics`, `avx`, `avx2`, `avx512bf16`, `avx512bitalg`, `avx512bw`, `avx512cd`, `avx512dq`, `avx512f`, `avx512fp16`, `avx512ifma`, `avx512vbmi`, `avx512vbmi2`, `avx512vl`, `avx512vnni`, `avx512vp2intersect`, and `avx512vpopcntdq` and 245 more = note: see for more information about checking conditional configuration warning: 27 warnings emitted diff --git a/tests/ui/check-cfg/well-known-values.stderr b/tests/ui/check-cfg/well-known-values.stderr index c6d403104ea..da790bbd528 100644 --- a/tests/ui/check-cfg/well-known-values.stderr +++ b/tests/ui/check-cfg/well-known-values.stderr @@ -174,7 +174,7 @@ warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE` LL | target_feature = "_UNEXPECTED_VALUE", | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | - = note: expected values for `target_feature` are: `10e60`, `2e3`, `3e3r1`, `3e3r2`, `3e3r3`, `3e7`, `7e10`, `a`, `aclass`, `adx`, `aes`, `altivec`, `alu32`, `amx-bf16`, `amx-complex`, `amx-fp16`, `amx-int8`, `amx-tile`, `atomics`, `avx`, `avx2`, `avx512bf16`, `avx512bitalg`, `avx512bw`, `avx512cd`, `avx512dq`, `avx512f`, `avx512fp16`, `avx512ifma`, `avx512vbmi`, `avx512vbmi2`, `avx512vl`, `avx512vnni`, `avx512vp2intersect`, `avx512vpopcntdq`, `avxifma`, `avxneconvert`, `avxvnni`, `avxvnniint16`, `avxvnniint8`, `backchain`, `bf16`, `bmi1`, `bmi2`, `bti`, `bulk-memory`, `c`, `cache`, `cmpxchg16b`, `crc`, `crt-static`, `cssc`, `d`, `d32`, `dit`, `doloop`, `dotprod`, `dpb`, `dpb2`, `dsp`, `dsp1e2`, `dspe60`, `e`, `e1`, `e2`, `ecv`, `edsp`, `elrw`, `ermsb`, `exception-handling`, `extended-const`, `f`, `f16c`, `f32mm`, `f64mm`, `faminmax`, `fcma`, `fdivdu`, `fhm`, `flagm`, `flagm2`, `float1e2`, `float1e3`, `float3e4`, `float7e60`, `floate1`, `fma`, `fp-armv8`, `fp16`, `fp64`, `fp8`, `fp8dot2`, `fp8dot4`, `fp8fma`, `fpuv2_df`, `fpuv2_sf`, `fpuv3_df`, `fpuv3_hf`, `fpuv3_hi`, `fpuv3_sf`, `frecipe`, `frintts`, `fxsr`, `gfni`, `hard-float`, `hard-float-abi`, `hard-tp`, `hbc`, `high-registers`, `hvx`, `hvx-length128b`, `hwdiv`, `i8mm`, `jsconv`, `lahfsahf`, `lasx`, `lbt`, `lor`, `lse`, `lse128`, `lse2`, `lsx`, `lut`, `lvz`, `lzcnt`, `m`, `mclass`, `mops`, `movbe`, `mp`, `mp1e2`, `msa`, `mte`, `multivalue`, `mutable-globals`, `neon`, `nontrapping-fptoint`, `nvic`, `paca`, `pacg`, `pan`, `partword-atomics`, `pclmulqdq`, `pmuv3`, `popcnt`, `power10-vector`, `power8-altivec`, `power8-vector`, `power9-altivec`, `power9-vector`, `prfchw`, `quadword-atomics`, `rand`, `ras`, `rclass`, `rcpc`, `rcpc2`, `rcpc3`, `rdm`, `rdrand`, `rdseed`, `reference-types`, `relax`, `relaxed-simd`, `rtm`, `sb`, `sha`, `sha2`, `sha3`, `sha512`, `sign-ext`, `simd128`, `sm3`, `sm4`, `sme`, `sme-f16f16`, `sme-f64f64`, `sme-f8f16`, `sme-f8f32`, `sme-fa64`, `sme-i16i64`, `sme-lutv2`, `sme2`, `sme2p1`, `spe`, `ssbs`, `sse`, `sse2`, `sse3`, `sse4.1`, `sse4.2`, `sse4a`, `ssse3`, `ssve-fp8dot2`, `ssve-fp8dot4`, `ssve-fp8fma`, `sve`, `sve-b16b16`, `sve2`, `sve2-aes`, `sve2-bitperm`, `sve2-sha3`, `sve2-sm4`, `sve2p1`, `tbm`, `thumb-mode`, `thumb2`, `tme`, `trust`, `trustzone`, `ual`, `unaligned-scalar-mem`, `v`, `v5te`, `v6`, `v6k`, `v6t2`, `v7`, `v8`, `v8.1a`, `v8.2a`, `v8.3a`, `v8.4a`, `v8.5a`, `v8.6a`, `v8.7a`, `v8.8a`, `v8.9a`, `v9.1a`, `v9.2a`, `v9.3a`, `v9.4a`, `v9.5a`, `v9a`, `vaes`, `vdsp2e60f`, `vdspv1`, `vdspv2`, `vector`, `vfp2`, `vfp3`, `vfp4`, `vh`, `virt`, `virtualization`, `vpclmulqdq`, `vsx`, `wfxt`, `xop`, `xsave`, `xsavec`, `xsaveopt`, `xsaves`, `zaamo`, `zabha`, `zalrsc`, `zba`, `zbb`, `zbc`, `zbkb`, `zbkc`, `zbkx`, `zbs`, `zdinx`, `zfh`, `zfhmin`, `zfinx`, `zhinx`, `zhinxmin`, `zk`, `zkn`, `zknd`, `zkne`, `zknh`, `zkr`, `zks`, `zksed`, `zksh`, and `zkt` + = note: expected values for `target_feature` are: `10e60`, `2e3`, `3e3r1`, `3e3r2`, `3e3r3`, `3e7`, `7e10`, `a`, `aclass`, `adx`, `aes`, `altivec`, `alu32`, `amx-bf16`, `amx-complex`, `amx-fp16`, `amx-int8`, `amx-tile`, `atomics`, `avx`, `avx2`, `avx512bf16`, `avx512bitalg`, `avx512bw`, `avx512cd`, `avx512dq`, `avx512f`, `avx512fp16`, `avx512ifma`, `avx512vbmi`, `avx512vbmi2`, `avx512vl`, `avx512vnni`, `avx512vp2intersect`, `avx512vpopcntdq`, `avxifma`, `avxneconvert`, `avxvnni`, `avxvnniint16`, `avxvnniint8`, `backchain`, `bf16`, `bmi1`, `bmi2`, `bti`, `bulk-memory`, `c`, `cache`, `cmpxchg16b`, `crc`, `crt-static`, `cssc`, `d`, `d32`, `dit`, `doloop`, `dotprod`, `dpb`, `dpb2`, `dsp`, `dsp1e2`, `dspe60`, `e`, `e1`, `e2`, `ecv`, `edsp`, `elrw`, `ermsb`, `exception-handling`, `extended-const`, `f`, `f16c`, `f32mm`, `f64mm`, `faminmax`, `fcma`, `fdivdu`, `fhm`, `flagm`, `flagm2`, `float1e2`, `float1e3`, `float3e4`, `float7e60`, `floate1`, `fma`, `fp-armv8`, `fp16`, `fp64`, `fp8`, `fp8dot2`, `fp8dot4`, `fp8fma`, `fpuv2_df`, `fpuv2_sf`, `fpuv3_df`, `fpuv3_hf`, `fpuv3_hi`, `fpuv3_sf`, `frecipe`, `frintts`, `fxsr`, `gfni`, `hard-float`, `hard-float-abi`, `hard-tp`, `hbc`, `high-registers`, `hvx`, `hvx-length128b`, `hwdiv`, `i8mm`, `jsconv`, `lahfsahf`, `lasx`, `lbt`, `lor`, `lse`, `lse128`, `lse2`, `lsx`, `lut`, `lvz`, `lzcnt`, `m`, `mclass`, `mops`, `movbe`, `mp`, `mp1e2`, `msa`, `mte`, `multivalue`, `mutable-globals`, `neon`, `nontrapping-fptoint`, `nvic`, `paca`, `pacg`, `pan`, `partword-atomics`, `pclmulqdq`, `pmuv3`, `popcnt`, `power10-vector`, `power8-altivec`, `power8-vector`, `power9-altivec`, `power9-vector`, `prfchw`, `quadword-atomics`, `rand`, `ras`, `rclass`, `rcpc`, `rcpc2`, `rcpc3`, `rdm`, `rdrand`, `rdseed`, `reference-types`, `relax`, `relaxed-simd`, `rtm`, `sb`, `sha`, `sha2`, `sha3`, `sha512`, `sign-ext`, `simd128`, `sm3`, `sm4`, `sme`, `sme-b16b16`, `sme-f16f16`, `sme-f64f64`, `sme-f8f16`, `sme-f8f32`, `sme-fa64`, `sme-i16i64`, `sme-lutv2`, `sme2`, `sme2p1`, `spe`, `ssbs`, `sse`, `sse2`, `sse3`, `sse4.1`, `sse4.2`, `sse4a`, `ssse3`, `ssve-fp8dot2`, `ssve-fp8dot4`, `ssve-fp8fma`, `sve`, `sve-b16b16`, `sve2`, `sve2-aes`, `sve2-bitperm`, `sve2-sha3`, `sve2-sm4`, `sve2p1`, `tbm`, `thumb-mode`, `thumb2`, `tme`, `trust`, `trustzone`, `ual`, `unaligned-scalar-mem`, `v`, `v5te`, `v6`, `v6k`, `v6t2`, `v7`, `v8`, `v8.1a`, `v8.2a`, `v8.3a`, `v8.4a`, `v8.5a`, `v8.6a`, `v8.7a`, `v8.8a`, `v8.9a`, `v9.1a`, `v9.2a`, `v9.3a`, `v9.4a`, `v9.5a`, `v9a`, `vaes`, `vdsp2e60f`, `vdspv1`, `vdspv2`, `vector`, `vfp2`, `vfp3`, `vfp4`, `vh`, `virt`, `virtualization`, `vpclmulqdq`, `vsx`, `wfxt`, `xop`, `xsave`, `xsavec`, `xsaveopt`, `xsaves`, `zaamo`, `zabha`, `zalrsc`, `zba`, `zbb`, `zbc`, `zbkb`, `zbkc`, `zbkx`, `zbs`, `zdinx`, `zfh`, `zfhmin`, `zfinx`, `zhinx`, `zhinxmin`, `zk`, `zkn`, `zknd`, `zkne`, `zknh`, `zkr`, `zks`, `zksed`, `zksh`, and `zkt` = note: see for more information about checking conditional configuration warning: unexpected `cfg` condition value: `_UNEXPECTED_VALUE` -- cgit 1.4.1-3-g733a5 From 559de745626901cf1fba5aaa5a2b96fbc92e09ff Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Tue, 8 Oct 2024 19:07:43 -0700 Subject: compiler: Move impl of ToJson for abi::Endian --- compiler/rustc_target/src/abi/mod.rs | 8 -------- compiler/rustc_target/src/json.rs | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs index b744d5ad4ed..9f26c98df86 100644 --- a/compiler/rustc_target/src/abi/mod.rs +++ b/compiler/rustc_target/src/abi/mod.rs @@ -6,19 +6,11 @@ use Primitive::*; use rustc_data_structures::intern::Interned; use rustc_macros::HashStable_Generic; -use crate::json::{Json, ToJson}; - pub mod call; // Explicitly import `Float` to avoid ambiguity with `Primitive::Float`. pub use rustc_abi::{Float, *}; -impl ToJson for Endian { - fn to_json(&self) -> Json { - self.as_str().to_json() - } -} - rustc_index::newtype_index! { /// The *source-order* index of a field in a variant. /// diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs index 2c367defe7b..b09d8d724ef 100644 --- a/compiler/rustc_target/src/json.rs +++ b/compiler/rustc_target/src/json.rs @@ -134,3 +134,9 @@ impl ToJson for TargetMetadata { }) } } + +impl ToJson for rustc_abi::Endian { + fn to_json(&self) -> Json { + self.as_str().to_json() + } +} -- cgit 1.4.1-3-g733a5 From 255bdd2f24de119d85073e0c48acdebca25af551 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 9 Oct 2024 11:07:22 -0700 Subject: compiler: Empty out rustc_target::abi --- compiler/rustc_abi/src/layout/ty.rs | 269 ++++++ compiler/rustc_target/src/abi/call/aarch64.rs | 127 --- compiler/rustc_target/src/abi/call/amdgpu.rs | 35 - compiler/rustc_target/src/abi/call/arm.rs | 105 --- compiler/rustc_target/src/abi/call/avr.rs | 59 -- compiler/rustc_target/src/abi/call/bpf.rs | 31 - compiler/rustc_target/src/abi/call/csky.rs | 61 -- compiler/rustc_target/src/abi/call/hexagon.rs | 30 - compiler/rustc_target/src/abi/call/loongarch.rs | 361 -------- compiler/rustc_target/src/abi/call/m68k.rs | 34 - compiler/rustc_target/src/abi/call/mips.rs | 53 -- compiler/rustc_target/src/abi/call/mips64.rs | 167 ---- compiler/rustc_target/src/abi/call/mod.rs | 1009 ----------------------- compiler/rustc_target/src/abi/call/msp430.rs | 39 - compiler/rustc_target/src/abi/call/nvptx64.rs | 102 --- compiler/rustc_target/src/abi/call/powerpc.rs | 38 - compiler/rustc_target/src/abi/call/powerpc64.rs | 118 --- compiler/rustc_target/src/abi/call/riscv.rs | 367 --------- compiler/rustc_target/src/abi/call/s390x.rs | 69 -- compiler/rustc_target/src/abi/call/sparc.rs | 53 -- compiler/rustc_target/src/abi/call/sparc64.rs | 234 ------ compiler/rustc_target/src/abi/call/wasm.rs | 103 --- compiler/rustc_target/src/abi/call/x86.rs | 185 ----- compiler/rustc_target/src/abi/call/x86_64.rs | 254 ------ compiler/rustc_target/src/abi/call/x86_win64.rs | 52 -- compiler/rustc_target/src/abi/call/xtensa.rs | 121 --- compiler/rustc_target/src/abi/mod.rs | 269 ------ compiler/rustc_target/src/callconv/aarch64.rs | 127 +++ compiler/rustc_target/src/callconv/amdgpu.rs | 35 + compiler/rustc_target/src/callconv/arm.rs | 105 +++ compiler/rustc_target/src/callconv/avr.rs | 59 ++ compiler/rustc_target/src/callconv/bpf.rs | 31 + compiler/rustc_target/src/callconv/csky.rs | 61 ++ compiler/rustc_target/src/callconv/hexagon.rs | 30 + compiler/rustc_target/src/callconv/loongarch.rs | 361 ++++++++ compiler/rustc_target/src/callconv/m68k.rs | 34 + compiler/rustc_target/src/callconv/mips.rs | 53 ++ compiler/rustc_target/src/callconv/mips64.rs | 167 ++++ compiler/rustc_target/src/callconv/mod.rs | 1009 +++++++++++++++++++++++ compiler/rustc_target/src/callconv/msp430.rs | 39 + compiler/rustc_target/src/callconv/nvptx64.rs | 102 +++ compiler/rustc_target/src/callconv/powerpc.rs | 38 + compiler/rustc_target/src/callconv/powerpc64.rs | 118 +++ compiler/rustc_target/src/callconv/riscv.rs | 367 +++++++++ compiler/rustc_target/src/callconv/s390x.rs | 69 ++ compiler/rustc_target/src/callconv/sparc.rs | 53 ++ compiler/rustc_target/src/callconv/sparc64.rs | 234 ++++++ compiler/rustc_target/src/callconv/wasm.rs | 103 +++ compiler/rustc_target/src/callconv/x86.rs | 185 +++++ compiler/rustc_target/src/callconv/x86_64.rs | 254 ++++++ compiler/rustc_target/src/callconv/x86_win64.rs | 52 ++ compiler/rustc_target/src/callconv/xtensa.rs | 121 +++ 52 files changed, 4076 insertions(+), 4076 deletions(-) create mode 100644 compiler/rustc_abi/src/layout/ty.rs delete mode 100644 compiler/rustc_target/src/abi/call/aarch64.rs delete mode 100644 compiler/rustc_target/src/abi/call/amdgpu.rs delete mode 100644 compiler/rustc_target/src/abi/call/arm.rs delete mode 100644 compiler/rustc_target/src/abi/call/avr.rs delete mode 100644 compiler/rustc_target/src/abi/call/bpf.rs delete mode 100644 compiler/rustc_target/src/abi/call/csky.rs delete mode 100644 compiler/rustc_target/src/abi/call/hexagon.rs delete mode 100644 compiler/rustc_target/src/abi/call/loongarch.rs delete mode 100644 compiler/rustc_target/src/abi/call/m68k.rs delete mode 100644 compiler/rustc_target/src/abi/call/mips.rs delete mode 100644 compiler/rustc_target/src/abi/call/mips64.rs delete mode 100644 compiler/rustc_target/src/abi/call/mod.rs delete mode 100644 compiler/rustc_target/src/abi/call/msp430.rs delete mode 100644 compiler/rustc_target/src/abi/call/nvptx64.rs delete mode 100644 compiler/rustc_target/src/abi/call/powerpc.rs delete mode 100644 compiler/rustc_target/src/abi/call/powerpc64.rs delete mode 100644 compiler/rustc_target/src/abi/call/riscv.rs delete mode 100644 compiler/rustc_target/src/abi/call/s390x.rs delete mode 100644 compiler/rustc_target/src/abi/call/sparc.rs delete mode 100644 compiler/rustc_target/src/abi/call/sparc64.rs delete mode 100644 compiler/rustc_target/src/abi/call/wasm.rs delete mode 100644 compiler/rustc_target/src/abi/call/x86.rs delete mode 100644 compiler/rustc_target/src/abi/call/x86_64.rs delete mode 100644 compiler/rustc_target/src/abi/call/x86_win64.rs delete mode 100644 compiler/rustc_target/src/abi/call/xtensa.rs delete mode 100644 compiler/rustc_target/src/abi/mod.rs create mode 100644 compiler/rustc_target/src/callconv/aarch64.rs create mode 100644 compiler/rustc_target/src/callconv/amdgpu.rs create mode 100644 compiler/rustc_target/src/callconv/arm.rs create mode 100644 compiler/rustc_target/src/callconv/avr.rs create mode 100644 compiler/rustc_target/src/callconv/bpf.rs create mode 100644 compiler/rustc_target/src/callconv/csky.rs create mode 100644 compiler/rustc_target/src/callconv/hexagon.rs create mode 100644 compiler/rustc_target/src/callconv/loongarch.rs create mode 100644 compiler/rustc_target/src/callconv/m68k.rs create mode 100644 compiler/rustc_target/src/callconv/mips.rs create mode 100644 compiler/rustc_target/src/callconv/mips64.rs create mode 100644 compiler/rustc_target/src/callconv/mod.rs create mode 100644 compiler/rustc_target/src/callconv/msp430.rs create mode 100644 compiler/rustc_target/src/callconv/nvptx64.rs create mode 100644 compiler/rustc_target/src/callconv/powerpc.rs create mode 100644 compiler/rustc_target/src/callconv/powerpc64.rs create mode 100644 compiler/rustc_target/src/callconv/riscv.rs create mode 100644 compiler/rustc_target/src/callconv/s390x.rs create mode 100644 compiler/rustc_target/src/callconv/sparc.rs create mode 100644 compiler/rustc_target/src/callconv/sparc64.rs create mode 100644 compiler/rustc_target/src/callconv/wasm.rs create mode 100644 compiler/rustc_target/src/callconv/x86.rs create mode 100644 compiler/rustc_target/src/callconv/x86_64.rs create mode 100644 compiler/rustc_target/src/callconv/x86_win64.rs create mode 100644 compiler/rustc_target/src/callconv/xtensa.rs (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs new file mode 100644 index 00000000000..9f26c98df86 --- /dev/null +++ b/compiler/rustc_abi/src/layout/ty.rs @@ -0,0 +1,269 @@ +use std::fmt; +use std::ops::Deref; + +use Float::*; +use Primitive::*; +use rustc_data_structures::intern::Interned; +use rustc_macros::HashStable_Generic; + +pub mod call; + +// Explicitly import `Float` to avoid ambiguity with `Primitive::Float`. +pub use rustc_abi::{Float, *}; + +rustc_index::newtype_index! { + /// The *source-order* index of a field in a variant. + /// + /// This is how most code after type checking refers to fields, rather than + /// using names (as names have hygiene complications and more complex lookup). + /// + /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order. + /// (It is for `repr(C)` `struct`s, however.) + /// + /// For example, in the following types, + /// ```rust + /// # enum Never {} + /// # #[repr(u16)] + /// enum Demo1 { + /// Variant0 { a: Never, b: i32 } = 100, + /// Variant1 { c: u8, d: u64 } = 10, + /// } + /// struct Demo2 { e: u8, f: u16, g: u8 } + /// ``` + /// `b` is `FieldIdx(1)` in `VariantIdx(0)`, + /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and + /// `f` is `FieldIdx(1)` in `VariantIdx(0)`. + #[derive(HashStable_Generic)] + #[encodable] + #[orderable] + pub struct FieldIdx {} +} + +rustc_index::newtype_index! { + /// The *source-order* index of a variant in a type. + /// + /// For enums, these are always `0..variant_count`, regardless of any + /// custom discriminants that may have been defined, and including any + /// variants that may end up uninhabited due to field types. (Some of the + /// variants may not be present in a monomorphized ABI [`Variants`], but + /// those skipped variants are always counted when determining the *index*.) + /// + /// `struct`s, `tuples`, and `unions`s are considered to have a single variant + /// with variant index zero, aka [`FIRST_VARIANT`]. + #[derive(HashStable_Generic)] + #[encodable] + #[orderable] + pub struct VariantIdx { + /// Equivalent to `VariantIdx(0)`. + const FIRST_VARIANT = 0; + } +} +#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)] +#[rustc_pass_by_value] +pub struct Layout<'a>(pub Interned<'a, LayoutS>); + +impl<'a> fmt::Debug for Layout<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // See comment on `::fmt` above. + self.0.0.fmt(f) + } +} + +impl<'a> Deref for Layout<'a> { + type Target = &'a LayoutS; + fn deref(&self) -> &&'a LayoutS { + &self.0.0 + } +} + +impl<'a> Layout<'a> { + pub fn fields(self) -> &'a FieldsShape { + &self.0.0.fields + } + + pub fn variants(self) -> &'a Variants { + &self.0.0.variants + } + + pub fn abi(self) -> Abi { + self.0.0.abi + } + + pub fn largest_niche(self) -> Option { + self.0.0.largest_niche + } + + pub fn align(self) -> AbiAndPrefAlign { + self.0.0.align + } + + pub fn size(self) -> Size { + self.0.0.size + } + + pub fn max_repr_align(self) -> Option { + self.0.0.max_repr_align + } + + pub fn unadjusted_abi_align(self) -> Align { + self.0.0.unadjusted_abi_align + } + + /// Whether the layout is from a type that implements [`std::marker::PointerLike`]. + /// + /// Currently, that means that the type is pointer-sized, pointer-aligned, + /// and has a initialized (non-union), scalar ABI. + pub fn is_pointer_like(self, data_layout: &TargetDataLayout) -> bool { + self.size() == data_layout.pointer_size + && self.align().abi == data_layout.pointer_align.abi + && matches!(self.abi(), Abi::Scalar(Scalar::Initialized { .. })) + } +} + +/// The layout of a type, alongside the type itself. +/// Provides various type traversal APIs (e.g., recursing into fields). +/// +/// Note that the layout is NOT guaranteed to always be identical +/// to that obtained from `layout_of(ty)`, as we need to produce +/// layouts for which Rust types do not exist, such as enum variants +/// or synthetic fields of enums (i.e., discriminants) and wide pointers. +#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)] +pub struct TyAndLayout<'a, Ty> { + pub ty: Ty, + pub layout: Layout<'a>, +} + +impl<'a, Ty: fmt::Display> fmt::Debug for TyAndLayout<'a, Ty> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Print the type in a readable way, not its debug representation. + f.debug_struct("TyAndLayout") + .field("ty", &format_args!("{}", self.ty)) + .field("layout", &self.layout) + .finish() + } +} + +impl<'a, Ty> Deref for TyAndLayout<'a, Ty> { + type Target = &'a LayoutS; + fn deref(&self) -> &&'a LayoutS { + &self.layout.0.0 + } +} + +/// Trait that needs to be implemented by the higher-level type representation +/// (e.g. `rustc_middle::ty::Ty`), to provide `rustc_target::abi` functionality. +pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug { + fn ty_and_layout_for_variant( + this: TyAndLayout<'a, Self>, + cx: &C, + variant_index: VariantIdx, + ) -> TyAndLayout<'a, Self>; + fn ty_and_layout_field(this: TyAndLayout<'a, Self>, cx: &C, i: usize) -> TyAndLayout<'a, Self>; + fn ty_and_layout_pointee_info_at( + this: TyAndLayout<'a, Self>, + cx: &C, + offset: Size, + ) -> Option; + fn is_adt(this: TyAndLayout<'a, Self>) -> bool; + fn is_never(this: TyAndLayout<'a, Self>) -> bool; + fn is_tuple(this: TyAndLayout<'a, Self>) -> bool; + fn is_unit(this: TyAndLayout<'a, Self>) -> bool; + fn is_transparent(this: TyAndLayout<'a, Self>) -> bool; +} + +impl<'a, Ty> TyAndLayout<'a, Ty> { + pub fn for_variant(self, cx: &C, variant_index: VariantIdx) -> Self + where + Ty: TyAbiInterface<'a, C>, + { + Ty::ty_and_layout_for_variant(self, cx, variant_index) + } + + pub fn field(self, cx: &C, i: usize) -> Self + where + Ty: TyAbiInterface<'a, C>, + { + Ty::ty_and_layout_field(self, cx, i) + } + + pub fn pointee_info_at(self, cx: &C, offset: Size) -> Option + where + Ty: TyAbiInterface<'a, C>, + { + Ty::ty_and_layout_pointee_info_at(self, cx, offset) + } + + pub fn is_single_fp_element(self, cx: &C) -> bool + where + Ty: TyAbiInterface<'a, C>, + C: HasDataLayout, + { + match self.abi { + Abi::Scalar(scalar) => matches!(scalar.primitive(), Float(F32 | F64)), + Abi::Aggregate { .. } => { + if self.fields.count() == 1 && self.fields.offset(0).bytes() == 0 { + self.field(cx, 0).is_single_fp_element(cx) + } else { + false + } + } + _ => false, + } + } + + pub fn is_adt(self) -> bool + where + Ty: TyAbiInterface<'a, C>, + { + Ty::is_adt(self) + } + + pub fn is_never(self) -> bool + where + Ty: TyAbiInterface<'a, C>, + { + Ty::is_never(self) + } + + pub fn is_tuple(self) -> bool + where + Ty: TyAbiInterface<'a, C>, + { + Ty::is_tuple(self) + } + + pub fn is_unit(self) -> bool + where + Ty: TyAbiInterface<'a, C>, + { + Ty::is_unit(self) + } + + pub fn is_transparent(self) -> bool + where + Ty: TyAbiInterface<'a, C>, + { + Ty::is_transparent(self) + } + + /// Finds the one field that is not a 1-ZST. + /// Returns `None` if there are multiple non-1-ZST fields or only 1-ZST-fields. + pub fn non_1zst_field(&self, cx: &C) -> Option<(usize, Self)> + where + Ty: TyAbiInterface<'a, C> + Copy, + { + let mut found = None; + for field_idx in 0..self.fields.count() { + let field = self.field(cx, field_idx); + if field.is_1zst() { + continue; + } + if found.is_some() { + // More than one non-1-ZST field. + return None; + } + found = Some((field_idx, field)); + } + found + } +} diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs deleted file mode 100644 index 55b65fb1caa..00000000000 --- a/compiler/rustc_target/src/abi/call/aarch64.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind, Uniform}; -use crate::abi::{HasDataLayout, TyAbiInterface}; - -/// Indicates the variant of the AArch64 ABI we are compiling for. -/// Used to accommodate Apple and Microsoft's deviations from the usual AAPCS ABI. -/// -/// Corresponds to Clang's `AArch64ABIInfo::ABIKind`. -#[derive(Copy, Clone, PartialEq)] -pub(crate) enum AbiKind { - AAPCS, - DarwinPCS, - Win64, -} - -fn is_homogeneous_aggregate<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -> Option -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { - let size = arg.layout.size; - - // Ensure we have at most four uniquely addressable members. - if size > unit.size.checked_mul(4, cx).unwrap() { - return None; - } - - let valid_unit = match unit.kind { - RegKind::Integer => false, - RegKind::Float => true, - RegKind::Vector => size.bits() == 64 || size.bits() == 128, - }; - - valid_unit.then_some(Uniform::consecutive(unit, size)) - }) -} - -fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, kind: AbiKind) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !ret.layout.is_sized() { - // Not touching this... - return; - } - if !ret.layout.is_aggregate() { - if kind == AbiKind::DarwinPCS { - // On Darwin, when returning an i8/i16, it must be sign-extended to 32 bits, - // and likewise a u8/u16 must be zero-extended to 32-bits. - // See also: - ret.extend_integer_width_to(32) - } - return; - } - if let Some(uniform) = is_homogeneous_aggregate(cx, ret) { - ret.cast_to(uniform); - return; - } - let size = ret.layout.size; - let bits = size.bits(); - if bits <= 128 { - ret.cast_to(Uniform::new(Reg::i64(), size)); - return; - } - ret.make_indirect(); -} - -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, kind: AbiKind) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - if !arg.layout.is_aggregate() { - if kind == AbiKind::DarwinPCS { - // On Darwin, when passing an i8/i16, it must be sign-extended to 32 bits, - // and likewise a u8/u16 must be zero-extended to 32-bits. - // See also: - arg.extend_integer_width_to(32); - } - return; - } - if let Some(uniform) = is_homogeneous_aggregate(cx, arg) { - arg.cast_to(uniform); - return; - } - let size = arg.layout.size; - let align = if kind == AbiKind::AAPCS { - // When passing small aggregates by value, the AAPCS ABI mandates using the unadjusted - // alignment of the type (not including `repr(align)`). - // This matches behavior of `AArch64ABIInfo::classifyArgumentType` in Clang. - // See: - arg.layout.unadjusted_abi_align - } else { - arg.layout.align.abi - }; - if size.bits() <= 128 { - if align.bits() == 128 { - arg.cast_to(Uniform::new(Reg::i128(), size)); - } else { - arg.cast_to(Uniform::new(Reg::i64(), size)); - } - return; - } - arg.make_indirect(); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, kind: AbiKind) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret, kind); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg, kind); - } -} diff --git a/compiler/rustc_target/src/abi/call/amdgpu.rs b/compiler/rustc_target/src/abi/call/amdgpu.rs deleted file mode 100644 index 3007a729a8b..00000000000 --- a/compiler/rustc_target/src/abi/call/amdgpu.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi}; -use crate::abi::{HasDataLayout, TyAbiInterface}; - -fn classify_ret<'a, Ty, C>(_cx: &C, ret: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - ret.extend_integer_width_to(32); -} - -fn classify_arg<'a, Ty, C>(_cx: &C, arg: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - arg.extend_integer_width_to(32); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs deleted file mode 100644 index bd6f781fb81..00000000000 --- a/compiler/rustc_target/src/abi/call/arm.rs +++ /dev/null @@ -1,105 +0,0 @@ -use crate::abi::call::{ArgAbi, Conv, FnAbi, Reg, RegKind, Uniform}; -use crate::abi::{HasDataLayout, TyAbiInterface}; -use crate::spec::HasTargetSpec; - -fn is_homogeneous_aggregate<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -> Option -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { - let size = arg.layout.size; - - // Ensure we have at most four uniquely addressable members. - if size > unit.size.checked_mul(4, cx).unwrap() { - return None; - } - - let valid_unit = match unit.kind { - RegKind::Integer => false, - RegKind::Float => true, - RegKind::Vector => size.bits() == 64 || size.bits() == 128, - }; - - valid_unit.then_some(Uniform::consecutive(unit, size)) - }) -} - -fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, vfp: bool) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !ret.layout.is_sized() { - // Not touching this... - return; - } - if !ret.layout.is_aggregate() { - ret.extend_integer_width_to(32); - return; - } - - if vfp { - if let Some(uniform) = is_homogeneous_aggregate(cx, ret) { - ret.cast_to(uniform); - return; - } - } - - let size = ret.layout.size; - let bits = size.bits(); - if bits <= 32 { - ret.cast_to(Uniform::new(Reg::i32(), size)); - return; - } - ret.make_indirect(); -} - -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, vfp: bool) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - if !arg.layout.is_aggregate() { - arg.extend_integer_width_to(32); - return; - } - - if vfp { - if let Some(uniform) = is_homogeneous_aggregate(cx, arg) { - arg.cast_to(uniform); - return; - } - } - - let align = arg.layout.align.abi.bytes(); - let total = arg.layout.size; - arg.cast_to(Uniform::consecutive(if align <= 4 { Reg::i32() } else { Reg::i64() }, total)); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - // If this is a target with a hard-float ABI, and the function is not explicitly - // `extern "aapcs"`, then we must use the VFP registers for homogeneous aggregates. - let vfp = cx.target_spec().llvm_target.ends_with("hf") - && fn_abi.conv != Conv::ArmAapcs - && !fn_abi.c_variadic; - - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret, vfp); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg, vfp); - } -} diff --git a/compiler/rustc_target/src/abi/call/avr.rs b/compiler/rustc_target/src/abi/call/avr.rs deleted file mode 100644 index dfc991e0954..00000000000 --- a/compiler/rustc_target/src/abi/call/avr.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! LLVM-frontend specific AVR calling convention implementation. -//! -//! # Current calling convention ABI -//! -//! Inherited from Clang's `clang::DefaultABIInfo` implementation - self described -//! as -//! -//! > the default implementation for ABI specific details. This implementation -//! > provides information which results in -//! > self-consistent and sensible LLVM IR generation, but does not -//! > conform to any particular ABI. -//! > -//! > - Doxygen Documentation of `clang::DefaultABIInfo` -//! -//! This calling convention may not match AVR-GCC in all cases. -//! -//! In the future, an AVR-GCC compatible argument classification ABI should be -//! adopted in both Rust and Clang. -//! -//! *NOTE*: Currently, this module implements the same calling convention -//! that clang with AVR currently does - the default, simple, unspecialized -//! ABI implementation available to all targets. This ABI is not -//! binary-compatible with AVR-GCC. Once LLVM [PR46140](https://bugs.llvm.org/show_bug.cgi?id=46140) -//! is completed, this module should be updated to match so that both Clang -//! and Rust emit code to the same AVR-GCC compatible ABI. -//! -//! In particular, both Clang and Rust may not have the same semantics -//! when promoting arguments to indirect references as AVR-GCC. It is important -//! to note that the core AVR ABI implementation within LLVM itself is ABI -//! compatible with AVR-GCC - Rust and AVR-GCC only differ in the small amount -//! of compiler frontend specific calling convention logic implemented here. - -use crate::abi::call::{ArgAbi, FnAbi}; - -fn classify_ret_ty(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() { - ret.make_indirect(); - } -} - -fn classify_arg_ty(arg: &mut ArgAbi<'_, Ty>) { - if arg.layout.is_aggregate() { - arg.make_indirect(); - } -} - -pub(crate) fn compute_abi_info(fty: &mut FnAbi<'_, Ty>) { - if !fty.ret.is_ignore() { - classify_ret_ty(&mut fty.ret); - } - - for arg in fty.args.iter_mut() { - if arg.is_ignore() { - continue; - } - - classify_arg_ty(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/bpf.rs b/compiler/rustc_target/src/abi/call/bpf.rs deleted file mode 100644 index f19772ac709..00000000000 --- a/compiler/rustc_target/src/abi/call/bpf.rs +++ /dev/null @@ -1,31 +0,0 @@ -// see https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/BPF/BPFCallingConv.td -use crate::abi::call::{ArgAbi, FnAbi}; - -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() || ret.layout.size.bits() > 64 { - ret.make_indirect(); - } else { - ret.extend_integer_width_to(32); - } -} - -fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { - if arg.layout.is_aggregate() || arg.layout.size.bits() > 64 { - arg.make_indirect(); - } else { - arg.extend_integer_width_to(32); - } -} - -pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/csky.rs b/compiler/rustc_target/src/abi/call/csky.rs deleted file mode 100644 index b1c1ae814a7..00000000000 --- a/compiler/rustc_target/src/abi/call/csky.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Reference: CSKY ABI Manual -// https://occ-oss-prod.oss-cn-hangzhou.aliyuncs.com/resource//1695027452256/T-HEAD_800_Series_ABI_Standards_Manual.pdf -// -// Reference: Clang CSKY lowering code -// https://github.com/llvm/llvm-project/blob/4a074f32a6914f2a8d7215d78758c24942dddc3d/clang/lib/CodeGen/Targets/CSKY.cpp#L76-L162 - -use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; - -fn classify_ret(arg: &mut ArgAbi<'_, Ty>) { - if !arg.layout.is_sized() { - // Not touching this... - return; - } - // For return type, aggregate which <= 2*XLen will be returned in registers. - // Otherwise, aggregate will be returned indirectly. - if arg.layout.is_aggregate() { - let total = arg.layout.size; - if total.bits() > 64 { - arg.make_indirect(); - } else if total.bits() > 32 { - arg.cast_to(Uniform::new(Reg::i32(), total)); - } else { - arg.cast_to(Reg::i32()); - } - } else { - arg.extend_integer_width_to(32); - } -} - -fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { - if !arg.layout.is_sized() { - // Not touching this... - return; - } - // For argument type, the first 4*XLen parts of aggregate will be passed - // in registers, and the rest will be passed in stack. - // So we can coerce to integers directly and let backend handle it correctly. - if arg.layout.is_aggregate() { - let total = arg.layout.size; - if total.bits() > 32 { - arg.cast_to(Uniform::new(Reg::i32(), total)); - } else { - arg.cast_to(Reg::i32()); - } - } else { - arg.extend_integer_width_to(32); - } -} - -pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/hexagon.rs b/compiler/rustc_target/src/abi/call/hexagon.rs deleted file mode 100644 index 0a0688880c0..00000000000 --- a/compiler/rustc_target/src/abi/call/hexagon.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi}; - -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() && ret.layout.size.bits() > 64 { - ret.make_indirect(); - } else { - ret.extend_integer_width_to(32); - } -} - -fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { - if arg.layout.is_aggregate() && arg.layout.size.bits() > 64 { - arg.make_indirect(); - } else { - arg.extend_integer_width_to(32); - } -} - -pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs deleted file mode 100644 index 4a21935623b..00000000000 --- a/compiler/rustc_target/src/abi/call/loongarch.rs +++ /dev/null @@ -1,361 +0,0 @@ -use crate::abi::call::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Reg, RegKind, Uniform}; -use crate::abi::{self, Abi, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; -use crate::spec::HasTargetSpec; - -#[derive(Copy, Clone)] -enum RegPassKind { - Float(Reg), - Integer(Reg), - Unknown, -} - -#[derive(Copy, Clone)] -enum FloatConv { - FloatPair(Reg, Reg), - Float(Reg), - MixedPair(Reg, Reg), -} - -#[derive(Copy, Clone)] -struct CannotUseFpConv; - -fn is_loongarch_aggregate(arg: &ArgAbi<'_, Ty>) -> bool { - match arg.layout.abi { - Abi::Vector { .. } => true, - _ => arg.layout.is_aggregate(), - } -} - -fn should_use_fp_conv_helper<'a, Ty, C>( - cx: &C, - arg_layout: &TyAndLayout<'a, Ty>, - xlen: u64, - flen: u64, - field1_kind: &mut RegPassKind, - field2_kind: &mut RegPassKind, -) -> Result<(), CannotUseFpConv> -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - match arg_layout.abi { - Abi::Scalar(scalar) => match scalar.primitive() { - abi::Int(..) | abi::Pointer(_) => { - if arg_layout.size.bits() > xlen { - return Err(CannotUseFpConv); - } - match (*field1_kind, *field2_kind) { - (RegPassKind::Unknown, _) => { - *field1_kind = RegPassKind::Integer(Reg { - kind: RegKind::Integer, - size: arg_layout.size, - }); - } - (RegPassKind::Float(_), RegPassKind::Unknown) => { - *field2_kind = RegPassKind::Integer(Reg { - kind: RegKind::Integer, - size: arg_layout.size, - }); - } - _ => return Err(CannotUseFpConv), - } - } - abi::Float(_) => { - if arg_layout.size.bits() > flen { - return Err(CannotUseFpConv); - } - match (*field1_kind, *field2_kind) { - (RegPassKind::Unknown, _) => { - *field1_kind = - RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); - } - (_, RegPassKind::Unknown) => { - *field2_kind = - RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); - } - _ => return Err(CannotUseFpConv), - } - } - }, - Abi::Vector { .. } | Abi::Uninhabited => return Err(CannotUseFpConv), - Abi::ScalarPair(..) | Abi::Aggregate { .. } => match arg_layout.fields { - FieldsShape::Primitive => { - unreachable!("aggregates can't have `FieldsShape::Primitive`") - } - FieldsShape::Union(_) => { - if !arg_layout.is_zst() { - if arg_layout.is_transparent() { - let non_1zst_elem = arg_layout.non_1zst_field(cx).expect("not exactly one non-1-ZST field in non-ZST repr(transparent) union").1; - return should_use_fp_conv_helper( - cx, - &non_1zst_elem, - xlen, - flen, - field1_kind, - field2_kind, - ); - } - return Err(CannotUseFpConv); - } - } - FieldsShape::Array { count, .. } => { - for _ in 0..count { - let elem_layout = arg_layout.field(cx, 0); - should_use_fp_conv_helper( - cx, - &elem_layout, - xlen, - flen, - field1_kind, - field2_kind, - )?; - } - } - FieldsShape::Arbitrary { .. } => { - match arg_layout.variants { - abi::Variants::Multiple { .. } => return Err(CannotUseFpConv), - abi::Variants::Single { .. } => (), - } - for i in arg_layout.fields.index_by_increasing_offset() { - let field = arg_layout.field(cx, i); - should_use_fp_conv_helper(cx, &field, xlen, flen, field1_kind, field2_kind)?; - } - } - }, - } - Ok(()) -} - -fn should_use_fp_conv<'a, Ty, C>( - cx: &C, - arg: &TyAndLayout<'a, Ty>, - xlen: u64, - flen: u64, -) -> Option -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - let mut field1_kind = RegPassKind::Unknown; - let mut field2_kind = RegPassKind::Unknown; - if should_use_fp_conv_helper(cx, arg, xlen, flen, &mut field1_kind, &mut field2_kind).is_err() { - return None; - } - match (field1_kind, field2_kind) { - (RegPassKind::Integer(l), RegPassKind::Float(r)) => Some(FloatConv::MixedPair(l, r)), - (RegPassKind::Float(l), RegPassKind::Integer(r)) => Some(FloatConv::MixedPair(l, r)), - (RegPassKind::Float(l), RegPassKind::Float(r)) => Some(FloatConv::FloatPair(l, r)), - (RegPassKind::Float(f), RegPassKind::Unknown) => Some(FloatConv::Float(f)), - _ => None, - } -} - -fn classify_ret<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, xlen: u64, flen: u64) -> bool -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - if !arg.layout.is_sized() { - // Not touching this... - return false; // I guess? return value of this function is not documented - } - if let Some(conv) = should_use_fp_conv(cx, &arg.layout, xlen, flen) { - match conv { - FloatConv::Float(f) => { - arg.cast_to(f); - } - FloatConv::FloatPair(l, r) => { - arg.cast_to(CastTarget::pair(l, r)); - } - FloatConv::MixedPair(l, r) => { - arg.cast_to(CastTarget::pair(l, r)); - } - } - return false; - } - - let total = arg.layout.size; - - // "Scalars wider than 2✕XLEN are passed by reference and are replaced in - // the argument list with the address." - // "Aggregates larger than 2✕XLEN bits are passed by reference and are - // replaced in the argument list with the address, as are C++ aggregates - // with nontrivial copy constructors, destructors, or vtables." - if total.bits() > 2 * xlen { - // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. - if is_loongarch_aggregate(arg) { - arg.make_indirect(); - } - return true; - } - - let xlen_reg = match xlen { - 32 => Reg::i32(), - 64 => Reg::i64(), - _ => unreachable!("Unsupported XLEN: {}", xlen), - }; - if is_loongarch_aggregate(arg) { - if total.bits() <= xlen { - arg.cast_to(xlen_reg); - } else { - arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2))); - } - return false; - } - - // "When passed in registers, scalars narrower than XLEN bits are widened - // according to the sign of their type up to 32 bits, then sign-extended to - // XLEN bits." - extend_integer_width(arg, xlen); - false -} - -fn classify_arg<'a, Ty, C>( - cx: &C, - arg: &mut ArgAbi<'a, Ty>, - xlen: u64, - flen: u64, - is_vararg: bool, - avail_gprs: &mut u64, - avail_fprs: &mut u64, -) where - Ty: TyAbiInterface<'a, C> + Copy, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - if !is_vararg { - match should_use_fp_conv(cx, &arg.layout, xlen, flen) { - Some(FloatConv::Float(f)) if *avail_fprs >= 1 => { - *avail_fprs -= 1; - arg.cast_to(f); - return; - } - Some(FloatConv::FloatPair(l, r)) if *avail_fprs >= 2 => { - *avail_fprs -= 2; - arg.cast_to(CastTarget::pair(l, r)); - return; - } - Some(FloatConv::MixedPair(l, r)) if *avail_fprs >= 1 && *avail_gprs >= 1 => { - *avail_gprs -= 1; - *avail_fprs -= 1; - arg.cast_to(CastTarget::pair(l, r)); - return; - } - _ => (), - } - } - - let total = arg.layout.size; - let align = arg.layout.align.abi.bits(); - - // "Scalars wider than 2✕XLEN are passed by reference and are replaced in - // the argument list with the address." - // "Aggregates larger than 2✕XLEN bits are passed by reference and are - // replaced in the argument list with the address, as are C++ aggregates - // with nontrivial copy constructors, destructors, or vtables." - if total.bits() > 2 * xlen { - // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. - if is_loongarch_aggregate(arg) { - arg.make_indirect(); - } - if *avail_gprs >= 1 { - *avail_gprs -= 1; - } - return; - } - - let double_xlen_reg = match xlen { - 32 => Reg::i64(), - 64 => Reg::i128(), - _ => unreachable!("Unsupported XLEN: {}", xlen), - }; - - let xlen_reg = match xlen { - 32 => Reg::i32(), - 64 => Reg::i64(), - _ => unreachable!("Unsupported XLEN: {}", xlen), - }; - - if total.bits() > xlen { - let align_regs = align > xlen; - if is_loongarch_aggregate(arg) { - arg.cast_to(Uniform::new( - if align_regs { double_xlen_reg } else { xlen_reg }, - Size::from_bits(xlen * 2), - )); - } - if align_regs && is_vararg { - *avail_gprs -= *avail_gprs % 2; - } - if *avail_gprs >= 2 { - *avail_gprs -= 2; - } else { - *avail_gprs = 0; - } - return; - } else if is_loongarch_aggregate(arg) { - arg.cast_to(xlen_reg); - if *avail_gprs >= 1 { - *avail_gprs -= 1; - } - return; - } - - // "When passed in registers, scalars narrower than XLEN bits are widened - // according to the sign of their type up to 32 bits, then sign-extended to - // XLEN bits." - if *avail_gprs >= 1 { - extend_integer_width(arg, xlen); - *avail_gprs -= 1; - } -} - -fn extend_integer_width(arg: &mut ArgAbi<'_, Ty>, xlen: u64) { - if let Abi::Scalar(scalar) = arg.layout.abi { - if let abi::Int(i, _) = scalar.primitive() { - // 32-bit integers are always sign-extended - if i.size().bits() == 32 && xlen > 32 { - if let PassMode::Direct(ref mut attrs) = arg.mode { - attrs.ext(ArgExtension::Sext); - return; - } - } - } - } - - arg.extend_integer_width_to(xlen); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - let xlen = cx.data_layout().pointer_size.bits(); - let flen = match &cx.target_spec().llvm_abiname[..] { - "ilp32f" | "lp64f" => 32, - "ilp32d" | "lp64d" => 64, - _ => 0, - }; - - let mut avail_gprs = 8; - let mut avail_fprs = 8; - - if !fn_abi.ret.is_ignore() && classify_ret(cx, &mut fn_abi.ret, xlen, flen) { - avail_gprs -= 1; - } - - for (i, arg) in fn_abi.args.iter_mut().enumerate() { - if arg.is_ignore() { - continue; - } - classify_arg( - cx, - arg, - xlen, - flen, - i >= fn_abi.fixed_count as usize, - &mut avail_gprs, - &mut avail_fprs, - ); - } -} diff --git a/compiler/rustc_target/src/abi/call/m68k.rs b/compiler/rustc_target/src/abi/call/m68k.rs deleted file mode 100644 index 82fe81f8c52..00000000000 --- a/compiler/rustc_target/src/abi/call/m68k.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi}; - -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() { - ret.make_indirect(); - } else { - ret.extend_integer_width_to(32); - } -} - -fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { - if !arg.layout.is_sized() { - // Not touching this... - return; - } - if arg.layout.is_aggregate() { - arg.pass_by_stack_offset(None); - } else { - arg.extend_integer_width_to(32); - } -} - -pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/mips.rs b/compiler/rustc_target/src/abi/call/mips.rs deleted file mode 100644 index 37980a91c76..00000000000 --- a/compiler/rustc_target/src/abi/call/mips.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; -use crate::abi::{HasDataLayout, Size}; - -fn classify_ret(cx: &C, ret: &mut ArgAbi<'_, Ty>, offset: &mut Size) -where - C: HasDataLayout, -{ - if !ret.layout.is_aggregate() { - ret.extend_integer_width_to(32); - } else { - ret.make_indirect(); - *offset += cx.data_layout().pointer_size; - } -} - -fn classify_arg(cx: &C, arg: &mut ArgAbi<'_, Ty>, offset: &mut Size) -where - C: HasDataLayout, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - let dl = cx.data_layout(); - let size = arg.layout.size; - let align = arg.layout.align.max(dl.i32_align).min(dl.i64_align).abi; - - if arg.layout.is_aggregate() { - let pad_i32 = !offset.is_aligned(align); - arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32); - } else { - arg.extend_integer_width_to(32); - } - - *offset = offset.align_to(align) + size.align_to(align); -} - -pub(crate) fn compute_abi_info(cx: &C, fn_abi: &mut FnAbi<'_, Ty>) -where - C: HasDataLayout, -{ - let mut offset = Size::ZERO; - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret, &mut offset); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg, &mut offset); - } -} diff --git a/compiler/rustc_target/src/abi/call/mips64.rs b/compiler/rustc_target/src/abi/call/mips64.rs deleted file mode 100644 index 2c3258c8d42..00000000000 --- a/compiler/rustc_target/src/abi/call/mips64.rs +++ /dev/null @@ -1,167 +0,0 @@ -use crate::abi::call::{ - ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, Reg, Uniform, -}; -use crate::abi::{self, HasDataLayout, Size, TyAbiInterface}; - -fn extend_integer_width_mips(arg: &mut ArgAbi<'_, Ty>, bits: u64) { - // Always sign extend u32 values on 64-bit mips - if let abi::Abi::Scalar(scalar) = arg.layout.abi { - if let abi::Int(i, signed) = scalar.primitive() { - if !signed && i.size().bits() == 32 { - if let PassMode::Direct(ref mut attrs) = arg.mode { - attrs.ext(ArgExtension::Sext); - return; - } - } - } - } - - arg.extend_integer_width_to(bits); -} - -fn float_reg<'a, Ty, C>(cx: &C, ret: &ArgAbi<'a, Ty>, i: usize) -> Option -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - match ret.layout.field(cx, i).abi { - abi::Abi::Scalar(scalar) => match scalar.primitive() { - abi::Float(abi::F32) => Some(Reg::f32()), - abi::Float(abi::F64) => Some(Reg::f64()), - _ => None, - }, - _ => None, - } -} - -fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !ret.layout.is_aggregate() { - extend_integer_width_mips(ret, 64); - return; - } - - let size = ret.layout.size; - let bits = size.bits(); - if bits <= 128 { - // Unlike other architectures which return aggregates in registers, MIPS n64 limits the - // use of float registers to structures (not unions) containing exactly one or two - // float fields. - - if let abi::FieldsShape::Arbitrary { .. } = ret.layout.fields { - if ret.layout.fields.count() == 1 { - if let Some(reg) = float_reg(cx, ret, 0) { - ret.cast_to(reg); - return; - } - } else if ret.layout.fields.count() == 2 { - if let Some(reg0) = float_reg(cx, ret, 0) { - if let Some(reg1) = float_reg(cx, ret, 1) { - ret.cast_to(CastTarget::pair(reg0, reg1)); - return; - } - } - } - } - - // Cast to a uniform int structure - ret.cast_to(Uniform::new(Reg::i64(), size)); - } else { - ret.make_indirect(); - } -} - -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !arg.layout.is_aggregate() { - extend_integer_width_mips(arg, 64); - return; - } - - let dl = cx.data_layout(); - let size = arg.layout.size; - let mut prefix = [None; 8]; - let mut prefix_index = 0; - - match arg.layout.fields { - abi::FieldsShape::Primitive => unreachable!(), - abi::FieldsShape::Array { .. } => { - // Arrays are passed indirectly - arg.make_indirect(); - return; - } - abi::FieldsShape::Union(_) => { - // Unions and are always treated as a series of 64-bit integer chunks - } - abi::FieldsShape::Arbitrary { .. } => { - // Structures are split up into a series of 64-bit integer chunks, but any aligned - // doubles not part of another aggregate are passed as floats. - let mut last_offset = Size::ZERO; - - for i in 0..arg.layout.fields.count() { - let field = arg.layout.field(cx, i); - let offset = arg.layout.fields.offset(i); - - // We only care about aligned doubles - if let abi::Abi::Scalar(scalar) = field.abi { - if scalar.primitive() == abi::Float(abi::F64) { - if offset.is_aligned(dl.f64_align.abi) { - // Insert enough integers to cover [last_offset, offset) - assert!(last_offset.is_aligned(dl.f64_align.abi)); - for _ in 0..((offset - last_offset).bits() / 64) - .min((prefix.len() - prefix_index) as u64) - { - prefix[prefix_index] = Some(Reg::i64()); - prefix_index += 1; - } - - if prefix_index == prefix.len() { - break; - } - - prefix[prefix_index] = Some(Reg::f64()); - prefix_index += 1; - last_offset = offset + Reg::f64().size; - } - } - } - } - } - }; - - // Extract first 8 chunks as the prefix - let rest_size = size - Size::from_bytes(8) * prefix_index as u64; - arg.cast_to(CastTarget { - prefix, - rest: Uniform::new(Reg::i64(), rest_size), - attrs: ArgAttributes { - regular: ArgAttribute::default(), - arg_ext: ArgExtension::None, - pointee_size: Size::ZERO, - pointee_align: None, - }, - }); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs deleted file mode 100644 index 352861c5ccb..00000000000 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ /dev/null @@ -1,1009 +0,0 @@ -use std::fmt; -use std::str::FromStr; - -use rustc_macros::HashStable_Generic; -use rustc_span::Symbol; - -use crate::abi::{self, Abi, Align, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; -use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi}; - -mod aarch64; -mod amdgpu; -mod arm; -mod avr; -mod bpf; -mod csky; -mod hexagon; -mod loongarch; -mod m68k; -mod mips; -mod mips64; -mod msp430; -mod nvptx64; -mod powerpc; -mod powerpc64; -mod riscv; -mod s390x; -mod sparc; -mod sparc64; -mod wasm; -mod x86; -mod x86_64; -mod x86_win64; -mod xtensa; - -#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub enum PassMode { - /// Ignore the argument. - /// - /// The argument is either uninhabited or a ZST. - Ignore, - /// Pass the argument directly. - /// - /// The argument has a layout abi of `Scalar` or `Vector`. - /// Unfortunately due to past mistakes, in rare cases on wasm, it can also be `Aggregate`. - /// This is bad since it leaks LLVM implementation details into the ABI. - /// (Also see .) - Direct(ArgAttributes), - /// Pass a pair's elements directly in two arguments. - /// - /// The argument has a layout abi of `ScalarPair`. - Pair(ArgAttributes, ArgAttributes), - /// Pass the argument after casting it. See the `CastTarget` docs for details. - /// - /// `pad_i32` indicates if a `Reg::i32()` dummy argument is emitted before the real argument. - Cast { pad_i32: bool, cast: Box }, - /// Pass the argument indirectly via a hidden pointer. - /// - /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized - /// argument. (This is the only mode that supports unsized arguments.) - /// - /// `on_stack` defines that the value should be passed at a fixed stack offset in accordance to - /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument - /// attribute. The `byval` argument will use a byte array with the same size as the Rust type - /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout), - /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's - /// alignment (if `None`). This means that the alignment will not always - /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info. - /// - /// `on_stack` cannot be true for unsized arguments, i.e., when `meta_attrs` is `Some`. - Indirect { attrs: ArgAttributes, meta_attrs: Option, on_stack: bool }, -} - -impl PassMode { - /// Checks if these two `PassMode` are equal enough to be considered "the same for all - /// function call ABIs". However, the `Layout` can also impact ABI decisions, - /// so that needs to be compared as well! - pub fn eq_abi(&self, other: &Self) -> bool { - match (self, other) { - (PassMode::Ignore, PassMode::Ignore) => true, - (PassMode::Direct(a1), PassMode::Direct(a2)) => a1.eq_abi(a2), - (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => a1.eq_abi(a2) && b1.eq_abi(b2), - ( - PassMode::Cast { cast: c1, pad_i32: pad1 }, - PassMode::Cast { cast: c2, pad_i32: pad2 }, - ) => c1.eq_abi(c2) && pad1 == pad2, - ( - PassMode::Indirect { attrs: a1, meta_attrs: None, on_stack: s1 }, - PassMode::Indirect { attrs: a2, meta_attrs: None, on_stack: s2 }, - ) => a1.eq_abi(a2) && s1 == s2, - ( - PassMode::Indirect { attrs: a1, meta_attrs: Some(e1), on_stack: s1 }, - PassMode::Indirect { attrs: a2, meta_attrs: Some(e2), on_stack: s2 }, - ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2, - _ => false, - } - } -} - -// Hack to disable non_upper_case_globals only for the bitflags! and not for the rest -// of this module -pub use attr_impl::ArgAttribute; - -#[allow(non_upper_case_globals)] -#[allow(unused)] -mod attr_impl { - use rustc_macros::HashStable_Generic; - - // The subset of llvm::Attribute needed for arguments, packed into a bitfield. - #[derive(Clone, Copy, Default, Hash, PartialEq, Eq, HashStable_Generic)] - pub struct ArgAttribute(u8); - bitflags::bitflags! { - impl ArgAttribute: u8 { - const NoAlias = 1 << 1; - const NoCapture = 1 << 2; - const NonNull = 1 << 3; - const ReadOnly = 1 << 4; - const InReg = 1 << 5; - const NoUndef = 1 << 6; - } - } - rustc_data_structures::external_bitflags_debug! { ArgAttribute } -} - -/// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum -/// defines if this extension should be zero-extension or sign-extension when necessary. When it is -/// not necessary to extend the argument, this enum is ignored. -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub enum ArgExtension { - None, - Zext, - Sext, -} - -/// A compact representation of LLVM attributes (at least those relevant for this module) -/// that can be manipulated without interacting with LLVM's Attribute machinery. -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub struct ArgAttributes { - pub regular: ArgAttribute, - pub arg_ext: ArgExtension, - /// The minimum size of the pointee, guaranteed to be valid for the duration of the whole call - /// (corresponding to LLVM's dereferenceable and dereferenceable_or_null attributes). - pub pointee_size: Size, - pub pointee_align: Option, -} - -impl ArgAttributes { - pub fn new() -> Self { - ArgAttributes { - regular: ArgAttribute::default(), - arg_ext: ArgExtension::None, - pointee_size: Size::ZERO, - pointee_align: None, - } - } - - pub fn ext(&mut self, ext: ArgExtension) -> &mut Self { - assert!( - self.arg_ext == ArgExtension::None || self.arg_ext == ext, - "cannot set {:?} when {:?} is already set", - ext, - self.arg_ext - ); - self.arg_ext = ext; - self - } - - pub fn set(&mut self, attr: ArgAttribute) -> &mut Self { - self.regular |= attr; - self - } - - pub fn contains(&self, attr: ArgAttribute) -> bool { - self.regular.contains(attr) - } - - /// Checks if these two `ArgAttributes` are equal enough to be considered "the same for all - /// function call ABIs". - pub fn eq_abi(&self, other: &Self) -> bool { - // There's only one regular attribute that matters for the call ABI: InReg. - // Everything else is things like noalias, dereferenceable, nonnull, ... - // (This also applies to pointee_size, pointee_align.) - if self.regular.contains(ArgAttribute::InReg) != other.regular.contains(ArgAttribute::InReg) - { - return false; - } - // We also compare the sign extension mode -- this could let the callee make assumptions - // about bits that conceptually were not even passed. - if self.arg_ext != other.arg_ext { - return false; - } - true - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub enum RegKind { - Integer, - Float, - Vector, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub struct Reg { - pub kind: RegKind, - pub size: Size, -} - -macro_rules! reg_ctor { - ($name:ident, $kind:ident, $bits:expr) => { - pub fn $name() -> Reg { - Reg { kind: RegKind::$kind, size: Size::from_bits($bits) } - } - }; -} - -impl Reg { - reg_ctor!(i8, Integer, 8); - reg_ctor!(i16, Integer, 16); - reg_ctor!(i32, Integer, 32); - reg_ctor!(i64, Integer, 64); - reg_ctor!(i128, Integer, 128); - - reg_ctor!(f32, Float, 32); - reg_ctor!(f64, Float, 64); -} - -impl Reg { - pub fn align(&self, cx: &C) -> Align { - let dl = cx.data_layout(); - match self.kind { - RegKind::Integer => match self.size.bits() { - 1 => dl.i1_align.abi, - 2..=8 => dl.i8_align.abi, - 9..=16 => dl.i16_align.abi, - 17..=32 => dl.i32_align.abi, - 33..=64 => dl.i64_align.abi, - 65..=128 => dl.i128_align.abi, - _ => panic!("unsupported integer: {self:?}"), - }, - RegKind::Float => match self.size.bits() { - 16 => dl.f16_align.abi, - 32 => dl.f32_align.abi, - 64 => dl.f64_align.abi, - 128 => dl.f128_align.abi, - _ => panic!("unsupported float: {self:?}"), - }, - RegKind::Vector => dl.vector_align(self.size).abi, - } - } -} - -/// An argument passed entirely registers with the -/// same kind (e.g., HFA / HVA on PPC64 and AArch64). -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub struct Uniform { - pub unit: Reg, - - /// The total size of the argument, which can be: - /// * equal to `unit.size` (one scalar/vector), - /// * a multiple of `unit.size` (an array of scalar/vectors), - /// * if `unit.kind` is `Integer`, the last element can be shorter, i.e., `{ i64, i64, i32 }` - /// for 64-bit integers with a total size of 20 bytes. When the argument is actually passed, - /// this size will be rounded up to the nearest multiple of `unit.size`. - pub total: Size, - - /// Indicate that the argument is consecutive, in the sense that either all values need to be - /// passed in register, or all on the stack. If they are passed on the stack, there should be - /// no additional padding between elements. - pub is_consecutive: bool, -} - -impl From for Uniform { - fn from(unit: Reg) -> Uniform { - Uniform { unit, total: unit.size, is_consecutive: false } - } -} - -impl Uniform { - pub fn align(&self, cx: &C) -> Align { - self.unit.align(cx) - } - - /// Pass using one or more values of the given type, without requiring them to be consecutive. - /// That is, some values may be passed in register and some on the stack. - pub fn new(unit: Reg, total: Size) -> Self { - Uniform { unit, total, is_consecutive: false } - } - - /// Pass using one or more consecutive values of the given type. Either all values will be - /// passed in registers, or all on the stack. - pub fn consecutive(unit: Reg, total: Size) -> Self { - Uniform { unit, total, is_consecutive: true } - } -} - -/// Describes the type used for `PassMode::Cast`. -/// -/// Passing arguments in this mode works as follows: the registers in the `prefix` (the ones that -/// are `Some`) get laid out one after the other (using `repr(C)` layout rules). Then the -/// `rest.unit` register type gets repeated often enough to cover `rest.size`. This describes the -/// actual type used for the call; the Rust type of the argument is then transmuted to this ABI type -/// (and all data in the padding between the registers is dropped). -#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub struct CastTarget { - pub prefix: [Option; 8], - pub rest: Uniform, - pub attrs: ArgAttributes, -} - -impl From for CastTarget { - fn from(unit: Reg) -> CastTarget { - CastTarget::from(Uniform::from(unit)) - } -} - -impl From for CastTarget { - fn from(uniform: Uniform) -> CastTarget { - CastTarget { - prefix: [None; 8], - rest: uniform, - attrs: ArgAttributes { - regular: ArgAttribute::default(), - arg_ext: ArgExtension::None, - pointee_size: Size::ZERO, - pointee_align: None, - }, - } - } -} - -impl CastTarget { - pub fn pair(a: Reg, b: Reg) -> CastTarget { - CastTarget { - prefix: [Some(a), None, None, None, None, None, None, None], - rest: Uniform::from(b), - attrs: ArgAttributes { - regular: ArgAttribute::default(), - arg_ext: ArgExtension::None, - pointee_size: Size::ZERO, - pointee_align: None, - }, - } - } - - /// When you only access the range containing valid data, you can use this unaligned size; - /// otherwise, use the safer `size` method. - pub fn unaligned_size(&self, _cx: &C) -> Size { - // Prefix arguments are passed in specific designated registers - let prefix_size = self - .prefix - .iter() - .filter_map(|x| x.map(|reg| reg.size)) - .fold(Size::ZERO, |acc, size| acc + size); - // Remaining arguments are passed in chunks of the unit size - let rest_size = - self.rest.unit.size * self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes()); - - prefix_size + rest_size - } - - pub fn size(&self, cx: &C) -> Size { - self.unaligned_size(cx).align_to(self.align(cx)) - } - - pub fn align(&self, cx: &C) -> Align { - self.prefix - .iter() - .filter_map(|x| x.map(|reg| reg.align(cx))) - .fold(cx.data_layout().aggregate_align.abi.max(self.rest.align(cx)), |acc, align| { - acc.max(align) - }) - } - - /// Checks if these two `CastTarget` are equal enough to be considered "the same for all - /// function call ABIs". - pub fn eq_abi(&self, other: &Self) -> bool { - let CastTarget { prefix: prefix_l, rest: rest_l, attrs: attrs_l } = self; - let CastTarget { prefix: prefix_r, rest: rest_r, attrs: attrs_r } = other; - prefix_l == prefix_r && rest_l == rest_r && attrs_l.eq_abi(attrs_r) - } -} - -/// Return value from the `homogeneous_aggregate` test function. -#[derive(Copy, Clone, Debug)] -pub enum HomogeneousAggregate { - /// Yes, all the "leaf fields" of this struct are passed in the - /// same way (specified in the `Reg` value). - Homogeneous(Reg), - - /// There are no leaf fields at all. - NoData, -} - -/// Error from the `homogeneous_aggregate` test function, indicating -/// there are distinct leaf fields passed in different ways, -/// or this is uninhabited. -#[derive(Copy, Clone, Debug)] -pub struct Heterogeneous; - -impl HomogeneousAggregate { - /// If this is a homogeneous aggregate, returns the homogeneous - /// unit, else `None`. - pub fn unit(self) -> Option { - match self { - HomogeneousAggregate::Homogeneous(reg) => Some(reg), - HomogeneousAggregate::NoData => None, - } - } - - /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in - /// the same `struct`. Only succeeds if only one of them has any data, - /// or both units are identical. - fn merge(self, other: HomogeneousAggregate) -> Result { - match (self, other) { - (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x), - - (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => { - if a != b { - return Err(Heterogeneous); - } - Ok(self) - } - } - } -} - -impl<'a, Ty> TyAndLayout<'a, Ty> { - /// Returns `true` if this is an aggregate type (including a ScalarPair!) - fn is_aggregate(&self) -> bool { - match self.abi { - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, - Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, - } - } - - /// Returns `Homogeneous` if this layout is an aggregate containing fields of - /// only a single type (e.g., `(u32, u32)`). Such aggregates are often - /// special-cased in ABIs. - /// - /// Note: We generally ignore 1-ZST fields when computing this value (see #56877). - /// - /// This is public so that it can be used in unit tests, but - /// should generally only be relevant to the ABI details of - /// specific targets. - pub fn homogeneous_aggregate(&self, cx: &C) -> Result - where - Ty: TyAbiInterface<'a, C> + Copy, - { - match self.abi { - Abi::Uninhabited => Err(Heterogeneous), - - // The primitive for this algorithm. - Abi::Scalar(scalar) => { - let kind = match scalar.primitive() { - abi::Int(..) | abi::Pointer(_) => RegKind::Integer, - abi::Float(_) => RegKind::Float, - }; - Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size })) - } - - Abi::Vector { .. } => { - assert!(!self.is_zst()); - Ok(HomogeneousAggregate::Homogeneous(Reg { - kind: RegKind::Vector, - size: self.size, - })) - } - - Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => { - // Helper for computing `homogeneous_aggregate`, allowing a custom - // starting offset (used below for handling variants). - let from_fields_at = - |layout: Self, - start: Size| - -> Result<(HomogeneousAggregate, Size), Heterogeneous> { - let is_union = match layout.fields { - FieldsShape::Primitive => { - unreachable!("aggregates can't have `FieldsShape::Primitive`") - } - FieldsShape::Array { count, .. } => { - assert_eq!(start, Size::ZERO); - - let result = if count > 0 { - layout.field(cx, 0).homogeneous_aggregate(cx)? - } else { - HomogeneousAggregate::NoData - }; - return Ok((result, layout.size)); - } - FieldsShape::Union(_) => true, - FieldsShape::Arbitrary { .. } => false, - }; - - let mut result = HomogeneousAggregate::NoData; - let mut total = start; - - for i in 0..layout.fields.count() { - let field = layout.field(cx, i); - if field.is_1zst() { - // No data here and no impact on layout, can be ignored. - // (We might be able to also ignore all aligned ZST but that's less clear.) - continue; - } - - if !is_union && total != layout.fields.offset(i) { - // This field isn't just after the previous one we considered, abort. - return Err(Heterogeneous); - } - - result = result.merge(field.homogeneous_aggregate(cx)?)?; - - // Keep track of the offset (without padding). - let size = field.size; - if is_union { - total = total.max(size); - } else { - total += size; - } - } - - Ok((result, total)) - }; - - let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?; - - match &self.variants { - abi::Variants::Single { .. } => {} - abi::Variants::Multiple { variants, .. } => { - // Treat enum variants like union members. - // HACK(eddyb) pretend the `enum` field (discriminant) - // is at the start of every variant (otherwise the gap - // at the start of all variants would disqualify them). - // - // NB: for all tagged `enum`s (which include all non-C-like - // `enum`s with defined FFI representation), this will - // match the homogeneous computation on the equivalent - // `struct { tag; union { variant1; ... } }` and/or - // `union { struct { tag; variant1; } ... }` - // (the offsets of variant fields should be identical - // between the two for either to be a homogeneous aggregate). - let variant_start = total; - for variant_idx in variants.indices() { - let (variant_result, variant_total) = - from_fields_at(self.for_variant(cx, variant_idx), variant_start)?; - - result = result.merge(variant_result)?; - total = total.max(variant_total); - } - } - } - - // There needs to be no padding. - if total != self.size { - Err(Heterogeneous) - } else { - match result { - HomogeneousAggregate::Homogeneous(_) => { - assert_ne!(total, Size::ZERO); - } - HomogeneousAggregate::NoData => { - assert_eq!(total, Size::ZERO); - } - } - Ok(result) - } - } - Abi::Aggregate { sized: false } => Err(Heterogeneous), - } - } -} - -/// Information about how to pass an argument to, -/// or return a value from, a function, under some ABI. -#[derive(Clone, PartialEq, Eq, Hash, HashStable_Generic)] -pub struct ArgAbi<'a, Ty> { - pub layout: TyAndLayout<'a, Ty>, - pub mode: PassMode, -} - -// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl. -impl<'a, Ty: fmt::Display> fmt::Debug for ArgAbi<'a, Ty> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let ArgAbi { layout, mode } = self; - f.debug_struct("ArgAbi").field("layout", layout).field("mode", mode).finish() - } -} - -impl<'a, Ty> ArgAbi<'a, Ty> { - /// This defines the "default ABI" for that type, that is then later adjusted in `fn_abi_adjust_for_abi`. - pub fn new( - cx: &impl HasDataLayout, - layout: TyAndLayout<'a, Ty>, - scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, abi::Scalar, Size) -> ArgAttributes, - ) -> Self { - let mode = match layout.abi { - Abi::Uninhabited => PassMode::Ignore, - Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)), - Abi::ScalarPair(a, b) => PassMode::Pair( - scalar_attrs(&layout, a, Size::ZERO), - scalar_attrs(&layout, b, a.size(cx).align_to(b.align(cx).abi)), - ), - Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()), - Abi::Aggregate { .. } => Self::indirect_pass_mode(&layout), - }; - ArgAbi { layout, mode } - } - - fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode { - let mut attrs = ArgAttributes::new(); - - // For non-immediate arguments the callee gets its own copy of - // the value on the stack, so there are no aliases. It's also - // program-invisible so can't possibly capture - attrs - .set(ArgAttribute::NoAlias) - .set(ArgAttribute::NoCapture) - .set(ArgAttribute::NonNull) - .set(ArgAttribute::NoUndef); - attrs.pointee_size = layout.size; - attrs.pointee_align = Some(layout.align.abi); - - let meta_attrs = layout.is_unsized().then_some(ArgAttributes::new()); - - PassMode::Indirect { attrs, meta_attrs, on_stack: false } - } - - /// Pass this argument directly instead. Should NOT be used! - /// Only exists because of past ABI mistakes that will take time to fix - /// (see ). - pub fn make_direct_deprecated(&mut self) { - match self.mode { - PassMode::Indirect { .. } => { - self.mode = PassMode::Direct(ArgAttributes::new()); - } - PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct - _ => panic!("Tried to make {:?} direct", self.mode), - } - } - - /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead. - /// This is valid for both sized and unsized arguments. - pub fn make_indirect(&mut self) { - match self.mode { - PassMode::Direct(_) | PassMode::Pair(_, _) => { - self.mode = Self::indirect_pass_mode(&self.layout); - } - PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => { - // already indirect - } - _ => panic!("Tried to make {:?} indirect", self.mode), - } - } - - /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass - /// ZSTs indirectly. - pub fn make_indirect_from_ignore(&mut self) { - match self.mode { - PassMode::Ignore => { - self.mode = Self::indirect_pass_mode(&self.layout); - } - PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => { - // already indirect - } - _ => panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode), - } - } - - /// Pass this argument indirectly, by placing it at a fixed stack offset. - /// This corresponds to the `byval` LLVM argument attribute. - /// This is only valid for sized arguments. - /// - /// `byval_align` specifies the alignment of the `byval` stack slot, which does not need to - /// correspond to the type's alignment. This will be `Some` if the target's ABI specifies that - /// stack slots used for arguments passed by-value have specific alignment requirements which - /// differ from the alignment used in other situations. - /// - /// If `None`, the type's alignment is used. - /// - /// If the resulting alignment differs from the type's alignment, - /// the argument will be copied to an alloca with sufficient alignment, - /// either in the caller (if the type's alignment is lower than the byval alignment) - /// or in the callee (if the type's alignment is higher than the byval alignment), - /// to ensure that Rust code never sees an underaligned pointer. - pub fn pass_by_stack_offset(&mut self, byval_align: Option) { - assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout"); - self.make_indirect(); - match self.mode { - PassMode::Indirect { ref mut attrs, meta_attrs: _, ref mut on_stack } => { - *on_stack = true; - - // Some platforms, like 32-bit x86, change the alignment of the type when passing - // `byval`. Account for that. - if let Some(byval_align) = byval_align { - // On all targets with byval align this is currently true, so let's assert it. - debug_assert!(byval_align >= Align::from_bytes(4).unwrap()); - attrs.pointee_align = Some(byval_align); - } - } - _ => unreachable!(), - } - } - - pub fn extend_integer_width_to(&mut self, bits: u64) { - // Only integers have signedness - if let Abi::Scalar(scalar) = self.layout.abi { - if let abi::Int(i, signed) = scalar.primitive() { - if i.size().bits() < bits { - if let PassMode::Direct(ref mut attrs) = self.mode { - if signed { - attrs.ext(ArgExtension::Sext) - } else { - attrs.ext(ArgExtension::Zext) - }; - } - } - } - } - } - - pub fn cast_to>(&mut self, target: T) { - self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32: false }; - } - - pub fn cast_to_and_pad_i32>(&mut self, target: T, pad_i32: bool) { - self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32 }; - } - - pub fn is_indirect(&self) -> bool { - matches!(self.mode, PassMode::Indirect { .. }) - } - - pub fn is_sized_indirect(&self) -> bool { - matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }) - } - - pub fn is_unsized_indirect(&self) -> bool { - matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }) - } - - pub fn is_ignore(&self) -> bool { - matches!(self.mode, PassMode::Ignore) - } - - /// Checks if these two `ArgAbi` are equal enough to be considered "the same for all - /// function call ABIs". - pub fn eq_abi(&self, other: &Self) -> bool - where - Ty: PartialEq, - { - // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will look - // at the type. - self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode) && { - // `fn_arg_sanity_check` accepts `PassMode::Direct` for some aggregates. - // That elevates any type difference to an ABI difference since we just use the - // full Rust type as the LLVM argument/return type. - if matches!(self.mode, PassMode::Direct(..)) - && matches!(self.layout.abi, Abi::Aggregate { .. }) - { - // For aggregates in `Direct` mode to be compatible, the types need to be equal. - self.layout.ty == other.layout.ty - } else { - true - } - } - } -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub enum Conv { - // General language calling conventions, for which every target - // should have its own backend (e.g. LLVM) support. - C, - Rust, - - Cold, - PreserveMost, - PreserveAll, - - // Target-specific calling conventions. - ArmAapcs, - CCmseNonSecureCall, - CCmseNonSecureEntry, - - Msp430Intr, - - PtxKernel, - - X86Fastcall, - X86Intr, - X86Stdcall, - X86ThisCall, - X86VectorCall, - - X86_64SysV, - X86_64Win64, - - AvrInterrupt, - AvrNonBlockingInterrupt, - - RiscvInterrupt { kind: RiscvInterruptKind }, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub enum RiscvInterruptKind { - Machine, - Supervisor, -} - -impl RiscvInterruptKind { - pub fn as_str(&self) -> &'static str { - match self { - Self::Machine => "machine", - Self::Supervisor => "supervisor", - } - } -} - -/// Metadata describing how the arguments to a native function -/// should be passed in order to respect the native ABI. -/// -/// The signature represented by this type may not match the MIR function signature. -/// Certain attributes, like `#[track_caller]` can introduce additional arguments, which are present in [`FnAbi`], but not in `FnSig`. -/// While this difference is rarely relevant, it should still be kept in mind. -/// -/// I will do my best to describe this structure, but these -/// comments are reverse-engineered and may be inaccurate. -NDM -#[derive(Clone, PartialEq, Eq, Hash, HashStable_Generic)] -pub struct FnAbi<'a, Ty> { - /// The type, layout, and information about how each argument is passed. - pub args: Box<[ArgAbi<'a, Ty>]>, - - /// The layout, type, and the way a value is returned from this function. - pub ret: ArgAbi<'a, Ty>, - - /// Marks this function as variadic (accepting a variable number of arguments). - pub c_variadic: bool, - - /// The count of non-variadic arguments. - /// - /// Should only be different from args.len() when c_variadic is true. - /// This can be used to know whether an argument is variadic or not. - pub fixed_count: u32, - /// The calling convention of this function. - pub conv: Conv, - /// Indicates if an unwind may happen across a call to this function. - pub can_unwind: bool, -} - -// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl. -impl<'a, Ty: fmt::Display> fmt::Debug for FnAbi<'a, Ty> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let FnAbi { args, ret, c_variadic, fixed_count, conv, can_unwind } = self; - f.debug_struct("FnAbi") - .field("args", args) - .field("ret", ret) - .field("c_variadic", c_variadic) - .field("fixed_count", fixed_count) - .field("conv", conv) - .field("can_unwind", can_unwind) - .finish() - } -} - -/// Error produced by attempting to adjust a `FnAbi`, for a "foreign" ABI. -#[derive(Copy, Clone, Debug, HashStable_Generic)] -pub enum AdjustForForeignAbiError { - /// Target architecture doesn't support "foreign" (i.e. non-Rust) ABIs. - Unsupported { arch: Symbol, abi: spec::abi::Abi }, -} - -impl<'a, Ty> FnAbi<'a, Ty> { - pub fn adjust_for_foreign_abi( - &mut self, - cx: &C, - abi: spec::abi::Abi, - ) -> Result<(), AdjustForForeignAbiError> - where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt, - { - if abi == spec::abi::Abi::X86Interrupt { - if let Some(arg) = self.args.first_mut() { - arg.pass_by_stack_offset(None); - } - return Ok(()); - } - - let spec = cx.target_spec(); - match &spec.arch[..] { - "x86" => { - let flavor = if let spec::abi::Abi::Fastcall { .. } - | spec::abi::Abi::Vectorcall { .. } = abi - { - x86::Flavor::FastcallOrVectorcall - } else { - x86::Flavor::General - }; - x86::compute_abi_info(cx, self, flavor); - } - "x86_64" => match abi { - spec::abi::Abi::SysV64 { .. } => x86_64::compute_abi_info(cx, self), - spec::abi::Abi::Win64 { .. } => x86_win64::compute_abi_info(cx, self), - _ => { - if cx.target_spec().is_like_windows { - x86_win64::compute_abi_info(cx, self) - } else { - x86_64::compute_abi_info(cx, self) - } - } - }, - "aarch64" | "arm64ec" => { - let kind = if cx.target_spec().is_like_osx { - aarch64::AbiKind::DarwinPCS - } else if cx.target_spec().is_like_windows { - aarch64::AbiKind::Win64 - } else { - aarch64::AbiKind::AAPCS - }; - aarch64::compute_abi_info(cx, self, kind) - } - "amdgpu" => amdgpu::compute_abi_info(cx, self), - "arm" => arm::compute_abi_info(cx, self), - "avr" => avr::compute_abi_info(self), - "loongarch64" => loongarch::compute_abi_info(cx, self), - "m68k" => m68k::compute_abi_info(self), - "csky" => csky::compute_abi_info(self), - "mips" | "mips32r6" => mips::compute_abi_info(cx, self), - "mips64" | "mips64r6" => mips64::compute_abi_info(cx, self), - "powerpc" => powerpc::compute_abi_info(cx, self), - "powerpc64" => powerpc64::compute_abi_info(cx, self), - "s390x" => s390x::compute_abi_info(cx, self), - "msp430" => msp430::compute_abi_info(self), - "sparc" => sparc::compute_abi_info(cx, self), - "sparc64" => sparc64::compute_abi_info(cx, self), - "nvptx64" => { - if cx.target_spec().adjust_abi(abi, self.c_variadic) == spec::abi::Abi::PtxKernel { - nvptx64::compute_ptx_kernel_abi_info(cx, self) - } else { - nvptx64::compute_abi_info(self) - } - } - "hexagon" => hexagon::compute_abi_info(self), - "xtensa" => xtensa::compute_abi_info(cx, self), - "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self), - "wasm32" => { - if spec.os == "unknown" && cx.wasm_c_abi_opt() == WasmCAbi::Legacy { - wasm::compute_wasm_abi_info(self) - } else { - wasm::compute_c_abi_info(cx, self) - } - } - "wasm64" => wasm::compute_c_abi_info(cx, self), - "bpf" => bpf::compute_abi_info(self), - arch => { - return Err(AdjustForForeignAbiError::Unsupported { - arch: Symbol::intern(arch), - abi, - }); - } - } - - Ok(()) - } -} - -impl FromStr for Conv { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "C" => Ok(Conv::C), - "Rust" => Ok(Conv::Rust), - "RustCold" => Ok(Conv::Rust), - "ArmAapcs" => Ok(Conv::ArmAapcs), - "CCmseNonSecureCall" => Ok(Conv::CCmseNonSecureCall), - "CCmseNonSecureEntry" => Ok(Conv::CCmseNonSecureEntry), - "Msp430Intr" => Ok(Conv::Msp430Intr), - "PtxKernel" => Ok(Conv::PtxKernel), - "X86Fastcall" => Ok(Conv::X86Fastcall), - "X86Intr" => Ok(Conv::X86Intr), - "X86Stdcall" => Ok(Conv::X86Stdcall), - "X86ThisCall" => Ok(Conv::X86ThisCall), - "X86VectorCall" => Ok(Conv::X86VectorCall), - "X86_64SysV" => Ok(Conv::X86_64SysV), - "X86_64Win64" => Ok(Conv::X86_64Win64), - "AvrInterrupt" => Ok(Conv::AvrInterrupt), - "AvrNonBlockingInterrupt" => Ok(Conv::AvrNonBlockingInterrupt), - "RiscvInterrupt(machine)" => { - Ok(Conv::RiscvInterrupt { kind: RiscvInterruptKind::Machine }) - } - "RiscvInterrupt(supervisor)" => { - Ok(Conv::RiscvInterrupt { kind: RiscvInterruptKind::Supervisor }) - } - _ => Err(format!("'{s}' is not a valid value for entry function call convention.")), - } - } -} - -// Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(target_pointer_width = "64")] -mod size_asserts { - use rustc_data_structures::static_assert_size; - - use super::*; - // tidy-alphabetical-start - static_assert_size!(ArgAbi<'_, usize>, 56); - static_assert_size!(FnAbi<'_, usize>, 80); - // tidy-alphabetical-end -} diff --git a/compiler/rustc_target/src/abi/call/msp430.rs b/compiler/rustc_target/src/abi/call/msp430.rs deleted file mode 100644 index 4f613aa6c15..00000000000 --- a/compiler/rustc_target/src/abi/call/msp430.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Reference: MSP430 Embedded Application Binary Interface -// https://www.ti.com/lit/an/slaa534a/slaa534a.pdf - -use crate::abi::call::{ArgAbi, FnAbi}; - -// 3.5 Structures or Unions Passed and Returned by Reference -// -// "Structures (including classes) and unions larger than 32 bits are passed and -// returned by reference. To pass a structure or union by reference, the caller -// places its address in the appropriate location: either in a register or on -// the stack, according to its position in the argument list. (..)" -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() && ret.layout.size.bits() > 32 { - ret.make_indirect(); - } else { - ret.extend_integer_width_to(16); - } -} - -fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { - if arg.layout.is_aggregate() && arg.layout.size.bits() > 32 { - arg.make_indirect(); - } else { - arg.extend_integer_width_to(16); - } -} - -pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/nvptx64.rs b/compiler/rustc_target/src/abi/call/nvptx64.rs deleted file mode 100644 index 2e8b16d3a93..00000000000 --- a/compiler/rustc_target/src/abi/call/nvptx64.rs +++ /dev/null @@ -1,102 +0,0 @@ -use super::{ArgAttribute, ArgAttributes, ArgExtension, CastTarget}; -use crate::abi::call::{ArgAbi, FnAbi, PassMode, Reg, Size, Uniform}; -use crate::abi::{HasDataLayout, TyAbiInterface}; - -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() && ret.layout.is_sized() { - classify_aggregate(ret) - } else if ret.layout.size.bits() < 32 && ret.layout.is_sized() { - ret.extend_integer_width_to(32); - } -} - -fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { - if arg.layout.is_aggregate() && arg.layout.is_sized() { - classify_aggregate(arg) - } else if arg.layout.size.bits() < 32 && arg.layout.is_sized() { - arg.extend_integer_width_to(32); - } -} - -/// the pass mode used for aggregates in arg and ret position -fn classify_aggregate(arg: &mut ArgAbi<'_, Ty>) { - let align_bytes = arg.layout.align.abi.bytes(); - let size = arg.layout.size; - - let reg = match align_bytes { - 1 => Reg::i8(), - 2 => Reg::i16(), - 4 => Reg::i32(), - 8 => Reg::i64(), - 16 => Reg::i128(), - _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"), - }; - - if align_bytes == size.bytes() { - arg.cast_to(CastTarget { - prefix: [Some(reg), None, None, None, None, None, None, None], - rest: Uniform::new(Reg::i8(), Size::from_bytes(0)), - attrs: ArgAttributes { - regular: ArgAttribute::default(), - arg_ext: ArgExtension::None, - pointee_size: Size::ZERO, - pointee_align: None, - }, - }); - } else { - arg.cast_to(Uniform::new(reg, size)); - } -} - -fn classify_arg_kernel<'a, Ty, C>(_cx: &C, arg: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if matches!(arg.mode, PassMode::Pair(..)) && (arg.layout.is_adt() || arg.layout.is_tuple()) { - let align_bytes = arg.layout.align.abi.bytes(); - - let unit = match align_bytes { - 1 => Reg::i8(), - 2 => Reg::i16(), - 4 => Reg::i32(), - 8 => Reg::i64(), - 16 => Reg::i128(), - _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"), - }; - arg.cast_to(Uniform::new(unit, Size::from_bytes(2 * align_bytes))); - } else { - // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271. - arg.make_direct_deprecated(); - } -} - -pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(arg); - } -} - -pub(crate) fn compute_ptx_kernel_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !fn_abi.ret.layout.is_unit() && !fn_abi.ret.layout.is_never() { - panic!("Kernels should not return anything other than () or !"); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg_kernel(cx, arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/powerpc.rs b/compiler/rustc_target/src/abi/call/powerpc.rs deleted file mode 100644 index f3b05c48173..00000000000 --- a/compiler/rustc_target/src/abi/call/powerpc.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi}; -use crate::spec::HasTargetSpec; - -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if ret.layout.is_aggregate() { - ret.make_indirect(); - } else { - ret.extend_integer_width_to(32); - } -} - -fn classify_arg(cx: &impl HasTargetSpec, arg: &mut ArgAbi<'_, Ty>) { - if arg.is_ignore() { - // powerpc-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs. - if cx.target_spec().os == "linux" - && matches!(&*cx.target_spec().env, "gnu" | "musl" | "uclibc") - && arg.layout.is_zst() - { - arg.make_indirect_from_ignore(); - } - return; - } - if arg.layout.is_aggregate() { - arg.make_indirect(); - } else { - arg.extend_integer_width_to(32); - } -} - -pub(crate) fn compute_abi_info(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - classify_arg(cx, arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs deleted file mode 100644 index 71e533b8cc5..00000000000 --- a/compiler/rustc_target/src/abi/call/powerpc64.rs +++ /dev/null @@ -1,118 +0,0 @@ -// FIXME: -// Alignment of 128 bit types is not currently handled, this will -// need to be fixed when PowerPC vector support is added. - -use crate::abi::call::{Align, ArgAbi, FnAbi, Reg, RegKind, Uniform}; -use crate::abi::{Endian, HasDataLayout, TyAbiInterface}; -use crate::spec::HasTargetSpec; - -#[derive(Debug, Clone, Copy, PartialEq)] -enum ABI { - ELFv1, // original ABI used for powerpc64 (big-endian) - ELFv2, // newer ABI used for powerpc64le and musl (both endians) - AIX, // used by AIX OS, big-endian only -} -use ABI::*; - -fn is_homogeneous_aggregate<'a, Ty, C>( - cx: &C, - arg: &mut ArgAbi<'a, Ty>, - abi: ABI, -) -> Option -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { - // ELFv1 and AIX only passes one-member aggregates transparently. - // ELFv2 passes up to eight uniquely addressable members. - if ((abi == ELFv1 || abi == AIX) && arg.layout.size > unit.size) - || arg.layout.size > unit.size.checked_mul(8, cx).unwrap() - { - return None; - } - - let valid_unit = match unit.kind { - RegKind::Integer => false, - RegKind::Float => true, - RegKind::Vector => arg.layout.size.bits() == 128, - }; - - valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size)) - }) -} - -fn classify<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, abi: ABI, is_ret: bool) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if arg.is_ignore() || !arg.layout.is_sized() { - // Not touching this... - return; - } - if !arg.layout.is_aggregate() { - arg.extend_integer_width_to(64); - return; - } - - // The AIX ABI expect byval for aggregates - // See https://github.com/llvm/llvm-project/blob/main/clang/lib/CodeGen/Targets/PPC.cpp. - if !is_ret && abi == AIX { - arg.pass_by_stack_offset(None); - return; - } - - // The ELFv1 ABI doesn't return aggregates in registers - if is_ret && (abi == ELFv1 || abi == AIX) { - arg.make_indirect(); - return; - } - - if let Some(uniform) = is_homogeneous_aggregate(cx, arg, abi) { - arg.cast_to(uniform); - return; - } - - let size = arg.layout.size; - if is_ret && size.bits() > 128 { - // Non-homogeneous aggregates larger than two doublewords are returned indirectly. - arg.make_indirect(); - } else if size.bits() <= 64 { - // Aggregates smaller than a doubleword should appear in - // the least-significant bits of the parameter doubleword. - arg.cast_to(Reg { kind: RegKind::Integer, size }) - } else { - // Aggregates larger than i64 should be padded at the tail to fill out a whole number - // of i64s or i128s, depending on the aggregate alignment. Always use an array for - // this, even if there is only a single element. - let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() }; - arg.cast_to(Uniform::consecutive( - reg, - size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()), - )) - }; -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - let abi = if cx.target_spec().env == "musl" { - ELFv2 - } else if cx.target_spec().os == "aix" { - AIX - } else { - match cx.data_layout().endian { - Endian::Big => ELFv1, - Endian::Little => ELFv2, - } - }; - - classify(cx, &mut fn_abi.ret, abi, true); - - for arg in fn_abi.args.iter_mut() { - classify(cx, arg, abi, false); - } -} diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs deleted file mode 100644 index be6bc701b49..00000000000 --- a/compiler/rustc_target/src/abi/call/riscv.rs +++ /dev/null @@ -1,367 +0,0 @@ -// Reference: RISC-V ELF psABI specification -// https://github.com/riscv/riscv-elf-psabi-doc -// -// Reference: Clang RISC-V ELF psABI lowering code -// https://github.com/llvm/llvm-project/blob/8e780252a7284be45cf1ba224cabd884847e8e92/clang/lib/CodeGen/TargetInfo.cpp#L9311-L9773 - -use crate::abi::call::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Reg, RegKind, Uniform}; -use crate::abi::{self, Abi, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; -use crate::spec::HasTargetSpec; - -#[derive(Copy, Clone)] -enum RegPassKind { - Float(Reg), - Integer(Reg), - Unknown, -} - -#[derive(Copy, Clone)] -enum FloatConv { - FloatPair(Reg, Reg), - Float(Reg), - MixedPair(Reg, Reg), -} - -#[derive(Copy, Clone)] -struct CannotUseFpConv; - -fn is_riscv_aggregate(arg: &ArgAbi<'_, Ty>) -> bool { - match arg.layout.abi { - Abi::Vector { .. } => true, - _ => arg.layout.is_aggregate(), - } -} - -fn should_use_fp_conv_helper<'a, Ty, C>( - cx: &C, - arg_layout: &TyAndLayout<'a, Ty>, - xlen: u64, - flen: u64, - field1_kind: &mut RegPassKind, - field2_kind: &mut RegPassKind, -) -> Result<(), CannotUseFpConv> -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - match arg_layout.abi { - Abi::Scalar(scalar) => match scalar.primitive() { - abi::Int(..) | abi::Pointer(_) => { - if arg_layout.size.bits() > xlen { - return Err(CannotUseFpConv); - } - match (*field1_kind, *field2_kind) { - (RegPassKind::Unknown, _) => { - *field1_kind = RegPassKind::Integer(Reg { - kind: RegKind::Integer, - size: arg_layout.size, - }); - } - (RegPassKind::Float(_), RegPassKind::Unknown) => { - *field2_kind = RegPassKind::Integer(Reg { - kind: RegKind::Integer, - size: arg_layout.size, - }); - } - _ => return Err(CannotUseFpConv), - } - } - abi::Float(_) => { - if arg_layout.size.bits() > flen { - return Err(CannotUseFpConv); - } - match (*field1_kind, *field2_kind) { - (RegPassKind::Unknown, _) => { - *field1_kind = - RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); - } - (_, RegPassKind::Unknown) => { - *field2_kind = - RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); - } - _ => return Err(CannotUseFpConv), - } - } - }, - Abi::Vector { .. } | Abi::Uninhabited => return Err(CannotUseFpConv), - Abi::ScalarPair(..) | Abi::Aggregate { .. } => match arg_layout.fields { - FieldsShape::Primitive => { - unreachable!("aggregates can't have `FieldsShape::Primitive`") - } - FieldsShape::Union(_) => { - if !arg_layout.is_zst() { - if arg_layout.is_transparent() { - let non_1zst_elem = arg_layout.non_1zst_field(cx).expect("not exactly one non-1-ZST field in non-ZST repr(transparent) union").1; - return should_use_fp_conv_helper( - cx, - &non_1zst_elem, - xlen, - flen, - field1_kind, - field2_kind, - ); - } - return Err(CannotUseFpConv); - } - } - FieldsShape::Array { count, .. } => { - for _ in 0..count { - let elem_layout = arg_layout.field(cx, 0); - should_use_fp_conv_helper( - cx, - &elem_layout, - xlen, - flen, - field1_kind, - field2_kind, - )?; - } - } - FieldsShape::Arbitrary { .. } => { - match arg_layout.variants { - abi::Variants::Multiple { .. } => return Err(CannotUseFpConv), - abi::Variants::Single { .. } => (), - } - for i in arg_layout.fields.index_by_increasing_offset() { - let field = arg_layout.field(cx, i); - should_use_fp_conv_helper(cx, &field, xlen, flen, field1_kind, field2_kind)?; - } - } - }, - } - Ok(()) -} - -fn should_use_fp_conv<'a, Ty, C>( - cx: &C, - arg: &TyAndLayout<'a, Ty>, - xlen: u64, - flen: u64, -) -> Option -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - let mut field1_kind = RegPassKind::Unknown; - let mut field2_kind = RegPassKind::Unknown; - if should_use_fp_conv_helper(cx, arg, xlen, flen, &mut field1_kind, &mut field2_kind).is_err() { - return None; - } - match (field1_kind, field2_kind) { - (RegPassKind::Integer(l), RegPassKind::Float(r)) => Some(FloatConv::MixedPair(l, r)), - (RegPassKind::Float(l), RegPassKind::Integer(r)) => Some(FloatConv::MixedPair(l, r)), - (RegPassKind::Float(l), RegPassKind::Float(r)) => Some(FloatConv::FloatPair(l, r)), - (RegPassKind::Float(f), RegPassKind::Unknown) => Some(FloatConv::Float(f)), - _ => None, - } -} - -fn classify_ret<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, xlen: u64, flen: u64) -> bool -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - if !arg.layout.is_sized() { - // Not touching this... - return false; // I guess? return value of this function is not documented - } - if let Some(conv) = should_use_fp_conv(cx, &arg.layout, xlen, flen) { - match conv { - FloatConv::Float(f) => { - arg.cast_to(f); - } - FloatConv::FloatPair(l, r) => { - arg.cast_to(CastTarget::pair(l, r)); - } - FloatConv::MixedPair(l, r) => { - arg.cast_to(CastTarget::pair(l, r)); - } - } - return false; - } - - let total = arg.layout.size; - - // "Scalars wider than 2✕XLEN are passed by reference and are replaced in - // the argument list with the address." - // "Aggregates larger than 2✕XLEN bits are passed by reference and are - // replaced in the argument list with the address, as are C++ aggregates - // with nontrivial copy constructors, destructors, or vtables." - if total.bits() > 2 * xlen { - // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. - if is_riscv_aggregate(arg) { - arg.make_indirect(); - } - return true; - } - - let xlen_reg = match xlen { - 32 => Reg::i32(), - 64 => Reg::i64(), - _ => unreachable!("Unsupported XLEN: {}", xlen), - }; - if is_riscv_aggregate(arg) { - if total.bits() <= xlen { - arg.cast_to(xlen_reg); - } else { - arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2))); - } - return false; - } - - // "When passed in registers, scalars narrower than XLEN bits are widened - // according to the sign of their type up to 32 bits, then sign-extended to - // XLEN bits." - extend_integer_width(arg, xlen); - false -} - -fn classify_arg<'a, Ty, C>( - cx: &C, - arg: &mut ArgAbi<'a, Ty>, - xlen: u64, - flen: u64, - is_vararg: bool, - avail_gprs: &mut u64, - avail_fprs: &mut u64, -) where - Ty: TyAbiInterface<'a, C> + Copy, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - if !is_vararg { - match should_use_fp_conv(cx, &arg.layout, xlen, flen) { - Some(FloatConv::Float(f)) if *avail_fprs >= 1 => { - *avail_fprs -= 1; - arg.cast_to(f); - return; - } - Some(FloatConv::FloatPair(l, r)) if *avail_fprs >= 2 => { - *avail_fprs -= 2; - arg.cast_to(CastTarget::pair(l, r)); - return; - } - Some(FloatConv::MixedPair(l, r)) if *avail_fprs >= 1 && *avail_gprs >= 1 => { - *avail_gprs -= 1; - *avail_fprs -= 1; - arg.cast_to(CastTarget::pair(l, r)); - return; - } - _ => (), - } - } - - let total = arg.layout.size; - let align = arg.layout.align.abi.bits(); - - // "Scalars wider than 2✕XLEN are passed by reference and are replaced in - // the argument list with the address." - // "Aggregates larger than 2✕XLEN bits are passed by reference and are - // replaced in the argument list with the address, as are C++ aggregates - // with nontrivial copy constructors, destructors, or vtables." - if total.bits() > 2 * xlen { - // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. - if is_riscv_aggregate(arg) { - arg.make_indirect(); - } - if *avail_gprs >= 1 { - *avail_gprs -= 1; - } - return; - } - - let double_xlen_reg = match xlen { - 32 => Reg::i64(), - 64 => Reg::i128(), - _ => unreachable!("Unsupported XLEN: {}", xlen), - }; - - let xlen_reg = match xlen { - 32 => Reg::i32(), - 64 => Reg::i64(), - _ => unreachable!("Unsupported XLEN: {}", xlen), - }; - - if total.bits() > xlen { - let align_regs = align > xlen; - if is_riscv_aggregate(arg) { - arg.cast_to(Uniform::new( - if align_regs { double_xlen_reg } else { xlen_reg }, - Size::from_bits(xlen * 2), - )); - } - if align_regs && is_vararg { - *avail_gprs -= *avail_gprs % 2; - } - if *avail_gprs >= 2 { - *avail_gprs -= 2; - } else { - *avail_gprs = 0; - } - return; - } else if is_riscv_aggregate(arg) { - arg.cast_to(xlen_reg); - if *avail_gprs >= 1 { - *avail_gprs -= 1; - } - return; - } - - // "When passed in registers, scalars narrower than XLEN bits are widened - // according to the sign of their type up to 32 bits, then sign-extended to - // XLEN bits." - if *avail_gprs >= 1 { - extend_integer_width(arg, xlen); - *avail_gprs -= 1; - } -} - -fn extend_integer_width(arg: &mut ArgAbi<'_, Ty>, xlen: u64) { - if let Abi::Scalar(scalar) = arg.layout.abi { - if let abi::Int(i, _) = scalar.primitive() { - // 32-bit integers are always sign-extended - if i.size().bits() == 32 && xlen > 32 { - if let PassMode::Direct(ref mut attrs) = arg.mode { - attrs.ext(ArgExtension::Sext); - return; - } - } - } - } - - arg.extend_integer_width_to(xlen); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - let flen = match &cx.target_spec().llvm_abiname[..] { - "ilp32f" | "lp64f" => 32, - "ilp32d" | "lp64d" => 64, - _ => 0, - }; - let xlen = cx.data_layout().pointer_size.bits(); - - let mut avail_gprs = 8; - let mut avail_fprs = 8; - - if !fn_abi.ret.is_ignore() && classify_ret(cx, &mut fn_abi.ret, xlen, flen) { - avail_gprs -= 1; - } - - for (i, arg) in fn_abi.args.iter_mut().enumerate() { - if arg.is_ignore() { - continue; - } - classify_arg( - cx, - arg, - xlen, - flen, - i >= fn_abi.fixed_count as usize, - &mut avail_gprs, - &mut avail_fprs, - ); - } -} diff --git a/compiler/rustc_target/src/abi/call/s390x.rs b/compiler/rustc_target/src/abi/call/s390x.rs deleted file mode 100644 index 502e7331267..00000000000 --- a/compiler/rustc_target/src/abi/call/s390x.rs +++ /dev/null @@ -1,69 +0,0 @@ -// FIXME: The assumes we're using the non-vector ABI, i.e., compiling -// for a pre-z13 machine or using -mno-vx. - -use crate::abi::call::{ArgAbi, FnAbi, Reg}; -use crate::abi::{HasDataLayout, TyAbiInterface}; -use crate::spec::HasTargetSpec; - -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if !ret.layout.is_aggregate() && ret.layout.size.bits() <= 64 { - ret.extend_integer_width_to(64); - } else { - ret.make_indirect(); - } -} - -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - if arg.is_ignore() { - // s390x-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs. - if cx.target_spec().os == "linux" - && matches!(&*cx.target_spec().env, "gnu" | "musl" | "uclibc") - && arg.layout.is_zst() - { - arg.make_indirect_from_ignore(); - } - return; - } - if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 { - arg.extend_integer_width_to(64); - return; - } - - if arg.layout.is_single_fp_element(cx) { - match arg.layout.size.bytes() { - 4 => arg.cast_to(Reg::f32()), - 8 => arg.cast_to(Reg::f64()), - _ => arg.make_indirect(), - } - } else { - match arg.layout.size.bytes() { - 1 => arg.cast_to(Reg::i8()), - 2 => arg.cast_to(Reg::i16()), - 4 => arg.cast_to(Reg::i32()), - 8 => arg.cast_to(Reg::i64()), - _ => arg.make_indirect(), - } - } -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - classify_arg(cx, arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/sparc.rs b/compiler/rustc_target/src/abi/call/sparc.rs deleted file mode 100644 index 37980a91c76..00000000000 --- a/compiler/rustc_target/src/abi/call/sparc.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; -use crate::abi::{HasDataLayout, Size}; - -fn classify_ret(cx: &C, ret: &mut ArgAbi<'_, Ty>, offset: &mut Size) -where - C: HasDataLayout, -{ - if !ret.layout.is_aggregate() { - ret.extend_integer_width_to(32); - } else { - ret.make_indirect(); - *offset += cx.data_layout().pointer_size; - } -} - -fn classify_arg(cx: &C, arg: &mut ArgAbi<'_, Ty>, offset: &mut Size) -where - C: HasDataLayout, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - let dl = cx.data_layout(); - let size = arg.layout.size; - let align = arg.layout.align.max(dl.i32_align).min(dl.i64_align).abi; - - if arg.layout.is_aggregate() { - let pad_i32 = !offset.is_aligned(align); - arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32); - } else { - arg.extend_integer_width_to(32); - } - - *offset = offset.align_to(align) + size.align_to(align); -} - -pub(crate) fn compute_abi_info(cx: &C, fn_abi: &mut FnAbi<'_, Ty>) -where - C: HasDataLayout, -{ - let mut offset = Size::ZERO; - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret, &mut offset); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg, &mut offset); - } -} diff --git a/compiler/rustc_target/src/abi/call/sparc64.rs b/compiler/rustc_target/src/abi/call/sparc64.rs deleted file mode 100644 index 835353f76fc..00000000000 --- a/compiler/rustc_target/src/abi/call/sparc64.rs +++ /dev/null @@ -1,234 +0,0 @@ -// FIXME: This needs an audit for correctness and completeness. - -use crate::abi::call::{ - ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, Reg, Uniform, -}; -use crate::abi::{self, HasDataLayout, Scalar, Size, TyAbiInterface, TyAndLayout}; -use crate::spec::HasTargetSpec; - -#[derive(Clone, Debug)] -struct Sdata { - pub prefix: [Option; 8], - pub prefix_index: usize, - pub last_offset: Size, - pub has_float: bool, - pub arg_attribute: ArgAttribute, -} - -fn arg_scalar(cx: &C, scalar: &Scalar, offset: Size, mut data: Sdata) -> Sdata -where - C: HasDataLayout, -{ - let dl = cx.data_layout(); - - if !matches!(scalar.primitive(), abi::Float(abi::F32 | abi::F64)) { - return data; - } - - data.has_float = true; - - if !data.last_offset.is_aligned(dl.f64_align.abi) && data.last_offset < offset { - if data.prefix_index == data.prefix.len() { - return data; - } - data.prefix[data.prefix_index] = Some(Reg::i32()); - data.prefix_index += 1; - data.last_offset = data.last_offset + Reg::i32().size; - } - - for _ in 0..((offset - data.last_offset).bits() / 64) - .min((data.prefix.len() - data.prefix_index) as u64) - { - data.prefix[data.prefix_index] = Some(Reg::i64()); - data.prefix_index += 1; - data.last_offset = data.last_offset + Reg::i64().size; - } - - if data.last_offset < offset { - if data.prefix_index == data.prefix.len() { - return data; - } - data.prefix[data.prefix_index] = Some(Reg::i32()); - data.prefix_index += 1; - data.last_offset = data.last_offset + Reg::i32().size; - } - - if data.prefix_index == data.prefix.len() { - return data; - } - - if scalar.primitive() == abi::Float(abi::F32) { - data.arg_attribute = ArgAttribute::InReg; - data.prefix[data.prefix_index] = Some(Reg::f32()); - data.last_offset = offset + Reg::f32().size; - } else { - data.prefix[data.prefix_index] = Some(Reg::f64()); - data.last_offset = offset + Reg::f64().size; - } - data.prefix_index += 1; - data -} - -fn arg_scalar_pair( - cx: &C, - scalar1: &Scalar, - scalar2: &Scalar, - mut offset: Size, - mut data: Sdata, -) -> Sdata -where - C: HasDataLayout, -{ - data = arg_scalar(cx, scalar1, offset, data); - match (scalar1.primitive(), scalar2.primitive()) { - (abi::Float(abi::F32), _) => offset += Reg::f32().size, - (_, abi::Float(abi::F64)) => offset += Reg::f64().size, - (abi::Int(i, _signed), _) => offset += i.size(), - (abi::Pointer(_), _) => offset += Reg::i64().size, - _ => {} - } - - if (offset.bytes() % 4) != 0 && matches!(scalar2.primitive(), abi::Float(abi::F32 | abi::F64)) { - offset += Size::from_bytes(4 - (offset.bytes() % 4)); - } - data = arg_scalar(cx, scalar2, offset, data); - data -} - -fn parse_structure<'a, Ty, C>( - cx: &C, - layout: TyAndLayout<'a, Ty>, - mut data: Sdata, - mut offset: Size, -) -> Sdata -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if let abi::FieldsShape::Union(_) = layout.fields { - return data; - } - - match layout.abi { - abi::Abi::Scalar(scalar) => { - data = arg_scalar(cx, &scalar, offset, data); - } - abi::Abi::Aggregate { .. } => { - for i in 0..layout.fields.count() { - if offset < layout.fields.offset(i) { - offset = layout.fields.offset(i); - } - data = parse_structure(cx, layout.field(cx, i), data.clone(), offset); - } - } - _ => { - if let abi::Abi::ScalarPair(scalar1, scalar2) = &layout.abi { - data = arg_scalar_pair(cx, scalar1, scalar2, offset, data); - } - } - } - - data -} - -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, in_registers_max: Size) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !arg.layout.is_aggregate() { - arg.extend_integer_width_to(64); - return; - } - - let total = arg.layout.size; - if total > in_registers_max { - arg.make_indirect(); - return; - } - - match arg.layout.fields { - abi::FieldsShape::Primitive => unreachable!(), - abi::FieldsShape::Array { .. } => { - // Arrays are passed indirectly - arg.make_indirect(); - return; - } - abi::FieldsShape::Union(_) => { - // Unions and are always treated as a series of 64-bit integer chunks - } - abi::FieldsShape::Arbitrary { .. } => { - // Structures with floating point numbers need special care. - - let mut data = parse_structure( - cx, - arg.layout, - Sdata { - prefix: [None; 8], - prefix_index: 0, - last_offset: Size::ZERO, - has_float: false, - arg_attribute: ArgAttribute::default(), - }, - Size::ZERO, - ); - - if data.has_float { - // Structure { float, int, int } doesn't like to be handled like - // { float, long int }. Other way around it doesn't mind. - if data.last_offset < arg.layout.size - && (data.last_offset.bytes() % 8) != 0 - && data.prefix_index < data.prefix.len() - { - data.prefix[data.prefix_index] = Some(Reg::i32()); - data.prefix_index += 1; - data.last_offset += Reg::i32().size; - } - - let mut rest_size = arg.layout.size - data.last_offset; - if (rest_size.bytes() % 8) != 0 && data.prefix_index < data.prefix.len() { - data.prefix[data.prefix_index] = Some(Reg::i32()); - rest_size = rest_size - Reg::i32().size; - } - - arg.cast_to(CastTarget { - prefix: data.prefix, - rest: Uniform::new(Reg::i64(), rest_size), - attrs: ArgAttributes { - regular: data.arg_attribute, - arg_ext: ArgExtension::None, - pointee_size: Size::ZERO, - pointee_align: None, - }, - }); - return; - } - } - } - - arg.cast_to(Uniform::new(Reg::i64(), total)); -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - if !fn_abi.ret.is_ignore() { - classify_arg(cx, &mut fn_abi.ret, Size::from_bytes(32)); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - // sparc64-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs. - if cx.target_spec().os == "linux" - && matches!(&*cx.target_spec().env, "gnu" | "musl" | "uclibc") - && arg.layout.is_zst() - { - arg.make_indirect_from_ignore(); - } - return; - } - classify_arg(cx, arg, Size::from_bytes(16)); - } -} diff --git a/compiler/rustc_target/src/abi/call/wasm.rs b/compiler/rustc_target/src/abi/call/wasm.rs deleted file mode 100644 index 3c4cd76a754..00000000000 --- a/compiler/rustc_target/src/abi/call/wasm.rs +++ /dev/null @@ -1,103 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi}; -use crate::abi::{HasDataLayout, TyAbiInterface}; - -fn unwrap_trivial_aggregate<'a, Ty, C>(cx: &C, val: &mut ArgAbi<'a, Ty>) -> bool -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if val.layout.is_aggregate() { - if let Some(unit) = val.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()) { - let size = val.layout.size; - if unit.size == size { - val.cast_to(unit); - return true; - } - } - } - false -} - -fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - ret.extend_integer_width_to(32); - if ret.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, ret) { - ret.make_indirect(); - } -} - -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !arg.layout.is_sized() { - // Not touching this... - return; - } - arg.extend_integer_width_to(32); - if arg.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, arg) { - arg.make_indirect(); - } -} - -/// The purpose of this ABI is to match the C ABI (aka clang) exactly. -pub(crate) fn compute_c_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - if !fn_abi.ret.is_ignore() { - classify_ret(cx, &mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg(cx, arg); - } -} - -/// The purpose of this ABI is for matching the WebAssembly standard. This -/// intentionally diverges from the C ABI and is specifically crafted to take -/// advantage of LLVM's support of multiple returns in WebAssembly. -/// -/// This ABI is *bad*! It uses `PassMode::Direct` for `abi::Aggregate` types, which leaks LLVM -/// implementation details into the ABI. It's just hard to fix because ABIs are hard to change. -/// Also see . -pub(crate) fn compute_wasm_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { - if !fn_abi.ret.is_ignore() { - classify_ret_wasm_abi(&mut fn_abi.ret); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg_wasm_abi(arg); - } - - fn classify_ret_wasm_abi(ret: &mut ArgAbi<'_, Ty>) { - if !ret.layout.is_sized() { - // Not touching this... - return; - } - // FIXME: this is bad! https://github.com/rust-lang/rust/issues/115666 - ret.make_direct_deprecated(); - ret.extend_integer_width_to(32); - } - - fn classify_arg_wasm_abi(arg: &mut ArgAbi<'_, Ty>) { - if !arg.layout.is_sized() { - // Not touching this... - return; - } - // FIXME: this is bad! https://github.com/rust-lang/rust/issues/115666 - arg.make_direct_deprecated(); - arg.extend_integer_width_to(32); - } -} diff --git a/compiler/rustc_target/src/abi/call/x86.rs b/compiler/rustc_target/src/abi/call/x86.rs deleted file mode 100644 index d9af83d3205..00000000000 --- a/compiler/rustc_target/src/abi/call/x86.rs +++ /dev/null @@ -1,185 +0,0 @@ -use crate::abi::call::{ArgAttribute, FnAbi, PassMode, Reg, RegKind}; -use crate::abi::{Abi, Align, HasDataLayout, TyAbiInterface, TyAndLayout}; -use crate::spec::HasTargetSpec; - -#[derive(PartialEq)] -pub(crate) enum Flavor { - General, - FastcallOrVectorcall, -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, flavor: Flavor) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - if !fn_abi.ret.is_ignore() { - if fn_abi.ret.layout.is_aggregate() && fn_abi.ret.layout.is_sized() { - // Returning a structure. Most often, this will use - // a hidden first argument. On some platforms, though, - // small structs are returned as integers. - // - // Some links: - // https://www.angelcode.com/dev/callconv/callconv.html - // Clang's ABI handling is in lib/CodeGen/TargetInfo.cpp - let t = cx.target_spec(); - if t.abi_return_struct_as_int { - // According to Clang, everyone but MSVC returns single-element - // float aggregates directly in a floating-point register. - if !t.is_like_msvc && fn_abi.ret.layout.is_single_fp_element(cx) { - match fn_abi.ret.layout.size.bytes() { - 4 => fn_abi.ret.cast_to(Reg::f32()), - 8 => fn_abi.ret.cast_to(Reg::f64()), - _ => fn_abi.ret.make_indirect(), - } - } else { - match fn_abi.ret.layout.size.bytes() { - 1 => fn_abi.ret.cast_to(Reg::i8()), - 2 => fn_abi.ret.cast_to(Reg::i16()), - 4 => fn_abi.ret.cast_to(Reg::i32()), - 8 => fn_abi.ret.cast_to(Reg::i64()), - _ => fn_abi.ret.make_indirect(), - } - } - } else { - fn_abi.ret.make_indirect(); - } - } else { - fn_abi.ret.extend_integer_width_to(32); - } - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() || !arg.layout.is_sized() { - continue; - } - - // FIXME: MSVC 2015+ will pass the first 3 vector arguments in [XYZ]MM0-2 - // See https://reviews.llvm.org/D72114 for Clang behavior - - let t = cx.target_spec(); - let align_4 = Align::from_bytes(4).unwrap(); - let align_16 = Align::from_bytes(16).unwrap(); - - if t.is_like_msvc - && arg.layout.is_adt() - && let Some(max_repr_align) = arg.layout.max_repr_align - && max_repr_align > align_4 - { - // MSVC has special rules for overaligned arguments: https://reviews.llvm.org/D72114. - // Summarized here: - // - Arguments with _requested_ alignment > 4 are passed indirectly. - // - For backwards compatibility, arguments with natural alignment > 4 are still passed - // on stack (via `byval`). For example, this includes `double`, `int64_t`, - // and structs containing them, provided they lack an explicit alignment attribute. - assert!( - arg.layout.align.abi >= max_repr_align, - "abi alignment {:?} less than requested alignment {max_repr_align:?}", - arg.layout.align.abi, - ); - arg.make_indirect(); - } else if arg.layout.is_aggregate() { - // We need to compute the alignment of the `byval` argument. The rules can be found in - // `X86_32ABIInfo::getTypeStackAlignInBytes` in Clang's `TargetInfo.cpp`. Summarized - // here, they are: - // - // 1. If the natural alignment of the type is <= 4, the alignment is 4. - // - // 2. Otherwise, on Linux, the alignment of any vector type is the natural alignment. - // This doesn't matter here because we only pass aggregates via `byval`, not vectors. - // - // 3. Otherwise, on Apple platforms, the alignment of anything that contains a vector - // type is 16. - // - // 4. If none of these conditions are true, the alignment is 4. - - fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>) -> bool - where - Ty: TyAbiInterface<'a, C> + Copy, - { - match layout.abi { - Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) => false, - Abi::Vector { .. } => true, - Abi::Aggregate { .. } => { - for i in 0..layout.fields.count() { - if contains_vector(cx, layout.field(cx, i)) { - return true; - } - } - false - } - } - } - - let byval_align = if arg.layout.align.abi < align_4 { - // (1.) - align_4 - } else if t.is_like_osx && contains_vector(cx, arg.layout) { - // (3.) - align_16 - } else { - // (4.) - align_4 - }; - - arg.pass_by_stack_offset(Some(byval_align)); - } else { - arg.extend_integer_width_to(32); - } - } - - if flavor == Flavor::FastcallOrVectorcall { - // Mark arguments as InReg like clang does it, - // so our fastcall/vectorcall is compatible with C/C++ fastcall/vectorcall. - - // Clang reference: lib/CodeGen/TargetInfo.cpp - // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs() - - // IsSoftFloatABI is only set to true on ARM platforms, - // which in turn can't be x86? - - let mut free_regs = 2; - - for arg in fn_abi.args.iter_mut() { - let attrs = match arg.mode { - PassMode::Ignore - | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { - continue; - } - PassMode::Direct(ref mut attrs) => attrs, - PassMode::Pair(..) - | PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } - | PassMode::Cast { .. } => { - unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode) - } - }; - - // At this point we know this must be a primitive of sorts. - let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap(); - assert_eq!(unit.size, arg.layout.size); - if unit.kind == RegKind::Float { - continue; - } - - let size_in_regs = (arg.layout.size.bits() + 31) / 32; - - if size_in_regs == 0 { - continue; - } - - if size_in_regs > free_regs { - break; - } - - free_regs -= size_in_regs; - - if arg.layout.size.bits() <= 32 && unit.kind == RegKind::Integer { - attrs.set(ArgAttribute::InReg); - } - - if free_regs == 0 { - break; - } - } - } -} diff --git a/compiler/rustc_target/src/abi/call/x86_64.rs b/compiler/rustc_target/src/abi/call/x86_64.rs deleted file mode 100644 index 9910e623ac9..00000000000 --- a/compiler/rustc_target/src/abi/call/x86_64.rs +++ /dev/null @@ -1,254 +0,0 @@ -// The classification code for the x86_64 ABI is taken from the clay language -// https://github.com/jckarter/clay/blob/db0bd2702ab0b6e48965cd85f8859bbd5f60e48e/compiler/externals.cpp - -use crate::abi::call::{ArgAbi, CastTarget, FnAbi, Reg, RegKind}; -use crate::abi::{self, Abi, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; - -/// Classification of "eightbyte" components. -// N.B., the order of the variants is from general to specific, -// such that `unify(a, b)` is the "smaller" of `a` and `b`. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] -enum Class { - Int, - Sse, - SseUp, -} - -#[derive(Clone, Copy, Debug)] -struct Memory; - -// Currently supported vector size (AVX-512). -const LARGEST_VECTOR_SIZE: usize = 512; -const MAX_EIGHTBYTES: usize = LARGEST_VECTOR_SIZE / 64; - -fn classify_arg<'a, Ty, C>( - cx: &C, - arg: &ArgAbi<'a, Ty>, -) -> Result<[Option; MAX_EIGHTBYTES], Memory> -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - fn classify<'a, Ty, C>( - cx: &C, - layout: TyAndLayout<'a, Ty>, - cls: &mut [Option], - off: Size, - ) -> Result<(), Memory> - where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, - { - if !off.is_aligned(layout.align.abi) { - if !layout.is_zst() { - return Err(Memory); - } - return Ok(()); - } - - let mut c = match layout.abi { - Abi::Uninhabited => return Ok(()), - - Abi::Scalar(scalar) => match scalar.primitive() { - abi::Int(..) | abi::Pointer(_) => Class::Int, - abi::Float(_) => Class::Sse, - }, - - Abi::Vector { .. } => Class::Sse, - - Abi::ScalarPair(..) | Abi::Aggregate { .. } => { - for i in 0..layout.fields.count() { - let field_off = off + layout.fields.offset(i); - classify(cx, layout.field(cx, i), cls, field_off)?; - } - - match &layout.variants { - abi::Variants::Single { .. } => {} - abi::Variants::Multiple { variants, .. } => { - // Treat enum variants like union members. - for variant_idx in variants.indices() { - classify(cx, layout.for_variant(cx, variant_idx), cls, off)?; - } - } - } - - return Ok(()); - } - }; - - // Fill in `cls` for scalars (Int/Sse) and vectors (Sse). - let first = (off.bytes() / 8) as usize; - let last = ((off.bytes() + layout.size.bytes() - 1) / 8) as usize; - for cls in &mut cls[first..=last] { - *cls = Some(cls.map_or(c, |old| old.min(c))); - - // Everything after the first Sse "eightbyte" - // component is the upper half of a register. - if c == Class::Sse { - c = Class::SseUp; - } - } - - Ok(()) - } - - let n = ((arg.layout.size.bytes() + 7) / 8) as usize; - if n > MAX_EIGHTBYTES { - return Err(Memory); - } - - let mut cls = [None; MAX_EIGHTBYTES]; - classify(cx, arg.layout, &mut cls, Size::ZERO)?; - if n > 2 { - if cls[0] != Some(Class::Sse) { - return Err(Memory); - } - if cls[1..n].iter().any(|&c| c != Some(Class::SseUp)) { - return Err(Memory); - } - } else { - let mut i = 0; - while i < n { - if cls[i] == Some(Class::SseUp) { - cls[i] = Some(Class::Sse); - } else if cls[i] == Some(Class::Sse) { - i += 1; - while i != n && cls[i] == Some(Class::SseUp) { - i += 1; - } - } else { - i += 1; - } - } - } - - Ok(cls) -} - -fn reg_component(cls: &[Option], i: &mut usize, size: Size) -> Option { - if *i >= cls.len() { - return None; - } - - match cls[*i] { - None => None, - Some(Class::Int) => { - *i += 1; - Some(if size.bytes() < 8 { Reg { kind: RegKind::Integer, size } } else { Reg::i64() }) - } - Some(Class::Sse) => { - let vec_len = - 1 + cls[*i + 1..].iter().take_while(|&&c| c == Some(Class::SseUp)).count(); - *i += vec_len; - Some(if vec_len == 1 { - match size.bytes() { - 4 => Reg::f32(), - _ => Reg::f64(), - } - } else { - Reg { kind: RegKind::Vector, size: Size::from_bytes(8) * (vec_len as u64) } - }) - } - Some(c) => unreachable!("reg_component: unhandled class {:?}", c), - } -} - -fn cast_target(cls: &[Option], size: Size) -> CastTarget { - let mut i = 0; - let lo = reg_component(cls, &mut i, size).unwrap(); - let offset = Size::from_bytes(8) * (i as u64); - let mut target = CastTarget::from(lo); - if size > offset { - if let Some(hi) = reg_component(cls, &mut i, size - offset) { - target = CastTarget::pair(lo, hi); - } - } - assert_eq!(reg_component(cls, &mut i, Size::ZERO), None); - target -} - -const MAX_INT_REGS: usize = 6; // RDI, RSI, RDX, RCX, R8, R9 -const MAX_SSE_REGS: usize = 8; // XMM0-7 - -pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout, -{ - let mut int_regs = MAX_INT_REGS; - let mut sse_regs = MAX_SSE_REGS; - - let mut x86_64_arg_or_ret = |arg: &mut ArgAbi<'a, Ty>, is_arg: bool| { - if !arg.layout.is_sized() { - // Not touching this... - return; - } - let mut cls_or_mem = classify_arg(cx, arg); - - if is_arg { - if let Ok(cls) = cls_or_mem { - let mut needed_int = 0; - let mut needed_sse = 0; - for c in cls { - match c { - Some(Class::Int) => needed_int += 1, - Some(Class::Sse) => needed_sse += 1, - _ => {} - } - } - match (int_regs.checked_sub(needed_int), sse_regs.checked_sub(needed_sse)) { - (Some(left_int), Some(left_sse)) => { - int_regs = left_int; - sse_regs = left_sse; - } - _ => { - // Not enough registers for this argument, so it will be - // passed on the stack, but we only mark aggregates - // explicitly as indirect `byval` arguments, as LLVM will - // automatically put immediates on the stack itself. - if arg.layout.is_aggregate() { - cls_or_mem = Err(Memory); - } - } - } - } - } - - match cls_or_mem { - Err(Memory) => { - if is_arg { - // The x86_64 ABI doesn't have any special requirements for `byval` alignment, - // the type's alignment is always used. - arg.pass_by_stack_offset(None); - } else { - // `sret` parameter thus one less integer register available - arg.make_indirect(); - // NOTE(eddyb) return is handled first, so no registers - // should've been used yet. - assert_eq!(int_regs, MAX_INT_REGS); - int_regs -= 1; - } - } - Ok(ref cls) => { - // split into sized chunks passed individually - if arg.layout.is_aggregate() { - let size = arg.layout.size; - arg.cast_to(cast_target(cls, size)); - } else { - arg.extend_integer_width_to(32); - } - } - } - }; - - if !fn_abi.ret.is_ignore() { - x86_64_arg_or_ret(&mut fn_abi.ret, false); - } - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - x86_64_arg_or_ret(arg, true); - } -} diff --git a/compiler/rustc_target/src/abi/call/x86_win64.rs b/compiler/rustc_target/src/abi/call/x86_win64.rs deleted file mode 100644 index e5a20b248e4..00000000000 --- a/compiler/rustc_target/src/abi/call/x86_win64.rs +++ /dev/null @@ -1,52 +0,0 @@ -use crate::abi::call::{ArgAbi, FnAbi, Reg}; -use crate::abi::{Abi, Float, Primitive}; -use crate::spec::HasTargetSpec; - -// Win64 ABI: https://docs.microsoft.com/en-us/cpp/build/parameter-passing - -pub(crate) fn compute_abi_info(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) { - let fixup = |a: &mut ArgAbi<'_, Ty>| { - match a.layout.abi { - Abi::Uninhabited | Abi::Aggregate { sized: false } => {} - Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => match a.layout.size.bits() { - 8 => a.cast_to(Reg::i8()), - 16 => a.cast_to(Reg::i16()), - 32 => a.cast_to(Reg::i32()), - 64 => a.cast_to(Reg::i64()), - _ => a.make_indirect(), - }, - Abi::Vector { .. } => { - // FIXME(eddyb) there should be a size cap here - // (probably what clang calls "illegal vectors"). - } - Abi::Scalar(scalar) => { - // Match what LLVM does for `f128` so that `compiler-builtins` builtins match up - // with what LLVM expects. - if a.layout.size.bytes() > 8 - && !matches!(scalar.primitive(), Primitive::Float(Float::F128)) - { - a.make_indirect(); - } else { - a.extend_integer_width_to(32); - } - } - } - }; - - if !fn_abi.ret.is_ignore() { - fixup(&mut fn_abi.ret); - } - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - // x86_64-pc-windows-gnu doesn't ignore ZSTs. - if cx.target_spec().os == "windows" - && cx.target_spec().env == "gnu" - && arg.layout.is_zst() - { - arg.make_indirect_from_ignore(); - } - continue; - } - fixup(arg); - } -} diff --git a/compiler/rustc_target/src/abi/call/xtensa.rs b/compiler/rustc_target/src/abi/call/xtensa.rs deleted file mode 100644 index e1728b08a39..00000000000 --- a/compiler/rustc_target/src/abi/call/xtensa.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! The Xtensa ABI implementation -//! -//! This ABI implementation is based on the following sources: -//! -//! Section 8.1.4 & 8.1.5 of the Xtensa ISA reference manual, as well as snippets from -//! Section 2.3 from the Xtensa programmers guide. - -use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; -use crate::abi::{Abi, HasDataLayout, Size, TyAbiInterface}; -use crate::spec::HasTargetSpec; - -const NUM_ARG_GPRS: u64 = 6; -const NUM_RET_GPRS: u64 = 4; -const MAX_ARG_IN_REGS_SIZE: u64 = NUM_ARG_GPRS * 32; -const MAX_RET_IN_REGS_SIZE: u64 = NUM_RET_GPRS * 32; - -fn classify_ret_ty<'a, Ty, C>(arg: &mut ArgAbi<'_, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - if arg.is_ignore() { - return; - } - - // The rules for return and argument types are the same, - // so defer to `classify_arg_ty`. - let mut arg_gprs_left = NUM_RET_GPRS; - classify_arg_ty(arg, &mut arg_gprs_left, MAX_RET_IN_REGS_SIZE); - // Ret args cannot be passed via stack, we lower to indirect and let the backend handle the invisible reference - match arg.mode { - super::PassMode::Indirect { attrs: _, meta_attrs: _, ref mut on_stack } => { - *on_stack = false; - } - _ => {} - } -} - -fn classify_arg_ty<'a, Ty, C>(arg: &mut ArgAbi<'_, Ty>, arg_gprs_left: &mut u64, max_size: u64) -where - Ty: TyAbiInterface<'a, C> + Copy, -{ - assert!(*arg_gprs_left <= NUM_ARG_GPRS, "Arg GPR tracking underflow"); - - // Ignore empty structs/unions. - if arg.layout.is_zst() { - return; - } - - let size = arg.layout.size.bits(); - let needed_align = arg.layout.align.abi.bits(); - let mut must_use_stack = false; - - // Determine the number of GPRs needed to pass the current argument - // according to the ABI. 2*XLen-aligned varargs are passed in "aligned" - // register pairs, so may consume 3 registers. - let mut needed_arg_gprs = (size + 32 - 1) / 32; - if needed_align == 64 { - needed_arg_gprs += *arg_gprs_left % 2; - } - - if needed_arg_gprs > *arg_gprs_left - || needed_align > 128 - || (*arg_gprs_left < (max_size / 32) && needed_align == 128) - { - must_use_stack = true; - needed_arg_gprs = *arg_gprs_left; - } - *arg_gprs_left -= needed_arg_gprs; - - if must_use_stack { - arg.pass_by_stack_offset(None); - } else if is_xtensa_aggregate(arg) { - // Aggregates which are <= max_size will be passed in - // registers if possible, so coerce to integers. - - // Use a single `xlen` int if possible, 2 * `xlen` if 2 * `xlen` alignment - // is required, and a 2-element `xlen` array if only `xlen` alignment is - // required. - if size <= 32 { - arg.cast_to(Reg::i32()); - } else { - let reg = if needed_align == 2 * 32 { Reg::i64() } else { Reg::i32() }; - let total = Size::from_bits(((size + 32 - 1) / 32) * 32); - arg.cast_to(Uniform::new(reg, total)); - } - } else { - // All integral types are promoted to `xlen` - // width. - // - // We let the LLVM backend handle integral types >= xlen. - if size < 32 { - arg.extend_integer_width_to(32); - } - } -} - -pub(crate) fn compute_abi_info<'a, Ty, C>(_cx: &C, fn_abi: &mut FnAbi<'a, Ty>) -where - Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, -{ - if !fn_abi.ret.is_ignore() { - classify_ret_ty(&mut fn_abi.ret); - } - - let mut arg_gprs_left = NUM_ARG_GPRS; - - for arg in fn_abi.args.iter_mut() { - if arg.is_ignore() { - continue; - } - classify_arg_ty(arg, &mut arg_gprs_left, MAX_ARG_IN_REGS_SIZE); - } -} - -fn is_xtensa_aggregate<'a, Ty>(arg: &ArgAbi<'a, Ty>) -> bool { - match arg.layout.abi { - Abi::Vector { .. } => true, - _ => arg.layout.is_aggregate(), - } -} diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs deleted file mode 100644 index 9f26c98df86..00000000000 --- a/compiler/rustc_target/src/abi/mod.rs +++ /dev/null @@ -1,269 +0,0 @@ -use std::fmt; -use std::ops::Deref; - -use Float::*; -use Primitive::*; -use rustc_data_structures::intern::Interned; -use rustc_macros::HashStable_Generic; - -pub mod call; - -// Explicitly import `Float` to avoid ambiguity with `Primitive::Float`. -pub use rustc_abi::{Float, *}; - -rustc_index::newtype_index! { - /// The *source-order* index of a field in a variant. - /// - /// This is how most code after type checking refers to fields, rather than - /// using names (as names have hygiene complications and more complex lookup). - /// - /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order. - /// (It is for `repr(C)` `struct`s, however.) - /// - /// For example, in the following types, - /// ```rust - /// # enum Never {} - /// # #[repr(u16)] - /// enum Demo1 { - /// Variant0 { a: Never, b: i32 } = 100, - /// Variant1 { c: u8, d: u64 } = 10, - /// } - /// struct Demo2 { e: u8, f: u16, g: u8 } - /// ``` - /// `b` is `FieldIdx(1)` in `VariantIdx(0)`, - /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and - /// `f` is `FieldIdx(1)` in `VariantIdx(0)`. - #[derive(HashStable_Generic)] - #[encodable] - #[orderable] - pub struct FieldIdx {} -} - -rustc_index::newtype_index! { - /// The *source-order* index of a variant in a type. - /// - /// For enums, these are always `0..variant_count`, regardless of any - /// custom discriminants that may have been defined, and including any - /// variants that may end up uninhabited due to field types. (Some of the - /// variants may not be present in a monomorphized ABI [`Variants`], but - /// those skipped variants are always counted when determining the *index*.) - /// - /// `struct`s, `tuples`, and `unions`s are considered to have a single variant - /// with variant index zero, aka [`FIRST_VARIANT`]. - #[derive(HashStable_Generic)] - #[encodable] - #[orderable] - pub struct VariantIdx { - /// Equivalent to `VariantIdx(0)`. - const FIRST_VARIANT = 0; - } -} -#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)] -#[rustc_pass_by_value] -pub struct Layout<'a>(pub Interned<'a, LayoutS>); - -impl<'a> fmt::Debug for Layout<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // See comment on `::fmt` above. - self.0.0.fmt(f) - } -} - -impl<'a> Deref for Layout<'a> { - type Target = &'a LayoutS; - fn deref(&self) -> &&'a LayoutS { - &self.0.0 - } -} - -impl<'a> Layout<'a> { - pub fn fields(self) -> &'a FieldsShape { - &self.0.0.fields - } - - pub fn variants(self) -> &'a Variants { - &self.0.0.variants - } - - pub fn abi(self) -> Abi { - self.0.0.abi - } - - pub fn largest_niche(self) -> Option { - self.0.0.largest_niche - } - - pub fn align(self) -> AbiAndPrefAlign { - self.0.0.align - } - - pub fn size(self) -> Size { - self.0.0.size - } - - pub fn max_repr_align(self) -> Option { - self.0.0.max_repr_align - } - - pub fn unadjusted_abi_align(self) -> Align { - self.0.0.unadjusted_abi_align - } - - /// Whether the layout is from a type that implements [`std::marker::PointerLike`]. - /// - /// Currently, that means that the type is pointer-sized, pointer-aligned, - /// and has a initialized (non-union), scalar ABI. - pub fn is_pointer_like(self, data_layout: &TargetDataLayout) -> bool { - self.size() == data_layout.pointer_size - && self.align().abi == data_layout.pointer_align.abi - && matches!(self.abi(), Abi::Scalar(Scalar::Initialized { .. })) - } -} - -/// The layout of a type, alongside the type itself. -/// Provides various type traversal APIs (e.g., recursing into fields). -/// -/// Note that the layout is NOT guaranteed to always be identical -/// to that obtained from `layout_of(ty)`, as we need to produce -/// layouts for which Rust types do not exist, such as enum variants -/// or synthetic fields of enums (i.e., discriminants) and wide pointers. -#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)] -pub struct TyAndLayout<'a, Ty> { - pub ty: Ty, - pub layout: Layout<'a>, -} - -impl<'a, Ty: fmt::Display> fmt::Debug for TyAndLayout<'a, Ty> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Print the type in a readable way, not its debug representation. - f.debug_struct("TyAndLayout") - .field("ty", &format_args!("{}", self.ty)) - .field("layout", &self.layout) - .finish() - } -} - -impl<'a, Ty> Deref for TyAndLayout<'a, Ty> { - type Target = &'a LayoutS; - fn deref(&self) -> &&'a LayoutS { - &self.layout.0.0 - } -} - -/// Trait that needs to be implemented by the higher-level type representation -/// (e.g. `rustc_middle::ty::Ty`), to provide `rustc_target::abi` functionality. -pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug { - fn ty_and_layout_for_variant( - this: TyAndLayout<'a, Self>, - cx: &C, - variant_index: VariantIdx, - ) -> TyAndLayout<'a, Self>; - fn ty_and_layout_field(this: TyAndLayout<'a, Self>, cx: &C, i: usize) -> TyAndLayout<'a, Self>; - fn ty_and_layout_pointee_info_at( - this: TyAndLayout<'a, Self>, - cx: &C, - offset: Size, - ) -> Option; - fn is_adt(this: TyAndLayout<'a, Self>) -> bool; - fn is_never(this: TyAndLayout<'a, Self>) -> bool; - fn is_tuple(this: TyAndLayout<'a, Self>) -> bool; - fn is_unit(this: TyAndLayout<'a, Self>) -> bool; - fn is_transparent(this: TyAndLayout<'a, Self>) -> bool; -} - -impl<'a, Ty> TyAndLayout<'a, Ty> { - pub fn for_variant(self, cx: &C, variant_index: VariantIdx) -> Self - where - Ty: TyAbiInterface<'a, C>, - { - Ty::ty_and_layout_for_variant(self, cx, variant_index) - } - - pub fn field(self, cx: &C, i: usize) -> Self - where - Ty: TyAbiInterface<'a, C>, - { - Ty::ty_and_layout_field(self, cx, i) - } - - pub fn pointee_info_at(self, cx: &C, offset: Size) -> Option - where - Ty: TyAbiInterface<'a, C>, - { - Ty::ty_and_layout_pointee_info_at(self, cx, offset) - } - - pub fn is_single_fp_element(self, cx: &C) -> bool - where - Ty: TyAbiInterface<'a, C>, - C: HasDataLayout, - { - match self.abi { - Abi::Scalar(scalar) => matches!(scalar.primitive(), Float(F32 | F64)), - Abi::Aggregate { .. } => { - if self.fields.count() == 1 && self.fields.offset(0).bytes() == 0 { - self.field(cx, 0).is_single_fp_element(cx) - } else { - false - } - } - _ => false, - } - } - - pub fn is_adt(self) -> bool - where - Ty: TyAbiInterface<'a, C>, - { - Ty::is_adt(self) - } - - pub fn is_never(self) -> bool - where - Ty: TyAbiInterface<'a, C>, - { - Ty::is_never(self) - } - - pub fn is_tuple(self) -> bool - where - Ty: TyAbiInterface<'a, C>, - { - Ty::is_tuple(self) - } - - pub fn is_unit(self) -> bool - where - Ty: TyAbiInterface<'a, C>, - { - Ty::is_unit(self) - } - - pub fn is_transparent(self) -> bool - where - Ty: TyAbiInterface<'a, C>, - { - Ty::is_transparent(self) - } - - /// Finds the one field that is not a 1-ZST. - /// Returns `None` if there are multiple non-1-ZST fields or only 1-ZST-fields. - pub fn non_1zst_field(&self, cx: &C) -> Option<(usize, Self)> - where - Ty: TyAbiInterface<'a, C> + Copy, - { - let mut found = None; - for field_idx in 0..self.fields.count() { - let field = self.field(cx, field_idx); - if field.is_1zst() { - continue; - } - if found.is_some() { - // More than one non-1-ZST field. - return None; - } - found = Some((field_idx, field)); - } - found - } -} diff --git a/compiler/rustc_target/src/callconv/aarch64.rs b/compiler/rustc_target/src/callconv/aarch64.rs new file mode 100644 index 00000000000..55b65fb1caa --- /dev/null +++ b/compiler/rustc_target/src/callconv/aarch64.rs @@ -0,0 +1,127 @@ +use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind, Uniform}; +use crate::abi::{HasDataLayout, TyAbiInterface}; + +/// Indicates the variant of the AArch64 ABI we are compiling for. +/// Used to accommodate Apple and Microsoft's deviations from the usual AAPCS ABI. +/// +/// Corresponds to Clang's `AArch64ABIInfo::ABIKind`. +#[derive(Copy, Clone, PartialEq)] +pub(crate) enum AbiKind { + AAPCS, + DarwinPCS, + Win64, +} + +fn is_homogeneous_aggregate<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -> Option +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { + let size = arg.layout.size; + + // Ensure we have at most four uniquely addressable members. + if size > unit.size.checked_mul(4, cx).unwrap() { + return None; + } + + let valid_unit = match unit.kind { + RegKind::Integer => false, + RegKind::Float => true, + RegKind::Vector => size.bits() == 64 || size.bits() == 128, + }; + + valid_unit.then_some(Uniform::consecutive(unit, size)) + }) +} + +fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, kind: AbiKind) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !ret.layout.is_sized() { + // Not touching this... + return; + } + if !ret.layout.is_aggregate() { + if kind == AbiKind::DarwinPCS { + // On Darwin, when returning an i8/i16, it must be sign-extended to 32 bits, + // and likewise a u8/u16 must be zero-extended to 32-bits. + // See also: + ret.extend_integer_width_to(32) + } + return; + } + if let Some(uniform) = is_homogeneous_aggregate(cx, ret) { + ret.cast_to(uniform); + return; + } + let size = ret.layout.size; + let bits = size.bits(); + if bits <= 128 { + ret.cast_to(Uniform::new(Reg::i64(), size)); + return; + } + ret.make_indirect(); +} + +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, kind: AbiKind) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + if !arg.layout.is_aggregate() { + if kind == AbiKind::DarwinPCS { + // On Darwin, when passing an i8/i16, it must be sign-extended to 32 bits, + // and likewise a u8/u16 must be zero-extended to 32-bits. + // See also: + arg.extend_integer_width_to(32); + } + return; + } + if let Some(uniform) = is_homogeneous_aggregate(cx, arg) { + arg.cast_to(uniform); + return; + } + let size = arg.layout.size; + let align = if kind == AbiKind::AAPCS { + // When passing small aggregates by value, the AAPCS ABI mandates using the unadjusted + // alignment of the type (not including `repr(align)`). + // This matches behavior of `AArch64ABIInfo::classifyArgumentType` in Clang. + // See: + arg.layout.unadjusted_abi_align + } else { + arg.layout.align.abi + }; + if size.bits() <= 128 { + if align.bits() == 128 { + arg.cast_to(Uniform::new(Reg::i128(), size)); + } else { + arg.cast_to(Uniform::new(Reg::i64(), size)); + } + return; + } + arg.make_indirect(); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, kind: AbiKind) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret, kind); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg, kind); + } +} diff --git a/compiler/rustc_target/src/callconv/amdgpu.rs b/compiler/rustc_target/src/callconv/amdgpu.rs new file mode 100644 index 00000000000..3007a729a8b --- /dev/null +++ b/compiler/rustc_target/src/callconv/amdgpu.rs @@ -0,0 +1,35 @@ +use crate::abi::call::{ArgAbi, FnAbi}; +use crate::abi::{HasDataLayout, TyAbiInterface}; + +fn classify_ret<'a, Ty, C>(_cx: &C, ret: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + ret.extend_integer_width_to(32); +} + +fn classify_arg<'a, Ty, C>(_cx: &C, arg: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + arg.extend_integer_width_to(32); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg); + } +} diff --git a/compiler/rustc_target/src/callconv/arm.rs b/compiler/rustc_target/src/callconv/arm.rs new file mode 100644 index 00000000000..bd6f781fb81 --- /dev/null +++ b/compiler/rustc_target/src/callconv/arm.rs @@ -0,0 +1,105 @@ +use crate::abi::call::{ArgAbi, Conv, FnAbi, Reg, RegKind, Uniform}; +use crate::abi::{HasDataLayout, TyAbiInterface}; +use crate::spec::HasTargetSpec; + +fn is_homogeneous_aggregate<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -> Option +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { + let size = arg.layout.size; + + // Ensure we have at most four uniquely addressable members. + if size > unit.size.checked_mul(4, cx).unwrap() { + return None; + } + + let valid_unit = match unit.kind { + RegKind::Integer => false, + RegKind::Float => true, + RegKind::Vector => size.bits() == 64 || size.bits() == 128, + }; + + valid_unit.then_some(Uniform::consecutive(unit, size)) + }) +} + +fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, vfp: bool) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !ret.layout.is_sized() { + // Not touching this... + return; + } + if !ret.layout.is_aggregate() { + ret.extend_integer_width_to(32); + return; + } + + if vfp { + if let Some(uniform) = is_homogeneous_aggregate(cx, ret) { + ret.cast_to(uniform); + return; + } + } + + let size = ret.layout.size; + let bits = size.bits(); + if bits <= 32 { + ret.cast_to(Uniform::new(Reg::i32(), size)); + return; + } + ret.make_indirect(); +} + +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, vfp: bool) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + if !arg.layout.is_aggregate() { + arg.extend_integer_width_to(32); + return; + } + + if vfp { + if let Some(uniform) = is_homogeneous_aggregate(cx, arg) { + arg.cast_to(uniform); + return; + } + } + + let align = arg.layout.align.abi.bytes(); + let total = arg.layout.size; + arg.cast_to(Uniform::consecutive(if align <= 4 { Reg::i32() } else { Reg::i64() }, total)); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + // If this is a target with a hard-float ABI, and the function is not explicitly + // `extern "aapcs"`, then we must use the VFP registers for homogeneous aggregates. + let vfp = cx.target_spec().llvm_target.ends_with("hf") + && fn_abi.conv != Conv::ArmAapcs + && !fn_abi.c_variadic; + + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret, vfp); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg, vfp); + } +} diff --git a/compiler/rustc_target/src/callconv/avr.rs b/compiler/rustc_target/src/callconv/avr.rs new file mode 100644 index 00000000000..dfc991e0954 --- /dev/null +++ b/compiler/rustc_target/src/callconv/avr.rs @@ -0,0 +1,59 @@ +//! LLVM-frontend specific AVR calling convention implementation. +//! +//! # Current calling convention ABI +//! +//! Inherited from Clang's `clang::DefaultABIInfo` implementation - self described +//! as +//! +//! > the default implementation for ABI specific details. This implementation +//! > provides information which results in +//! > self-consistent and sensible LLVM IR generation, but does not +//! > conform to any particular ABI. +//! > +//! > - Doxygen Documentation of `clang::DefaultABIInfo` +//! +//! This calling convention may not match AVR-GCC in all cases. +//! +//! In the future, an AVR-GCC compatible argument classification ABI should be +//! adopted in both Rust and Clang. +//! +//! *NOTE*: Currently, this module implements the same calling convention +//! that clang with AVR currently does - the default, simple, unspecialized +//! ABI implementation available to all targets. This ABI is not +//! binary-compatible with AVR-GCC. Once LLVM [PR46140](https://bugs.llvm.org/show_bug.cgi?id=46140) +//! is completed, this module should be updated to match so that both Clang +//! and Rust emit code to the same AVR-GCC compatible ABI. +//! +//! In particular, both Clang and Rust may not have the same semantics +//! when promoting arguments to indirect references as AVR-GCC. It is important +//! to note that the core AVR ABI implementation within LLVM itself is ABI +//! compatible with AVR-GCC - Rust and AVR-GCC only differ in the small amount +//! of compiler frontend specific calling convention logic implemented here. + +use crate::abi::call::{ArgAbi, FnAbi}; + +fn classify_ret_ty(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() { + ret.make_indirect(); + } +} + +fn classify_arg_ty(arg: &mut ArgAbi<'_, Ty>) { + if arg.layout.is_aggregate() { + arg.make_indirect(); + } +} + +pub(crate) fn compute_abi_info(fty: &mut FnAbi<'_, Ty>) { + if !fty.ret.is_ignore() { + classify_ret_ty(&mut fty.ret); + } + + for arg in fty.args.iter_mut() { + if arg.is_ignore() { + continue; + } + + classify_arg_ty(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/bpf.rs b/compiler/rustc_target/src/callconv/bpf.rs new file mode 100644 index 00000000000..f19772ac709 --- /dev/null +++ b/compiler/rustc_target/src/callconv/bpf.rs @@ -0,0 +1,31 @@ +// see https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/BPF/BPFCallingConv.td +use crate::abi::call::{ArgAbi, FnAbi}; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() || ret.layout.size.bits() > 64 { + ret.make_indirect(); + } else { + ret.extend_integer_width_to(32); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if arg.layout.is_aggregate() || arg.layout.size.bits() > 64 { + arg.make_indirect(); + } else { + arg.extend_integer_width_to(32); + } +} + +pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/csky.rs b/compiler/rustc_target/src/callconv/csky.rs new file mode 100644 index 00000000000..b1c1ae814a7 --- /dev/null +++ b/compiler/rustc_target/src/callconv/csky.rs @@ -0,0 +1,61 @@ +// Reference: CSKY ABI Manual +// https://occ-oss-prod.oss-cn-hangzhou.aliyuncs.com/resource//1695027452256/T-HEAD_800_Series_ABI_Standards_Manual.pdf +// +// Reference: Clang CSKY lowering code +// https://github.com/llvm/llvm-project/blob/4a074f32a6914f2a8d7215d78758c24942dddc3d/clang/lib/CodeGen/Targets/CSKY.cpp#L76-L162 + +use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; + +fn classify_ret(arg: &mut ArgAbi<'_, Ty>) { + if !arg.layout.is_sized() { + // Not touching this... + return; + } + // For return type, aggregate which <= 2*XLen will be returned in registers. + // Otherwise, aggregate will be returned indirectly. + if arg.layout.is_aggregate() { + let total = arg.layout.size; + if total.bits() > 64 { + arg.make_indirect(); + } else if total.bits() > 32 { + arg.cast_to(Uniform::new(Reg::i32(), total)); + } else { + arg.cast_to(Reg::i32()); + } + } else { + arg.extend_integer_width_to(32); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if !arg.layout.is_sized() { + // Not touching this... + return; + } + // For argument type, the first 4*XLen parts of aggregate will be passed + // in registers, and the rest will be passed in stack. + // So we can coerce to integers directly and let backend handle it correctly. + if arg.layout.is_aggregate() { + let total = arg.layout.size; + if total.bits() > 32 { + arg.cast_to(Uniform::new(Reg::i32(), total)); + } else { + arg.cast_to(Reg::i32()); + } + } else { + arg.extend_integer_width_to(32); + } +} + +pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/hexagon.rs b/compiler/rustc_target/src/callconv/hexagon.rs new file mode 100644 index 00000000000..0a0688880c0 --- /dev/null +++ b/compiler/rustc_target/src/callconv/hexagon.rs @@ -0,0 +1,30 @@ +use crate::abi::call::{ArgAbi, FnAbi}; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() && ret.layout.size.bits() > 64 { + ret.make_indirect(); + } else { + ret.extend_integer_width_to(32); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if arg.layout.is_aggregate() && arg.layout.size.bits() > 64 { + arg.make_indirect(); + } else { + arg.extend_integer_width_to(32); + } +} + +pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/loongarch.rs b/compiler/rustc_target/src/callconv/loongarch.rs new file mode 100644 index 00000000000..4a21935623b --- /dev/null +++ b/compiler/rustc_target/src/callconv/loongarch.rs @@ -0,0 +1,361 @@ +use crate::abi::call::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Reg, RegKind, Uniform}; +use crate::abi::{self, Abi, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; +use crate::spec::HasTargetSpec; + +#[derive(Copy, Clone)] +enum RegPassKind { + Float(Reg), + Integer(Reg), + Unknown, +} + +#[derive(Copy, Clone)] +enum FloatConv { + FloatPair(Reg, Reg), + Float(Reg), + MixedPair(Reg, Reg), +} + +#[derive(Copy, Clone)] +struct CannotUseFpConv; + +fn is_loongarch_aggregate(arg: &ArgAbi<'_, Ty>) -> bool { + match arg.layout.abi { + Abi::Vector { .. } => true, + _ => arg.layout.is_aggregate(), + } +} + +fn should_use_fp_conv_helper<'a, Ty, C>( + cx: &C, + arg_layout: &TyAndLayout<'a, Ty>, + xlen: u64, + flen: u64, + field1_kind: &mut RegPassKind, + field2_kind: &mut RegPassKind, +) -> Result<(), CannotUseFpConv> +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + match arg_layout.abi { + Abi::Scalar(scalar) => match scalar.primitive() { + abi::Int(..) | abi::Pointer(_) => { + if arg_layout.size.bits() > xlen { + return Err(CannotUseFpConv); + } + match (*field1_kind, *field2_kind) { + (RegPassKind::Unknown, _) => { + *field1_kind = RegPassKind::Integer(Reg { + kind: RegKind::Integer, + size: arg_layout.size, + }); + } + (RegPassKind::Float(_), RegPassKind::Unknown) => { + *field2_kind = RegPassKind::Integer(Reg { + kind: RegKind::Integer, + size: arg_layout.size, + }); + } + _ => return Err(CannotUseFpConv), + } + } + abi::Float(_) => { + if arg_layout.size.bits() > flen { + return Err(CannotUseFpConv); + } + match (*field1_kind, *field2_kind) { + (RegPassKind::Unknown, _) => { + *field1_kind = + RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); + } + (_, RegPassKind::Unknown) => { + *field2_kind = + RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); + } + _ => return Err(CannotUseFpConv), + } + } + }, + Abi::Vector { .. } | Abi::Uninhabited => return Err(CannotUseFpConv), + Abi::ScalarPair(..) | Abi::Aggregate { .. } => match arg_layout.fields { + FieldsShape::Primitive => { + unreachable!("aggregates can't have `FieldsShape::Primitive`") + } + FieldsShape::Union(_) => { + if !arg_layout.is_zst() { + if arg_layout.is_transparent() { + let non_1zst_elem = arg_layout.non_1zst_field(cx).expect("not exactly one non-1-ZST field in non-ZST repr(transparent) union").1; + return should_use_fp_conv_helper( + cx, + &non_1zst_elem, + xlen, + flen, + field1_kind, + field2_kind, + ); + } + return Err(CannotUseFpConv); + } + } + FieldsShape::Array { count, .. } => { + for _ in 0..count { + let elem_layout = arg_layout.field(cx, 0); + should_use_fp_conv_helper( + cx, + &elem_layout, + xlen, + flen, + field1_kind, + field2_kind, + )?; + } + } + FieldsShape::Arbitrary { .. } => { + match arg_layout.variants { + abi::Variants::Multiple { .. } => return Err(CannotUseFpConv), + abi::Variants::Single { .. } => (), + } + for i in arg_layout.fields.index_by_increasing_offset() { + let field = arg_layout.field(cx, i); + should_use_fp_conv_helper(cx, &field, xlen, flen, field1_kind, field2_kind)?; + } + } + }, + } + Ok(()) +} + +fn should_use_fp_conv<'a, Ty, C>( + cx: &C, + arg: &TyAndLayout<'a, Ty>, + xlen: u64, + flen: u64, +) -> Option +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + let mut field1_kind = RegPassKind::Unknown; + let mut field2_kind = RegPassKind::Unknown; + if should_use_fp_conv_helper(cx, arg, xlen, flen, &mut field1_kind, &mut field2_kind).is_err() { + return None; + } + match (field1_kind, field2_kind) { + (RegPassKind::Integer(l), RegPassKind::Float(r)) => Some(FloatConv::MixedPair(l, r)), + (RegPassKind::Float(l), RegPassKind::Integer(r)) => Some(FloatConv::MixedPair(l, r)), + (RegPassKind::Float(l), RegPassKind::Float(r)) => Some(FloatConv::FloatPair(l, r)), + (RegPassKind::Float(f), RegPassKind::Unknown) => Some(FloatConv::Float(f)), + _ => None, + } +} + +fn classify_ret<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, xlen: u64, flen: u64) -> bool +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + if !arg.layout.is_sized() { + // Not touching this... + return false; // I guess? return value of this function is not documented + } + if let Some(conv) = should_use_fp_conv(cx, &arg.layout, xlen, flen) { + match conv { + FloatConv::Float(f) => { + arg.cast_to(f); + } + FloatConv::FloatPair(l, r) => { + arg.cast_to(CastTarget::pair(l, r)); + } + FloatConv::MixedPair(l, r) => { + arg.cast_to(CastTarget::pair(l, r)); + } + } + return false; + } + + let total = arg.layout.size; + + // "Scalars wider than 2✕XLEN are passed by reference and are replaced in + // the argument list with the address." + // "Aggregates larger than 2✕XLEN bits are passed by reference and are + // replaced in the argument list with the address, as are C++ aggregates + // with nontrivial copy constructors, destructors, or vtables." + if total.bits() > 2 * xlen { + // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. + if is_loongarch_aggregate(arg) { + arg.make_indirect(); + } + return true; + } + + let xlen_reg = match xlen { + 32 => Reg::i32(), + 64 => Reg::i64(), + _ => unreachable!("Unsupported XLEN: {}", xlen), + }; + if is_loongarch_aggregate(arg) { + if total.bits() <= xlen { + arg.cast_to(xlen_reg); + } else { + arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2))); + } + return false; + } + + // "When passed in registers, scalars narrower than XLEN bits are widened + // according to the sign of their type up to 32 bits, then sign-extended to + // XLEN bits." + extend_integer_width(arg, xlen); + false +} + +fn classify_arg<'a, Ty, C>( + cx: &C, + arg: &mut ArgAbi<'a, Ty>, + xlen: u64, + flen: u64, + is_vararg: bool, + avail_gprs: &mut u64, + avail_fprs: &mut u64, +) where + Ty: TyAbiInterface<'a, C> + Copy, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + if !is_vararg { + match should_use_fp_conv(cx, &arg.layout, xlen, flen) { + Some(FloatConv::Float(f)) if *avail_fprs >= 1 => { + *avail_fprs -= 1; + arg.cast_to(f); + return; + } + Some(FloatConv::FloatPair(l, r)) if *avail_fprs >= 2 => { + *avail_fprs -= 2; + arg.cast_to(CastTarget::pair(l, r)); + return; + } + Some(FloatConv::MixedPair(l, r)) if *avail_fprs >= 1 && *avail_gprs >= 1 => { + *avail_gprs -= 1; + *avail_fprs -= 1; + arg.cast_to(CastTarget::pair(l, r)); + return; + } + _ => (), + } + } + + let total = arg.layout.size; + let align = arg.layout.align.abi.bits(); + + // "Scalars wider than 2✕XLEN are passed by reference and are replaced in + // the argument list with the address." + // "Aggregates larger than 2✕XLEN bits are passed by reference and are + // replaced in the argument list with the address, as are C++ aggregates + // with nontrivial copy constructors, destructors, or vtables." + if total.bits() > 2 * xlen { + // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. + if is_loongarch_aggregate(arg) { + arg.make_indirect(); + } + if *avail_gprs >= 1 { + *avail_gprs -= 1; + } + return; + } + + let double_xlen_reg = match xlen { + 32 => Reg::i64(), + 64 => Reg::i128(), + _ => unreachable!("Unsupported XLEN: {}", xlen), + }; + + let xlen_reg = match xlen { + 32 => Reg::i32(), + 64 => Reg::i64(), + _ => unreachable!("Unsupported XLEN: {}", xlen), + }; + + if total.bits() > xlen { + let align_regs = align > xlen; + if is_loongarch_aggregate(arg) { + arg.cast_to(Uniform::new( + if align_regs { double_xlen_reg } else { xlen_reg }, + Size::from_bits(xlen * 2), + )); + } + if align_regs && is_vararg { + *avail_gprs -= *avail_gprs % 2; + } + if *avail_gprs >= 2 { + *avail_gprs -= 2; + } else { + *avail_gprs = 0; + } + return; + } else if is_loongarch_aggregate(arg) { + arg.cast_to(xlen_reg); + if *avail_gprs >= 1 { + *avail_gprs -= 1; + } + return; + } + + // "When passed in registers, scalars narrower than XLEN bits are widened + // according to the sign of their type up to 32 bits, then sign-extended to + // XLEN bits." + if *avail_gprs >= 1 { + extend_integer_width(arg, xlen); + *avail_gprs -= 1; + } +} + +fn extend_integer_width(arg: &mut ArgAbi<'_, Ty>, xlen: u64) { + if let Abi::Scalar(scalar) = arg.layout.abi { + if let abi::Int(i, _) = scalar.primitive() { + // 32-bit integers are always sign-extended + if i.size().bits() == 32 && xlen > 32 { + if let PassMode::Direct(ref mut attrs) = arg.mode { + attrs.ext(ArgExtension::Sext); + return; + } + } + } + } + + arg.extend_integer_width_to(xlen); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + let xlen = cx.data_layout().pointer_size.bits(); + let flen = match &cx.target_spec().llvm_abiname[..] { + "ilp32f" | "lp64f" => 32, + "ilp32d" | "lp64d" => 64, + _ => 0, + }; + + let mut avail_gprs = 8; + let mut avail_fprs = 8; + + if !fn_abi.ret.is_ignore() && classify_ret(cx, &mut fn_abi.ret, xlen, flen) { + avail_gprs -= 1; + } + + for (i, arg) in fn_abi.args.iter_mut().enumerate() { + if arg.is_ignore() { + continue; + } + classify_arg( + cx, + arg, + xlen, + flen, + i >= fn_abi.fixed_count as usize, + &mut avail_gprs, + &mut avail_fprs, + ); + } +} diff --git a/compiler/rustc_target/src/callconv/m68k.rs b/compiler/rustc_target/src/callconv/m68k.rs new file mode 100644 index 00000000000..82fe81f8c52 --- /dev/null +++ b/compiler/rustc_target/src/callconv/m68k.rs @@ -0,0 +1,34 @@ +use crate::abi::call::{ArgAbi, FnAbi}; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() { + ret.make_indirect(); + } else { + ret.extend_integer_width_to(32); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if !arg.layout.is_sized() { + // Not touching this... + return; + } + if arg.layout.is_aggregate() { + arg.pass_by_stack_offset(None); + } else { + arg.extend_integer_width_to(32); + } +} + +pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/mips.rs b/compiler/rustc_target/src/callconv/mips.rs new file mode 100644 index 00000000000..37980a91c76 --- /dev/null +++ b/compiler/rustc_target/src/callconv/mips.rs @@ -0,0 +1,53 @@ +use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; +use crate::abi::{HasDataLayout, Size}; + +fn classify_ret(cx: &C, ret: &mut ArgAbi<'_, Ty>, offset: &mut Size) +where + C: HasDataLayout, +{ + if !ret.layout.is_aggregate() { + ret.extend_integer_width_to(32); + } else { + ret.make_indirect(); + *offset += cx.data_layout().pointer_size; + } +} + +fn classify_arg(cx: &C, arg: &mut ArgAbi<'_, Ty>, offset: &mut Size) +where + C: HasDataLayout, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + let dl = cx.data_layout(); + let size = arg.layout.size; + let align = arg.layout.align.max(dl.i32_align).min(dl.i64_align).abi; + + if arg.layout.is_aggregate() { + let pad_i32 = !offset.is_aligned(align); + arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32); + } else { + arg.extend_integer_width_to(32); + } + + *offset = offset.align_to(align) + size.align_to(align); +} + +pub(crate) fn compute_abi_info(cx: &C, fn_abi: &mut FnAbi<'_, Ty>) +where + C: HasDataLayout, +{ + let mut offset = Size::ZERO; + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret, &mut offset); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg, &mut offset); + } +} diff --git a/compiler/rustc_target/src/callconv/mips64.rs b/compiler/rustc_target/src/callconv/mips64.rs new file mode 100644 index 00000000000..2c3258c8d42 --- /dev/null +++ b/compiler/rustc_target/src/callconv/mips64.rs @@ -0,0 +1,167 @@ +use crate::abi::call::{ + ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, Reg, Uniform, +}; +use crate::abi::{self, HasDataLayout, Size, TyAbiInterface}; + +fn extend_integer_width_mips(arg: &mut ArgAbi<'_, Ty>, bits: u64) { + // Always sign extend u32 values on 64-bit mips + if let abi::Abi::Scalar(scalar) = arg.layout.abi { + if let abi::Int(i, signed) = scalar.primitive() { + if !signed && i.size().bits() == 32 { + if let PassMode::Direct(ref mut attrs) = arg.mode { + attrs.ext(ArgExtension::Sext); + return; + } + } + } + } + + arg.extend_integer_width_to(bits); +} + +fn float_reg<'a, Ty, C>(cx: &C, ret: &ArgAbi<'a, Ty>, i: usize) -> Option +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + match ret.layout.field(cx, i).abi { + abi::Abi::Scalar(scalar) => match scalar.primitive() { + abi::Float(abi::F32) => Some(Reg::f32()), + abi::Float(abi::F64) => Some(Reg::f64()), + _ => None, + }, + _ => None, + } +} + +fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !ret.layout.is_aggregate() { + extend_integer_width_mips(ret, 64); + return; + } + + let size = ret.layout.size; + let bits = size.bits(); + if bits <= 128 { + // Unlike other architectures which return aggregates in registers, MIPS n64 limits the + // use of float registers to structures (not unions) containing exactly one or two + // float fields. + + if let abi::FieldsShape::Arbitrary { .. } = ret.layout.fields { + if ret.layout.fields.count() == 1 { + if let Some(reg) = float_reg(cx, ret, 0) { + ret.cast_to(reg); + return; + } + } else if ret.layout.fields.count() == 2 { + if let Some(reg0) = float_reg(cx, ret, 0) { + if let Some(reg1) = float_reg(cx, ret, 1) { + ret.cast_to(CastTarget::pair(reg0, reg1)); + return; + } + } + } + } + + // Cast to a uniform int structure + ret.cast_to(Uniform::new(Reg::i64(), size)); + } else { + ret.make_indirect(); + } +} + +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !arg.layout.is_aggregate() { + extend_integer_width_mips(arg, 64); + return; + } + + let dl = cx.data_layout(); + let size = arg.layout.size; + let mut prefix = [None; 8]; + let mut prefix_index = 0; + + match arg.layout.fields { + abi::FieldsShape::Primitive => unreachable!(), + abi::FieldsShape::Array { .. } => { + // Arrays are passed indirectly + arg.make_indirect(); + return; + } + abi::FieldsShape::Union(_) => { + // Unions and are always treated as a series of 64-bit integer chunks + } + abi::FieldsShape::Arbitrary { .. } => { + // Structures are split up into a series of 64-bit integer chunks, but any aligned + // doubles not part of another aggregate are passed as floats. + let mut last_offset = Size::ZERO; + + for i in 0..arg.layout.fields.count() { + let field = arg.layout.field(cx, i); + let offset = arg.layout.fields.offset(i); + + // We only care about aligned doubles + if let abi::Abi::Scalar(scalar) = field.abi { + if scalar.primitive() == abi::Float(abi::F64) { + if offset.is_aligned(dl.f64_align.abi) { + // Insert enough integers to cover [last_offset, offset) + assert!(last_offset.is_aligned(dl.f64_align.abi)); + for _ in 0..((offset - last_offset).bits() / 64) + .min((prefix.len() - prefix_index) as u64) + { + prefix[prefix_index] = Some(Reg::i64()); + prefix_index += 1; + } + + if prefix_index == prefix.len() { + break; + } + + prefix[prefix_index] = Some(Reg::f64()); + prefix_index += 1; + last_offset = offset + Reg::f64().size; + } + } + } + } + } + }; + + // Extract first 8 chunks as the prefix + let rest_size = size - Size::from_bytes(8) * prefix_index as u64; + arg.cast_to(CastTarget { + prefix, + rest: Uniform::new(Reg::i64(), rest_size), + attrs: ArgAttributes { + regular: ArgAttribute::default(), + arg_ext: ArgExtension::None, + pointee_size: Size::ZERO, + pointee_align: None, + }, + }); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg); + } +} diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs new file mode 100644 index 00000000000..352861c5ccb --- /dev/null +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -0,0 +1,1009 @@ +use std::fmt; +use std::str::FromStr; + +use rustc_macros::HashStable_Generic; +use rustc_span::Symbol; + +use crate::abi::{self, Abi, Align, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; +use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi}; + +mod aarch64; +mod amdgpu; +mod arm; +mod avr; +mod bpf; +mod csky; +mod hexagon; +mod loongarch; +mod m68k; +mod mips; +mod mips64; +mod msp430; +mod nvptx64; +mod powerpc; +mod powerpc64; +mod riscv; +mod s390x; +mod sparc; +mod sparc64; +mod wasm; +mod x86; +mod x86_64; +mod x86_win64; +mod xtensa; + +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub enum PassMode { + /// Ignore the argument. + /// + /// The argument is either uninhabited or a ZST. + Ignore, + /// Pass the argument directly. + /// + /// The argument has a layout abi of `Scalar` or `Vector`. + /// Unfortunately due to past mistakes, in rare cases on wasm, it can also be `Aggregate`. + /// This is bad since it leaks LLVM implementation details into the ABI. + /// (Also see .) + Direct(ArgAttributes), + /// Pass a pair's elements directly in two arguments. + /// + /// The argument has a layout abi of `ScalarPair`. + Pair(ArgAttributes, ArgAttributes), + /// Pass the argument after casting it. See the `CastTarget` docs for details. + /// + /// `pad_i32` indicates if a `Reg::i32()` dummy argument is emitted before the real argument. + Cast { pad_i32: bool, cast: Box }, + /// Pass the argument indirectly via a hidden pointer. + /// + /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized + /// argument. (This is the only mode that supports unsized arguments.) + /// + /// `on_stack` defines that the value should be passed at a fixed stack offset in accordance to + /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument + /// attribute. The `byval` argument will use a byte array with the same size as the Rust type + /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout), + /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's + /// alignment (if `None`). This means that the alignment will not always + /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info. + /// + /// `on_stack` cannot be true for unsized arguments, i.e., when `meta_attrs` is `Some`. + Indirect { attrs: ArgAttributes, meta_attrs: Option, on_stack: bool }, +} + +impl PassMode { + /// Checks if these two `PassMode` are equal enough to be considered "the same for all + /// function call ABIs". However, the `Layout` can also impact ABI decisions, + /// so that needs to be compared as well! + pub fn eq_abi(&self, other: &Self) -> bool { + match (self, other) { + (PassMode::Ignore, PassMode::Ignore) => true, + (PassMode::Direct(a1), PassMode::Direct(a2)) => a1.eq_abi(a2), + (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => a1.eq_abi(a2) && b1.eq_abi(b2), + ( + PassMode::Cast { cast: c1, pad_i32: pad1 }, + PassMode::Cast { cast: c2, pad_i32: pad2 }, + ) => c1.eq_abi(c2) && pad1 == pad2, + ( + PassMode::Indirect { attrs: a1, meta_attrs: None, on_stack: s1 }, + PassMode::Indirect { attrs: a2, meta_attrs: None, on_stack: s2 }, + ) => a1.eq_abi(a2) && s1 == s2, + ( + PassMode::Indirect { attrs: a1, meta_attrs: Some(e1), on_stack: s1 }, + PassMode::Indirect { attrs: a2, meta_attrs: Some(e2), on_stack: s2 }, + ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2, + _ => false, + } + } +} + +// Hack to disable non_upper_case_globals only for the bitflags! and not for the rest +// of this module +pub use attr_impl::ArgAttribute; + +#[allow(non_upper_case_globals)] +#[allow(unused)] +mod attr_impl { + use rustc_macros::HashStable_Generic; + + // The subset of llvm::Attribute needed for arguments, packed into a bitfield. + #[derive(Clone, Copy, Default, Hash, PartialEq, Eq, HashStable_Generic)] + pub struct ArgAttribute(u8); + bitflags::bitflags! { + impl ArgAttribute: u8 { + const NoAlias = 1 << 1; + const NoCapture = 1 << 2; + const NonNull = 1 << 3; + const ReadOnly = 1 << 4; + const InReg = 1 << 5; + const NoUndef = 1 << 6; + } + } + rustc_data_structures::external_bitflags_debug! { ArgAttribute } +} + +/// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum +/// defines if this extension should be zero-extension or sign-extension when necessary. When it is +/// not necessary to extend the argument, this enum is ignored. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub enum ArgExtension { + None, + Zext, + Sext, +} + +/// A compact representation of LLVM attributes (at least those relevant for this module) +/// that can be manipulated without interacting with LLVM's Attribute machinery. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct ArgAttributes { + pub regular: ArgAttribute, + pub arg_ext: ArgExtension, + /// The minimum size of the pointee, guaranteed to be valid for the duration of the whole call + /// (corresponding to LLVM's dereferenceable and dereferenceable_or_null attributes). + pub pointee_size: Size, + pub pointee_align: Option, +} + +impl ArgAttributes { + pub fn new() -> Self { + ArgAttributes { + regular: ArgAttribute::default(), + arg_ext: ArgExtension::None, + pointee_size: Size::ZERO, + pointee_align: None, + } + } + + pub fn ext(&mut self, ext: ArgExtension) -> &mut Self { + assert!( + self.arg_ext == ArgExtension::None || self.arg_ext == ext, + "cannot set {:?} when {:?} is already set", + ext, + self.arg_ext + ); + self.arg_ext = ext; + self + } + + pub fn set(&mut self, attr: ArgAttribute) -> &mut Self { + self.regular |= attr; + self + } + + pub fn contains(&self, attr: ArgAttribute) -> bool { + self.regular.contains(attr) + } + + /// Checks if these two `ArgAttributes` are equal enough to be considered "the same for all + /// function call ABIs". + pub fn eq_abi(&self, other: &Self) -> bool { + // There's only one regular attribute that matters for the call ABI: InReg. + // Everything else is things like noalias, dereferenceable, nonnull, ... + // (This also applies to pointee_size, pointee_align.) + if self.regular.contains(ArgAttribute::InReg) != other.regular.contains(ArgAttribute::InReg) + { + return false; + } + // We also compare the sign extension mode -- this could let the callee make assumptions + // about bits that conceptually were not even passed. + if self.arg_ext != other.arg_ext { + return false; + } + true + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub enum RegKind { + Integer, + Float, + Vector, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct Reg { + pub kind: RegKind, + pub size: Size, +} + +macro_rules! reg_ctor { + ($name:ident, $kind:ident, $bits:expr) => { + pub fn $name() -> Reg { + Reg { kind: RegKind::$kind, size: Size::from_bits($bits) } + } + }; +} + +impl Reg { + reg_ctor!(i8, Integer, 8); + reg_ctor!(i16, Integer, 16); + reg_ctor!(i32, Integer, 32); + reg_ctor!(i64, Integer, 64); + reg_ctor!(i128, Integer, 128); + + reg_ctor!(f32, Float, 32); + reg_ctor!(f64, Float, 64); +} + +impl Reg { + pub fn align(&self, cx: &C) -> Align { + let dl = cx.data_layout(); + match self.kind { + RegKind::Integer => match self.size.bits() { + 1 => dl.i1_align.abi, + 2..=8 => dl.i8_align.abi, + 9..=16 => dl.i16_align.abi, + 17..=32 => dl.i32_align.abi, + 33..=64 => dl.i64_align.abi, + 65..=128 => dl.i128_align.abi, + _ => panic!("unsupported integer: {self:?}"), + }, + RegKind::Float => match self.size.bits() { + 16 => dl.f16_align.abi, + 32 => dl.f32_align.abi, + 64 => dl.f64_align.abi, + 128 => dl.f128_align.abi, + _ => panic!("unsupported float: {self:?}"), + }, + RegKind::Vector => dl.vector_align(self.size).abi, + } + } +} + +/// An argument passed entirely registers with the +/// same kind (e.g., HFA / HVA on PPC64 and AArch64). +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct Uniform { + pub unit: Reg, + + /// The total size of the argument, which can be: + /// * equal to `unit.size` (one scalar/vector), + /// * a multiple of `unit.size` (an array of scalar/vectors), + /// * if `unit.kind` is `Integer`, the last element can be shorter, i.e., `{ i64, i64, i32 }` + /// for 64-bit integers with a total size of 20 bytes. When the argument is actually passed, + /// this size will be rounded up to the nearest multiple of `unit.size`. + pub total: Size, + + /// Indicate that the argument is consecutive, in the sense that either all values need to be + /// passed in register, or all on the stack. If they are passed on the stack, there should be + /// no additional padding between elements. + pub is_consecutive: bool, +} + +impl From for Uniform { + fn from(unit: Reg) -> Uniform { + Uniform { unit, total: unit.size, is_consecutive: false } + } +} + +impl Uniform { + pub fn align(&self, cx: &C) -> Align { + self.unit.align(cx) + } + + /// Pass using one or more values of the given type, without requiring them to be consecutive. + /// That is, some values may be passed in register and some on the stack. + pub fn new(unit: Reg, total: Size) -> Self { + Uniform { unit, total, is_consecutive: false } + } + + /// Pass using one or more consecutive values of the given type. Either all values will be + /// passed in registers, or all on the stack. + pub fn consecutive(unit: Reg, total: Size) -> Self { + Uniform { unit, total, is_consecutive: true } + } +} + +/// Describes the type used for `PassMode::Cast`. +/// +/// Passing arguments in this mode works as follows: the registers in the `prefix` (the ones that +/// are `Some`) get laid out one after the other (using `repr(C)` layout rules). Then the +/// `rest.unit` register type gets repeated often enough to cover `rest.size`. This describes the +/// actual type used for the call; the Rust type of the argument is then transmuted to this ABI type +/// (and all data in the padding between the registers is dropped). +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct CastTarget { + pub prefix: [Option; 8], + pub rest: Uniform, + pub attrs: ArgAttributes, +} + +impl From for CastTarget { + fn from(unit: Reg) -> CastTarget { + CastTarget::from(Uniform::from(unit)) + } +} + +impl From for CastTarget { + fn from(uniform: Uniform) -> CastTarget { + CastTarget { + prefix: [None; 8], + rest: uniform, + attrs: ArgAttributes { + regular: ArgAttribute::default(), + arg_ext: ArgExtension::None, + pointee_size: Size::ZERO, + pointee_align: None, + }, + } + } +} + +impl CastTarget { + pub fn pair(a: Reg, b: Reg) -> CastTarget { + CastTarget { + prefix: [Some(a), None, None, None, None, None, None, None], + rest: Uniform::from(b), + attrs: ArgAttributes { + regular: ArgAttribute::default(), + arg_ext: ArgExtension::None, + pointee_size: Size::ZERO, + pointee_align: None, + }, + } + } + + /// When you only access the range containing valid data, you can use this unaligned size; + /// otherwise, use the safer `size` method. + pub fn unaligned_size(&self, _cx: &C) -> Size { + // Prefix arguments are passed in specific designated registers + let prefix_size = self + .prefix + .iter() + .filter_map(|x| x.map(|reg| reg.size)) + .fold(Size::ZERO, |acc, size| acc + size); + // Remaining arguments are passed in chunks of the unit size + let rest_size = + self.rest.unit.size * self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes()); + + prefix_size + rest_size + } + + pub fn size(&self, cx: &C) -> Size { + self.unaligned_size(cx).align_to(self.align(cx)) + } + + pub fn align(&self, cx: &C) -> Align { + self.prefix + .iter() + .filter_map(|x| x.map(|reg| reg.align(cx))) + .fold(cx.data_layout().aggregate_align.abi.max(self.rest.align(cx)), |acc, align| { + acc.max(align) + }) + } + + /// Checks if these two `CastTarget` are equal enough to be considered "the same for all + /// function call ABIs". + pub fn eq_abi(&self, other: &Self) -> bool { + let CastTarget { prefix: prefix_l, rest: rest_l, attrs: attrs_l } = self; + let CastTarget { prefix: prefix_r, rest: rest_r, attrs: attrs_r } = other; + prefix_l == prefix_r && rest_l == rest_r && attrs_l.eq_abi(attrs_r) + } +} + +/// Return value from the `homogeneous_aggregate` test function. +#[derive(Copy, Clone, Debug)] +pub enum HomogeneousAggregate { + /// Yes, all the "leaf fields" of this struct are passed in the + /// same way (specified in the `Reg` value). + Homogeneous(Reg), + + /// There are no leaf fields at all. + NoData, +} + +/// Error from the `homogeneous_aggregate` test function, indicating +/// there are distinct leaf fields passed in different ways, +/// or this is uninhabited. +#[derive(Copy, Clone, Debug)] +pub struct Heterogeneous; + +impl HomogeneousAggregate { + /// If this is a homogeneous aggregate, returns the homogeneous + /// unit, else `None`. + pub fn unit(self) -> Option { + match self { + HomogeneousAggregate::Homogeneous(reg) => Some(reg), + HomogeneousAggregate::NoData => None, + } + } + + /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in + /// the same `struct`. Only succeeds if only one of them has any data, + /// or both units are identical. + fn merge(self, other: HomogeneousAggregate) -> Result { + match (self, other) { + (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x), + + (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => { + if a != b { + return Err(Heterogeneous); + } + Ok(self) + } + } + } +} + +impl<'a, Ty> TyAndLayout<'a, Ty> { + /// Returns `true` if this is an aggregate type (including a ScalarPair!) + fn is_aggregate(&self) -> bool { + match self.abi { + Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, + Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, + } + } + + /// Returns `Homogeneous` if this layout is an aggregate containing fields of + /// only a single type (e.g., `(u32, u32)`). Such aggregates are often + /// special-cased in ABIs. + /// + /// Note: We generally ignore 1-ZST fields when computing this value (see #56877). + /// + /// This is public so that it can be used in unit tests, but + /// should generally only be relevant to the ABI details of + /// specific targets. + pub fn homogeneous_aggregate(&self, cx: &C) -> Result + where + Ty: TyAbiInterface<'a, C> + Copy, + { + match self.abi { + Abi::Uninhabited => Err(Heterogeneous), + + // The primitive for this algorithm. + Abi::Scalar(scalar) => { + let kind = match scalar.primitive() { + abi::Int(..) | abi::Pointer(_) => RegKind::Integer, + abi::Float(_) => RegKind::Float, + }; + Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size })) + } + + Abi::Vector { .. } => { + assert!(!self.is_zst()); + Ok(HomogeneousAggregate::Homogeneous(Reg { + kind: RegKind::Vector, + size: self.size, + })) + } + + Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => { + // Helper for computing `homogeneous_aggregate`, allowing a custom + // starting offset (used below for handling variants). + let from_fields_at = + |layout: Self, + start: Size| + -> Result<(HomogeneousAggregate, Size), Heterogeneous> { + let is_union = match layout.fields { + FieldsShape::Primitive => { + unreachable!("aggregates can't have `FieldsShape::Primitive`") + } + FieldsShape::Array { count, .. } => { + assert_eq!(start, Size::ZERO); + + let result = if count > 0 { + layout.field(cx, 0).homogeneous_aggregate(cx)? + } else { + HomogeneousAggregate::NoData + }; + return Ok((result, layout.size)); + } + FieldsShape::Union(_) => true, + FieldsShape::Arbitrary { .. } => false, + }; + + let mut result = HomogeneousAggregate::NoData; + let mut total = start; + + for i in 0..layout.fields.count() { + let field = layout.field(cx, i); + if field.is_1zst() { + // No data here and no impact on layout, can be ignored. + // (We might be able to also ignore all aligned ZST but that's less clear.) + continue; + } + + if !is_union && total != layout.fields.offset(i) { + // This field isn't just after the previous one we considered, abort. + return Err(Heterogeneous); + } + + result = result.merge(field.homogeneous_aggregate(cx)?)?; + + // Keep track of the offset (without padding). + let size = field.size; + if is_union { + total = total.max(size); + } else { + total += size; + } + } + + Ok((result, total)) + }; + + let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?; + + match &self.variants { + abi::Variants::Single { .. } => {} + abi::Variants::Multiple { variants, .. } => { + // Treat enum variants like union members. + // HACK(eddyb) pretend the `enum` field (discriminant) + // is at the start of every variant (otherwise the gap + // at the start of all variants would disqualify them). + // + // NB: for all tagged `enum`s (which include all non-C-like + // `enum`s with defined FFI representation), this will + // match the homogeneous computation on the equivalent + // `struct { tag; union { variant1; ... } }` and/or + // `union { struct { tag; variant1; } ... }` + // (the offsets of variant fields should be identical + // between the two for either to be a homogeneous aggregate). + let variant_start = total; + for variant_idx in variants.indices() { + let (variant_result, variant_total) = + from_fields_at(self.for_variant(cx, variant_idx), variant_start)?; + + result = result.merge(variant_result)?; + total = total.max(variant_total); + } + } + } + + // There needs to be no padding. + if total != self.size { + Err(Heterogeneous) + } else { + match result { + HomogeneousAggregate::Homogeneous(_) => { + assert_ne!(total, Size::ZERO); + } + HomogeneousAggregate::NoData => { + assert_eq!(total, Size::ZERO); + } + } + Ok(result) + } + } + Abi::Aggregate { sized: false } => Err(Heterogeneous), + } + } +} + +/// Information about how to pass an argument to, +/// or return a value from, a function, under some ABI. +#[derive(Clone, PartialEq, Eq, Hash, HashStable_Generic)] +pub struct ArgAbi<'a, Ty> { + pub layout: TyAndLayout<'a, Ty>, + pub mode: PassMode, +} + +// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl. +impl<'a, Ty: fmt::Display> fmt::Debug for ArgAbi<'a, Ty> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let ArgAbi { layout, mode } = self; + f.debug_struct("ArgAbi").field("layout", layout).field("mode", mode).finish() + } +} + +impl<'a, Ty> ArgAbi<'a, Ty> { + /// This defines the "default ABI" for that type, that is then later adjusted in `fn_abi_adjust_for_abi`. + pub fn new( + cx: &impl HasDataLayout, + layout: TyAndLayout<'a, Ty>, + scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, abi::Scalar, Size) -> ArgAttributes, + ) -> Self { + let mode = match layout.abi { + Abi::Uninhabited => PassMode::Ignore, + Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)), + Abi::ScalarPair(a, b) => PassMode::Pair( + scalar_attrs(&layout, a, Size::ZERO), + scalar_attrs(&layout, b, a.size(cx).align_to(b.align(cx).abi)), + ), + Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()), + Abi::Aggregate { .. } => Self::indirect_pass_mode(&layout), + }; + ArgAbi { layout, mode } + } + + fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode { + let mut attrs = ArgAttributes::new(); + + // For non-immediate arguments the callee gets its own copy of + // the value on the stack, so there are no aliases. It's also + // program-invisible so can't possibly capture + attrs + .set(ArgAttribute::NoAlias) + .set(ArgAttribute::NoCapture) + .set(ArgAttribute::NonNull) + .set(ArgAttribute::NoUndef); + attrs.pointee_size = layout.size; + attrs.pointee_align = Some(layout.align.abi); + + let meta_attrs = layout.is_unsized().then_some(ArgAttributes::new()); + + PassMode::Indirect { attrs, meta_attrs, on_stack: false } + } + + /// Pass this argument directly instead. Should NOT be used! + /// Only exists because of past ABI mistakes that will take time to fix + /// (see ). + pub fn make_direct_deprecated(&mut self) { + match self.mode { + PassMode::Indirect { .. } => { + self.mode = PassMode::Direct(ArgAttributes::new()); + } + PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct + _ => panic!("Tried to make {:?} direct", self.mode), + } + } + + /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead. + /// This is valid for both sized and unsized arguments. + pub fn make_indirect(&mut self) { + match self.mode { + PassMode::Direct(_) | PassMode::Pair(_, _) => { + self.mode = Self::indirect_pass_mode(&self.layout); + } + PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => { + // already indirect + } + _ => panic!("Tried to make {:?} indirect", self.mode), + } + } + + /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass + /// ZSTs indirectly. + pub fn make_indirect_from_ignore(&mut self) { + match self.mode { + PassMode::Ignore => { + self.mode = Self::indirect_pass_mode(&self.layout); + } + PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => { + // already indirect + } + _ => panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode), + } + } + + /// Pass this argument indirectly, by placing it at a fixed stack offset. + /// This corresponds to the `byval` LLVM argument attribute. + /// This is only valid for sized arguments. + /// + /// `byval_align` specifies the alignment of the `byval` stack slot, which does not need to + /// correspond to the type's alignment. This will be `Some` if the target's ABI specifies that + /// stack slots used for arguments passed by-value have specific alignment requirements which + /// differ from the alignment used in other situations. + /// + /// If `None`, the type's alignment is used. + /// + /// If the resulting alignment differs from the type's alignment, + /// the argument will be copied to an alloca with sufficient alignment, + /// either in the caller (if the type's alignment is lower than the byval alignment) + /// or in the callee (if the type's alignment is higher than the byval alignment), + /// to ensure that Rust code never sees an underaligned pointer. + pub fn pass_by_stack_offset(&mut self, byval_align: Option) { + assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout"); + self.make_indirect(); + match self.mode { + PassMode::Indirect { ref mut attrs, meta_attrs: _, ref mut on_stack } => { + *on_stack = true; + + // Some platforms, like 32-bit x86, change the alignment of the type when passing + // `byval`. Account for that. + if let Some(byval_align) = byval_align { + // On all targets with byval align this is currently true, so let's assert it. + debug_assert!(byval_align >= Align::from_bytes(4).unwrap()); + attrs.pointee_align = Some(byval_align); + } + } + _ => unreachable!(), + } + } + + pub fn extend_integer_width_to(&mut self, bits: u64) { + // Only integers have signedness + if let Abi::Scalar(scalar) = self.layout.abi { + if let abi::Int(i, signed) = scalar.primitive() { + if i.size().bits() < bits { + if let PassMode::Direct(ref mut attrs) = self.mode { + if signed { + attrs.ext(ArgExtension::Sext) + } else { + attrs.ext(ArgExtension::Zext) + }; + } + } + } + } + } + + pub fn cast_to>(&mut self, target: T) { + self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32: false }; + } + + pub fn cast_to_and_pad_i32>(&mut self, target: T, pad_i32: bool) { + self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32 }; + } + + pub fn is_indirect(&self) -> bool { + matches!(self.mode, PassMode::Indirect { .. }) + } + + pub fn is_sized_indirect(&self) -> bool { + matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }) + } + + pub fn is_unsized_indirect(&self) -> bool { + matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }) + } + + pub fn is_ignore(&self) -> bool { + matches!(self.mode, PassMode::Ignore) + } + + /// Checks if these two `ArgAbi` are equal enough to be considered "the same for all + /// function call ABIs". + pub fn eq_abi(&self, other: &Self) -> bool + where + Ty: PartialEq, + { + // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will look + // at the type. + self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode) && { + // `fn_arg_sanity_check` accepts `PassMode::Direct` for some aggregates. + // That elevates any type difference to an ABI difference since we just use the + // full Rust type as the LLVM argument/return type. + if matches!(self.mode, PassMode::Direct(..)) + && matches!(self.layout.abi, Abi::Aggregate { .. }) + { + // For aggregates in `Direct` mode to be compatible, the types need to be equal. + self.layout.ty == other.layout.ty + } else { + true + } + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub enum Conv { + // General language calling conventions, for which every target + // should have its own backend (e.g. LLVM) support. + C, + Rust, + + Cold, + PreserveMost, + PreserveAll, + + // Target-specific calling conventions. + ArmAapcs, + CCmseNonSecureCall, + CCmseNonSecureEntry, + + Msp430Intr, + + PtxKernel, + + X86Fastcall, + X86Intr, + X86Stdcall, + X86ThisCall, + X86VectorCall, + + X86_64SysV, + X86_64Win64, + + AvrInterrupt, + AvrNonBlockingInterrupt, + + RiscvInterrupt { kind: RiscvInterruptKind }, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub enum RiscvInterruptKind { + Machine, + Supervisor, +} + +impl RiscvInterruptKind { + pub fn as_str(&self) -> &'static str { + match self { + Self::Machine => "machine", + Self::Supervisor => "supervisor", + } + } +} + +/// Metadata describing how the arguments to a native function +/// should be passed in order to respect the native ABI. +/// +/// The signature represented by this type may not match the MIR function signature. +/// Certain attributes, like `#[track_caller]` can introduce additional arguments, which are present in [`FnAbi`], but not in `FnSig`. +/// While this difference is rarely relevant, it should still be kept in mind. +/// +/// I will do my best to describe this structure, but these +/// comments are reverse-engineered and may be inaccurate. -NDM +#[derive(Clone, PartialEq, Eq, Hash, HashStable_Generic)] +pub struct FnAbi<'a, Ty> { + /// The type, layout, and information about how each argument is passed. + pub args: Box<[ArgAbi<'a, Ty>]>, + + /// The layout, type, and the way a value is returned from this function. + pub ret: ArgAbi<'a, Ty>, + + /// Marks this function as variadic (accepting a variable number of arguments). + pub c_variadic: bool, + + /// The count of non-variadic arguments. + /// + /// Should only be different from args.len() when c_variadic is true. + /// This can be used to know whether an argument is variadic or not. + pub fixed_count: u32, + /// The calling convention of this function. + pub conv: Conv, + /// Indicates if an unwind may happen across a call to this function. + pub can_unwind: bool, +} + +// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl. +impl<'a, Ty: fmt::Display> fmt::Debug for FnAbi<'a, Ty> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let FnAbi { args, ret, c_variadic, fixed_count, conv, can_unwind } = self; + f.debug_struct("FnAbi") + .field("args", args) + .field("ret", ret) + .field("c_variadic", c_variadic) + .field("fixed_count", fixed_count) + .field("conv", conv) + .field("can_unwind", can_unwind) + .finish() + } +} + +/// Error produced by attempting to adjust a `FnAbi`, for a "foreign" ABI. +#[derive(Copy, Clone, Debug, HashStable_Generic)] +pub enum AdjustForForeignAbiError { + /// Target architecture doesn't support "foreign" (i.e. non-Rust) ABIs. + Unsupported { arch: Symbol, abi: spec::abi::Abi }, +} + +impl<'a, Ty> FnAbi<'a, Ty> { + pub fn adjust_for_foreign_abi( + &mut self, + cx: &C, + abi: spec::abi::Abi, + ) -> Result<(), AdjustForForeignAbiError> + where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt, + { + if abi == spec::abi::Abi::X86Interrupt { + if let Some(arg) = self.args.first_mut() { + arg.pass_by_stack_offset(None); + } + return Ok(()); + } + + let spec = cx.target_spec(); + match &spec.arch[..] { + "x86" => { + let flavor = if let spec::abi::Abi::Fastcall { .. } + | spec::abi::Abi::Vectorcall { .. } = abi + { + x86::Flavor::FastcallOrVectorcall + } else { + x86::Flavor::General + }; + x86::compute_abi_info(cx, self, flavor); + } + "x86_64" => match abi { + spec::abi::Abi::SysV64 { .. } => x86_64::compute_abi_info(cx, self), + spec::abi::Abi::Win64 { .. } => x86_win64::compute_abi_info(cx, self), + _ => { + if cx.target_spec().is_like_windows { + x86_win64::compute_abi_info(cx, self) + } else { + x86_64::compute_abi_info(cx, self) + } + } + }, + "aarch64" | "arm64ec" => { + let kind = if cx.target_spec().is_like_osx { + aarch64::AbiKind::DarwinPCS + } else if cx.target_spec().is_like_windows { + aarch64::AbiKind::Win64 + } else { + aarch64::AbiKind::AAPCS + }; + aarch64::compute_abi_info(cx, self, kind) + } + "amdgpu" => amdgpu::compute_abi_info(cx, self), + "arm" => arm::compute_abi_info(cx, self), + "avr" => avr::compute_abi_info(self), + "loongarch64" => loongarch::compute_abi_info(cx, self), + "m68k" => m68k::compute_abi_info(self), + "csky" => csky::compute_abi_info(self), + "mips" | "mips32r6" => mips::compute_abi_info(cx, self), + "mips64" | "mips64r6" => mips64::compute_abi_info(cx, self), + "powerpc" => powerpc::compute_abi_info(cx, self), + "powerpc64" => powerpc64::compute_abi_info(cx, self), + "s390x" => s390x::compute_abi_info(cx, self), + "msp430" => msp430::compute_abi_info(self), + "sparc" => sparc::compute_abi_info(cx, self), + "sparc64" => sparc64::compute_abi_info(cx, self), + "nvptx64" => { + if cx.target_spec().adjust_abi(abi, self.c_variadic) == spec::abi::Abi::PtxKernel { + nvptx64::compute_ptx_kernel_abi_info(cx, self) + } else { + nvptx64::compute_abi_info(self) + } + } + "hexagon" => hexagon::compute_abi_info(self), + "xtensa" => xtensa::compute_abi_info(cx, self), + "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self), + "wasm32" => { + if spec.os == "unknown" && cx.wasm_c_abi_opt() == WasmCAbi::Legacy { + wasm::compute_wasm_abi_info(self) + } else { + wasm::compute_c_abi_info(cx, self) + } + } + "wasm64" => wasm::compute_c_abi_info(cx, self), + "bpf" => bpf::compute_abi_info(self), + arch => { + return Err(AdjustForForeignAbiError::Unsupported { + arch: Symbol::intern(arch), + abi, + }); + } + } + + Ok(()) + } +} + +impl FromStr for Conv { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "C" => Ok(Conv::C), + "Rust" => Ok(Conv::Rust), + "RustCold" => Ok(Conv::Rust), + "ArmAapcs" => Ok(Conv::ArmAapcs), + "CCmseNonSecureCall" => Ok(Conv::CCmseNonSecureCall), + "CCmseNonSecureEntry" => Ok(Conv::CCmseNonSecureEntry), + "Msp430Intr" => Ok(Conv::Msp430Intr), + "PtxKernel" => Ok(Conv::PtxKernel), + "X86Fastcall" => Ok(Conv::X86Fastcall), + "X86Intr" => Ok(Conv::X86Intr), + "X86Stdcall" => Ok(Conv::X86Stdcall), + "X86ThisCall" => Ok(Conv::X86ThisCall), + "X86VectorCall" => Ok(Conv::X86VectorCall), + "X86_64SysV" => Ok(Conv::X86_64SysV), + "X86_64Win64" => Ok(Conv::X86_64Win64), + "AvrInterrupt" => Ok(Conv::AvrInterrupt), + "AvrNonBlockingInterrupt" => Ok(Conv::AvrNonBlockingInterrupt), + "RiscvInterrupt(machine)" => { + Ok(Conv::RiscvInterrupt { kind: RiscvInterruptKind::Machine }) + } + "RiscvInterrupt(supervisor)" => { + Ok(Conv::RiscvInterrupt { kind: RiscvInterruptKind::Supervisor }) + } + _ => Err(format!("'{s}' is not a valid value for entry function call convention.")), + } + } +} + +// Some types are used a lot. Make sure they don't unintentionally get bigger. +#[cfg(target_pointer_width = "64")] +mod size_asserts { + use rustc_data_structures::static_assert_size; + + use super::*; + // tidy-alphabetical-start + static_assert_size!(ArgAbi<'_, usize>, 56); + static_assert_size!(FnAbi<'_, usize>, 80); + // tidy-alphabetical-end +} diff --git a/compiler/rustc_target/src/callconv/msp430.rs b/compiler/rustc_target/src/callconv/msp430.rs new file mode 100644 index 00000000000..4f613aa6c15 --- /dev/null +++ b/compiler/rustc_target/src/callconv/msp430.rs @@ -0,0 +1,39 @@ +// Reference: MSP430 Embedded Application Binary Interface +// https://www.ti.com/lit/an/slaa534a/slaa534a.pdf + +use crate::abi::call::{ArgAbi, FnAbi}; + +// 3.5 Structures or Unions Passed and Returned by Reference +// +// "Structures (including classes) and unions larger than 32 bits are passed and +// returned by reference. To pass a structure or union by reference, the caller +// places its address in the appropriate location: either in a register or on +// the stack, according to its position in the argument list. (..)" +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() && ret.layout.size.bits() > 32 { + ret.make_indirect(); + } else { + ret.extend_integer_width_to(16); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if arg.layout.is_aggregate() && arg.layout.size.bits() > 32 { + arg.make_indirect(); + } else { + arg.extend_integer_width_to(16); + } +} + +pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/nvptx64.rs b/compiler/rustc_target/src/callconv/nvptx64.rs new file mode 100644 index 00000000000..2e8b16d3a93 --- /dev/null +++ b/compiler/rustc_target/src/callconv/nvptx64.rs @@ -0,0 +1,102 @@ +use super::{ArgAttribute, ArgAttributes, ArgExtension, CastTarget}; +use crate::abi::call::{ArgAbi, FnAbi, PassMode, Reg, Size, Uniform}; +use crate::abi::{HasDataLayout, TyAbiInterface}; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() && ret.layout.is_sized() { + classify_aggregate(ret) + } else if ret.layout.size.bits() < 32 && ret.layout.is_sized() { + ret.extend_integer_width_to(32); + } +} + +fn classify_arg(arg: &mut ArgAbi<'_, Ty>) { + if arg.layout.is_aggregate() && arg.layout.is_sized() { + classify_aggregate(arg) + } else if arg.layout.size.bits() < 32 && arg.layout.is_sized() { + arg.extend_integer_width_to(32); + } +} + +/// the pass mode used for aggregates in arg and ret position +fn classify_aggregate(arg: &mut ArgAbi<'_, Ty>) { + let align_bytes = arg.layout.align.abi.bytes(); + let size = arg.layout.size; + + let reg = match align_bytes { + 1 => Reg::i8(), + 2 => Reg::i16(), + 4 => Reg::i32(), + 8 => Reg::i64(), + 16 => Reg::i128(), + _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"), + }; + + if align_bytes == size.bytes() { + arg.cast_to(CastTarget { + prefix: [Some(reg), None, None, None, None, None, None, None], + rest: Uniform::new(Reg::i8(), Size::from_bytes(0)), + attrs: ArgAttributes { + regular: ArgAttribute::default(), + arg_ext: ArgExtension::None, + pointee_size: Size::ZERO, + pointee_align: None, + }, + }); + } else { + arg.cast_to(Uniform::new(reg, size)); + } +} + +fn classify_arg_kernel<'a, Ty, C>(_cx: &C, arg: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if matches!(arg.mode, PassMode::Pair(..)) && (arg.layout.is_adt() || arg.layout.is_tuple()) { + let align_bytes = arg.layout.align.abi.bytes(); + + let unit = match align_bytes { + 1 => Reg::i8(), + 2 => Reg::i16(), + 4 => Reg::i32(), + 8 => Reg::i64(), + 16 => Reg::i128(), + _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"), + }; + arg.cast_to(Uniform::new(unit, Size::from_bytes(2 * align_bytes))); + } else { + // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271. + arg.make_direct_deprecated(); + } +} + +pub(crate) fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(arg); + } +} + +pub(crate) fn compute_ptx_kernel_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !fn_abi.ret.layout.is_unit() && !fn_abi.ret.layout.is_never() { + panic!("Kernels should not return anything other than () or !"); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg_kernel(cx, arg); + } +} diff --git a/compiler/rustc_target/src/callconv/powerpc.rs b/compiler/rustc_target/src/callconv/powerpc.rs new file mode 100644 index 00000000000..f3b05c48173 --- /dev/null +++ b/compiler/rustc_target/src/callconv/powerpc.rs @@ -0,0 +1,38 @@ +use crate::abi::call::{ArgAbi, FnAbi}; +use crate::spec::HasTargetSpec; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if ret.layout.is_aggregate() { + ret.make_indirect(); + } else { + ret.extend_integer_width_to(32); + } +} + +fn classify_arg(cx: &impl HasTargetSpec, arg: &mut ArgAbi<'_, Ty>) { + if arg.is_ignore() { + // powerpc-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs. + if cx.target_spec().os == "linux" + && matches!(&*cx.target_spec().env, "gnu" | "musl" | "uclibc") + && arg.layout.is_zst() + { + arg.make_indirect_from_ignore(); + } + return; + } + if arg.layout.is_aggregate() { + arg.make_indirect(); + } else { + arg.extend_integer_width_to(32); + } +} + +pub(crate) fn compute_abi_info(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + classify_arg(cx, arg); + } +} diff --git a/compiler/rustc_target/src/callconv/powerpc64.rs b/compiler/rustc_target/src/callconv/powerpc64.rs new file mode 100644 index 00000000000..71e533b8cc5 --- /dev/null +++ b/compiler/rustc_target/src/callconv/powerpc64.rs @@ -0,0 +1,118 @@ +// FIXME: +// Alignment of 128 bit types is not currently handled, this will +// need to be fixed when PowerPC vector support is added. + +use crate::abi::call::{Align, ArgAbi, FnAbi, Reg, RegKind, Uniform}; +use crate::abi::{Endian, HasDataLayout, TyAbiInterface}; +use crate::spec::HasTargetSpec; + +#[derive(Debug, Clone, Copy, PartialEq)] +enum ABI { + ELFv1, // original ABI used for powerpc64 (big-endian) + ELFv2, // newer ABI used for powerpc64le and musl (both endians) + AIX, // used by AIX OS, big-endian only +} +use ABI::*; + +fn is_homogeneous_aggregate<'a, Ty, C>( + cx: &C, + arg: &mut ArgAbi<'a, Ty>, + abi: ABI, +) -> Option +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + arg.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()).and_then(|unit| { + // ELFv1 and AIX only passes one-member aggregates transparently. + // ELFv2 passes up to eight uniquely addressable members. + if ((abi == ELFv1 || abi == AIX) && arg.layout.size > unit.size) + || arg.layout.size > unit.size.checked_mul(8, cx).unwrap() + { + return None; + } + + let valid_unit = match unit.kind { + RegKind::Integer => false, + RegKind::Float => true, + RegKind::Vector => arg.layout.size.bits() == 128, + }; + + valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size)) + }) +} + +fn classify<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, abi: ABI, is_ret: bool) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if arg.is_ignore() || !arg.layout.is_sized() { + // Not touching this... + return; + } + if !arg.layout.is_aggregate() { + arg.extend_integer_width_to(64); + return; + } + + // The AIX ABI expect byval for aggregates + // See https://github.com/llvm/llvm-project/blob/main/clang/lib/CodeGen/Targets/PPC.cpp. + if !is_ret && abi == AIX { + arg.pass_by_stack_offset(None); + return; + } + + // The ELFv1 ABI doesn't return aggregates in registers + if is_ret && (abi == ELFv1 || abi == AIX) { + arg.make_indirect(); + return; + } + + if let Some(uniform) = is_homogeneous_aggregate(cx, arg, abi) { + arg.cast_to(uniform); + return; + } + + let size = arg.layout.size; + if is_ret && size.bits() > 128 { + // Non-homogeneous aggregates larger than two doublewords are returned indirectly. + arg.make_indirect(); + } else if size.bits() <= 64 { + // Aggregates smaller than a doubleword should appear in + // the least-significant bits of the parameter doubleword. + arg.cast_to(Reg { kind: RegKind::Integer, size }) + } else { + // Aggregates larger than i64 should be padded at the tail to fill out a whole number + // of i64s or i128s, depending on the aggregate alignment. Always use an array for + // this, even if there is only a single element. + let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() }; + arg.cast_to(Uniform::consecutive( + reg, + size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()), + )) + }; +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + let abi = if cx.target_spec().env == "musl" { + ELFv2 + } else if cx.target_spec().os == "aix" { + AIX + } else { + match cx.data_layout().endian { + Endian::Big => ELFv1, + Endian::Little => ELFv2, + } + }; + + classify(cx, &mut fn_abi.ret, abi, true); + + for arg in fn_abi.args.iter_mut() { + classify(cx, arg, abi, false); + } +} diff --git a/compiler/rustc_target/src/callconv/riscv.rs b/compiler/rustc_target/src/callconv/riscv.rs new file mode 100644 index 00000000000..be6bc701b49 --- /dev/null +++ b/compiler/rustc_target/src/callconv/riscv.rs @@ -0,0 +1,367 @@ +// Reference: RISC-V ELF psABI specification +// https://github.com/riscv/riscv-elf-psabi-doc +// +// Reference: Clang RISC-V ELF psABI lowering code +// https://github.com/llvm/llvm-project/blob/8e780252a7284be45cf1ba224cabd884847e8e92/clang/lib/CodeGen/TargetInfo.cpp#L9311-L9773 + +use crate::abi::call::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Reg, RegKind, Uniform}; +use crate::abi::{self, Abi, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; +use crate::spec::HasTargetSpec; + +#[derive(Copy, Clone)] +enum RegPassKind { + Float(Reg), + Integer(Reg), + Unknown, +} + +#[derive(Copy, Clone)] +enum FloatConv { + FloatPair(Reg, Reg), + Float(Reg), + MixedPair(Reg, Reg), +} + +#[derive(Copy, Clone)] +struct CannotUseFpConv; + +fn is_riscv_aggregate(arg: &ArgAbi<'_, Ty>) -> bool { + match arg.layout.abi { + Abi::Vector { .. } => true, + _ => arg.layout.is_aggregate(), + } +} + +fn should_use_fp_conv_helper<'a, Ty, C>( + cx: &C, + arg_layout: &TyAndLayout<'a, Ty>, + xlen: u64, + flen: u64, + field1_kind: &mut RegPassKind, + field2_kind: &mut RegPassKind, +) -> Result<(), CannotUseFpConv> +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + match arg_layout.abi { + Abi::Scalar(scalar) => match scalar.primitive() { + abi::Int(..) | abi::Pointer(_) => { + if arg_layout.size.bits() > xlen { + return Err(CannotUseFpConv); + } + match (*field1_kind, *field2_kind) { + (RegPassKind::Unknown, _) => { + *field1_kind = RegPassKind::Integer(Reg { + kind: RegKind::Integer, + size: arg_layout.size, + }); + } + (RegPassKind::Float(_), RegPassKind::Unknown) => { + *field2_kind = RegPassKind::Integer(Reg { + kind: RegKind::Integer, + size: arg_layout.size, + }); + } + _ => return Err(CannotUseFpConv), + } + } + abi::Float(_) => { + if arg_layout.size.bits() > flen { + return Err(CannotUseFpConv); + } + match (*field1_kind, *field2_kind) { + (RegPassKind::Unknown, _) => { + *field1_kind = + RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); + } + (_, RegPassKind::Unknown) => { + *field2_kind = + RegPassKind::Float(Reg { kind: RegKind::Float, size: arg_layout.size }); + } + _ => return Err(CannotUseFpConv), + } + } + }, + Abi::Vector { .. } | Abi::Uninhabited => return Err(CannotUseFpConv), + Abi::ScalarPair(..) | Abi::Aggregate { .. } => match arg_layout.fields { + FieldsShape::Primitive => { + unreachable!("aggregates can't have `FieldsShape::Primitive`") + } + FieldsShape::Union(_) => { + if !arg_layout.is_zst() { + if arg_layout.is_transparent() { + let non_1zst_elem = arg_layout.non_1zst_field(cx).expect("not exactly one non-1-ZST field in non-ZST repr(transparent) union").1; + return should_use_fp_conv_helper( + cx, + &non_1zst_elem, + xlen, + flen, + field1_kind, + field2_kind, + ); + } + return Err(CannotUseFpConv); + } + } + FieldsShape::Array { count, .. } => { + for _ in 0..count { + let elem_layout = arg_layout.field(cx, 0); + should_use_fp_conv_helper( + cx, + &elem_layout, + xlen, + flen, + field1_kind, + field2_kind, + )?; + } + } + FieldsShape::Arbitrary { .. } => { + match arg_layout.variants { + abi::Variants::Multiple { .. } => return Err(CannotUseFpConv), + abi::Variants::Single { .. } => (), + } + for i in arg_layout.fields.index_by_increasing_offset() { + let field = arg_layout.field(cx, i); + should_use_fp_conv_helper(cx, &field, xlen, flen, field1_kind, field2_kind)?; + } + } + }, + } + Ok(()) +} + +fn should_use_fp_conv<'a, Ty, C>( + cx: &C, + arg: &TyAndLayout<'a, Ty>, + xlen: u64, + flen: u64, +) -> Option +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + let mut field1_kind = RegPassKind::Unknown; + let mut field2_kind = RegPassKind::Unknown; + if should_use_fp_conv_helper(cx, arg, xlen, flen, &mut field1_kind, &mut field2_kind).is_err() { + return None; + } + match (field1_kind, field2_kind) { + (RegPassKind::Integer(l), RegPassKind::Float(r)) => Some(FloatConv::MixedPair(l, r)), + (RegPassKind::Float(l), RegPassKind::Integer(r)) => Some(FloatConv::MixedPair(l, r)), + (RegPassKind::Float(l), RegPassKind::Float(r)) => Some(FloatConv::FloatPair(l, r)), + (RegPassKind::Float(f), RegPassKind::Unknown) => Some(FloatConv::Float(f)), + _ => None, + } +} + +fn classify_ret<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, xlen: u64, flen: u64) -> bool +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + if !arg.layout.is_sized() { + // Not touching this... + return false; // I guess? return value of this function is not documented + } + if let Some(conv) = should_use_fp_conv(cx, &arg.layout, xlen, flen) { + match conv { + FloatConv::Float(f) => { + arg.cast_to(f); + } + FloatConv::FloatPair(l, r) => { + arg.cast_to(CastTarget::pair(l, r)); + } + FloatConv::MixedPair(l, r) => { + arg.cast_to(CastTarget::pair(l, r)); + } + } + return false; + } + + let total = arg.layout.size; + + // "Scalars wider than 2✕XLEN are passed by reference and are replaced in + // the argument list with the address." + // "Aggregates larger than 2✕XLEN bits are passed by reference and are + // replaced in the argument list with the address, as are C++ aggregates + // with nontrivial copy constructors, destructors, or vtables." + if total.bits() > 2 * xlen { + // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. + if is_riscv_aggregate(arg) { + arg.make_indirect(); + } + return true; + } + + let xlen_reg = match xlen { + 32 => Reg::i32(), + 64 => Reg::i64(), + _ => unreachable!("Unsupported XLEN: {}", xlen), + }; + if is_riscv_aggregate(arg) { + if total.bits() <= xlen { + arg.cast_to(xlen_reg); + } else { + arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2))); + } + return false; + } + + // "When passed in registers, scalars narrower than XLEN bits are widened + // according to the sign of their type up to 32 bits, then sign-extended to + // XLEN bits." + extend_integer_width(arg, xlen); + false +} + +fn classify_arg<'a, Ty, C>( + cx: &C, + arg: &mut ArgAbi<'a, Ty>, + xlen: u64, + flen: u64, + is_vararg: bool, + avail_gprs: &mut u64, + avail_fprs: &mut u64, +) where + Ty: TyAbiInterface<'a, C> + Copy, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + if !is_vararg { + match should_use_fp_conv(cx, &arg.layout, xlen, flen) { + Some(FloatConv::Float(f)) if *avail_fprs >= 1 => { + *avail_fprs -= 1; + arg.cast_to(f); + return; + } + Some(FloatConv::FloatPair(l, r)) if *avail_fprs >= 2 => { + *avail_fprs -= 2; + arg.cast_to(CastTarget::pair(l, r)); + return; + } + Some(FloatConv::MixedPair(l, r)) if *avail_fprs >= 1 && *avail_gprs >= 1 => { + *avail_gprs -= 1; + *avail_fprs -= 1; + arg.cast_to(CastTarget::pair(l, r)); + return; + } + _ => (), + } + } + + let total = arg.layout.size; + let align = arg.layout.align.abi.bits(); + + // "Scalars wider than 2✕XLEN are passed by reference and are replaced in + // the argument list with the address." + // "Aggregates larger than 2✕XLEN bits are passed by reference and are + // replaced in the argument list with the address, as are C++ aggregates + // with nontrivial copy constructors, destructors, or vtables." + if total.bits() > 2 * xlen { + // We rely on the LLVM backend lowering code to lower passing a scalar larger than 2*XLEN. + if is_riscv_aggregate(arg) { + arg.make_indirect(); + } + if *avail_gprs >= 1 { + *avail_gprs -= 1; + } + return; + } + + let double_xlen_reg = match xlen { + 32 => Reg::i64(), + 64 => Reg::i128(), + _ => unreachable!("Unsupported XLEN: {}", xlen), + }; + + let xlen_reg = match xlen { + 32 => Reg::i32(), + 64 => Reg::i64(), + _ => unreachable!("Unsupported XLEN: {}", xlen), + }; + + if total.bits() > xlen { + let align_regs = align > xlen; + if is_riscv_aggregate(arg) { + arg.cast_to(Uniform::new( + if align_regs { double_xlen_reg } else { xlen_reg }, + Size::from_bits(xlen * 2), + )); + } + if align_regs && is_vararg { + *avail_gprs -= *avail_gprs % 2; + } + if *avail_gprs >= 2 { + *avail_gprs -= 2; + } else { + *avail_gprs = 0; + } + return; + } else if is_riscv_aggregate(arg) { + arg.cast_to(xlen_reg); + if *avail_gprs >= 1 { + *avail_gprs -= 1; + } + return; + } + + // "When passed in registers, scalars narrower than XLEN bits are widened + // according to the sign of their type up to 32 bits, then sign-extended to + // XLEN bits." + if *avail_gprs >= 1 { + extend_integer_width(arg, xlen); + *avail_gprs -= 1; + } +} + +fn extend_integer_width(arg: &mut ArgAbi<'_, Ty>, xlen: u64) { + if let Abi::Scalar(scalar) = arg.layout.abi { + if let abi::Int(i, _) = scalar.primitive() { + // 32-bit integers are always sign-extended + if i.size().bits() == 32 && xlen > 32 { + if let PassMode::Direct(ref mut attrs) = arg.mode { + attrs.ext(ArgExtension::Sext); + return; + } + } + } + } + + arg.extend_integer_width_to(xlen); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + let flen = match &cx.target_spec().llvm_abiname[..] { + "ilp32f" | "lp64f" => 32, + "ilp32d" | "lp64d" => 64, + _ => 0, + }; + let xlen = cx.data_layout().pointer_size.bits(); + + let mut avail_gprs = 8; + let mut avail_fprs = 8; + + if !fn_abi.ret.is_ignore() && classify_ret(cx, &mut fn_abi.ret, xlen, flen) { + avail_gprs -= 1; + } + + for (i, arg) in fn_abi.args.iter_mut().enumerate() { + if arg.is_ignore() { + continue; + } + classify_arg( + cx, + arg, + xlen, + flen, + i >= fn_abi.fixed_count as usize, + &mut avail_gprs, + &mut avail_fprs, + ); + } +} diff --git a/compiler/rustc_target/src/callconv/s390x.rs b/compiler/rustc_target/src/callconv/s390x.rs new file mode 100644 index 00000000000..502e7331267 --- /dev/null +++ b/compiler/rustc_target/src/callconv/s390x.rs @@ -0,0 +1,69 @@ +// FIXME: The assumes we're using the non-vector ABI, i.e., compiling +// for a pre-z13 machine or using -mno-vx. + +use crate::abi::call::{ArgAbi, FnAbi, Reg}; +use crate::abi::{HasDataLayout, TyAbiInterface}; +use crate::spec::HasTargetSpec; + +fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { + if !ret.layout.is_aggregate() && ret.layout.size.bits() <= 64 { + ret.extend_integer_width_to(64); + } else { + ret.make_indirect(); + } +} + +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + if arg.is_ignore() { + // s390x-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs. + if cx.target_spec().os == "linux" + && matches!(&*cx.target_spec().env, "gnu" | "musl" | "uclibc") + && arg.layout.is_zst() + { + arg.make_indirect_from_ignore(); + } + return; + } + if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 { + arg.extend_integer_width_to(64); + return; + } + + if arg.layout.is_single_fp_element(cx) { + match arg.layout.size.bytes() { + 4 => arg.cast_to(Reg::f32()), + 8 => arg.cast_to(Reg::f64()), + _ => arg.make_indirect(), + } + } else { + match arg.layout.size.bytes() { + 1 => arg.cast_to(Reg::i8()), + 2 => arg.cast_to(Reg::i16()), + 4 => arg.cast_to(Reg::i32()), + 8 => arg.cast_to(Reg::i64()), + _ => arg.make_indirect(), + } + } +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + if !fn_abi.ret.is_ignore() { + classify_ret(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + classify_arg(cx, arg); + } +} diff --git a/compiler/rustc_target/src/callconv/sparc.rs b/compiler/rustc_target/src/callconv/sparc.rs new file mode 100644 index 00000000000..37980a91c76 --- /dev/null +++ b/compiler/rustc_target/src/callconv/sparc.rs @@ -0,0 +1,53 @@ +use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; +use crate::abi::{HasDataLayout, Size}; + +fn classify_ret(cx: &C, ret: &mut ArgAbi<'_, Ty>, offset: &mut Size) +where + C: HasDataLayout, +{ + if !ret.layout.is_aggregate() { + ret.extend_integer_width_to(32); + } else { + ret.make_indirect(); + *offset += cx.data_layout().pointer_size; + } +} + +fn classify_arg(cx: &C, arg: &mut ArgAbi<'_, Ty>, offset: &mut Size) +where + C: HasDataLayout, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + let dl = cx.data_layout(); + let size = arg.layout.size; + let align = arg.layout.align.max(dl.i32_align).min(dl.i64_align).abi; + + if arg.layout.is_aggregate() { + let pad_i32 = !offset.is_aligned(align); + arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32); + } else { + arg.extend_integer_width_to(32); + } + + *offset = offset.align_to(align) + size.align_to(align); +} + +pub(crate) fn compute_abi_info(cx: &C, fn_abi: &mut FnAbi<'_, Ty>) +where + C: HasDataLayout, +{ + let mut offset = Size::ZERO; + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret, &mut offset); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg, &mut offset); + } +} diff --git a/compiler/rustc_target/src/callconv/sparc64.rs b/compiler/rustc_target/src/callconv/sparc64.rs new file mode 100644 index 00000000000..835353f76fc --- /dev/null +++ b/compiler/rustc_target/src/callconv/sparc64.rs @@ -0,0 +1,234 @@ +// FIXME: This needs an audit for correctness and completeness. + +use crate::abi::call::{ + ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, Reg, Uniform, +}; +use crate::abi::{self, HasDataLayout, Scalar, Size, TyAbiInterface, TyAndLayout}; +use crate::spec::HasTargetSpec; + +#[derive(Clone, Debug)] +struct Sdata { + pub prefix: [Option; 8], + pub prefix_index: usize, + pub last_offset: Size, + pub has_float: bool, + pub arg_attribute: ArgAttribute, +} + +fn arg_scalar(cx: &C, scalar: &Scalar, offset: Size, mut data: Sdata) -> Sdata +where + C: HasDataLayout, +{ + let dl = cx.data_layout(); + + if !matches!(scalar.primitive(), abi::Float(abi::F32 | abi::F64)) { + return data; + } + + data.has_float = true; + + if !data.last_offset.is_aligned(dl.f64_align.abi) && data.last_offset < offset { + if data.prefix_index == data.prefix.len() { + return data; + } + data.prefix[data.prefix_index] = Some(Reg::i32()); + data.prefix_index += 1; + data.last_offset = data.last_offset + Reg::i32().size; + } + + for _ in 0..((offset - data.last_offset).bits() / 64) + .min((data.prefix.len() - data.prefix_index) as u64) + { + data.prefix[data.prefix_index] = Some(Reg::i64()); + data.prefix_index += 1; + data.last_offset = data.last_offset + Reg::i64().size; + } + + if data.last_offset < offset { + if data.prefix_index == data.prefix.len() { + return data; + } + data.prefix[data.prefix_index] = Some(Reg::i32()); + data.prefix_index += 1; + data.last_offset = data.last_offset + Reg::i32().size; + } + + if data.prefix_index == data.prefix.len() { + return data; + } + + if scalar.primitive() == abi::Float(abi::F32) { + data.arg_attribute = ArgAttribute::InReg; + data.prefix[data.prefix_index] = Some(Reg::f32()); + data.last_offset = offset + Reg::f32().size; + } else { + data.prefix[data.prefix_index] = Some(Reg::f64()); + data.last_offset = offset + Reg::f64().size; + } + data.prefix_index += 1; + data +} + +fn arg_scalar_pair( + cx: &C, + scalar1: &Scalar, + scalar2: &Scalar, + mut offset: Size, + mut data: Sdata, +) -> Sdata +where + C: HasDataLayout, +{ + data = arg_scalar(cx, scalar1, offset, data); + match (scalar1.primitive(), scalar2.primitive()) { + (abi::Float(abi::F32), _) => offset += Reg::f32().size, + (_, abi::Float(abi::F64)) => offset += Reg::f64().size, + (abi::Int(i, _signed), _) => offset += i.size(), + (abi::Pointer(_), _) => offset += Reg::i64().size, + _ => {} + } + + if (offset.bytes() % 4) != 0 && matches!(scalar2.primitive(), abi::Float(abi::F32 | abi::F64)) { + offset += Size::from_bytes(4 - (offset.bytes() % 4)); + } + data = arg_scalar(cx, scalar2, offset, data); + data +} + +fn parse_structure<'a, Ty, C>( + cx: &C, + layout: TyAndLayout<'a, Ty>, + mut data: Sdata, + mut offset: Size, +) -> Sdata +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if let abi::FieldsShape::Union(_) = layout.fields { + return data; + } + + match layout.abi { + abi::Abi::Scalar(scalar) => { + data = arg_scalar(cx, &scalar, offset, data); + } + abi::Abi::Aggregate { .. } => { + for i in 0..layout.fields.count() { + if offset < layout.fields.offset(i) { + offset = layout.fields.offset(i); + } + data = parse_structure(cx, layout.field(cx, i), data.clone(), offset); + } + } + _ => { + if let abi::Abi::ScalarPair(scalar1, scalar2) = &layout.abi { + data = arg_scalar_pair(cx, scalar1, scalar2, offset, data); + } + } + } + + data +} + +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, in_registers_max: Size) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !arg.layout.is_aggregate() { + arg.extend_integer_width_to(64); + return; + } + + let total = arg.layout.size; + if total > in_registers_max { + arg.make_indirect(); + return; + } + + match arg.layout.fields { + abi::FieldsShape::Primitive => unreachable!(), + abi::FieldsShape::Array { .. } => { + // Arrays are passed indirectly + arg.make_indirect(); + return; + } + abi::FieldsShape::Union(_) => { + // Unions and are always treated as a series of 64-bit integer chunks + } + abi::FieldsShape::Arbitrary { .. } => { + // Structures with floating point numbers need special care. + + let mut data = parse_structure( + cx, + arg.layout, + Sdata { + prefix: [None; 8], + prefix_index: 0, + last_offset: Size::ZERO, + has_float: false, + arg_attribute: ArgAttribute::default(), + }, + Size::ZERO, + ); + + if data.has_float { + // Structure { float, int, int } doesn't like to be handled like + // { float, long int }. Other way around it doesn't mind. + if data.last_offset < arg.layout.size + && (data.last_offset.bytes() % 8) != 0 + && data.prefix_index < data.prefix.len() + { + data.prefix[data.prefix_index] = Some(Reg::i32()); + data.prefix_index += 1; + data.last_offset += Reg::i32().size; + } + + let mut rest_size = arg.layout.size - data.last_offset; + if (rest_size.bytes() % 8) != 0 && data.prefix_index < data.prefix.len() { + data.prefix[data.prefix_index] = Some(Reg::i32()); + rest_size = rest_size - Reg::i32().size; + } + + arg.cast_to(CastTarget { + prefix: data.prefix, + rest: Uniform::new(Reg::i64(), rest_size), + attrs: ArgAttributes { + regular: data.arg_attribute, + arg_ext: ArgExtension::None, + pointee_size: Size::ZERO, + pointee_align: None, + }, + }); + return; + } + } + } + + arg.cast_to(Uniform::new(Reg::i64(), total)); +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + if !fn_abi.ret.is_ignore() { + classify_arg(cx, &mut fn_abi.ret, Size::from_bytes(32)); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + // sparc64-unknown-linux-{gnu,musl,uclibc} doesn't ignore ZSTs. + if cx.target_spec().os == "linux" + && matches!(&*cx.target_spec().env, "gnu" | "musl" | "uclibc") + && arg.layout.is_zst() + { + arg.make_indirect_from_ignore(); + } + return; + } + classify_arg(cx, arg, Size::from_bytes(16)); + } +} diff --git a/compiler/rustc_target/src/callconv/wasm.rs b/compiler/rustc_target/src/callconv/wasm.rs new file mode 100644 index 00000000000..3c4cd76a754 --- /dev/null +++ b/compiler/rustc_target/src/callconv/wasm.rs @@ -0,0 +1,103 @@ +use crate::abi::call::{ArgAbi, FnAbi}; +use crate::abi::{HasDataLayout, TyAbiInterface}; + +fn unwrap_trivial_aggregate<'a, Ty, C>(cx: &C, val: &mut ArgAbi<'a, Ty>) -> bool +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if val.layout.is_aggregate() { + if let Some(unit) = val.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()) { + let size = val.layout.size; + if unit.size == size { + val.cast_to(unit); + return true; + } + } + } + false +} + +fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + ret.extend_integer_width_to(32); + if ret.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, ret) { + ret.make_indirect(); + } +} + +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !arg.layout.is_sized() { + // Not touching this... + return; + } + arg.extend_integer_width_to(32); + if arg.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, arg) { + arg.make_indirect(); + } +} + +/// The purpose of this ABI is to match the C ABI (aka clang) exactly. +pub(crate) fn compute_c_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + if !fn_abi.ret.is_ignore() { + classify_ret(cx, &mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg(cx, arg); + } +} + +/// The purpose of this ABI is for matching the WebAssembly standard. This +/// intentionally diverges from the C ABI and is specifically crafted to take +/// advantage of LLVM's support of multiple returns in WebAssembly. +/// +/// This ABI is *bad*! It uses `PassMode::Direct` for `abi::Aggregate` types, which leaks LLVM +/// implementation details into the ABI. It's just hard to fix because ABIs are hard to change. +/// Also see . +pub(crate) fn compute_wasm_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { + if !fn_abi.ret.is_ignore() { + classify_ret_wasm_abi(&mut fn_abi.ret); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg_wasm_abi(arg); + } + + fn classify_ret_wasm_abi(ret: &mut ArgAbi<'_, Ty>) { + if !ret.layout.is_sized() { + // Not touching this... + return; + } + // FIXME: this is bad! https://github.com/rust-lang/rust/issues/115666 + ret.make_direct_deprecated(); + ret.extend_integer_width_to(32); + } + + fn classify_arg_wasm_abi(arg: &mut ArgAbi<'_, Ty>) { + if !arg.layout.is_sized() { + // Not touching this... + return; + } + // FIXME: this is bad! https://github.com/rust-lang/rust/issues/115666 + arg.make_direct_deprecated(); + arg.extend_integer_width_to(32); + } +} diff --git a/compiler/rustc_target/src/callconv/x86.rs b/compiler/rustc_target/src/callconv/x86.rs new file mode 100644 index 00000000000..d9af83d3205 --- /dev/null +++ b/compiler/rustc_target/src/callconv/x86.rs @@ -0,0 +1,185 @@ +use crate::abi::call::{ArgAttribute, FnAbi, PassMode, Reg, RegKind}; +use crate::abi::{Abi, Align, HasDataLayout, TyAbiInterface, TyAndLayout}; +use crate::spec::HasTargetSpec; + +#[derive(PartialEq)] +pub(crate) enum Flavor { + General, + FastcallOrVectorcall, +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, flavor: Flavor) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + if !fn_abi.ret.is_ignore() { + if fn_abi.ret.layout.is_aggregate() && fn_abi.ret.layout.is_sized() { + // Returning a structure. Most often, this will use + // a hidden first argument. On some platforms, though, + // small structs are returned as integers. + // + // Some links: + // https://www.angelcode.com/dev/callconv/callconv.html + // Clang's ABI handling is in lib/CodeGen/TargetInfo.cpp + let t = cx.target_spec(); + if t.abi_return_struct_as_int { + // According to Clang, everyone but MSVC returns single-element + // float aggregates directly in a floating-point register. + if !t.is_like_msvc && fn_abi.ret.layout.is_single_fp_element(cx) { + match fn_abi.ret.layout.size.bytes() { + 4 => fn_abi.ret.cast_to(Reg::f32()), + 8 => fn_abi.ret.cast_to(Reg::f64()), + _ => fn_abi.ret.make_indirect(), + } + } else { + match fn_abi.ret.layout.size.bytes() { + 1 => fn_abi.ret.cast_to(Reg::i8()), + 2 => fn_abi.ret.cast_to(Reg::i16()), + 4 => fn_abi.ret.cast_to(Reg::i32()), + 8 => fn_abi.ret.cast_to(Reg::i64()), + _ => fn_abi.ret.make_indirect(), + } + } + } else { + fn_abi.ret.make_indirect(); + } + } else { + fn_abi.ret.extend_integer_width_to(32); + } + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() || !arg.layout.is_sized() { + continue; + } + + // FIXME: MSVC 2015+ will pass the first 3 vector arguments in [XYZ]MM0-2 + // See https://reviews.llvm.org/D72114 for Clang behavior + + let t = cx.target_spec(); + let align_4 = Align::from_bytes(4).unwrap(); + let align_16 = Align::from_bytes(16).unwrap(); + + if t.is_like_msvc + && arg.layout.is_adt() + && let Some(max_repr_align) = arg.layout.max_repr_align + && max_repr_align > align_4 + { + // MSVC has special rules for overaligned arguments: https://reviews.llvm.org/D72114. + // Summarized here: + // - Arguments with _requested_ alignment > 4 are passed indirectly. + // - For backwards compatibility, arguments with natural alignment > 4 are still passed + // on stack (via `byval`). For example, this includes `double`, `int64_t`, + // and structs containing them, provided they lack an explicit alignment attribute. + assert!( + arg.layout.align.abi >= max_repr_align, + "abi alignment {:?} less than requested alignment {max_repr_align:?}", + arg.layout.align.abi, + ); + arg.make_indirect(); + } else if arg.layout.is_aggregate() { + // We need to compute the alignment of the `byval` argument. The rules can be found in + // `X86_32ABIInfo::getTypeStackAlignInBytes` in Clang's `TargetInfo.cpp`. Summarized + // here, they are: + // + // 1. If the natural alignment of the type is <= 4, the alignment is 4. + // + // 2. Otherwise, on Linux, the alignment of any vector type is the natural alignment. + // This doesn't matter here because we only pass aggregates via `byval`, not vectors. + // + // 3. Otherwise, on Apple platforms, the alignment of anything that contains a vector + // type is 16. + // + // 4. If none of these conditions are true, the alignment is 4. + + fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>) -> bool + where + Ty: TyAbiInterface<'a, C> + Copy, + { + match layout.abi { + Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) => false, + Abi::Vector { .. } => true, + Abi::Aggregate { .. } => { + for i in 0..layout.fields.count() { + if contains_vector(cx, layout.field(cx, i)) { + return true; + } + } + false + } + } + } + + let byval_align = if arg.layout.align.abi < align_4 { + // (1.) + align_4 + } else if t.is_like_osx && contains_vector(cx, arg.layout) { + // (3.) + align_16 + } else { + // (4.) + align_4 + }; + + arg.pass_by_stack_offset(Some(byval_align)); + } else { + arg.extend_integer_width_to(32); + } + } + + if flavor == Flavor::FastcallOrVectorcall { + // Mark arguments as InReg like clang does it, + // so our fastcall/vectorcall is compatible with C/C++ fastcall/vectorcall. + + // Clang reference: lib/CodeGen/TargetInfo.cpp + // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs() + + // IsSoftFloatABI is only set to true on ARM platforms, + // which in turn can't be x86? + + let mut free_regs = 2; + + for arg in fn_abi.args.iter_mut() { + let attrs = match arg.mode { + PassMode::Ignore + | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + continue; + } + PassMode::Direct(ref mut attrs) => attrs, + PassMode::Pair(..) + | PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } + | PassMode::Cast { .. } => { + unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode) + } + }; + + // At this point we know this must be a primitive of sorts. + let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap(); + assert_eq!(unit.size, arg.layout.size); + if unit.kind == RegKind::Float { + continue; + } + + let size_in_regs = (arg.layout.size.bits() + 31) / 32; + + if size_in_regs == 0 { + continue; + } + + if size_in_regs > free_regs { + break; + } + + free_regs -= size_in_regs; + + if arg.layout.size.bits() <= 32 && unit.kind == RegKind::Integer { + attrs.set(ArgAttribute::InReg); + } + + if free_regs == 0 { + break; + } + } + } +} diff --git a/compiler/rustc_target/src/callconv/x86_64.rs b/compiler/rustc_target/src/callconv/x86_64.rs new file mode 100644 index 00000000000..9910e623ac9 --- /dev/null +++ b/compiler/rustc_target/src/callconv/x86_64.rs @@ -0,0 +1,254 @@ +// The classification code for the x86_64 ABI is taken from the clay language +// https://github.com/jckarter/clay/blob/db0bd2702ab0b6e48965cd85f8859bbd5f60e48e/compiler/externals.cpp + +use crate::abi::call::{ArgAbi, CastTarget, FnAbi, Reg, RegKind}; +use crate::abi::{self, Abi, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; + +/// Classification of "eightbyte" components. +// N.B., the order of the variants is from general to specific, +// such that `unify(a, b)` is the "smaller" of `a` and `b`. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +enum Class { + Int, + Sse, + SseUp, +} + +#[derive(Clone, Copy, Debug)] +struct Memory; + +// Currently supported vector size (AVX-512). +const LARGEST_VECTOR_SIZE: usize = 512; +const MAX_EIGHTBYTES: usize = LARGEST_VECTOR_SIZE / 64; + +fn classify_arg<'a, Ty, C>( + cx: &C, + arg: &ArgAbi<'a, Ty>, +) -> Result<[Option; MAX_EIGHTBYTES], Memory> +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + fn classify<'a, Ty, C>( + cx: &C, + layout: TyAndLayout<'a, Ty>, + cls: &mut [Option], + off: Size, + ) -> Result<(), Memory> + where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, + { + if !off.is_aligned(layout.align.abi) { + if !layout.is_zst() { + return Err(Memory); + } + return Ok(()); + } + + let mut c = match layout.abi { + Abi::Uninhabited => return Ok(()), + + Abi::Scalar(scalar) => match scalar.primitive() { + abi::Int(..) | abi::Pointer(_) => Class::Int, + abi::Float(_) => Class::Sse, + }, + + Abi::Vector { .. } => Class::Sse, + + Abi::ScalarPair(..) | Abi::Aggregate { .. } => { + for i in 0..layout.fields.count() { + let field_off = off + layout.fields.offset(i); + classify(cx, layout.field(cx, i), cls, field_off)?; + } + + match &layout.variants { + abi::Variants::Single { .. } => {} + abi::Variants::Multiple { variants, .. } => { + // Treat enum variants like union members. + for variant_idx in variants.indices() { + classify(cx, layout.for_variant(cx, variant_idx), cls, off)?; + } + } + } + + return Ok(()); + } + }; + + // Fill in `cls` for scalars (Int/Sse) and vectors (Sse). + let first = (off.bytes() / 8) as usize; + let last = ((off.bytes() + layout.size.bytes() - 1) / 8) as usize; + for cls in &mut cls[first..=last] { + *cls = Some(cls.map_or(c, |old| old.min(c))); + + // Everything after the first Sse "eightbyte" + // component is the upper half of a register. + if c == Class::Sse { + c = Class::SseUp; + } + } + + Ok(()) + } + + let n = ((arg.layout.size.bytes() + 7) / 8) as usize; + if n > MAX_EIGHTBYTES { + return Err(Memory); + } + + let mut cls = [None; MAX_EIGHTBYTES]; + classify(cx, arg.layout, &mut cls, Size::ZERO)?; + if n > 2 { + if cls[0] != Some(Class::Sse) { + return Err(Memory); + } + if cls[1..n].iter().any(|&c| c != Some(Class::SseUp)) { + return Err(Memory); + } + } else { + let mut i = 0; + while i < n { + if cls[i] == Some(Class::SseUp) { + cls[i] = Some(Class::Sse); + } else if cls[i] == Some(Class::Sse) { + i += 1; + while i != n && cls[i] == Some(Class::SseUp) { + i += 1; + } + } else { + i += 1; + } + } + } + + Ok(cls) +} + +fn reg_component(cls: &[Option], i: &mut usize, size: Size) -> Option { + if *i >= cls.len() { + return None; + } + + match cls[*i] { + None => None, + Some(Class::Int) => { + *i += 1; + Some(if size.bytes() < 8 { Reg { kind: RegKind::Integer, size } } else { Reg::i64() }) + } + Some(Class::Sse) => { + let vec_len = + 1 + cls[*i + 1..].iter().take_while(|&&c| c == Some(Class::SseUp)).count(); + *i += vec_len; + Some(if vec_len == 1 { + match size.bytes() { + 4 => Reg::f32(), + _ => Reg::f64(), + } + } else { + Reg { kind: RegKind::Vector, size: Size::from_bytes(8) * (vec_len as u64) } + }) + } + Some(c) => unreachable!("reg_component: unhandled class {:?}", c), + } +} + +fn cast_target(cls: &[Option], size: Size) -> CastTarget { + let mut i = 0; + let lo = reg_component(cls, &mut i, size).unwrap(); + let offset = Size::from_bytes(8) * (i as u64); + let mut target = CastTarget::from(lo); + if size > offset { + if let Some(hi) = reg_component(cls, &mut i, size - offset) { + target = CastTarget::pair(lo, hi); + } + } + assert_eq!(reg_component(cls, &mut i, Size::ZERO), None); + target +} + +const MAX_INT_REGS: usize = 6; // RDI, RSI, RDX, RCX, R8, R9 +const MAX_SSE_REGS: usize = 8; // XMM0-7 + +pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout, +{ + let mut int_regs = MAX_INT_REGS; + let mut sse_regs = MAX_SSE_REGS; + + let mut x86_64_arg_or_ret = |arg: &mut ArgAbi<'a, Ty>, is_arg: bool| { + if !arg.layout.is_sized() { + // Not touching this... + return; + } + let mut cls_or_mem = classify_arg(cx, arg); + + if is_arg { + if let Ok(cls) = cls_or_mem { + let mut needed_int = 0; + let mut needed_sse = 0; + for c in cls { + match c { + Some(Class::Int) => needed_int += 1, + Some(Class::Sse) => needed_sse += 1, + _ => {} + } + } + match (int_regs.checked_sub(needed_int), sse_regs.checked_sub(needed_sse)) { + (Some(left_int), Some(left_sse)) => { + int_regs = left_int; + sse_regs = left_sse; + } + _ => { + // Not enough registers for this argument, so it will be + // passed on the stack, but we only mark aggregates + // explicitly as indirect `byval` arguments, as LLVM will + // automatically put immediates on the stack itself. + if arg.layout.is_aggregate() { + cls_or_mem = Err(Memory); + } + } + } + } + } + + match cls_or_mem { + Err(Memory) => { + if is_arg { + // The x86_64 ABI doesn't have any special requirements for `byval` alignment, + // the type's alignment is always used. + arg.pass_by_stack_offset(None); + } else { + // `sret` parameter thus one less integer register available + arg.make_indirect(); + // NOTE(eddyb) return is handled first, so no registers + // should've been used yet. + assert_eq!(int_regs, MAX_INT_REGS); + int_regs -= 1; + } + } + Ok(ref cls) => { + // split into sized chunks passed individually + if arg.layout.is_aggregate() { + let size = arg.layout.size; + arg.cast_to(cast_target(cls, size)); + } else { + arg.extend_integer_width_to(32); + } + } + } + }; + + if !fn_abi.ret.is_ignore() { + x86_64_arg_or_ret(&mut fn_abi.ret, false); + } + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + x86_64_arg_or_ret(arg, true); + } +} diff --git a/compiler/rustc_target/src/callconv/x86_win64.rs b/compiler/rustc_target/src/callconv/x86_win64.rs new file mode 100644 index 00000000000..e5a20b248e4 --- /dev/null +++ b/compiler/rustc_target/src/callconv/x86_win64.rs @@ -0,0 +1,52 @@ +use crate::abi::call::{ArgAbi, FnAbi, Reg}; +use crate::abi::{Abi, Float, Primitive}; +use crate::spec::HasTargetSpec; + +// Win64 ABI: https://docs.microsoft.com/en-us/cpp/build/parameter-passing + +pub(crate) fn compute_abi_info(cx: &impl HasTargetSpec, fn_abi: &mut FnAbi<'_, Ty>) { + let fixup = |a: &mut ArgAbi<'_, Ty>| { + match a.layout.abi { + Abi::Uninhabited | Abi::Aggregate { sized: false } => {} + Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => match a.layout.size.bits() { + 8 => a.cast_to(Reg::i8()), + 16 => a.cast_to(Reg::i16()), + 32 => a.cast_to(Reg::i32()), + 64 => a.cast_to(Reg::i64()), + _ => a.make_indirect(), + }, + Abi::Vector { .. } => { + // FIXME(eddyb) there should be a size cap here + // (probably what clang calls "illegal vectors"). + } + Abi::Scalar(scalar) => { + // Match what LLVM does for `f128` so that `compiler-builtins` builtins match up + // with what LLVM expects. + if a.layout.size.bytes() > 8 + && !matches!(scalar.primitive(), Primitive::Float(Float::F128)) + { + a.make_indirect(); + } else { + a.extend_integer_width_to(32); + } + } + } + }; + + if !fn_abi.ret.is_ignore() { + fixup(&mut fn_abi.ret); + } + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + // x86_64-pc-windows-gnu doesn't ignore ZSTs. + if cx.target_spec().os == "windows" + && cx.target_spec().env == "gnu" + && arg.layout.is_zst() + { + arg.make_indirect_from_ignore(); + } + continue; + } + fixup(arg); + } +} diff --git a/compiler/rustc_target/src/callconv/xtensa.rs b/compiler/rustc_target/src/callconv/xtensa.rs new file mode 100644 index 00000000000..e1728b08a39 --- /dev/null +++ b/compiler/rustc_target/src/callconv/xtensa.rs @@ -0,0 +1,121 @@ +//! The Xtensa ABI implementation +//! +//! This ABI implementation is based on the following sources: +//! +//! Section 8.1.4 & 8.1.5 of the Xtensa ISA reference manual, as well as snippets from +//! Section 2.3 from the Xtensa programmers guide. + +use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform}; +use crate::abi::{Abi, HasDataLayout, Size, TyAbiInterface}; +use crate::spec::HasTargetSpec; + +const NUM_ARG_GPRS: u64 = 6; +const NUM_RET_GPRS: u64 = 4; +const MAX_ARG_IN_REGS_SIZE: u64 = NUM_ARG_GPRS * 32; +const MAX_RET_IN_REGS_SIZE: u64 = NUM_RET_GPRS * 32; + +fn classify_ret_ty<'a, Ty, C>(arg: &mut ArgAbi<'_, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + if arg.is_ignore() { + return; + } + + // The rules for return and argument types are the same, + // so defer to `classify_arg_ty`. + let mut arg_gprs_left = NUM_RET_GPRS; + classify_arg_ty(arg, &mut arg_gprs_left, MAX_RET_IN_REGS_SIZE); + // Ret args cannot be passed via stack, we lower to indirect and let the backend handle the invisible reference + match arg.mode { + super::PassMode::Indirect { attrs: _, meta_attrs: _, ref mut on_stack } => { + *on_stack = false; + } + _ => {} + } +} + +fn classify_arg_ty<'a, Ty, C>(arg: &mut ArgAbi<'_, Ty>, arg_gprs_left: &mut u64, max_size: u64) +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + assert!(*arg_gprs_left <= NUM_ARG_GPRS, "Arg GPR tracking underflow"); + + // Ignore empty structs/unions. + if arg.layout.is_zst() { + return; + } + + let size = arg.layout.size.bits(); + let needed_align = arg.layout.align.abi.bits(); + let mut must_use_stack = false; + + // Determine the number of GPRs needed to pass the current argument + // according to the ABI. 2*XLen-aligned varargs are passed in "aligned" + // register pairs, so may consume 3 registers. + let mut needed_arg_gprs = (size + 32 - 1) / 32; + if needed_align == 64 { + needed_arg_gprs += *arg_gprs_left % 2; + } + + if needed_arg_gprs > *arg_gprs_left + || needed_align > 128 + || (*arg_gprs_left < (max_size / 32) && needed_align == 128) + { + must_use_stack = true; + needed_arg_gprs = *arg_gprs_left; + } + *arg_gprs_left -= needed_arg_gprs; + + if must_use_stack { + arg.pass_by_stack_offset(None); + } else if is_xtensa_aggregate(arg) { + // Aggregates which are <= max_size will be passed in + // registers if possible, so coerce to integers. + + // Use a single `xlen` int if possible, 2 * `xlen` if 2 * `xlen` alignment + // is required, and a 2-element `xlen` array if only `xlen` alignment is + // required. + if size <= 32 { + arg.cast_to(Reg::i32()); + } else { + let reg = if needed_align == 2 * 32 { Reg::i64() } else { Reg::i32() }; + let total = Size::from_bits(((size + 32 - 1) / 32) * 32); + arg.cast_to(Uniform::new(reg, total)); + } + } else { + // All integral types are promoted to `xlen` + // width. + // + // We let the LLVM backend handle integral types >= xlen. + if size < 32 { + arg.extend_integer_width_to(32); + } + } +} + +pub(crate) fn compute_abi_info<'a, Ty, C>(_cx: &C, fn_abi: &mut FnAbi<'a, Ty>) +where + Ty: TyAbiInterface<'a, C> + Copy, + C: HasDataLayout + HasTargetSpec, +{ + if !fn_abi.ret.is_ignore() { + classify_ret_ty(&mut fn_abi.ret); + } + + let mut arg_gprs_left = NUM_ARG_GPRS; + + for arg in fn_abi.args.iter_mut() { + if arg.is_ignore() { + continue; + } + classify_arg_ty(arg, &mut arg_gprs_left, MAX_ARG_IN_REGS_SIZE); + } +} + +fn is_xtensa_aggregate<'a, Ty>(arg: &ArgAbi<'a, Ty>) -> bool { + match arg.layout.abi { + Abi::Vector { .. } => true, + _ => arg.layout.is_aggregate(), + } +} -- cgit 1.4.1-3-g733a5 From 10721909f29024ba56a9ce667da8652a99a04784 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 9 Oct 2024 12:20:28 -0700 Subject: compiler: Wire `{TyAnd,}Layout` into `rustc_abi` This finally unites TyAndLayout, Layout, and LayoutS into the same crate, as one might imagine they would be placed. No functional changes. --- compiler/rustc_abi/src/callconv.rs | 254 ++++++++++++++++++++++++++++++ compiler/rustc_abi/src/layout.rs | 4 + compiler/rustc_abi/src/layout/ty.rs | 4 +- compiler/rustc_abi/src/lib.rs | 8 +- compiler/rustc_middle/src/ty/context.rs | 2 +- compiler/rustc_target/src/callconv/mod.rs | 249 +---------------------------- compiler/rustc_target/src/lib.rs | 11 +- 7 files changed, 279 insertions(+), 253 deletions(-) create mode 100644 compiler/rustc_abi/src/callconv.rs (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_abi/src/callconv.rs b/compiler/rustc_abi/src/callconv.rs new file mode 100644 index 00000000000..2ecac8a9df1 --- /dev/null +++ b/compiler/rustc_abi/src/callconv.rs @@ -0,0 +1,254 @@ +mod abi { + pub(crate) use crate::Primitive::*; + pub(crate) use crate::Variants; +} + +use rustc_macros::HashStable_Generic; + +use crate::{Abi, Align, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub enum RegKind { + Integer, + Float, + Vector, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct Reg { + pub kind: RegKind, + pub size: Size, +} + +macro_rules! reg_ctor { + ($name:ident, $kind:ident, $bits:expr) => { + pub fn $name() -> Reg { + Reg { kind: RegKind::$kind, size: Size::from_bits($bits) } + } + }; +} + +impl Reg { + reg_ctor!(i8, Integer, 8); + reg_ctor!(i16, Integer, 16); + reg_ctor!(i32, Integer, 32); + reg_ctor!(i64, Integer, 64); + reg_ctor!(i128, Integer, 128); + + reg_ctor!(f32, Float, 32); + reg_ctor!(f64, Float, 64); +} + +impl Reg { + pub fn align(&self, cx: &C) -> Align { + let dl = cx.data_layout(); + match self.kind { + RegKind::Integer => match self.size.bits() { + 1 => dl.i1_align.abi, + 2..=8 => dl.i8_align.abi, + 9..=16 => dl.i16_align.abi, + 17..=32 => dl.i32_align.abi, + 33..=64 => dl.i64_align.abi, + 65..=128 => dl.i128_align.abi, + _ => panic!("unsupported integer: {self:?}"), + }, + RegKind::Float => match self.size.bits() { + 16 => dl.f16_align.abi, + 32 => dl.f32_align.abi, + 64 => dl.f64_align.abi, + 128 => dl.f128_align.abi, + _ => panic!("unsupported float: {self:?}"), + }, + RegKind::Vector => dl.vector_align(self.size).abi, + } + } +} + +/// Return value from the `homogeneous_aggregate` test function. +#[derive(Copy, Clone, Debug)] +pub enum HomogeneousAggregate { + /// Yes, all the "leaf fields" of this struct are passed in the + /// same way (specified in the `Reg` value). + Homogeneous(Reg), + + /// There are no leaf fields at all. + NoData, +} + +/// Error from the `homogeneous_aggregate` test function, indicating +/// there are distinct leaf fields passed in different ways, +/// or this is uninhabited. +#[derive(Copy, Clone, Debug)] +pub struct Heterogeneous; + +impl HomogeneousAggregate { + /// If this is a homogeneous aggregate, returns the homogeneous + /// unit, else `None`. + pub fn unit(self) -> Option { + match self { + HomogeneousAggregate::Homogeneous(reg) => Some(reg), + HomogeneousAggregate::NoData => None, + } + } + + /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in + /// the same `struct`. Only succeeds if only one of them has any data, + /// or both units are identical. + fn merge(self, other: HomogeneousAggregate) -> Result { + match (self, other) { + (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x), + + (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => { + if a != b { + return Err(Heterogeneous); + } + Ok(self) + } + } + } +} + +impl<'a, Ty> TyAndLayout<'a, Ty> { + /// Returns `true` if this is an aggregate type (including a ScalarPair!) + pub fn is_aggregate(&self) -> bool { + match self.abi { + Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, + Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, + } + } + + /// Returns `Homogeneous` if this layout is an aggregate containing fields of + /// only a single type (e.g., `(u32, u32)`). Such aggregates are often + /// special-cased in ABIs. + /// + /// Note: We generally ignore 1-ZST fields when computing this value (see #56877). + /// + /// This is public so that it can be used in unit tests, but + /// should generally only be relevant to the ABI details of + /// specific targets. + pub fn homogeneous_aggregate(&self, cx: &C) -> Result + where + Ty: TyAbiInterface<'a, C> + Copy, + { + match self.abi { + Abi::Uninhabited => Err(Heterogeneous), + + // The primitive for this algorithm. + Abi::Scalar(scalar) => { + let kind = match scalar.primitive() { + abi::Int(..) | abi::Pointer(_) => RegKind::Integer, + abi::Float(_) => RegKind::Float, + }; + Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size })) + } + + Abi::Vector { .. } => { + assert!(!self.is_zst()); + Ok(HomogeneousAggregate::Homogeneous(Reg { + kind: RegKind::Vector, + size: self.size, + })) + } + + Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => { + // Helper for computing `homogeneous_aggregate`, allowing a custom + // starting offset (used below for handling variants). + let from_fields_at = + |layout: Self, + start: Size| + -> Result<(HomogeneousAggregate, Size), Heterogeneous> { + let is_union = match layout.fields { + FieldsShape::Primitive => { + unreachable!("aggregates can't have `FieldsShape::Primitive`") + } + FieldsShape::Array { count, .. } => { + assert_eq!(start, Size::ZERO); + + let result = if count > 0 { + layout.field(cx, 0).homogeneous_aggregate(cx)? + } else { + HomogeneousAggregate::NoData + }; + return Ok((result, layout.size)); + } + FieldsShape::Union(_) => true, + FieldsShape::Arbitrary { .. } => false, + }; + + let mut result = HomogeneousAggregate::NoData; + let mut total = start; + + for i in 0..layout.fields.count() { + let field = layout.field(cx, i); + if field.is_1zst() { + // No data here and no impact on layout, can be ignored. + // (We might be able to also ignore all aligned ZST but that's less clear.) + continue; + } + + if !is_union && total != layout.fields.offset(i) { + // This field isn't just after the previous one we considered, abort. + return Err(Heterogeneous); + } + + result = result.merge(field.homogeneous_aggregate(cx)?)?; + + // Keep track of the offset (without padding). + let size = field.size; + if is_union { + total = total.max(size); + } else { + total += size; + } + } + + Ok((result, total)) + }; + + let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?; + + match &self.variants { + abi::Variants::Single { .. } => {} + abi::Variants::Multiple { variants, .. } => { + // Treat enum variants like union members. + // HACK(eddyb) pretend the `enum` field (discriminant) + // is at the start of every variant (otherwise the gap + // at the start of all variants would disqualify them). + // + // NB: for all tagged `enum`s (which include all non-C-like + // `enum`s with defined FFI representation), this will + // match the homogeneous computation on the equivalent + // `struct { tag; union { variant1; ... } }` and/or + // `union { struct { tag; variant1; } ... }` + // (the offsets of variant fields should be identical + // between the two for either to be a homogeneous aggregate). + let variant_start = total; + for variant_idx in variants.indices() { + let (variant_result, variant_total) = + from_fields_at(self.for_variant(cx, variant_idx), variant_start)?; + + result = result.merge(variant_result)?; + total = total.max(variant_total); + } + } + } + + // There needs to be no padding. + if total != self.size { + Err(Heterogeneous) + } else { + match result { + HomogeneousAggregate::Homogeneous(_) => { + assert_ne!(total, Size::ZERO); + } + HomogeneousAggregate::NoData => { + assert_eq!(total, Size::ZERO); + } + } + Ok(result) + } + } + Abi::Aggregate { sized: false } => Err(Heterogeneous), + } + } +} diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 620a051eeb7..6e1299944a0 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -11,6 +11,10 @@ use crate::{ Variants, WrappingRange, }; +mod ty; + +pub use ty::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx}; + // A variant is absent if it's uninhabited and only has ZST fields. // Present uninhabited variants only require space for their fields, // but *not* an encoding of the discriminant (e.g., a tag value). diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs index 9f26c98df86..c6812c4d4c0 100644 --- a/compiler/rustc_abi/src/layout/ty.rs +++ b/compiler/rustc_abi/src/layout/ty.rs @@ -6,10 +6,8 @@ use Primitive::*; use rustc_data_structures::intern::Interned; use rustc_macros::HashStable_Generic; -pub mod call; - // Explicitly import `Float` to avoid ambiguity with `Primitive::Float`. -pub use rustc_abi::{Float, *}; +use crate::{Float, *}; rustc_index::newtype_index! { /// The *source-order* index of a field in a variant. diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index fa7c98a7fa0..84d756b6d51 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1,6 +1,7 @@ // tidy-alphabetical-start #![cfg_attr(feature = "nightly", allow(internal_features))] #![cfg_attr(feature = "nightly", doc(rust_logo))] +#![cfg_attr(feature = "nightly", feature(rustc_attrs))] #![cfg_attr(feature = "nightly", feature(rustdoc_internals))] #![cfg_attr(feature = "nightly", feature(step_trait))] #![warn(unreachable_pub)] @@ -22,11 +23,16 @@ use rustc_macros::HashStable_Generic; #[cfg(feature = "nightly")] use rustc_macros::{Decodable_Generic, Encodable_Generic}; +mod callconv; mod layout; #[cfg(test)] mod tests; -pub use layout::{LayoutCalculator, LayoutCalculatorError}; +pub use callconv::{Heterogeneous, HomogeneousAggregate, Reg, RegKind}; +pub use layout::{ + FIRST_VARIANT, FieldIdx, Layout, LayoutCalculator, LayoutCalculatorError, TyAbiInterface, + TyAndLayout, VariantIdx, +}; /// Requirements for a `StableHashingContext` to be used in this crate. /// This is a hack to allow using the `HashStable_Generic` derive macro diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 50e7be82a79..1bdcae4dfb4 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -12,6 +12,7 @@ use std::marker::PhantomData; use std::ops::{Bound, Deref}; use std::{fmt, iter, mem}; +use rustc_abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx}; use rustc_ast::{self as ast, attr}; use rustc_data_structures::defer; use rustc_data_structures::fingerprint::Fingerprint; @@ -48,7 +49,6 @@ use rustc_session::{Limit, MetadataKind, Session}; use rustc_span::def_id::{CRATE_DEF_ID, DefPathHash, StableCrateId}; use rustc_span::symbol::{Ident, Symbol, kw, sym}; use rustc_span::{DUMMY_SP, Span}; -use rustc_target::abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx}; use rustc_target::spec::abi; use rustc_type_ir::TyKind::*; use rustc_type_ir::fold::TypeFoldable; diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 352861c5ccb..832246495bc 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -1,10 +1,11 @@ use std::fmt; use std::str::FromStr; +pub use rustc_abi::{Reg, RegKind}; use rustc_macros::HashStable_Generic; use rustc_span::Symbol; -use crate::abi::{self, Abi, Align, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; +use crate::abi::{self, Abi, Align, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi}; mod aarch64; @@ -192,63 +193,6 @@ impl ArgAttributes { } } -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub enum RegKind { - Integer, - Float, - Vector, -} - -#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub struct Reg { - pub kind: RegKind, - pub size: Size, -} - -macro_rules! reg_ctor { - ($name:ident, $kind:ident, $bits:expr) => { - pub fn $name() -> Reg { - Reg { kind: RegKind::$kind, size: Size::from_bits($bits) } - } - }; -} - -impl Reg { - reg_ctor!(i8, Integer, 8); - reg_ctor!(i16, Integer, 16); - reg_ctor!(i32, Integer, 32); - reg_ctor!(i64, Integer, 64); - reg_ctor!(i128, Integer, 128); - - reg_ctor!(f32, Float, 32); - reg_ctor!(f64, Float, 64); -} - -impl Reg { - pub fn align(&self, cx: &C) -> Align { - let dl = cx.data_layout(); - match self.kind { - RegKind::Integer => match self.size.bits() { - 1 => dl.i1_align.abi, - 2..=8 => dl.i8_align.abi, - 9..=16 => dl.i16_align.abi, - 17..=32 => dl.i32_align.abi, - 33..=64 => dl.i64_align.abi, - 65..=128 => dl.i128_align.abi, - _ => panic!("unsupported integer: {self:?}"), - }, - RegKind::Float => match self.size.bits() { - 16 => dl.f16_align.abi, - 32 => dl.f32_align.abi, - 64 => dl.f64_align.abi, - 128 => dl.f128_align.abi, - _ => panic!("unsupported float: {self:?}"), - }, - RegKind::Vector => dl.vector_align(self.size).abi, - } - } -} - /// An argument passed entirely registers with the /// same kind (e.g., HFA / HVA on PPC64 and AArch64). #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] @@ -380,195 +324,6 @@ impl CastTarget { } } -/// Return value from the `homogeneous_aggregate` test function. -#[derive(Copy, Clone, Debug)] -pub enum HomogeneousAggregate { - /// Yes, all the "leaf fields" of this struct are passed in the - /// same way (specified in the `Reg` value). - Homogeneous(Reg), - - /// There are no leaf fields at all. - NoData, -} - -/// Error from the `homogeneous_aggregate` test function, indicating -/// there are distinct leaf fields passed in different ways, -/// or this is uninhabited. -#[derive(Copy, Clone, Debug)] -pub struct Heterogeneous; - -impl HomogeneousAggregate { - /// If this is a homogeneous aggregate, returns the homogeneous - /// unit, else `None`. - pub fn unit(self) -> Option { - match self { - HomogeneousAggregate::Homogeneous(reg) => Some(reg), - HomogeneousAggregate::NoData => None, - } - } - - /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in - /// the same `struct`. Only succeeds if only one of them has any data, - /// or both units are identical. - fn merge(self, other: HomogeneousAggregate) -> Result { - match (self, other) { - (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x), - - (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => { - if a != b { - return Err(Heterogeneous); - } - Ok(self) - } - } - } -} - -impl<'a, Ty> TyAndLayout<'a, Ty> { - /// Returns `true` if this is an aggregate type (including a ScalarPair!) - fn is_aggregate(&self) -> bool { - match self.abi { - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, - Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, - } - } - - /// Returns `Homogeneous` if this layout is an aggregate containing fields of - /// only a single type (e.g., `(u32, u32)`). Such aggregates are often - /// special-cased in ABIs. - /// - /// Note: We generally ignore 1-ZST fields when computing this value (see #56877). - /// - /// This is public so that it can be used in unit tests, but - /// should generally only be relevant to the ABI details of - /// specific targets. - pub fn homogeneous_aggregate(&self, cx: &C) -> Result - where - Ty: TyAbiInterface<'a, C> + Copy, - { - match self.abi { - Abi::Uninhabited => Err(Heterogeneous), - - // The primitive for this algorithm. - Abi::Scalar(scalar) => { - let kind = match scalar.primitive() { - abi::Int(..) | abi::Pointer(_) => RegKind::Integer, - abi::Float(_) => RegKind::Float, - }; - Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size })) - } - - Abi::Vector { .. } => { - assert!(!self.is_zst()); - Ok(HomogeneousAggregate::Homogeneous(Reg { - kind: RegKind::Vector, - size: self.size, - })) - } - - Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => { - // Helper for computing `homogeneous_aggregate`, allowing a custom - // starting offset (used below for handling variants). - let from_fields_at = - |layout: Self, - start: Size| - -> Result<(HomogeneousAggregate, Size), Heterogeneous> { - let is_union = match layout.fields { - FieldsShape::Primitive => { - unreachable!("aggregates can't have `FieldsShape::Primitive`") - } - FieldsShape::Array { count, .. } => { - assert_eq!(start, Size::ZERO); - - let result = if count > 0 { - layout.field(cx, 0).homogeneous_aggregate(cx)? - } else { - HomogeneousAggregate::NoData - }; - return Ok((result, layout.size)); - } - FieldsShape::Union(_) => true, - FieldsShape::Arbitrary { .. } => false, - }; - - let mut result = HomogeneousAggregate::NoData; - let mut total = start; - - for i in 0..layout.fields.count() { - let field = layout.field(cx, i); - if field.is_1zst() { - // No data here and no impact on layout, can be ignored. - // (We might be able to also ignore all aligned ZST but that's less clear.) - continue; - } - - if !is_union && total != layout.fields.offset(i) { - // This field isn't just after the previous one we considered, abort. - return Err(Heterogeneous); - } - - result = result.merge(field.homogeneous_aggregate(cx)?)?; - - // Keep track of the offset (without padding). - let size = field.size; - if is_union { - total = total.max(size); - } else { - total += size; - } - } - - Ok((result, total)) - }; - - let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?; - - match &self.variants { - abi::Variants::Single { .. } => {} - abi::Variants::Multiple { variants, .. } => { - // Treat enum variants like union members. - // HACK(eddyb) pretend the `enum` field (discriminant) - // is at the start of every variant (otherwise the gap - // at the start of all variants would disqualify them). - // - // NB: for all tagged `enum`s (which include all non-C-like - // `enum`s with defined FFI representation), this will - // match the homogeneous computation on the equivalent - // `struct { tag; union { variant1; ... } }` and/or - // `union { struct { tag; variant1; } ... }` - // (the offsets of variant fields should be identical - // between the two for either to be a homogeneous aggregate). - let variant_start = total; - for variant_idx in variants.indices() { - let (variant_result, variant_total) = - from_fields_at(self.for_variant(cx, variant_idx), variant_start)?; - - result = result.merge(variant_result)?; - total = total.max(variant_total); - } - } - } - - // There needs to be no padding. - if total != self.size { - Err(Heterogeneous) - } else { - match result { - HomogeneousAggregate::Homogeneous(_) => { - assert_ne!(total, Size::ZERO); - } - HomogeneousAggregate::NoData => { - assert_eq!(total, Size::ZERO); - } - } - Ok(result) - } - } - Abi::Aggregate { sized: false } => Err(Heterogeneous), - } - } -} - /// Information about how to pass an argument to, /// or return a value from, a function, under some ABI. #[derive(Clone, PartialEq, Eq, Hash, HashStable_Generic)] diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs index 2121c4110dd..50679ab8cc8 100644 --- a/compiler/rustc_target/src/lib.rs +++ b/compiler/rustc_target/src/lib.rs @@ -21,8 +21,8 @@ use std::path::{Path, PathBuf}; -pub mod abi; pub mod asm; +pub mod callconv; pub mod json; pub mod spec; pub mod target_features; @@ -30,6 +30,15 @@ pub mod target_features; #[cfg(test)] mod tests; +pub mod abi { + pub(crate) use Float::*; + pub(crate) use Primitive::*; + // Explicitly import `Float` to avoid ambiguity with `Primitive::Float`. + pub use rustc_abi::{Float, *}; + + pub use crate::callconv as call; +} + pub use rustc_abi::HashStableContext; /// The name of rustc's own place to organize libraries. -- cgit 1.4.1-3-g733a5 From d858dfedbbde119f2ea707fea253e99c824ec252 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Mon, 14 Oct 2024 05:30:45 +0900 Subject: Fix clobber_abi and disallow SVE-related registers in Arm64EC inline assembly --- compiler/rustc_target/src/asm/aarch64.rs | 37 ++++++++++++++++---------------- compiler/rustc_target/src/asm/mod.rs | 17 +++++++++++++-- tests/codegen/asm-arm64ec-clobbers.rs | 36 +++++++++++++++++++++++++++++++ tests/ui/asm/aarch64/aarch64-sve.rs | 28 ++++++++++++++++++++++++ tests/ui/asm/aarch64/arm64ec-sve.rs | 31 ++++++++++++++++++++++++++ tests/ui/asm/aarch64/arm64ec-sve.stderr | 14 ++++++++++++ 6 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 tests/codegen/asm-arm64ec-clobbers.rs create mode 100644 tests/ui/asm/aarch64/aarch64-sve.rs create mode 100644 tests/ui/asm/aarch64/arm64ec-sve.rs create mode 100644 tests/ui/asm/aarch64/arm64ec-sve.stderr (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs index 74970a26b23..fdd9adabc8e 100644 --- a/compiler/rustc_target/src/asm/aarch64.rs +++ b/compiler/rustc_target/src/asm/aarch64.rs @@ -64,6 +64,7 @@ impl AArch64InlineAsmRegClass { neon: I8, I16, I32, I64, F16, F32, F64, F128, VecI8(8), VecI16(4), VecI32(2), VecI64(1), VecF16(4), VecF32(2), VecF64(1), VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF16(8), VecF32(4), VecF64(2); + // Note: When adding support for SVE vector types, they must be rejected for Arm64EC. }, Self::preg => &[], } @@ -96,7 +97,7 @@ fn restricted_for_arm64ec( _is_clobber: bool, ) -> Result<(), &'static str> { if arch == InlineAsmArch::Arm64EC { - Err("x13, x14, x23, x24, x28, v16-v31 cannot be used for Arm64EC") + Err("x13, x14, x23, x24, x28, v16-v31, p*, ffr cannot be used for Arm64EC") } else { Ok(()) } @@ -165,23 +166,23 @@ def_regs! { v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"] % restricted_for_arm64ec, v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"] % restricted_for_arm64ec, v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"] % restricted_for_arm64ec, - p0: preg = ["p0"], - p1: preg = ["p1"], - p2: preg = ["p2"], - p3: preg = ["p3"], - p4: preg = ["p4"], - p5: preg = ["p5"], - p6: preg = ["p6"], - p7: preg = ["p7"], - p8: preg = ["p8"], - p9: preg = ["p9"], - p10: preg = ["p10"], - p11: preg = ["p11"], - p12: preg = ["p12"], - p13: preg = ["p13"], - p14: preg = ["p14"], - p15: preg = ["p15"], - ffr: preg = ["ffr"], + p0: preg = ["p0"] % restricted_for_arm64ec, + p1: preg = ["p1"] % restricted_for_arm64ec, + p2: preg = ["p2"] % restricted_for_arm64ec, + p3: preg = ["p3"] % restricted_for_arm64ec, + p4: preg = ["p4"] % restricted_for_arm64ec, + p5: preg = ["p5"] % restricted_for_arm64ec, + p6: preg = ["p6"] % restricted_for_arm64ec, + p7: preg = ["p7"] % restricted_for_arm64ec, + p8: preg = ["p8"] % restricted_for_arm64ec, + p9: preg = ["p9"] % restricted_for_arm64ec, + p10: preg = ["p10"] % restricted_for_arm64ec, + p11: preg = ["p11"] % restricted_for_arm64ec, + p12: preg = ["p12"] % restricted_for_arm64ec, + p13: preg = ["p13"] % restricted_for_arm64ec, + p14: preg = ["p14"] % restricted_for_arm64ec, + p15: preg = ["p15"] % restricted_for_arm64ec, + ffr: preg = ["ffr"] % restricted_for_arm64ec, #error = ["x19", "w19"] => "x19 is used internally by LLVM and cannot be used as an operand for inline asm", #error = ["x29", "w29", "fp", "wfp"] => diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index 73ae1ae96ae..4b539eb8e20 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -890,6 +890,7 @@ pub enum InlineAsmClobberAbi { Arm, AArch64, AArch64NoX18, + Arm64EC, RiscV, LoongArch, S390x, @@ -932,7 +933,7 @@ impl InlineAsmClobberAbi { _ => Err(&["C", "system", "efiapi"]), }, InlineAsmArch::Arm64EC => match name { - "C" | "system" => Ok(InlineAsmClobberAbi::AArch64NoX18), + "C" | "system" => Ok(InlineAsmClobberAbi::Arm64EC), _ => Err(&["C", "system"]), }, InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => match name { @@ -1033,7 +1034,6 @@ impl InlineAsmClobberAbi { p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, ffr, - } }, InlineAsmClobberAbi::AArch64NoX18 => clobbered_regs! { @@ -1052,7 +1052,20 @@ impl InlineAsmClobberAbi { p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, ffr, + } + }, + InlineAsmClobberAbi::Arm64EC => clobbered_regs! { + AArch64 AArch64InlineAsmReg { + // x13 and x14 cannot be used in Arm64EC. + x0, x1, x2, x3, x4, x5, x6, x7, + x8, x9, x10, x11, x12, x15, + x16, x17, x30, + // Technically the low 64 bits of v8-v15 are preserved, but + // we have no way of expressing this using clobbers. + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + // v16-v31, p*, and ffr cannot be used in Arm64EC. } }, InlineAsmClobberAbi::Arm => clobbered_regs! { diff --git a/tests/codegen/asm-arm64ec-clobbers.rs b/tests/codegen/asm-arm64ec-clobbers.rs new file mode 100644 index 00000000000..2ec61907947 --- /dev/null +++ b/tests/codegen/asm-arm64ec-clobbers.rs @@ -0,0 +1,36 @@ +//@ assembly-output: emit-asm +//@ compile-flags: --target arm64ec-pc-windows-msvc +//@ needs-llvm-components: aarch64 + +#![crate_type = "rlib"] +#![feature(no_core, rustc_attrs, lang_items, asm_experimental_arch)] +#![no_core] + +#[lang = "sized"] +trait Sized {} + +#[rustc_builtin_macro] +macro_rules! asm { + () => {}; +} + +// CHECK-LABEL: @cc_clobber +// CHECK: call void asm sideeffect "", "~{cc}"() +#[no_mangle] +pub unsafe fn cc_clobber() { + asm!("", options(nostack, nomem)); +} + +// CHECK-LABEL: @no_clobber +// CHECK: call void asm sideeffect "", ""() +#[no_mangle] +pub unsafe fn no_clobber() { + asm!("", options(nostack, nomem, preserves_flags)); +} + +// CHECK-LABEL: @clobber_abi +// CHECK: asm sideeffect "", "={w0},={w1},={w2},={w3},={w4},={w5},={w6},={w7},={w8},={w9},={w10},={w11},={w12},={w15},={w16},={w17},={w30},={q0},={q1},={q2},={q3},={q4},={q5},={q6},={q7},={q8},={q9},={q10},={q11},={q12},={q13},={q14},={q15}"() +#[no_mangle] +pub unsafe fn clobber_abi() { + asm!("", clobber_abi("C"), options(nostack, nomem, preserves_flags)); +} diff --git a/tests/ui/asm/aarch64/aarch64-sve.rs b/tests/ui/asm/aarch64/aarch64-sve.rs new file mode 100644 index 00000000000..8cdc9dd4266 --- /dev/null +++ b/tests/ui/asm/aarch64/aarch64-sve.rs @@ -0,0 +1,28 @@ +//@ only-aarch64 +//@ build-pass +//@ needs-asm-support + +#![crate_type = "rlib"] +#![feature(no_core, rustc_attrs, lang_items)] +#![no_core] + +// AArch64 test corresponding to arm64ec-sve.rs. + +#[lang = "sized"] +trait Sized {} +#[lang = "copy"] +trait Copy {} + +impl Copy for f64 {} + +#[rustc_builtin_macro] +macro_rules! asm { + () => {}; +} + +fn f(x: f64) { + unsafe { + asm!("", out("p0") _); + asm!("", out("ffr") _); + } +} diff --git a/tests/ui/asm/aarch64/arm64ec-sve.rs b/tests/ui/asm/aarch64/arm64ec-sve.rs new file mode 100644 index 00000000000..389b365a754 --- /dev/null +++ b/tests/ui/asm/aarch64/arm64ec-sve.rs @@ -0,0 +1,31 @@ +//@ compile-flags: --target arm64ec-pc-windows-msvc +//@ needs-asm-support +//@ needs-llvm-components: aarch64 + +#![crate_type = "rlib"] +#![feature(no_core, rustc_attrs, lang_items, asm_experimental_arch)] +#![no_core] + +// SVE cannot be used for Arm64EC +// https://github.com/rust-lang/rust/pull/131332#issuecomment-2401189142 + +#[lang = "sized"] +trait Sized {} +#[lang = "copy"] +trait Copy {} + +impl Copy for f64 {} + +#[rustc_builtin_macro] +macro_rules! asm { + () => {}; +} + +fn f(x: f64) { + unsafe { + asm!("", out("p0") _); + //~^ ERROR cannot use register `p0` + asm!("", out("ffr") _); + //~^ ERROR cannot use register `ffr` + } +} diff --git a/tests/ui/asm/aarch64/arm64ec-sve.stderr b/tests/ui/asm/aarch64/arm64ec-sve.stderr new file mode 100644 index 00000000000..3e1a5d57004 --- /dev/null +++ b/tests/ui/asm/aarch64/arm64ec-sve.stderr @@ -0,0 +1,14 @@ +error: cannot use register `p0`: x13, x14, x23, x24, x28, v16-v31, p*, ffr cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:26:18 + | +LL | asm!("", out("p0") _); + | ^^^^^^^^^^^ + +error: cannot use register `ffr`: x13, x14, x23, x24, x28, v16-v31, p*, ffr cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:28:18 + | +LL | asm!("", out("ffr") _); + | ^^^^^^^^^^^^ + +error: aborting due to 2 previous errors + -- cgit 1.4.1-3-g733a5 From 67ebb6c20b0cf8dfb587fd1085413232ccd8260c Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Mon, 14 Oct 2024 06:04:07 +0900 Subject: Fix AArch64InlineAsmReg::emit --- compiler/rustc_codegen_llvm/src/asm.rs | 51 +++------------------------ compiler/rustc_target/src/asm/aarch64.rs | 60 ++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 49 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 298cac2fd6e..d1d7d0cf4ce 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -542,57 +542,16 @@ fn xmm_reg_index(reg: InlineAsmReg) -> Option { /// If the register is an AArch64 integer register then return its index. fn a64_reg_index(reg: InlineAsmReg) -> Option { - use AArch64InlineAsmReg::*; - // Unlike `a64_vreg_index`, we can't subtract `x0` to get the u32 because - // `x19` and `x29` are missing and the integer constants for the - // `x0`..`x30` enum variants don't all match the register number. E.g. the - // integer constant for `x18` is 18, but the constant for `x20` is 19. - Some(match reg { - InlineAsmReg::AArch64(r) => match r { - x0 => 0, - x1 => 1, - x2 => 2, - x3 => 3, - x4 => 4, - x5 => 5, - x6 => 6, - x7 => 7, - x8 => 8, - x9 => 9, - x10 => 10, - x11 => 11, - x12 => 12, - x13 => 13, - x14 => 14, - x15 => 15, - x16 => 16, - x17 => 17, - x18 => 18, - // x19 is reserved - x20 => 20, - x21 => 21, - x22 => 22, - x23 => 23, - x24 => 24, - x25 => 25, - x26 => 26, - x27 => 27, - x28 => 28, - // x29 is reserved - x30 => 30, - _ => return None, - }, - _ => return None, - }) + match reg { + InlineAsmReg::AArch64(r) => r.reg_index(), + _ => None, + } } /// If the register is an AArch64 vector register then return its index. fn a64_vreg_index(reg: InlineAsmReg) -> Option { - use AArch64InlineAsmReg::*; match reg { - InlineAsmReg::AArch64(reg) if reg as u32 >= v0 as u32 && reg as u32 <= v31 as u32 => { - Some(reg as u32 - v0 as u32) - } + InlineAsmReg::AArch64(reg) => reg.vreg_index(), _ => None, } } diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs index 74970a26b23..8b173f9bb99 100644 --- a/compiler/rustc_target/src/asm/aarch64.rs +++ b/compiler/rustc_target/src/asm/aarch64.rs @@ -200,12 +200,66 @@ impl AArch64InlineAsmReg { _arch: InlineAsmArch, modifier: Option, ) -> fmt::Result { - let (prefix, index) = if (self as u32) < Self::v0 as u32 { - (modifier.unwrap_or('x'), self as u32 - Self::x0 as u32) + let (prefix, index) = if let Some(index) = self.reg_index() { + (modifier.unwrap_or('x'), index) + } else if let Some(index) = self.vreg_index() { + (modifier.unwrap_or('v'), index) } else { - (modifier.unwrap_or('v'), self as u32 - Self::v0 as u32) + return out.write_str(self.name()); }; assert!(index < 32); write!(out, "{prefix}{index}") } + + /// If the register is an integer register then return its index. + pub fn reg_index(self) -> Option { + // Unlike `vreg_index`, we can't subtract `x0` to get the u32 because + // `x19` and `x29` are missing and the integer constants for the + // `x0`..`x30` enum variants don't all match the register number. E.g. the + // integer constant for `x18` is 18, but the constant for `x20` is 19. + use AArch64InlineAsmReg::*; + Some(match self { + x0 => 0, + x1 => 1, + x2 => 2, + x3 => 3, + x4 => 4, + x5 => 5, + x6 => 6, + x7 => 7, + x8 => 8, + x9 => 9, + x10 => 10, + x11 => 11, + x12 => 12, + x13 => 13, + x14 => 14, + x15 => 15, + x16 => 16, + x17 => 17, + x18 => 18, + // x19 is reserved + x20 => 20, + x21 => 21, + x22 => 22, + x23 => 23, + x24 => 24, + x25 => 25, + x26 => 26, + x27 => 27, + x28 => 28, + // x29 is reserved + x30 => 30, + _ => return None, + }) + } + + /// If the register is a vector register then return its index. + pub fn vreg_index(self) -> Option { + use AArch64InlineAsmReg::*; + if self as u32 >= v0 as u32 && self as u32 <= v31 as u32 { + return Some(self as u32 - v0 as u32); + } + None + } } -- cgit 1.4.1-3-g733a5 From e9853961452b56997cc127b51308879b9cd09482 Mon Sep 17 00:00:00 2001 From: Matthew Maurer Date: Tue, 15 Oct 2024 20:56:20 +0000 Subject: llvm: Match aarch64 data layout to new LLVM layout LLVM has added 3 new address spaces to support special Windows use cases. These shouldn't trouble us for now, but LLVM requires matching data layouts. See llvm/llvm-project#111879 for details --- compiler/rustc_codegen_llvm/src/context.rs | 10 ++++++++++ compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs | 3 ++- compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs | 3 ++- compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_apple_visionos.rs | 3 ++- .../src/spec/targets/aarch64_apple_visionos_sim.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_apple_watchos.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs | 3 ++- .../rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_linux_android.rs | 2 +- .../src/spec/targets/aarch64_nintendo_switch_freestanding.rs | 2 +- .../src/spec/targets/aarch64_pc_windows_gnullvm.rs | 4 +++- .../rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs | 4 +++- .../rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_hermit.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_illumos.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs | 2 +- .../src/spec/targets/aarch64_unknown_linux_musl.rs | 2 +- .../src/spec/targets/aarch64_unknown_linux_ohos.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs | 2 +- compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs | 2 +- .../src/spec/targets/aarch64_unknown_none_softfloat.rs | 2 +- .../src/spec/targets/aarch64_unknown_nto_qnx700.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_redox.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_teeos.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_unknown_trusty.rs | 2 +- compiler/rustc_target/src/spec/targets/aarch64_unknown_uefi.rs | 4 +++- .../rustc_target/src/spec/targets/aarch64_uwp_windows_msvc.rs | 4 +++- compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs | 2 +- .../rustc_target/src/spec/targets/arm64_32_apple_watchos.rs | 3 ++- compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs | 3 ++- compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs | 3 ++- compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs | 3 ++- .../rustc_target/src/spec/targets/arm64ec_pc_windows_msvc.rs | 4 +++- 39 files changed, 72 insertions(+), 38 deletions(-) (limited to 'compiler/rustc_target/src') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index c836dd5473f..2f830d6f941 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -138,6 +138,16 @@ pub(crate) unsafe fn create_module<'ll>( } } + if llvm_version < (20, 0, 0) { + if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { + // LLVM 20 defines three additional address spaces for alternate + // pointer kinds used in Windows. + // See https://github.com/llvm/llvm-project/pull/111879 + target_data_layout = + target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); + } + } + // Ensure the data-layout values hardcoded remain the defaults. { let tm = crate::back::write::create_informational_target_machine(tcx.sess, false); diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs index e6ce50a005f..adee6f5fe99 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { mcount: "\u{1}mcount".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs index 78441ede333..efc42b909e4 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs index 857abd21faf..be503d18bf1 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a12".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs index cbc8b7cd95e..04bbee45cd3 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs index f5176d35f80..fa0bc130e1c 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs index d6738514145..428045da493 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs index dc5dac4fd8f..62d6ffbd34f 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(false), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a16".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs index e9fe786f65e..a66c4f6e96b 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(false), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a16".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs index 4f5ce2f12e7..abd924b5934 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+v8a,+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs index 952c98c324f..ba85647fddc 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs b/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs index 97ce53b2d8f..58fc703946e 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs @@ -11,7 +11,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { linker: Some("aarch64-kmc-elf-gcc".into()), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs index a8ef5544a24..a021d317cc8 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs @@ -13,7 +13,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { max_atomic_width: Some(128), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs b/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs index 1f1cd966326..6ac69e0f57f 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs @@ -15,7 +15,7 @@ pub(crate) fn target() -> Target { std: Some(false), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs index 7a56809dcfc..8b96f589c74 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs @@ -15,7 +15,9 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: + "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs index e1133a4ce05..14ce5edd2f3 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs @@ -14,7 +14,9 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: + "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs index 2ca2f403e17..dd90161f440 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs @@ -10,7 +10,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs index 26a736a2712..df13d52a223 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs @@ -10,7 +10,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs index 3989cc13fba..459e888eb94 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs @@ -11,7 +11,7 @@ pub(crate) fn target() -> Target { }, pointer_width: 64, arch: "aarch64".into(), - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), options: TargetOptions { features: "+v8a,+strict-align,+neon,+fp-armv8".into(), max_atomic_width: Some(128), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_illumos.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_illumos.rs index b76f10b8efd..699376a7928 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_illumos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_illumos.rs @@ -18,7 +18,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs index 96c25af0a7f..18711cb399d 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs @@ -10,7 +10,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a,+outline-atomics".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs index 197ff2d74e5..bb65048a56d 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs @@ -21,7 +21,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { mcount: "\u{1}_mcount".into(), ..base }, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs index 9d5bce05350..22b3a5f8842 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs @@ -13,7 +13,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+reserve-x18".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs index d1769d1476c..0ec76e4b42f 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs @@ -10,7 +10,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs index d124f8a3d29..05fe32734a7 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs @@ -37,7 +37,7 @@ pub(crate) fn target() -> Target { std: Some(false), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: opts, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs index 7933041b584..d6b77ffd091 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs @@ -32,7 +32,7 @@ pub(crate) fn target() -> Target { std: Some(false), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: opts, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_nto_qnx700.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_nto_qnx700.rs index 6c061112b16..441ed7cf153 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_nto_qnx700.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_nto_qnx700.rs @@ -22,7 +22,7 @@ pub(crate) fn target() -> Target { // i128:128 = 128-bit-integer, minimum_alignment=128, preferred_alignment=128 // n32:64 = 32 and 64 are native integer widths; Elements of this set are considered to support most general arithmetic operations efficiently. // S128 = 128 bits are the natural alignment of the stack in bits. - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs index 31c8daa98b0..0fcf5c34bb0 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs @@ -10,7 +10,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs index 6bd8c306abb..7ff99c574ad 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs @@ -15,7 +15,7 @@ pub(crate) fn target() -> Target { std: None, // ? }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs index 4bffef4758d..fb8b59f7729 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs @@ -15,7 +15,7 @@ pub(crate) fn target() -> Target { std: None, // ? }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_trusty.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_trusty.rs index 69ab992e12c..cebd8ff2f68 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_trusty.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_trusty.rs @@ -12,7 +12,7 @@ pub(crate) fn target() -> Target { std: Some(false), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+neon,+fp-armv8,+reserve-x18".into(), diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_uefi.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_uefi.rs index bde1ce95b3f..9656024ddaa 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_uefi.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_uefi.rs @@ -19,7 +19,9 @@ pub(crate) fn target() -> Target { std: None, // ? }, pointer_width: 64, - data_layout: "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: + "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_uwp_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/aarch64_uwp_windows_msvc.rs index c916aa6b23c..3d7c0269808 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_uwp_windows_msvc.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_uwp_windows_msvc.rs @@ -14,7 +14,9 @@ pub(crate) fn target() -> Target { std: None, // ? }, pointer_width: 64, - data_layout: "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: + "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch: "aarch64".into(), options: base, } diff --git a/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs index dd5298944e0..d5e78d03076 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs @@ -10,7 +10,7 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { features: "+v8a".into(), diff --git a/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs index 4f7a1857e76..a5a6f772ac8 100644 --- a/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs +++ b/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 32, - data_layout: "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: + "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32".into(), arch, options: TargetOptions { features: "+v8a,+neon,+fp-armv8,+apple-a7".into(), diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs index 04906fb0460..744d95445b8 100644 --- a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs +++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { mcount: "\u{1}mcount".into(), diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs index 3264669c169..dace11dae24 100644 --- a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs +++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(), diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs index 4c231cb519c..2ccdc76c52e 100644 --- a/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs +++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs @@ -12,7 +12,8 @@ pub(crate) fn target() -> Target { std: Some(true), }, pointer_width: 64, - data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch, options: TargetOptions { features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(), diff --git a/compiler/rustc_target/src/spec/targets/arm64ec_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/arm64ec_pc_windows_msvc.rs index a6744b95a3e..03c96fbfdb0 100644 --- a/compiler/rustc_target/src/spec/targets/arm64ec_pc_windows_msvc.rs +++ b/compiler/rustc_target/src/spec/targets/arm64ec_pc_windows_msvc.rs @@ -18,7 +18,9 @@ pub(crate) fn target() -> Target { std: None, // ? }, pointer_width: 64, - data_layout: "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32".into(), + data_layout: + "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32" + .into(), arch: "arm64ec".into(), options: base, } -- cgit 1.4.1-3-g733a5