document ABI compatibility

3 files changed, 113 insertions, 8 deletions

diff --git a/library/core/src/option.rs b/library/core/src/option.rs
index acf3dfbdf4c..ba1367cde31 100644
--- a/library/core/src/option.rs
+++ b/library/core/src/option.rs
@@ -119,7 +119,7 @@
 //! # Representation
 //!
 //! Rust guarantees to optimize the following types `T` such that
-//! [`Option<T>`] has the same size and alignment as `T`. In some
+//! [`Option<T>`] has the same size, alignment, and [function call ABI] as `T`. In some
 //! of these cases, Rust further guarantees that
 //! `transmute::<_, Option<T>>([0u8; size_of::<T>()])` is sound and
 //! produces `Option::<T>::None`. These cases are identified by the
@@ -127,7 +127,7 @@
 //!
 //! | `T`                                                                 | `transmute::<_, Option<T>>([0u8; size_of::<T>()])` sound? |
 //! |---------------------------------------------------------------------|----------------------------------------------------------------------|
-//! | [`Box<U>`]                                                          | when `U: Sized`                                                      |
+//! | [`Box<U>`] (specifically, only `Box<U, Global>`)                    | when `U: Sized`                                                      |
 //! | `&U`                                                                | when `U: Sized`                                                      |
 //! | `&mut U`                                                            | when `U: Sized`                                                      |
 //! | `fn`, `extern "C" fn`[^extern_fn]                                   | always                                                               |
@@ -135,11 +135,12 @@
 //! | [`ptr::NonNull<U>`]                                                 | when `U: Sized`                                                      |
 //! | `#[repr(transparent)]` struct around one of the types in this list. | when it holds for the inner type                                     |
 //!
-//! [^extern_fn]: this remains true for any other ABI: `extern "abi" fn` (_e.g._, `extern "system" fn`)
+//! [^extern_fn]: this remains true for any argument/return types and any other ABI: `extern "abi" fn` (_e.g._, `extern "system" fn`)
 //!
 //! [`Box<U>`]: ../../std/boxed/struct.Box.html
 //! [`num::NonZero*`]: crate::num
 //! [`ptr::NonNull<U>`]: crate::ptr::NonNull
+//! [function call ABI]: ../primitive.fn.html#abi-compatibility
 //!
 //! This is called the "null pointer optimization" or NPO.
 //!
diff --git a/library/core/src/primitive_docs.rs b/library/core/src/primitive_docs.rs
index f3695d16d7a..694ea8c703d 100644
--- a/library/core/src/primitive_docs.rs
+++ b/library/core/src/primitive_docs.rs
@@ -1480,7 +1480,7 @@ mod prim_ref {}
 ///
 /// ### Casting to and from integers
 ///
-/// You cast function pointers directly to integers:
+/// You can cast function pointers directly to integers:
 ///
 /// ```rust
 /// let fnptr: fn(i32) -> i32 = |x| x+2;
@@ -1506,6 +1506,110 @@ mod prim_ref {}
 /// Note that all of this is not portable to platforms where function pointers and data pointers
 /// have different sizes.
 ///
+/// ### ABI compatibility
+///
+/// Generally, when a function is declared with one signature and called via a function pointer with
+/// a different signature, the two signatures must be *ABI-compatible* or else calling the function
+/// via that function pointer is Undefined Behavior. ABI compatibility is a lot stricter than merely
+/// having the same memory layout; for example, even if `i32` and `f32` have the same size and
+/// alignment, they might be passed in different registers and hence not be ABI-compatible.
+///
+/// ABI compatibility as a concern only arises in code that alters the type of function pointers,
+/// and in code that combines `#[target_feature]` with `extern fn`. Altering the type of
+/// function pointers is wildly unsafe (as in, a lot more unsafe than even
+/// [`transmute_copy`][mem::transmute_copy]), and should only occur in the most exceptional
+/// circumstances. `#[target_feature]` is also used rarely. But assuming such circumstances, what
+/// are the rules?
+///
+/// For two signatures to be considered *ABI-compatible*, they must use a compatible ABI string,
+/// must take the same number of arguments, the individual argument types and the return types must
+/// be ABI-compatible, and the target feature requirements must be met (see the subsection below for
+/// the last point). The ABI string is declared via `extern "ABI" fn(...) -> ...`; note that
+/// `fn name(...) -> ...` implicitly uses the `"Rust"` ABI string and `extern fn name(...) -> ...`
+/// implicitly uses the `"C"` ABI string.
+///
+/// The ABI strings are guaranteed to be compatible if they are the same, or if the caller ABI
+/// string is `$X-unwind` and the callee ABI string is `$X`, where `$X` is one of the following:
+/// "C", "aapcs", "fastcall", "stdcall", "system", "sysv64", "thiscall", "vectorcall", "win64".
+///
+/// The following types are guaranteed to be ABI-compatible:
+///
+/// - `*const T`, `*mut T`, `&T`, `&mut T`, `Box<T>` (specifically, only `Box<T, Global>`),
+///   `NonNull<T>` are all ABI-compatible with each other for all `T`. Two of these pointer types
+///   with different `T` are ABI-compatible if they have the same metadata type (`<T as
+///   Pointee>::Metadata`).
+/// - `usize` is ABI-compatible with the `uN` integer type of the same size, and likewise `isize` is
+///   ABI-compatible with the `iN` integer type of the same size.
+/// - Any two `fn` types are ABI-compatible with each other if they have the same ABI string or the
+///   ABI string only differs in a trailing `-unwind`, independent of the rest of their signature.
+///   (Note that this is about the case of passing a function pointer as an argument to a function.
+///   The two pointers being ABI-compatible here means that the call successfully passes the
+///   pointer. When actually calling the pointer, of course the rest of the signature becomes
+///   relevant as well, according to the rules in this section.)
+/// - Any two types with size 0 and alignment 1 are ABI-compatible.
+/// - A `repr(transparent)` type `T` is ABI-compatible with its unique non-trivial field, i.e., the
+///   unique field that doesn't have size 0 and alignment 1 (if there is such a field).
+/// - `i32` is ABI-compatible with `NonZeroI32`, and similar for all other integer types with their
+///   matching `NonZero*` type.
+/// - If `T` is guaranteed to be subject to the [null pointer
+///   optimization](option/index.html#representation), then `T` and `Option<T>` are ABI-compatible.
+///
+/// Furthermore, ABI compatibility satisfies the following general properties:
+///
+/// - Every type is ABI-compatible with itself.
+/// - If `T1` and `T2` are ABI-compatible, then two `repr(C)` types that only differ because one
+///   field type was changed from `T1` to `T2` are ABI-compatible.
+/// - If `T1` and `T2` are ABI-compatible and `T2` and `T3` are ABI-compatible, then so are `T1` and
+///   `T3` (i.e., ABI-compatibility is transitive).
+/// - If `T1` and `T2` are ABI-compatible, then so are `T2` and `T1` (i.e., ABI-compatibility is
+///   symmetric).
+///
+/// More signatures can be ABI-compatible on specific targets, but that should not be relied upon
+/// since it is not portable and not a stable guarantee.
+///
+/// Noteworthy cases of types *not* being ABI-compatible in general are:
+/// * `bool` vs `u8`, and `i32` vs `u32`: on some targets, the calling conventions for these types
+///   differ in terms of what they guarantee for the remaining bits in the register that are not
+///   used by the value.
+/// * `i32` vs `f32` are not compatible either, as has already been mentioned above.
+/// * `struct Foo(u32)` and `u32` are not compatible (without `repr(transparent)`) since structs are
+///   aggregate types and often passed in a different way than primitives like `i32`.
+///
+/// Note that these rules describe when two completely known types are ABI-compatible. When
+/// considering ABI compatibility of a type declared in another crate (including the standard
+/// library), consider that any type that has a private field or the `#[non_exhaustive]` attribute
+/// may change its layout as a non-breaking update unless documented otherwise -- so for instance,
+/// even if such a type is a 1-ZST or `repr(transparent)` right now, this might change with any
+/// library version bump.
+///
+/// If the declared signature and the signature of the function pointer are ABI-compatible, then the
+/// function call behaves as if every argument was [`transmute`d][mem::transmute] from the
+/// type in the function pointer to the type at the function declaration, and the return value is
+/// [`transmute`d][mem::transmute] from the type in the declaration to the type in the
+/// pointer. All the usual caveats and concerns around transmutation apply; for instance, if the
+/// function expects a `NonNullI32` and the function pointer uses the ABI-compatible type
+/// `Option<NonNullI32>`, and the value used for the argument is `None`, then this call is Undefined
+/// Behavior since transmuting `None::<NonNullI32>` to `NonNullI32` violates the non-null
+/// requirement.
+///
+/// #### Requirements concerning target features
+///
+/// Under some conditions, the signature used by the caller and the callee can be ABI-incompatible
+/// even if the exact same ABI string and types are being used. As an example, the
+/// `std::arch::x86_64::__m256` type has a different `extern "C"` ABI when the `avx` feature is
+/// enabled vs when it is not enabled.
+///
+/// Therefore, to ensure ABI compatibility when code using different target features is combined
+/// (such as via `#[target_feature]`), we further require that one of the following conditions is
+/// met:
+///
+/// - The function uses the `"Rust"` ABI string (which is the default without `extern`).
+/// - Caller and callee are using the exact same set of target features. For the callee we consider
+///   the features enabled (via `#[target_feature]` and `-C target-feature`/`-C target-cpu`) at the
+///   declaration site; for the caller we consider the features enabled at the call site.
+/// - Neither any argument nor the return value involves a SIMD type (`#[repr(simd)]`) that is not
+///   behind a pointer indirection (i.e., `*mut __m256` is fine, but `(i32, __m256)` is not).
+///
 /// ### Trait implementations
 ///
 /// In this documentation the shorthand `fn (T₁, T₂, …, Tₙ)` is used to represent non-variadic
diff --git a/tests/ui/abi/compatibility.rs b/tests/ui/abi/compatibility.rs
index 0cdf229711a..53e1eff9d72 100644
--- a/tests/ui/abi/compatibility.rs
+++ b/tests/ui/abi/compatibility.rs
@@ -231,8 +231,7 @@ macro_rules! test_abi_compatible {
     };
 }
 
-// Compatibility of pointers is probably de-facto guaranteed,
-// but that does not seem to be documented.
+// Compatibility of pointers.
 test_abi_compatible!(ptr_mut, *const i32, *mut i32);
 test_abi_compatible!(ptr_pointee, *const i32, *const Vec<i32>);
 test_abi_compatible!(ref_mut, &i32, &mut i32);
@@ -241,14 +240,15 @@ test_abi_compatible!(box_ptr, Box<i32>, *const i32);
 test_abi_compatible!(nonnull_ptr, NonNull<i32>, *const i32);
 test_abi_compatible!(fn_fn, fn(), fn(i32) -> i32);
 
-// Some further guarantees we will likely (have to) make.
+// Compatibility of 1-ZST.
 test_abi_compatible!(zst_unit, Zst, ());
 #[cfg(not(any(target_arch = "sparc64")))]
 test_abi_compatible!(zst_array, Zst, [u8; 0]);
 test_abi_compatible!(nonzero_int, NonZeroI32, i32);
 
 // `DispatchFromDyn` relies on ABI compatibility.
-// This is interesting since these types are not `repr(transparent)`.
+// This is interesting since these types are not `repr(transparent)`. So this is not part of our
+// public ABI guarantees, but is relied on by the compiler.
 test_abi_compatible!(rc, Rc<i32>, *mut i32);
 test_abi_compatible!(arc, Arc<i32>, *mut i32);