Rollup merge of #128784 - tdittr:check-abi-on-fn-ptr, r=compiler-errors

Check ABI target compatibility for function pointers

Tracking issue: https://github.com/rust-lang/rust/issues/130260
Related tracking issue: #87678

Compatibility of an ABI for a target was previously only performed on function definitions and `extern` blocks. This PR adds it also to function pointers to be consistent.

This might have broken some of the `tests/ui/` depending on the platform, so a try run seems like a good idea.

Also this might break existing code, because we now emit extra errors. Does this require a crater run?

# Example
```rust
// build with: --target=x86_64-unknown-linux-gnu

// These raise E0570
extern "thiscall" fn foo() {}
extern "thiscall" { fn bar() }

// This did not raise any error
fn baz(f: extern "thiscall" fn()) { f() }
```

# Open Questions
* [x] Should this report a future incompatibility warning like #87678 ?
* [ ] Is this the best place to perform the check?

5 files changed, 82 insertions, 15 deletions

diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index dd13c3d8fc1..4429f6346e8 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -8,7 +8,9 @@ use rustc_hir::Node;
 use rustc_hir::def::{CtorKind, DefKind};
 use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt};
 use rustc_infer::traits::Obligation;
-use rustc_lint_defs::builtin::REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS;
+use rustc_lint_defs::builtin::{
+    REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
+};
 use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::span_bug;
@@ -52,16 +54,18 @@ pub fn check_abi(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: Abi) {
             });
         }
     }
+}
 
-    // This ABI is only allowed on function pointers
-    if abi == Abi::CCmseNonSecureCall {
-        struct_span_code_err!(
-            tcx.dcx(),
-            span,
-            E0781,
-            "the `\"C-cmse-nonsecure-call\"` ABI is only allowed on function pointers"
-        )
-        .emit();
+pub fn check_abi_fn_ptr(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: Abi) {
+    match tcx.sess.target.is_abi_supported(abi) {
+        Some(true) => (),
+        Some(false) | None => {
+            tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
+                lint.primary_message(
+                    "use of calling convention not supported on this target on function pointer",
+                );
+            });
+        }
     }
 }
 
diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs
index d3d88919d87..004540b2643 100644
--- a/compiler/rustc_hir_analysis/src/check/mod.rs
+++ b/compiler/rustc_hir_analysis/src/check/mod.rs
@@ -73,7 +73,7 @@ pub mod wfcheck;
 
 use std::num::NonZero;
 
-pub use check::check_abi;
+pub use check::{check_abi, check_abi_fn_ptr};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_errors::{Diag, ErrorGuaranteed, pluralize, struct_span_code_err};
 use rustc_hir::def_id::{DefId, LocalDefId};
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
index 5150db7f51b..a562759da11 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
@@ -1,6 +1,5 @@
-use rustc_errors::DiagCtxtHandle;
-use rustc_hir as hir;
-use rustc_hir::HirId;
+use rustc_errors::{DiagCtxtHandle, E0781, struct_span_code_err};
+use rustc_hir::{self as hir, HirId};
 use rustc_middle::ty::layout::LayoutError;
 use rustc_middle::ty::{self, ParamEnv, TyCtxt};
 use rustc_span::Span;
@@ -26,7 +25,19 @@ pub(crate) fn validate_cmse_abi<'tcx>(
             ..
         }) = hir_node
         else {
-            // might happen when this ABI is used incorrectly. That will be handled elsewhere
+            let span = match tcx.parent_hir_node(hir_id) {
+                hir::Node::Item(hir::Item {
+                    kind: hir::ItemKind::ForeignMod { .. }, span, ..
+                }) => *span,
+                _ => tcx.hir().span(hir_id),
+            };
+            struct_span_code_err!(
+                tcx.dcx(),
+                span,
+                E0781,
+                "the `\"C-cmse-nonsecure-call\"` ABI is only allowed on function pointers"
+            )
+            .emit();
             return;
         };
 
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index 28a1fc88741..9148b0e681e 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -53,6 +53,7 @@ use rustc_trait_selection::traits::{self, ObligationCtxt};
 use tracing::{debug, debug_span, instrument};
 
 use crate::bounds::Bounds;
+use crate::check::check_abi_fn_ptr;
 use crate::errors::{AmbiguousLifetimeBound, BadReturnTypeNotation, WildPatTy};
 use crate::hir_ty_lowering::errors::{GenericsArgsErrExtend, prohibit_assoc_item_constraint};
 use crate::hir_ty_lowering::generics::{check_generic_arg_count, lower_generic_args};
@@ -2337,6 +2338,12 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         let fn_ty = tcx.mk_fn_sig(input_tys, output_ty, decl.c_variadic, safety, abi);
         let bare_fn_ty = ty::Binder::bind_with_vars(fn_ty, bound_vars);
 
+        if let hir::Node::Ty(hir::Ty { kind: hir::TyKind::BareFn(bare_fn_ty), span, .. }) =
+            tcx.hir_node(hir_id)
+        {
+            check_abi_fn_ptr(tcx, hir_id, *span, bare_fn_ty.abi);
+        }
+
         // reject function types that violate cmse ABI requirements
         cmse::validate_cmse_abi(self.tcx(), self.dcx(), hir_id, abi, bare_fn_ty);
 
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index 827791c54be..23dc5214fe2 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -124,6 +124,7 @@ declare_lint_pass! {
         UNSTABLE_NAME_COLLISIONS,
         UNSTABLE_SYNTAX_PRE_EXPANSION,
         UNSUPPORTED_CALLING_CONVENTIONS,
+        UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
         UNUSED_ASSIGNMENTS,
         UNUSED_ASSOCIATED_TYPE_BOUNDS,
         UNUSED_ATTRIBUTES,
@@ -3840,6 +3841,50 @@ declare_lint! {
 }
 
 declare_lint! {
+    /// The `unsupported_fn_ptr_calling_conventions` lint is output whenever there is a use of
+    /// a target dependent calling convention on a target that does not support this calling
+    /// convention on a function pointer.
+    ///
+    /// For example `stdcall` does not make much sense for a x86_64 or, more apparently, powerpc
+    /// code, because this calling convention was never specified for those targets.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (needs specific targets)
+    /// fn stdcall_ptr(f: extern "stdcall" fn ()) {
+    ///     f()
+    /// }
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    /// warning: use of calling convention not supported on this target on function pointer
+    ///   --> $DIR/unsupported.rs:34:15
+    ///    |
+    /// LL | fn stdcall_ptr(f: extern "stdcall" fn()) {
+    ///    |               ^^^^^^^^^^^^^^^^^^^^^^^^
+    ///    |
+    ///    = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+    ///    = note: for more information, see issue #130260 <https://github.com/rust-lang/rust/issues/130260>
+    ///    = note: `#[warn(unsupported_fn_ptr_calling_conventions)]` on by default
+    /// ```
+    ///
+    /// ### Explanation
+    ///
+    /// On most of the targets the behaviour of `stdcall` and similar calling conventions is not
+    /// defined at all, but was previously accepted due to a bug in the implementation of the
+    /// compiler.
+    pub UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
+    Warn,
+    "use of unsupported calling convention for function pointer",
+    @future_incompatible = FutureIncompatibleInfo {
+        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reference: "issue #130260 <https://github.com/rust-lang/rust/issues/130260>",
+    };
+}
+
+declare_lint! {
     /// The `break_with_label_and_loop` lint detects labeled `break` expressions with
     /// an unlabeled loop as their value expression.
     ///