Rollup merge of #145932 - JamieCunliffe:target-feature-inlining, r=jackh726

Allow `inline(always)` with a target feature behind a unstable feature `target_feature_inline_always`.

Rather than adding the inline always attribute to the function definition, we add it to the callsite. We can then check that the target features match and that the call would be safe to inline. If the function isn't inlined due to a mismatch, we emit a warning informing the user that the function can't be inlined due to the target feature mismatch.

See tracking issue rust-lang/rust#145574

16 files changed, 400 insertions, 27 deletions

diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 5affb26483a..9f2d37d39d8 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -29,8 +29,18 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[
 }
 
 /// Get LLVM attribute for the provided inline heuristic.
-#[inline]
-fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> {
+pub(crate) fn inline_attr<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    instance: ty::Instance<'tcx>,
+) -> Option<&'ll Attribute> {
+    // `optnone` requires `noinline`
+    let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id());
+    let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) {
+        (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never,
+        (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint,
+        (inline, _) => inline,
+    };
+
     if !cx.tcx.sess.opts.unstable_opts.inline_llvm {
         // disable LLVM inlining
         return Some(AttributeKind::NoInline.create_attr(cx.llcx));
@@ -346,14 +356,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         OptimizeAttr::Speed => {}
     }
 
-    // `optnone` requires `noinline`
-    let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) {
-        (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never,
-        (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint,
-        (inline, _) => inline,
-    };
-    to_add.extend(inline_attr(cx, inline));
-
     if cx.sess().must_emit_unwind_tables() {
         to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind));
     }
@@ -488,6 +490,14 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     let function_features =
         codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>();
 
+    // Apply function attributes as per usual if there are no user defined
+    // target features otherwise this will get applied at the callsite.
+    if function_features.is_empty() {
+        if let Some(inline_attr) = inline_attr(cx, instance) {
+            to_add.push(inline_attr);
+        }
+    }
+
     let function_features = function_features
         .iter()
         // Convert to LLVMFeatures and filter out unavailable ones
@@ -517,6 +527,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     let function_features = function_features.iter().map(|s| s.as_str());
     let target_features: String =
         global_features.chain(function_features).intersperse(",").collect();
+
     if !target_features.is_empty() {
         to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features));
     }
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 37379586d58..7d0691366e6 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -1392,7 +1392,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     fn call(
         &mut self,
         llty: &'ll Type,
-        fn_attrs: Option<&CodegenFnAttrs>,
+        fn_call_attrs: Option<&CodegenFnAttrs>,
         fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
         llfn: &'ll Value,
         args: &[&'ll Value],
@@ -1409,10 +1409,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
 
         // Emit CFI pointer type membership test
-        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
+        self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn);
 
         // Emit KCFI operand bundle
-        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
+        let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn);
         if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
             bundles.push(kcfi_bundle);
         }
@@ -1429,6 +1429,29 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 c"".as_ptr(),
             )
         };
+
+        if let Some(instance) = instance {
+            // Attributes on the function definition being called
+            let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id());
+            if let Some(fn_call_attrs) = fn_call_attrs
+                && !fn_call_attrs.target_features.is_empty()
+                // If there is an inline attribute and a target feature that matches
+                // we will add the attribute to the callsite otherwise we'll omit
+                // this and not add the attribute to prevent soundness issues.
+                && let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance)
+                && self.cx.tcx.is_target_feature_call_safe(
+                    &fn_call_attrs.target_features,
+                    &fn_defn_attrs.target_features,
+                )
+            {
+                attributes::apply_to_callsite(
+                    call,
+                    llvm::AttributePlace::Function,
+                    &[inlining_rule],
+                );
+            }
+        }
+
         if let Some(fn_abi) = fn_abi {
             fn_abi.apply_attrs_callsite(self, call);
         }
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 961bb788149..008340e614d 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -428,9 +428,16 @@ fn check_result(
     // llvm/llvm-project#70563).
     if !codegen_fn_attrs.target_features.is_empty()
         && matches!(codegen_fn_attrs.inline, InlineAttr::Always)
+        && !tcx.features().target_feature_inline_always()
         && let Some(span) = interesting_spans.inline
     {
-        tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`");
+        feature_err(
+            tcx.sess,
+            sym::target_feature_inline_always,
+            span,
+            "cannot use `#[inline(always)]` with `#[target_feature]`",
+        )
+        .emit();
     }
 
     // warn that inline has no effect when no_sanitize is present
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 573e9aab055..4f35bf63a1a 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -642,6 +642,8 @@ declare_features! (
     (unstable, super_let, "1.88.0", Some(139076)),
     /// Allows subtrait items to shadow supertrait items.
     (unstable, supertrait_item_shadowing, "1.86.0", Some(89151)),
+    /// Allows the use of target_feature when a function is marked inline(always).
+    (unstable, target_feature_inline_always, "CURRENT_RUSTC_VERSION", Some(145574)),
     /// Allows using `#[thread_local]` on `static` items.
     (unstable, thread_local, "1.0.0", Some(29594)),
     /// Allows defining `trait X = A + B;` alias items.
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index ce92263babd..b79075ac09b 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -5141,3 +5141,57 @@ declare_lint! {
     "detects tail calls of functions marked with `#[track_caller]`",
     @feature_gate = explicit_tail_calls;
 }
+declare_lint! {
+    /// The `inline_always_mismatching_target_features` lint will trigger when a
+    /// function with the `#[inline(always)]` and `#[target_feature(enable = "...")]`
+    /// attributes is called and cannot be inlined due to missing target features in the caller.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (fails on x86_64)
+    /// #[inline(always)]
+    /// #[target_feature(enable = "fp16")]
+    /// unsafe fn callee() {
+    ///     // operations using fp16 types
+    /// }
+    ///
+    /// // Caller does not enable the required target feature
+    /// fn caller() {
+    ///     unsafe { callee(); }
+    /// }
+    ///
+    /// fn main() {
+    ///     caller();
+    /// }
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    /// warning: call to `#[inline(always)]`-annotated `callee` requires the same target features. Function will not have `alwaysinline` attribute applied
+    ///   --> $DIR/builtin.rs:5192:14
+    ///    |
+    /// 10 |     unsafe { callee(); }
+    ///    |              ^^^^^^^^
+    ///    |
+    /// note: `fp16` target feature enabled in `callee` here but missing from `caller`
+    ///   --> $DIR/builtin.rs:5185:1
+    ///    |
+    /// 3  | #[target_feature(enable = "fp16")]
+    ///    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    /// 4  | unsafe fn callee() {
+    ///    | ------------------
+    ///    = note: `#[warn(inline_always_mismatching_target_features)]` on by default
+    /// warning: 1 warning emitted
+    /// ```
+    ///
+    /// ### Explanation
+    ///
+    /// Inlining a function with a target feature attribute into a caller that
+    /// lacks the corresponding target feature can lead to unsound behavior.
+    /// LLVM may select the wrong instructions or registers, or reorder
+    /// operations, potentially resulting in runtime errors.
+    pub INLINE_ALWAYS_MISMATCHING_TARGET_FEATURES,
+    Warn,
+    r#"detects when a function annotated with `#[inline(always)]` and `#[target_feature(enable = "..")]` is inlined into a caller without the required target feature"#,
+}
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index 866736f74a0..8b4503073b0 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -82,7 +82,7 @@ pub enum TargetFeatureKind {
     Forced,
 }
 
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, HashStable)]
 pub struct TargetFeature {
     /// The name of the target feature (e.g. "avx")
     pub name: Symbol,
diff --git a/compiler/rustc_mir_transform/src/check_inline_always_target_features.rs b/compiler/rustc_mir_transform/src/check_inline_always_target_features.rs
new file mode 100644
index 00000000000..abad28f0a8f
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/check_inline_always_target_features.rs
@@ -0,0 +1,88 @@
+use rustc_hir::attrs::InlineAttr;
+use rustc_middle::middle::codegen_fn_attrs::{TargetFeature, TargetFeatureKind};
+use rustc_middle::mir::{Body, TerminatorKind};
+use rustc_middle::ty::{self, TyCtxt};
+
+use crate::pass_manager::MirLint;
+
+pub(super) struct CheckInlineAlwaysTargetFeature;
+
+impl<'tcx> MirLint<'tcx> for CheckInlineAlwaysTargetFeature {
+    fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
+        check_inline_always_target_features(tcx, body)
+    }
+}
+
+/// `#[target_feature]`-annotated functions can be marked `#[inline]` and will only be inlined if
+/// the target features match (as well as all of the other inlining heuristics). `#[inline(always)]`
+/// will always inline regardless of matching target features, which can result in errors from LLVM.
+/// However, it is desirable to be able to always annotate certain functions (e.g. SIMD intrinsics)
+/// as `#[inline(always)]` but check the target features match in Rust to avoid the LLVM errors.
+///
+/// We check the caller and callee target features to ensure that this can
+/// be done or emit a lint.
+#[inline]
+fn check_inline_always_target_features<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
+    let caller_def_id = body.source.def_id().expect_local();
+    if !tcx.def_kind(caller_def_id).has_codegen_attrs() {
+        return;
+    }
+
+    let caller_codegen_fn_attrs = tcx.codegen_fn_attrs(caller_def_id);
+
+    for bb in body.basic_blocks.iter() {
+        let terminator = bb.terminator();
+        match &terminator.kind {
+            TerminatorKind::Call { func, .. } | TerminatorKind::TailCall { func, .. } => {
+                let fn_ty = func.ty(body, tcx);
+                let ty::FnDef(callee_def_id, _) = *fn_ty.kind() else {
+                    continue;
+                };
+
+                if !tcx.def_kind(callee_def_id).has_codegen_attrs() {
+                    continue;
+                }
+                let callee_codegen_fn_attrs = tcx.codegen_fn_attrs(callee_def_id);
+                if callee_codegen_fn_attrs.inline != InlineAttr::Always
+                    || callee_codegen_fn_attrs.target_features.is_empty()
+                {
+                    continue;
+                }
+
+                // Scan the users defined target features and ensure they
+                // match the caller.
+                if tcx.is_target_feature_call_safe(
+                    &callee_codegen_fn_attrs.target_features,
+                    &caller_codegen_fn_attrs
+                        .target_features
+                        .iter()
+                        .cloned()
+                        .chain(tcx.sess.target_features.iter().map(|feat| TargetFeature {
+                            name: *feat,
+                            kind: TargetFeatureKind::Implied,
+                        }))
+                        .collect::<Vec<_>>(),
+                ) {
+                    continue;
+                }
+
+                let callee_only: Vec<_> = callee_codegen_fn_attrs
+                    .target_features
+                    .iter()
+                    .filter(|it| !caller_codegen_fn_attrs.target_features.contains(it))
+                    .filter(|it| !matches!(it.kind, TargetFeatureKind::Implied))
+                    .map(|it| it.name.as_str())
+                    .collect();
+
+                crate::errors::emit_inline_always_target_feature_diagnostic(
+                    tcx,
+                    terminator.source_info.span,
+                    callee_def_id,
+                    caller_def_id.into(),
+                    &callee_only,
+                );
+            }
+            _ => (),
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs
index ad9635aae33..775f5f9a7cf 100644
--- a/compiler/rustc_mir_transform/src/errors.rs
+++ b/compiler/rustc_mir_transform/src/errors.rs
@@ -2,6 +2,7 @@ use rustc_errors::codes::*;
 use rustc_errors::{Diag, LintDiagnostic};
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::mir::AssertKind;
+use rustc_middle::query::Key;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::{self, Lint};
 use rustc_span::def_id::DefId;
@@ -9,6 +10,46 @@ use rustc_span::{Ident, Span, Symbol};
 
 use crate::fluent_generated as fluent;
 
+/// Emit diagnostic for calls to `#[inline(always)]`-annotated functions with a
+/// `#[target_feature]` attribute where the caller enables a different set of target features.
+pub(crate) fn emit_inline_always_target_feature_diagnostic<'a, 'tcx>(
+    tcx: TyCtxt<'tcx>,
+    call_span: Span,
+    callee_def_id: DefId,
+    caller_def_id: DefId,
+    callee_only: &[&'a str],
+) {
+    let callee = tcx.def_path_str(callee_def_id);
+    let caller = tcx.def_path_str(caller_def_id);
+
+    tcx.node_span_lint(
+        lint::builtin::INLINE_ALWAYS_MISMATCHING_TARGET_FEATURES,
+        tcx.local_def_id_to_hir_id(caller_def_id.as_local().unwrap()),
+        call_span,
+        |lint| {
+            lint.primary_message(format!(
+                "call to `#[inline(always)]`-annotated `{callee}` \
+                requires the same target features to be inlined"
+            ));
+            lint.note("function will not be inlined");
+
+            lint.note(format!(
+                "the following target features are on `{callee}` but missing from `{caller}`: {}",
+                callee_only.join(", ")
+            ));
+            lint.span_note(callee_def_id.default_span(tcx), format!("`{callee}` is defined here"));
+
+            let feats = callee_only.join(",");
+            lint.span_suggestion(
+                tcx.def_span(caller_def_id).shrink_to_lo(),
+                format!("add `#[target_feature]` attribute to `{caller}`"),
+                format!("#[target_feature(enable = \"{feats}\")]\n"),
+                lint::Applicability::MaybeIncorrect,
+            );
+        },
+    );
+}
+
 #[derive(LintDiagnostic)]
 #[diag(mir_transform_unconditional_recursion)]
 #[help]
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 8f319e64916..1663dfa744f 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -117,6 +117,7 @@ declare_passes! {
     mod add_subtyping_projections : Subtyper;
     mod check_inline : CheckForceInline;
     mod check_call_recursion : CheckCallRecursion, CheckDropRecursion;
+    mod check_inline_always_target_features: CheckInlineAlwaysTargetFeature;
     mod check_alignment : CheckAlignment;
     mod check_enums : CheckEnums;
     mod check_const_item_mutation : CheckConstItemMutation;
@@ -384,6 +385,9 @@ fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
             // MIR-level lints.
             &Lint(check_inline::CheckForceInline),
             &Lint(check_call_recursion::CheckCallRecursion),
+            // Check callee's target features match callers target features when
+            // using `#[inline(always)]`
+            &Lint(check_inline_always_target_features::CheckInlineAlwaysTargetFeature),
             &Lint(check_packed_ref::CheckPackedRef),
             &Lint(check_const_item_mutation::CheckConstItemMutation),
             &Lint(function_item_references::FunctionItemReferences),
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 77260d07c99..ff5cf34cf94 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -2162,6 +2162,7 @@ symbols! {
         target_family,
         target_feature,
         target_feature_11,
+        target_feature_inline_always,
         target_has_atomic,
         target_has_atomic_equal_alignment,
         target_has_atomic_load_store,
diff --git a/tests/ui/feature-gates/feature-gate-target-feature-inline-always.rs b/tests/ui/feature-gates/feature-gate-target-feature-inline-always.rs
new file mode 100644
index 00000000000..181f9a21000
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-target-feature-inline-always.rs
@@ -0,0 +1,9 @@
+//@ only-aarch64
+#[inline(always)]
+//~^ ERROR cannot use `#[inline(always)]` with `#[target_feature]`
+#[target_feature(enable="fp16")]
+fn test() {
+
+}
+
+fn main() { }
diff --git a/tests/ui/feature-gates/feature-gate-target-feature-inline-always.stderr b/tests/ui/feature-gates/feature-gate-target-feature-inline-always.stderr
new file mode 100644
index 00000000000..de54844bc29
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-target-feature-inline-always.stderr
@@ -0,0 +1,13 @@
+error[E0658]: cannot use `#[inline(always)]` with `#[target_feature]`
+  --> $DIR/feature-gate-target-feature-inline-always.rs:2:1
+   |
+LL | #[inline(always)]
+   | ^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #145574 <https://github.com/rust-lang/rust/issues/145574> for more information
+   = help: add `#![feature(target_feature_inline_always)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/target-feature/inline-always.aarch64.stderr b/tests/ui/target-feature/inline-always.aarch64.stderr
new file mode 100644
index 00000000000..a9ffb425c52
--- /dev/null
+++ b/tests/ui/target-feature/inline-always.aarch64.stderr
@@ -0,0 +1,60 @@
+warning: call to `#[inline(always)]`-annotated `target_feature_identity` requires the same target features to be inlined
+  --> $DIR/inline-always.rs:19:5
+   |
+LL |     target_feature_identity();
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: function will not be inlined
+   = note: the following target features are on `target_feature_identity` but missing from `call_no_target_features`: neon, fp16
+note: `target_feature_identity` is defined here
+  --> $DIR/inline-always.rs:16:1
+   |
+LL | pub unsafe fn target_feature_identity() {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   = note: `#[warn(inline_always_mismatching_target_features)]` on by default
+help: add `#[target_feature]` attribute to `call_no_target_features`
+   |
+LL + #[target_feature(enable = "neon,fp16")]
+LL | unsafe fn call_no_target_features() {
+   |
+
+warning: call to `#[inline(always)]`-annotated `multiple_target_features` requires the same target features to be inlined
+  --> $DIR/inline-always.rs:22:5
+   |
+LL |     multiple_target_features();
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: function will not be inlined
+   = note: the following target features are on `multiple_target_features` but missing from `call_no_target_features`: fp16, sve, rdm
+note: `multiple_target_features` is defined here
+  --> $DIR/inline-always.rs:52:1
+   |
+LL | fn multiple_target_features() {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+help: add `#[target_feature]` attribute to `call_no_target_features`
+   |
+LL + #[target_feature(enable = "fp16,sve,rdm")]
+LL | unsafe fn call_no_target_features() {
+   |
+
+warning: call to `#[inline(always)]`-annotated `multiple_target_features` requires the same target features to be inlined
+  --> $DIR/inline-always.rs:28:5
+   |
+LL |     multiple_target_features();
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: function will not be inlined
+   = note: the following target features are on `multiple_target_features` but missing from `call_to_first_set`: rdm
+note: `multiple_target_features` is defined here
+  --> $DIR/inline-always.rs:52:1
+   |
+LL | fn multiple_target_features() {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+help: add `#[target_feature]` attribute to `call_to_first_set`
+   |
+LL + #[target_feature(enable = "rdm")]
+LL | unsafe fn call_to_first_set() {
+   |
+
+warning: 3 warnings emitted
+
diff --git a/tests/ui/target-feature/inline-always.rs b/tests/ui/target-feature/inline-always.rs
new file mode 100644
index 00000000000..dbf46537caa
--- /dev/null
+++ b/tests/ui/target-feature/inline-always.rs
@@ -0,0 +1,54 @@
+//@ add-core-stubs
+//@ build-pass
+//@ compile-flags: --crate-type=lib
+//@ revisions: aarch64
+//@[aarch64] compile-flags: --target aarch64-unknown-linux-gnu
+//@[aarch64] needs-llvm-components: aarch64
+
+#![feature(no_core, target_feature_inline_always)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[inline(always)]
+#[target_feature(enable = "neon,fp16")]
+pub unsafe fn target_feature_identity() {}
+
+unsafe fn call_no_target_features() {
+    target_feature_identity();
+    //~^ WARNING call to `#[inline(always)]`-annotated `target_feature_identity` requires the same target features to be inlined [inline_always_mismatching_target_features]
+    global_feature_enabled();
+    multiple_target_features();
+    //~^ WARNING call to `#[inline(always)]`-annotated `multiple_target_features` requires the same target features to be inlined [inline_always_mismatching_target_features]
+}
+
+#[target_feature(enable = "fp16,sve")]
+unsafe fn call_to_first_set() {
+    multiple_target_features();
+    //~^ WARNING call to `#[inline(always)]`-annotated `multiple_target_features` requires the same target features to be inlined [inline_always_mismatching_target_features]
+}
+
+/* You can't have "fhm" without "fp16" */
+#[target_feature(enable = "fhm")]
+unsafe fn mismatching_features() {
+    target_feature_identity()
+}
+
+#[target_feature(enable = "fp16")]
+unsafe fn matching_target_features() {
+    target_feature_identity()
+}
+
+#[inline(always)]
+#[target_feature(enable = "neon")]
+unsafe fn global_feature_enabled() {
+
+}
+
+#[inline(always)]
+#[target_feature(enable = "fp16,sve")]
+#[target_feature(enable="rdm")]
+fn multiple_target_features() {
+
+ }
diff --git a/tests/ui/target-feature/invalid-attribute.rs b/tests/ui/target-feature/invalid-attribute.rs
index b34a48aba26..a958700231a 100644
--- a/tests/ui/target-feature/invalid-attribute.rs
+++ b/tests/ui/target-feature/invalid-attribute.rs
@@ -61,6 +61,8 @@ trait Baz {}
 
 #[inline(always)]
 //~^ ERROR: cannot use `#[inline(always)]`
+//~| NOTE: see issue #145574 <https://github.com/rust-lang/rust/issues/145574> for more information
+//~| NOTE: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 #[target_feature(enable = "sse2")]
 unsafe fn test() {}
 
diff --git a/tests/ui/target-feature/invalid-attribute.stderr b/tests/ui/target-feature/invalid-attribute.stderr
index 7b75367b48c..d85bccce441 100644
--- a/tests/ui/target-feature/invalid-attribute.stderr
+++ b/tests/ui/target-feature/invalid-attribute.stderr
@@ -106,7 +106,7 @@ LL | #[target_feature(enable = "sse2")]
    = help: `#[target_feature]` can only be applied to functions
 
 error: `#[target_feature]` attribute cannot be used on statics
-  --> $DIR/invalid-attribute.rs:67:1
+  --> $DIR/invalid-attribute.rs:69:1
    |
 LL | #[target_feature(enable = "sse2")]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -114,7 +114,7 @@ LL | #[target_feature(enable = "sse2")]
    = help: `#[target_feature]` can only be applied to functions
 
 error: `#[target_feature]` attribute cannot be used on trait impl blocks
-  --> $DIR/invalid-attribute.rs:71:1
+  --> $DIR/invalid-attribute.rs:73:1
    |
 LL | #[target_feature(enable = "sse2")]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -122,7 +122,7 @@ LL | #[target_feature(enable = "sse2")]
    = help: `#[target_feature]` can only be applied to functions
 
 error: `#[target_feature]` attribute cannot be used on inherent impl blocks
-  --> $DIR/invalid-attribute.rs:77:1
+  --> $DIR/invalid-attribute.rs:79:1
    |
 LL | #[target_feature(enable = "sse2")]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -130,7 +130,7 @@ LL | #[target_feature(enable = "sse2")]
    = help: `#[target_feature]` can only be applied to functions
 
 error: `#[target_feature]` attribute cannot be used on expressions
-  --> $DIR/invalid-attribute.rs:98:5
+  --> $DIR/invalid-attribute.rs:100:5
    |
 LL |     #[target_feature(enable = "sse2")]
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -138,18 +138,22 @@ LL |     #[target_feature(enable = "sse2")]
    = help: `#[target_feature]` can only be applied to functions
 
 error: `#[target_feature]` attribute cannot be used on closures
-  --> $DIR/invalid-attribute.rs:104:5
+  --> $DIR/invalid-attribute.rs:106:5
    |
 LL |     #[target_feature(enable = "sse2")]
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = help: `#[target_feature]` can be applied to methods and functions
 
-error: cannot use `#[inline(always)]` with `#[target_feature]`
+error[E0658]: cannot use `#[inline(always)]` with `#[target_feature]`
   --> $DIR/invalid-attribute.rs:62:1
    |
 LL | #[inline(always)]
    | ^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #145574 <https://github.com/rust-lang/rust/issues/145574> for more information
+   = help: add `#![feature(target_feature_inline_always)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
 error: the feature named `foo` is not valid for this target
   --> $DIR/invalid-attribute.rs:20:18
@@ -158,7 +162,7 @@ LL | #[target_feature(enable = "foo")]
    |                  ^^^^^^^^^^^^^^ `foo` is not valid for this target
 
 error[E0046]: not all trait items implemented, missing: `foo`
-  --> $DIR/invalid-attribute.rs:73:1
+  --> $DIR/invalid-attribute.rs:75:1
    |
 LL | impl Quux for u8 {}
    | ^^^^^^^^^^^^^^^^ missing `foo` in implementation
@@ -167,7 +171,7 @@ LL |     fn foo();
    |     --------- `foo` from trait
 
 error: `#[target_feature(..)]` cannot be applied to safe trait method
-  --> $DIR/invalid-attribute.rs:87:5
+  --> $DIR/invalid-attribute.rs:89:5
    |
 LL |     #[target_feature(enable = "sse2")]
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot be applied to safe trait method
@@ -176,13 +180,13 @@ LL |     fn foo() {}
    |     -------- not an `unsafe` function
 
 error[E0053]: method `foo` has an incompatible type for trait
-  --> $DIR/invalid-attribute.rs:90:5
+  --> $DIR/invalid-attribute.rs:92:5
    |
 LL |     fn foo() {}
    |     ^^^^^^^^ expected safe fn, found unsafe fn
    |
 note: type in trait
-  --> $DIR/invalid-attribute.rs:82:5
+  --> $DIR/invalid-attribute.rs:84:5
    |
 LL |     fn foo();
    |     ^^^^^^^^^
@@ -190,7 +194,7 @@ LL |     fn foo();
               found signature `#[target_features] fn()`
 
 error: the feature named `+sse2` is not valid for this target
-  --> $DIR/invalid-attribute.rs:109:18
+  --> $DIR/invalid-attribute.rs:111:18
    |
 LL | #[target_feature(enable = "+sse2")]
    |                  ^^^^^^^^^^^^^^^^ `+sse2` is not valid for this target
@@ -199,5 +203,5 @@ LL | #[target_feature(enable = "+sse2")]
 
 error: aborting due to 24 previous errors
 
-Some errors have detailed explanations: E0046, E0053, E0539.
+Some errors have detailed explanations: E0046, E0053, E0539, E0658.
 For more information about an error, try `rustc --explain E0046`.