Auto merge of #138791 - matthiaskrgr:rollup-ev46cqr, r=matthiaskrgr

Rollup of 9 pull requests

Successful merges:

 - #138364 (ports the compiler test cases to new rust_intrinsic format)
 - #138570 (add `naked_functions_target_feature` unstable feature)
 - #138623 ([bootstrap] Use llvm_runtimes for compiler-rt)
 - #138627 (Autodiff cleanups)
 - #138669 (tests: accept some noise from LLVM 21 in symbols-all-mangled)
 - #138706 (Improve bootstrap git modified path handling)
 - #138709 (Update GCC submodule)
 - #138717 (Add an attribute that makes the spans from a macro edition 2021, and fix pin on edition 2024 with it)
 - #138721 (Use explicit cpu in some asm and codegen tests.)

r? `@ghost`
`@rustbot` modify labels: rollup

17 files changed, 284 insertions, 202 deletions

diff --git a/compiler/rustc_attr_data_structures/src/attributes.rs b/compiler/rustc_attr_data_structures/src/attributes.rs
index d2d1285b075..969bce7ae20 100644
--- a/compiler/rustc_attr_data_structures/src/attributes.rs
+++ b/compiler/rustc_attr_data_structures/src/attributes.rs
@@ -191,6 +191,7 @@ pub enum AttributeKind {
     },
     MacroTransparency(Transparency),
     Repr(ThinVec<(ReprAttr, Span)>),
+    RustcMacroEdition2021,
     Stability {
         stability: Stability,
         /// Span of the `#[stable(...)]` or `#[unstable(...)]` attribute
diff --git a/compiler/rustc_attr_data_structures/src/lib.rs b/compiler/rustc_attr_data_structures/src/lib.rs
index bbd3308809c..c61b44b273d 100644
--- a/compiler/rustc_attr_data_structures/src/lib.rs
+++ b/compiler/rustc_attr_data_structures/src/lib.rs
@@ -182,21 +182,18 @@ macro_rules! find_attr {
     }};
 
     ($attributes_list: expr, $pattern: pat $(if $guard: expr)? => $e: expr) => {{
-        fn check_attribute_iterator<'a>(_: &'_ impl IntoIterator<Item = &'a rustc_hir::Attribute>) {}
-        check_attribute_iterator(&$attributes_list);
-
-        let find_attribute = |iter| {
+        'done: {
             for i in $attributes_list {
+                let i: &rustc_hir::Attribute = i;
                 match i {
                     rustc_hir::Attribute::Parsed($pattern) $(if $guard)? => {
-                        return Some($e);
+                        break 'done Some($e);
                     }
                     _ => {}
                 }
             }
 
             None
-        };
-        find_attribute($attributes_list)
+        }
     }};
 }
diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs
index 6ecd6b4d7db..bac111159db 100644
--- a/compiler/rustc_attr_parsing/src/attributes/mod.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs
@@ -28,6 +28,7 @@ pub(crate) mod cfg;
 pub(crate) mod confusables;
 pub(crate) mod deprecation;
 pub(crate) mod repr;
+pub(crate) mod rustc;
 pub(crate) mod stability;
 pub(crate) mod transparency;
 pub(crate) mod util;
diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc.rs b/compiler/rustc_attr_parsing/src/attributes/rustc.rs
new file mode 100644
index 00000000000..bdd3bef2834
--- /dev/null
+++ b/compiler/rustc_attr_parsing/src/attributes/rustc.rs
@@ -0,0 +1,19 @@
+use rustc_attr_data_structures::AttributeKind;
+use rustc_span::sym;
+
+use super::{AcceptContext, SingleAttributeParser};
+use crate::parser::ArgParser;
+
+pub(crate) struct RustcMacroEdition2021Parser;
+
+// FIXME(jdonszelmann): make these proper diagnostics
+impl SingleAttributeParser for RustcMacroEdition2021Parser {
+    const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_macro_edition_2021];
+
+    fn on_duplicate(_cx: &crate::context::AcceptContext<'_>, _first_span: rustc_span::Span) {}
+
+    fn convert(_cx: &AcceptContext<'_>, args: &ArgParser<'_>) -> Option<AttributeKind> {
+        assert!(args.no_args());
+        Some(AttributeKind::RustcMacroEdition2021)
+    }
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 35541bb04bd..a68d4578b40 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -15,6 +15,7 @@ use crate::attributes::allow_unstable::{AllowConstFnUnstableParser, AllowInterna
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::deprecation::DeprecationParser;
 use crate::attributes::repr::ReprParser;
+use crate::attributes::rustc::RustcMacroEdition2021Parser;
 use crate::attributes::stability::{
     BodyStabilityParser, ConstStabilityIndirectParser, ConstStabilityParser, StabilityParser,
 };
@@ -76,6 +77,7 @@ attribute_groups!(
         // tidy-alphabetical-start
         Single<ConstStabilityIndirectParser>,
         Single<DeprecationParser>,
+        Single<RustcMacroEdition2021Parser>,
         Single<TransparencyParser>,
         // tidy-alphabetical-end
     ];
diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs
index dcd3c1ce8d9..6591ed151cf 100644
--- a/compiler/rustc_builtin_macros/src/autodiff.rs
+++ b/compiler/rustc_builtin_macros/src/autodiff.rs
@@ -26,6 +26,16 @@ mod llvm_enzyme {
 
     use crate::errors;
 
+    pub(crate) fn outer_normal_attr(
+        kind: &P<rustc_ast::NormalAttr>,
+        id: rustc_ast::AttrId,
+        span: Span,
+    ) -> rustc_ast::Attribute {
+        let style = rustc_ast::AttrStyle::Outer;
+        let kind = rustc_ast::AttrKind::Normal(kind.clone());
+        rustc_ast::Attribute { kind, id, style, span }
+    }
+
     // If we have a default `()` return type or explicitley `()` return type,
     // then we often can skip doing some work.
     fn has_ret(ty: &FnRetTy) -> bool {
@@ -224,20 +234,8 @@ mod llvm_enzyme {
             .filter(|a| **a == DiffActivity::Active || **a == DiffActivity::ActiveOnly)
             .count() as u32;
         let (d_sig, new_args, idents, errored) = gen_enzyme_decl(ecx, &sig, &x, span);
-        let new_decl_span = d_sig.span;
         let d_body = gen_enzyme_body(
-            ecx,
-            &x,
-            n_active,
-            &sig,
-            &d_sig,
-            primal,
-            &new_args,
-            span,
-            sig_span,
-            new_decl_span,
-            idents,
-            errored,
+            ecx, &x, n_active, &sig, &d_sig, primal, &new_args, span, sig_span, idents, errored,
         );
         let d_ident = first_ident(&meta_item_vec[0]);
 
@@ -270,36 +268,39 @@ mod llvm_enzyme {
         };
         let inline_never_attr = P(ast::NormalAttr { item: inline_item, tokens: None });
         let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id();
-        let attr: ast::Attribute = ast::Attribute {
-            kind: ast::AttrKind::Normal(rustc_ad_attr.clone()),
-            id: new_id,
-            style: ast::AttrStyle::Outer,
-            span,
-        };
+        let attr = outer_normal_attr(&rustc_ad_attr, new_id, span);
         let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id();
-        let inline_never: ast::Attribute = ast::Attribute {
-            kind: ast::AttrKind::Normal(inline_never_attr),
-            id: new_id,
-            style: ast::AttrStyle::Outer,
-            span,
-        };
+        let inline_never = outer_normal_attr(&inline_never_attr, new_id, span);
+
+        // We're avoid duplicating the attributes `#[rustc_autodiff]` and `#[inline(never)]`.
+        fn same_attribute(attr: &ast::AttrKind, item: &ast::AttrKind) -> bool {
+            match (attr, item) {
+                (ast::AttrKind::Normal(a), ast::AttrKind::Normal(b)) => {
+                    let a = &a.item.path;
+                    let b = &b.item.path;
+                    a.segments.len() == b.segments.len()
+                        && a.segments.iter().zip(b.segments.iter()).all(|(a, b)| a.ident == b.ident)
+                }
+                _ => false,
+            }
+        }
 
         // Don't add it multiple times:
         let orig_annotatable: Annotatable = match item {
             Annotatable::Item(ref mut iitem) => {
-                if !iitem.attrs.iter().any(|a| a.id == attr.id) {
+                if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) {
                     iitem.attrs.push(attr);
                 }
-                if !iitem.attrs.iter().any(|a| a.id == inline_never.id) {
+                if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) {
                     iitem.attrs.push(inline_never.clone());
                 }
                 Annotatable::Item(iitem.clone())
             }
             Annotatable::AssocItem(ref mut assoc_item, i @ Impl) => {
-                if !assoc_item.attrs.iter().any(|a| a.id == attr.id) {
+                if !assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) {
                     assoc_item.attrs.push(attr);
                 }
-                if !assoc_item.attrs.iter().any(|a| a.id == inline_never.id) {
+                if !assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) {
                     assoc_item.attrs.push(inline_never.clone());
                 }
                 Annotatable::AssocItem(assoc_item.clone(), i)
@@ -314,13 +315,7 @@ mod llvm_enzyme {
             delim: rustc_ast::token::Delimiter::Parenthesis,
             tokens: ts,
         });
-        let d_attr: ast::Attribute = ast::Attribute {
-            kind: ast::AttrKind::Normal(rustc_ad_attr.clone()),
-            id: new_id,
-            style: ast::AttrStyle::Outer,
-            span,
-        };
-
+        let d_attr = outer_normal_attr(&rustc_ad_attr, new_id, span);
         let d_annotatable = if is_impl {
             let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(asdf);
             let d_fn = P(ast::AssocItem {
@@ -361,30 +356,27 @@ mod llvm_enzyme {
         ty
     }
 
-    /// We only want this function to type-check, since we will replace the body
-    /// later on llvm level. Using `loop {}` does not cover all return types anymore,
-    /// so instead we build something that should pass. We also add a inline_asm
-    /// line, as one more barrier for rustc to prevent inlining of this function.
-    /// FIXME(ZuseZ4): We still have cases of incorrect inlining across modules, see
-    /// <https://github.com/EnzymeAD/rust/issues/173>, so this isn't sufficient.
-    /// It also triggers an Enzyme crash if we due to a bug ever try to differentiate
-    /// this function (which should never happen, since it is only a placeholder).
-    /// Finally, we also add back_box usages of all input arguments, to prevent rustc
-    /// from optimizing any arguments away.
-    fn gen_enzyme_body(
+    // Will generate a body of the type:
+    // ```
+    // {
+    //   unsafe {
+    //   asm!("NOP");
+    //   }
+    //   ::core::hint::black_box(primal(args));
+    //   ::core::hint::black_box((args, ret));
+    //   <This part remains to be done by following function>
+    // }
+    // ```
+    fn init_body_helper(
         ecx: &ExtCtxt<'_>,
-        x: &AutoDiffAttrs,
-        n_active: u32,
-        sig: &ast::FnSig,
-        d_sig: &ast::FnSig,
+        span: Span,
         primal: Ident,
         new_names: &[String],
-        span: Span,
         sig_span: Span,
         new_decl_span: Span,
-        idents: Vec<Ident>,
+        idents: &[Ident],
         errored: bool,
-    ) -> P<ast::Block> {
+    ) -> (P<ast::Block>, P<ast::Expr>, P<ast::Expr>, P<ast::Expr>) {
         let blackbox_path = ecx.std_path(&[sym::hint, sym::black_box]);
         let noop = ast::InlineAsm {
             asm_macro: ast::AsmMacro::Asm,
@@ -433,6 +425,51 @@ mod llvm_enzyme {
         }
         body.stmts.push(ecx.stmt_semi(black_box_remaining_args));
 
+        (body, primal_call, black_box_primal_call, blackbox_call_expr)
+    }
+
+    /// We only want this function to type-check, since we will replace the body
+    /// later on llvm level. Using `loop {}` does not cover all return types anymore,
+    /// so instead we manually build something that should pass the type checker.
+    /// We also add a inline_asm line, as one more barrier for rustc to prevent inlining
+    /// or const propagation. inline_asm will also triggers an Enzyme crash if due to another
+    /// bug would ever try to accidentially differentiate this placeholder function body.
+    /// Finally, we also add back_box usages of all input arguments, to prevent rustc
+    /// from optimizing any arguments away.
+    fn gen_enzyme_body(
+        ecx: &ExtCtxt<'_>,
+        x: &AutoDiffAttrs,
+        n_active: u32,
+        sig: &ast::FnSig,
+        d_sig: &ast::FnSig,
+        primal: Ident,
+        new_names: &[String],
+        span: Span,
+        sig_span: Span,
+        idents: Vec<Ident>,
+        errored: bool,
+    ) -> P<ast::Block> {
+        let new_decl_span = d_sig.span;
+
+        // Just adding some default inline-asm and black_box usages to prevent early inlining
+        // and optimizations which alter the function signature.
+        //
+        // The bb_primal_call is the black_box call of the primal function. We keep it around,
+        // since it has the convenient property of returning the type of the primal function,
+        // Remember, we only care to match types here.
+        // No matter which return we pick, we always wrap it into a std::hint::black_box call,
+        // to prevent rustc from propagating it into the caller.
+        let (mut body, primal_call, bb_primal_call, bb_call_expr) = init_body_helper(
+            ecx,
+            span,
+            primal,
+            new_names,
+            sig_span,
+            new_decl_span,
+            &idents,
+            errored,
+        );
+
         if !has_ret(&d_sig.decl.output) {
             // there is no return type that we have to match, () works fine.
             return body;
@@ -444,7 +481,7 @@ mod llvm_enzyme {
 
         if primal_ret && n_active == 0 && x.mode.is_rev() {
             // We only have the primal ret.
-            body.stmts.push(ecx.stmt_expr(black_box_primal_call));
+            body.stmts.push(ecx.stmt_expr(bb_primal_call));
             return body;
         }
 
@@ -536,11 +573,11 @@ mod llvm_enzyme {
                 return body;
             }
             [arg] => {
-                ret = ecx.expr_call(new_decl_span, blackbox_call_expr, thin_vec![arg.clone()]);
+                ret = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![arg.clone()]);
             }
             args => {
                 let ret_tuple: P<ast::Expr> = ecx.expr_tuple(span, args.into());
-                ret = ecx.expr_call(new_decl_span, blackbox_call_expr, thin_vec![ret_tuple]);
+                ret = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![ret_tuple]);
             }
         }
         assert!(has_ret(&d_sig.decl.output));
@@ -553,7 +590,7 @@ mod llvm_enzyme {
         ecx: &ExtCtxt<'_>,
         span: Span,
         primal: Ident,
-        idents: Vec<Ident>,
+        idents: &[Ident],
     ) -> P<ast::Expr> {
         let has_self = idents.len() > 0 && idents[0].name == kw::SelfLower;
         if has_self {
diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
index 71705ecb4d0..7cd4ee539d8 100644
--- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
+++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
@@ -28,6 +28,113 @@ fn get_params(fnc: &Value) -> Vec<&Value> {
     }
 }
 
+fn match_args_from_caller_to_enzyme<'ll>(
+    cx: &SimpleCx<'ll>,
+    args: &mut Vec<&'ll llvm::Value>,
+    inputs: &[DiffActivity],
+    outer_args: &[&'ll llvm::Value],
+) {
+    debug!("matching autodiff arguments");
+    // We now handle the issue that Rust level arguments not always match the llvm-ir level
+    // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on
+    // llvm-ir level. The number of activities matches the number of Rust level arguments, so we
+    // need to match those.
+    // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it
+    // using iterators and peek()?
+    let mut outer_pos: usize = 0;
+    let mut activity_pos = 0;
+
+    let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap();
+    let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap();
+    let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap();
+    let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap();
+
+    while activity_pos < inputs.len() {
+        let diff_activity = inputs[activity_pos as usize];
+        // Duplicated arguments received a shadow argument, into which enzyme will write the
+        // gradient.
+        let (activity, duplicated): (&Metadata, bool) = match diff_activity {
+            DiffActivity::None => panic!("not a valid input activity"),
+            DiffActivity::Const => (enzyme_const, false),
+            DiffActivity::Active => (enzyme_out, false),
+            DiffActivity::ActiveOnly => (enzyme_out, false),
+            DiffActivity::Dual => (enzyme_dup, true),
+            DiffActivity::DualOnly => (enzyme_dupnoneed, true),
+            DiffActivity::Duplicated => (enzyme_dup, true),
+            DiffActivity::DuplicatedOnly => (enzyme_dupnoneed, true),
+            DiffActivity::FakeActivitySize => (enzyme_const, false),
+        };
+        let outer_arg = outer_args[outer_pos];
+        args.push(cx.get_metadata_value(activity));
+        args.push(outer_arg);
+        if duplicated {
+            // We know that duplicated args by construction have a following argument,
+            // so this can not be out of bounds.
+            let next_outer_arg = outer_args[outer_pos + 1];
+            let next_outer_ty = cx.val_ty(next_outer_arg);
+            // FIXME(ZuseZ4): We should add support for Vec here too, but it's less urgent since
+            // vectors behind references (&Vec<T>) are already supported. Users can not pass a
+            // Vec by value for reverse mode, so this would only help forward mode autodiff.
+            let slice = {
+                if activity_pos + 1 >= inputs.len() {
+                    // If there is no arg following our ptr, it also can't be a slice,
+                    // since that would lead to a ptr, int pair.
+                    false
+                } else {
+                    let next_activity = inputs[activity_pos + 1];
+                    // We analyze the MIR types and add this dummy activity if we visit a slice.
+                    next_activity == DiffActivity::FakeActivitySize
+                }
+            };
+            if slice {
+                // A duplicated slice will have the following two outer_fn arguments:
+                // (..., ptr1, int1, ptr2, int2, ...). We add the following llvm-ir to our __enzyme call:
+                // (..., metadata! enzyme_dup, ptr, ptr, int1, ...).
+                // FIXME(ZuseZ4): We will upstream a safety check later which asserts that
+                // int2 >= int1, which means the shadow vector is large enough to store the gradient.
+                assert!(unsafe {
+                    llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Integer
+                });
+                let next_outer_arg2 = outer_args[outer_pos + 2];
+                let next_outer_ty2 = cx.val_ty(next_outer_arg2);
+                assert!(unsafe {
+                    llvm::LLVMRustGetTypeKind(next_outer_ty2) == llvm::TypeKind::Pointer
+                });
+                let next_outer_arg3 = outer_args[outer_pos + 3];
+                let next_outer_ty3 = cx.val_ty(next_outer_arg3);
+                assert!(unsafe {
+                    llvm::LLVMRustGetTypeKind(next_outer_ty3) == llvm::TypeKind::Integer
+                });
+                args.push(next_outer_arg2);
+                args.push(cx.get_metadata_value(enzyme_const));
+                args.push(next_outer_arg);
+                outer_pos += 4;
+                activity_pos += 2;
+            } else {
+                // A duplicated pointer will have the following two outer_fn arguments:
+                // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call:
+                // (..., metadata! enzyme_dup, ptr, ptr, ...).
+                if matches!(diff_activity, DiffActivity::Duplicated | DiffActivity::DuplicatedOnly)
+                {
+                    assert!(
+                        unsafe { llvm::LLVMRustGetTypeKind(next_outer_ty) }
+                            == llvm::TypeKind::Pointer
+                    );
+                }
+                // In the case of Dual we don't have assumptions, e.g. f32 would be valid.
+                args.push(next_outer_arg);
+                outer_pos += 2;
+                activity_pos += 1;
+            }
+        } else {
+            // We do not differentiate with resprect to this argument.
+            // We already added the metadata and argument above, so just increase the counters.
+            outer_pos += 1;
+            activity_pos += 1;
+        }
+    }
+}
+
 /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another
 /// function with expected naming and calling conventions[^1] which will be
 /// discovered by the enzyme LLVM pass and its body populated with the differentiated
@@ -43,9 +150,6 @@ fn generate_enzyme_call<'ll>(
     outer_fn: &'ll Value,
     attrs: AutoDiffAttrs,
 ) {
-    let inputs = attrs.input_activity;
-    let output = attrs.ret_activity;
-
     // We have to pick the name depending on whether we want forward or reverse mode autodiff.
     let mut ad_name: String = match attrs.mode {
         DiffMode::Forward => "__enzyme_fwddiff",
@@ -132,111 +236,13 @@ fn generate_enzyme_call<'ll>(
         let mut args = Vec::with_capacity(num_args as usize + 1);
         args.push(fn_to_diff);
 
-        let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap();
-        let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap();
-        let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap();
-        let enzyme_dupnoneed = cx.create_metadata("enzyme_dupnoneed".to_string()).unwrap();
         let enzyme_primal_ret = cx.create_metadata("enzyme_primal_return".to_string()).unwrap();
-
-        match output {
-            DiffActivity::Dual => {
-                args.push(cx.get_metadata_value(enzyme_primal_ret));
-            }
-            DiffActivity::Active => {
-                args.push(cx.get_metadata_value(enzyme_primal_ret));
-            }
-            _ => {}
+        if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) {
+            args.push(cx.get_metadata_value(enzyme_primal_ret));
         }
 
-        debug!("matching autodiff arguments");
-        // We now handle the issue that Rust level arguments not always match the llvm-ir level
-        // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on
-        // llvm-ir level. The number of activities matches the number of Rust level arguments, so we
-        // need to match those.
-        // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it
-        // using iterators and peek()?
-        let mut outer_pos: usize = 0;
-        let mut activity_pos = 0;
         let outer_args: Vec<&llvm::Value> = get_params(outer_fn);
-        while activity_pos < inputs.len() {
-            let diff_activity = inputs[activity_pos as usize];
-            // Duplicated arguments received a shadow argument, into which enzyme will write the
-            // gradient.
-            let (activity, duplicated): (&Metadata, bool) = match diff_activity {
-                DiffActivity::None => panic!("not a valid input activity"),
-                DiffActivity::Const => (enzyme_const, false),
-                DiffActivity::Active => (enzyme_out, false),
-                DiffActivity::ActiveOnly => (enzyme_out, false),
-                DiffActivity::Dual => (enzyme_dup, true),
-                DiffActivity::DualOnly => (enzyme_dupnoneed, true),
-                DiffActivity::Duplicated => (enzyme_dup, true),
-                DiffActivity::DuplicatedOnly => (enzyme_dupnoneed, true),
-                DiffActivity::FakeActivitySize => (enzyme_const, false),
-            };
-            let outer_arg = outer_args[outer_pos];
-            args.push(cx.get_metadata_value(activity));
-            args.push(outer_arg);
-            if duplicated {
-                // We know that duplicated args by construction have a following argument,
-                // so this can not be out of bounds.
-                let next_outer_arg = outer_args[outer_pos + 1];
-                let next_outer_ty = cx.val_ty(next_outer_arg);
-                // FIXME(ZuseZ4): We should add support for Vec here too, but it's less urgent since
-                // vectors behind references (&Vec<T>) are already supported. Users can not pass a
-                // Vec by value for reverse mode, so this would only help forward mode autodiff.
-                let slice = {
-                    if activity_pos + 1 >= inputs.len() {
-                        // If there is no arg following our ptr, it also can't be a slice,
-                        // since that would lead to a ptr, int pair.
-                        false
-                    } else {
-                        let next_activity = inputs[activity_pos + 1];
-                        // We analyze the MIR types and add this dummy activity if we visit a slice.
-                        next_activity == DiffActivity::FakeActivitySize
-                    }
-                };
-                if slice {
-                    // A duplicated slice will have the following two outer_fn arguments:
-                    // (..., ptr1, int1, ptr2, int2, ...). We add the following llvm-ir to our __enzyme call:
-                    // (..., metadata! enzyme_dup, ptr, ptr, int1, ...).
-                    // FIXME(ZuseZ4): We will upstream a safety check later which asserts that
-                    // int2 >= int1, which means the shadow vector is large enough to store the gradient.
-                    assert!(llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Integer);
-                    let next_outer_arg2 = outer_args[outer_pos + 2];
-                    let next_outer_ty2 = cx.val_ty(next_outer_arg2);
-                    assert!(llvm::LLVMRustGetTypeKind(next_outer_ty2) == llvm::TypeKind::Pointer);
-                    let next_outer_arg3 = outer_args[outer_pos + 3];
-                    let next_outer_ty3 = cx.val_ty(next_outer_arg3);
-                    assert!(llvm::LLVMRustGetTypeKind(next_outer_ty3) == llvm::TypeKind::Integer);
-                    args.push(next_outer_arg2);
-                    args.push(cx.get_metadata_value(enzyme_const));
-                    args.push(next_outer_arg);
-                    outer_pos += 4;
-                    activity_pos += 2;
-                } else {
-                    // A duplicated pointer will have the following two outer_fn arguments:
-                    // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call:
-                    // (..., metadata! enzyme_dup, ptr, ptr, ...).
-                    if matches!(
-                        diff_activity,
-                        DiffActivity::Duplicated | DiffActivity::DuplicatedOnly
-                    ) {
-                        assert!(
-                            llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer
-                        );
-                    }
-                    // In the case of Dual we don't have assumptions, e.g. f32 would be valid.
-                    args.push(next_outer_arg);
-                    outer_pos += 2;
-                    activity_pos += 1;
-                }
-            } else {
-                // We do not differentiate with resprect to this argument.
-                // We already added the metadata and argument above, so just increase the counters.
-                outer_pos += 1;
-                activity_pos += 1;
-            }
-        }
+        match_args_from_caller_to_enzyme(&cx, &mut args, &attrs.input_activity, &outer_args);
 
         let call = builder.call(enzyme_ty, ad_fn, &args, None);
 
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index d5972b5c9e6..d16d4ed2283 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -790,16 +790,10 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
 
     // check for exactly one autodiff attribute on placeholder functions.
     // There should only be one, since we generate a new placeholder per ad macro.
-    // FIXME(ZuseZ4): re-enable this check. Currently we add multiple, which doesn't cause harm but
-    // looks strange e.g. under cargo-expand.
     let attr = match &attrs[..] {
         [] => return None,
         [attr] => attr,
-        // These two attributes are the same and unfortunately duplicated due to a previous bug.
-        [attr, _attr2] => attr,
         _ => {
-            //FIXME(ZuseZ4): Once we fixed our parser, we should also prohibit the two-attribute
-            //branch above.
             span_bug!(attrs[1].span(), "cg_ssa: rustc_autodiff should only exist once per source");
         }
     };
diff --git a/compiler/rustc_error_codes/src/error_codes/E0092.md b/compiler/rustc_error_codes/src/error_codes/E0092.md
index 84ec0656d1a..be459d040c2 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0092.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0092.md
@@ -6,10 +6,9 @@ Erroneous code example:
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
-extern "rust-intrinsic" {
-    fn atomic_foo(); // error: unrecognized atomic operation
-                     //        function
-}
+#[rustc_intrinsic]
+unsafe fn atomic_foo(); // error: unrecognized atomic operation
+                        //        function
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
@@ -20,7 +19,6 @@ functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
-extern "rust-intrinsic" {
-    fn atomic_fence_seqcst(); // ok!
-}
+#[rustc_intrinsic]
+unsafe fn atomic_fence_seqcst(); // ok!
 ```
diff --git a/compiler/rustc_error_codes/src/error_codes/E0093.md b/compiler/rustc_error_codes/src/error_codes/E0093.md
index 2bda4d74f72..9929a069927 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0093.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0093.md
@@ -6,9 +6,8 @@ Erroneous code example:
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
-extern "rust-intrinsic" {
-    fn foo(); // error: unrecognized intrinsic function: `foo`
-}
+#[rustc_intrinsic]
+unsafe fn foo(); // error: unrecognized intrinsic function: `foo`
 
 fn main() {
     unsafe {
@@ -25,9 +24,8 @@ functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
-extern "rust-intrinsic" {
-    fn atomic_fence_seqcst(); // ok!
-}
+#[rustc_intrinsic]
+unsafe fn atomic_fence_seqcst(); // ok!
 
 fn main() {
     unsafe {
diff --git a/compiler/rustc_error_codes/src/error_codes/E0211.md b/compiler/rustc_error_codes/src/error_codes/E0211.md
index 7aa42628549..c702f14d4d6 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0211.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0211.md
@@ -7,9 +7,8 @@ used. Erroneous code examples:
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
-extern "rust-intrinsic" {
-    fn unreachable(); // error: intrinsic has wrong type
-}
+#[rustc_intrinsic]
+unsafe fn unreachable(); // error: intrinsic has wrong type
 
 // or:
 
@@ -43,9 +42,8 @@ For the first code example, please check the function definition. Example:
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
-extern "rust-intrinsic" {
-    fn unreachable() -> !; // ok!
-}
+#[rustc_intrinsic]
+unsafe fn unreachable() -> !; // ok!
 ```
 
 The second case example is a bit particular: the main function must always
diff --git a/compiler/rustc_error_codes/src/error_codes/E0511.md b/compiler/rustc_error_codes/src/error_codes/E0511.md
index 45ff49bdebb..0fb1cfda67d 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0511.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0511.md
@@ -5,9 +5,8 @@ Erroneous code example:
 ```compile_fail,E0511
 #![feature(intrinsics)]
 
-extern "rust-intrinsic" {
-    fn simd_add<T>(a: T, b: T) -> T;
-}
+#[rustc_intrinsic]
+unsafe fn simd_add<T>(a: T, b: T) -> T;
 
 fn main() {
     unsafe { simd_add(0, 1); }
@@ -25,9 +24,8 @@ The generic type has to be a SIMD type. Example:
 #[derive(Copy, Clone)]
 struct i32x2([i32; 2]);
 
-extern "rust-intrinsic" {
-    fn simd_add<T>(a: T, b: T) -> T;
-}
+#[rustc_intrinsic]
+unsafe fn simd_add<T>(a: T, b: T) -> T;
 
 unsafe { simd_add(i32x2([0, 0]), i32x2([1, 2])); } // ok!
 ```
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 5430bfde785..fd936458f11 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -661,6 +661,14 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         "`rustc_never_type_options` is used to experiment with never type fallback and work on \
          never type stabilization, and will never be stable"
     ),
+    rustc_attr!(
+        rustc_macro_edition_2021,
+        Normal,
+        template!(Word),
+        ErrorFollowing,
+        EncodeCrossCrate::No,
+        "makes spans in this macro edition 2021"
+    ),
 
     // ==========================================================================
     // Internal attributes: Runtime related:
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 3b75c69132c..4707daea089 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -566,6 +566,8 @@ declare_features! (
     (incomplete, mut_ref, "1.79.0", Some(123076)),
     /// Allows using `#[naked]` on functions.
     (unstable, naked_functions, "1.9.0", Some(90957)),
+    /// Allows using `#[target_feature(enable = "...")]` on `#[naked]` on functions.
+    (unstable, naked_functions_target_feature, "1.86.0", Some(138568)),
     /// Allows specifying the as-needed link modifier
     (unstable, native_link_modifiers_as_needed, "1.53.0", Some(81490)),
     /// Allow negative trait implementations.
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index e0739c342df..1ace98c91e6 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -600,7 +600,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
             sym::repr,
             // code generation
             sym::cold,
-            sym::target_feature,
             // documentation
             sym::doc,
         ];
@@ -626,6 +625,21 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                         _ => {}
                     }
 
+                    if other_attr.has_name(sym::target_feature) {
+                        if !self.tcx.features().naked_functions_target_feature() {
+                            feature_err(
+                                &self.tcx.sess,
+                                sym::naked_functions_target_feature,
+                                other_attr.span(),
+                                "`#[target_feature(/* ... */)]` is currently unstable on `#[naked]` functions",
+                            ).emit();
+
+                            return;
+                        } else {
+                            continue;
+                        }
+                    }
+
                     if !ALLOW_LIST.iter().any(|name| other_attr.has_name(*name)) {
                         self.dcx().emit_err(errors::NakedFunctionIncompatibleAttribute {
                             span: other_attr.span(),
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index c94846e95d4..c4304a7a6df 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -8,7 +8,7 @@ use std::sync::Arc;
 use rustc_ast::expand::StrippedCfgItem;
 use rustc_ast::{self as ast, Crate, NodeId, attr};
 use rustc_ast_pretty::pprust;
-use rustc_attr_parsing::StabilityLevel;
+use rustc_attr_parsing::{AttributeKind, StabilityLevel, find_attr};
 use rustc_data_structures::intern::Interned;
 use rustc_errors::{Applicability, StashKey};
 use rustc_expand::base::{
@@ -1125,6 +1125,13 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             edition,
         );
 
+        // The #[rustc_macro_edition_2021] attribute is used by the pin!() macro
+        // as a temporary workaround for a regression in expressiveness in Rust 2024.
+        // See https://github.com/rust-lang/rust/issues/138718.
+        if find_attr!(attrs.iter(), AttributeKind::RustcMacroEdition2021) {
+            ext.edition = Edition::Edition2021;
+        }
+
         if let Some(builtin_name) = ext.builtin_name {
             // The macro was marked with `#[rustc_builtin_macro]`.
             if let Some(builtin_ext_kind) = self.builtin_macros.get(&builtin_name) {
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index d83863b1f7a..015ddc6e273 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1376,6 +1376,7 @@ symbols! {
         naked,
         naked_asm,
         naked_functions,
+        naked_functions_target_feature,
         name,
         names,
         native_link_modifiers,
@@ -1794,6 +1795,7 @@ symbols! {
         rustc_lint_opt_ty,
         rustc_lint_query_instability,
         rustc_lint_untracked_query_information,
+        rustc_macro_edition_2021,
         rustc_macro_transparency,
         rustc_main,
         rustc_mir,