Auto merge of #62580 - Centril:rollup-remihe0, r=Centril

Rollup of 7 pull requests

Successful merges:

 - #61665 (core: check for pointer equality when comparing Eq slices)
 - #61923 (Prerequisites from dep graph refactoring #2)
 - #62270 (Move async-await tests from run-pass to ui)
 - #62425 (filedesc: don't use ioctl(FIOCLEX) on Linux)
 - #62476 (Continue refactoring macro expansion and resolution)
 - #62519 (Regression test for HRTB bug (issue 30786).)
 - #62557 (Fix typo in libcore/intrinsics.rs)

Failed merges:

r? @ghost

8 files changed, 234 insertions, 348 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index c0ba41b8af4..37d5885db60 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -10,12 +10,12 @@ use crate::parse::token;
 use crate::ptr::P;
 use crate::symbol::{kw, sym, Ident, Symbol};
 use crate::{ThinVec, MACRO_ARGUMENTS};
-use crate::tokenstream::{self, TokenStream};
+use crate::tokenstream::{self, TokenStream, TokenTree};
 
 use errors::{DiagnosticBuilder, DiagnosticId};
 use smallvec::{smallvec, SmallVec};
 use syntax_pos::{Span, MultiSpan, DUMMY_SP};
-use syntax_pos::hygiene::{ExpnInfo, ExpnFormat};
+use syntax_pos::hygiene::{ExpnInfo, ExpnKind};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
@@ -24,6 +24,7 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::default::Default;
 
+pub use syntax_pos::hygiene::MacroKind;
 
 #[derive(Debug,Clone)]
 pub enum Annotatable {
@@ -218,7 +219,6 @@ pub trait TTMacroExpander {
         ecx: &'cx mut ExtCtxt<'_>,
         span: Span,
         input: TokenStream,
-        def_span: Option<Span>,
     ) -> Box<dyn MacResult+'cx>;
 }
 
@@ -235,7 +235,6 @@ impl<F> TTMacroExpander for F
         ecx: &'cx mut ExtCtxt<'_>,
         span: Span,
         input: TokenStream,
-        _def_span: Option<Span>,
     ) -> Box<dyn MacResult+'cx> {
         struct AvoidInterpolatedIdents;
 
@@ -518,37 +517,6 @@ impl MacResult for DummyResult {
     }
 }
 
-/// Represents different kinds of macro invocations that can be resolved.
-#[derive(Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
-pub enum MacroKind {
-    /// A bang macro - foo!()
-    Bang,
-    /// An attribute macro - #[foo]
-    Attr,
-    /// A derive attribute macro - #[derive(Foo)]
-    Derive,
-    /// A view of a procedural macro from the same crate that defines it.
-    ProcMacroStub,
-}
-
-impl MacroKind {
-    pub fn descr(self) -> &'static str {
-        match self {
-            MacroKind::Bang => "macro",
-            MacroKind::Attr => "attribute macro",
-            MacroKind::Derive => "derive macro",
-            MacroKind::ProcMacroStub => "crate-local procedural macro",
-        }
-    }
-
-    pub fn article(self) -> &'static str {
-        match self {
-            MacroKind::Attr => "an",
-            _ => "a",
-        }
-    }
-}
-
 /// A syntax extension kind.
 pub enum SyntaxExtensionKind {
     /// A token-based function-like macro.
@@ -672,19 +640,31 @@ impl SyntaxExtension {
         }
     }
 
-    fn expn_format(&self, symbol: Symbol) -> ExpnFormat {
-        match self.kind {
-            SyntaxExtensionKind::Bang(..) |
-            SyntaxExtensionKind::LegacyBang(..) => ExpnFormat::MacroBang(symbol),
-            _ => ExpnFormat::MacroAttribute(symbol),
+    pub fn dummy_bang(edition: Edition) -> SyntaxExtension {
+        fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: &[TokenTree])
+                         -> Box<dyn MacResult + 'cx> {
+            DummyResult::any(span)
+        }
+        SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition)
+    }
+
+    pub fn dummy_derive(edition: Edition) -> SyntaxExtension {
+        fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable)
+                    -> Vec<Annotatable> {
+            Vec::new()
         }
+        SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition)
+    }
+
+    pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension {
+        SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition)
     }
 
-    pub fn expn_info(&self, call_site: Span, format: &str) -> ExpnInfo {
+    pub fn expn_info(&self, call_site: Span, descr: Symbol) -> ExpnInfo {
         ExpnInfo {
             call_site,
-            format: self.expn_format(Symbol::intern(format)),
-            def_site: Some(self.span),
+            kind: ExpnKind::Macro(self.macro_kind(), descr),
+            def_site: self.span,
             default_transparency: self.default_transparency,
             allow_internal_unstable: self.allow_internal_unstable.clone(),
             allow_internal_unsafe: self.allow_internal_unsafe,
@@ -696,6 +676,9 @@ impl SyntaxExtension {
 
 pub type NamedSyntaxExtension = (Name, SyntaxExtension);
 
+/// Error type that denotes indeterminacy.
+pub struct Indeterminate;
+
 pub trait Resolver {
     fn next_node_id(&mut self) -> ast::NodeId;
 
@@ -709,26 +692,11 @@ pub trait Resolver {
     fn resolve_imports(&mut self);
 
     fn resolve_macro_invocation(&mut self, invoc: &Invocation, invoc_id: Mark, force: bool)
-                                -> Result<Option<Lrc<SyntaxExtension>>, Determinacy>;
-    fn resolve_macro_path(&mut self, path: &ast::Path, kind: MacroKind, invoc_id: Mark,
-                          derives_in_scope: Vec<ast::Path>, force: bool)
-                          -> Result<Lrc<SyntaxExtension>, Determinacy>;
+                                -> Result<Option<Lrc<SyntaxExtension>>, Indeterminate>;
 
     fn check_unused_macros(&self);
 }
 
-#[derive(Copy, Clone, PartialEq, Debug)]
-pub enum Determinacy {
-    Determined,
-    Undetermined,
-}
-
-impl Determinacy {
-    pub fn determined(determined: bool) -> Determinacy {
-        if determined { Determinacy::Determined } else { Determinacy::Undetermined }
-    }
-}
-
 #[derive(Clone)]
 pub struct ModuleData {
     pub mod_path: Vec<ast::Ident>,
@@ -753,6 +721,7 @@ pub struct ExtCtxt<'a> {
     pub resolver: &'a mut dyn Resolver,
     pub current_expansion: ExpansionData,
     pub expansions: FxHashMap<Span, Vec<String>>,
+    pub allow_derive_markers: Lrc<[Symbol]>,
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -772,6 +741,7 @@ impl<'a> ExtCtxt<'a> {
                 directory_ownership: DirectoryOwnership::Owned { relative: None },
             },
             expansions: FxHashMap::default(),
+            allow_derive_markers: [sym::rustc_attrs, sym::structural_match][..].into(),
         }
     }
 
@@ -810,7 +780,7 @@ impl<'a> ExtCtxt<'a> {
         let mut last_macro = None;
         loop {
             if ctxt.outer_expn_info().map_or(None, |info| {
-                if info.format.name() == sym::include {
+                if info.kind.descr() == sym::include {
                     // Stop going up the backtrace once include! is encountered
                     return None;
                 }
diff --git a/src/libsyntax/ext/derive.rs b/src/libsyntax/ext/derive.rs
index 2a56f3dd756..1c15deab373 100644
--- a/src/libsyntax/ext/derive.rs
+++ b/src/libsyntax/ext/derive.rs
@@ -1,14 +1,13 @@
 use crate::attr::HasAttrs;
 use crate::ast;
-use crate::source_map::{ExpnInfo, ExpnFormat};
-use crate::ext::base::ExtCtxt;
+use crate::source_map::{ExpnInfo, ExpnKind};
+use crate::ext::base::{ExtCtxt, MacroKind};
 use crate::ext::build::AstBuilder;
 use crate::parse::parser::PathStyle;
 use crate::symbol::{Symbol, sym};
 use crate::errors::Applicability;
 
 use syntax_pos::Span;
-
 use rustc_data_structures::fx::FxHashSet;
 
 pub fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> {
@@ -46,7 +45,7 @@ pub fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) ->
 pub fn add_derived_markers<T>(cx: &mut ExtCtxt<'_>, span: Span, traits: &[ast::Path], item: &mut T)
     where T: HasAttrs,
 {
-    let (mut names, mut pretty_name) = (FxHashSet::default(), "derive(".to_owned());
+    let (mut names, mut pretty_name) = (FxHashSet::default(), String::new());
     for (i, path) in traits.iter().enumerate() {
         if i > 0 {
             pretty_name.push_str(", ");
@@ -54,14 +53,12 @@ pub fn add_derived_markers<T>(cx: &mut ExtCtxt<'_>, span: Span, traits: &[ast::P
         pretty_name.push_str(&path.to_string());
         names.insert(unwrap_or!(path.segments.get(0), continue).ident.name);
     }
-    pretty_name.push(')');
 
-    cx.current_expansion.mark.set_expn_info(ExpnInfo::with_unstable(
-        ExpnFormat::MacroAttribute(Symbol::intern(&pretty_name)), span, cx.parse_sess.edition,
-        &[sym::rustc_attrs, sym::structural_match],
+    let span = span.fresh_expansion(cx.current_expansion.mark, ExpnInfo::allow_unstable(
+        ExpnKind::Macro(MacroKind::Derive, Symbol::intern(&pretty_name)), span,
+        cx.parse_sess.edition, cx.allow_derive_markers.clone(),
     ));
 
-    let span = span.with_ctxt(cx.backtrace());
     item.visit_attrs(|attrs| {
         if names.contains(&sym::Eq) && names.contains(&sym::PartialEq) {
             let meta = cx.meta_word(span, sym::structural_match);
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 053686b8b1f..7fc62e357c5 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -5,7 +5,7 @@ use crate::source_map::{dummy_spanned, respan};
 use crate::config::StripUnconfigured;
 use crate::ext::base::*;
 use crate::ext::derive::{add_derived_markers, collect_derives};
-use crate::ext::hygiene::{Mark, SyntaxContext};
+use crate::ext::hygiene::{Mark, SyntaxContext, ExpnInfo, ExpnKind};
 use crate::ext::placeholders::{placeholder, PlaceholderExpander};
 use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
 use crate::mut_visit::*;
@@ -158,8 +158,8 @@ ast_fragments! {
 }
 
 impl AstFragmentKind {
-    fn dummy(self, span: Span) -> Option<AstFragment> {
-        self.make_from(DummyResult::any(span))
+    fn dummy(self, span: Span) -> AstFragment {
+        self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment")
     }
 
     fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I)
@@ -199,25 +199,34 @@ pub enum InvocationKind {
         span: Span,
     },
     Attr {
-        attr: Option<ast::Attribute>,
-        traits: Vec<Path>,
+        attr: ast::Attribute,
         item: Annotatable,
+        // Required for resolving derive helper attributes.
+        derives: Vec<Path>,
         // We temporarily report errors for attribute macros placed after derives
         after_derive: bool,
     },
     Derive {
         path: Path,
         item: Annotatable,
+        item_with_markers: Annotatable,
+    },
+    /// "Invocation" that contains all derives from an item,
+    /// broken into multiple `Derive` invocations when expanded.
+    /// FIXME: Find a way to remove it.
+    DeriveContainer {
+        derives: Vec<Path>,
+        item: Annotatable,
     },
 }
 
 impl Invocation {
     pub fn span(&self) -> Span {
-        match self.kind {
-            InvocationKind::Bang { span, .. } => span,
-            InvocationKind::Attr { attr: Some(ref attr), .. } => attr.span,
-            InvocationKind::Attr { attr: None, .. } => DUMMY_SP,
-            InvocationKind::Derive { ref path, .. } => path.span,
+        match &self.kind {
+            InvocationKind::Bang { span, .. } => *span,
+            InvocationKind::Attr { attr, .. } => attr.span,
+            InvocationKind::Derive { path, .. } => path.span,
+            InvocationKind::DeriveContainer { item, .. } => item.span(),
         }
     }
 }
@@ -312,9 +321,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             let scope =
                 if self.monotonic { invoc.expansion_data.mark } else { orig_expansion_data.mark };
             let ext = match self.cx.resolver.resolve_macro_invocation(&invoc, scope, force) {
-                Ok(ext) => Some(ext),
-                Err(Determinacy::Determined) => None,
-                Err(Determinacy::Undetermined) => {
+                Ok(ext) => ext,
+                Err(Indeterminate) => {
                     undetermined_invocations.push(invoc);
                     continue
                 }
@@ -323,73 +331,62 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             progress = true;
             let ExpansionData { depth, mark, .. } = invoc.expansion_data;
             self.cx.current_expansion = invoc.expansion_data.clone();
-
             self.cx.current_expansion.mark = scope;
+
             // FIXME(jseyfried): Refactor out the following logic
             let (expanded_fragment, new_invocations) = if let Some(ext) = ext {
-                if let Some(ext) = ext {
-                    let (invoc_fragment_kind, invoc_span) = (invoc.fragment_kind, invoc.span());
-                    let fragment = self.expand_invoc(invoc, &*ext).unwrap_or_else(|| {
-                        invoc_fragment_kind.dummy(invoc_span).unwrap()
-                    });
-                    self.collect_invocations(fragment, &[])
-                } else if let InvocationKind::Attr { attr: None, traits, item, .. } = invoc.kind {
-                    if !item.derive_allowed() {
-                        let attr = attr::find_by_name(item.attrs(), sym::derive)
-                            .expect("`derive` attribute should exist");
-                        let span = attr.span;
-                        let mut err = self.cx.mut_span_err(span,
-                                                           "`derive` may only be applied to \
-                                                            structs, enums and unions");
-                        if let ast::AttrStyle::Inner = attr.style {
-                            let trait_list = traits.iter()
-                                .map(|t| t.to_string()).collect::<Vec<_>>();
-                            let suggestion = format!("#[derive({})]", trait_list.join(", "));
-                            err.span_suggestion(
-                                span, "try an outer attribute", suggestion,
-                                // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
-                                Applicability::MaybeIncorrect
-                            );
-                        }
-                        err.emit();
+                let fragment = self.expand_invoc(invoc, &ext.kind);
+                self.collect_invocations(fragment, &[])
+            } else if let InvocationKind::DeriveContainer { derives: traits, item } = invoc.kind {
+                if !item.derive_allowed() {
+                    let attr = attr::find_by_name(item.attrs(), sym::derive)
+                        .expect("`derive` attribute should exist");
+                    let span = attr.span;
+                    let mut err = self.cx.mut_span_err(span,
+                                                        "`derive` may only be applied to \
+                                                        structs, enums and unions");
+                    if let ast::AttrStyle::Inner = attr.style {
+                        let trait_list = traits.iter()
+                            .map(|t| t.to_string()).collect::<Vec<_>>();
+                        let suggestion = format!("#[derive({})]", trait_list.join(", "));
+                        err.span_suggestion(
+                            span, "try an outer attribute", suggestion,
+                            // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
+                            Applicability::MaybeIncorrect
+                        );
                     }
+                    err.emit();
+                }
 
-                    let mut item = self.fully_configure(item);
-                    item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
-                    let mut item_with_markers = item.clone();
-                    add_derived_markers(&mut self.cx, item.span(), &traits, &mut item_with_markers);
-                    let derives = derives.entry(invoc.expansion_data.mark).or_default();
-
-                    derives.reserve(traits.len());
-                    invocations.reserve(traits.len());
-                    for path in &traits {
-                        let mark = Mark::fresh(self.cx.current_expansion.mark);
-                        derives.push(mark);
-                        let item = match self.cx.resolver.resolve_macro_path(
-                                path, MacroKind::Derive, Mark::root(), Vec::new(), false) {
-                            Ok(ext) => match ext.kind {
-                                SyntaxExtensionKind::LegacyDerive(..) => item_with_markers.clone(),
-                                _ => item.clone(),
-                            },
-                            _ => item.clone(),
-                        };
-                        invocations.push(Invocation {
-                            kind: InvocationKind::Derive { path: path.clone(), item },
-                            fragment_kind: invoc.fragment_kind,
-                            expansion_data: ExpansionData {
-                                mark,
-                                ..invoc.expansion_data.clone()
-                            },
-                        });
-                    }
-                    let fragment = invoc.fragment_kind
-                        .expect_from_annotatables(::std::iter::once(item_with_markers));
-                    self.collect_invocations(fragment, derives)
-                } else {
-                    unreachable!()
+                let mut item = self.fully_configure(item);
+                item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
+                let mut item_with_markers = item.clone();
+                add_derived_markers(&mut self.cx, item.span(), &traits, &mut item_with_markers);
+                let derives = derives.entry(invoc.expansion_data.mark).or_default();
+
+                derives.reserve(traits.len());
+                invocations.reserve(traits.len());
+                for path in traits {
+                    let mark = Mark::fresh(self.cx.current_expansion.mark, None);
+                    derives.push(mark);
+                    invocations.push(Invocation {
+                        kind: InvocationKind::Derive {
+                            path,
+                            item: item.clone(),
+                            item_with_markers: item_with_markers.clone(),
+                        },
+                        fragment_kind: invoc.fragment_kind,
+                        expansion_data: ExpansionData {
+                            mark,
+                            ..invoc.expansion_data.clone()
+                        },
+                    });
                 }
+                let fragment = invoc.fragment_kind
+                    .expect_from_annotatables(::std::iter::once(item_with_markers));
+                self.collect_invocations(fragment, derives)
             } else {
-                self.collect_invocations(invoc.fragment_kind.dummy(invoc.span()).unwrap(), &[])
+                unreachable!()
             };
 
             if expanded_fragments.len() < depth {
@@ -485,28 +482,22 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         }
     }
 
-    fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtension) -> Option<AstFragment> {
-        if invoc.fragment_kind == AstFragmentKind::ForeignItems &&
-           !self.cx.ecfg.macros_in_extern() {
-            if let SyntaxExtensionKind::NonMacroAttr { .. } = ext.kind {} else {
+    fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment {
+        let (fragment_kind, span) = (invoc.fragment_kind, invoc.span());
+        if fragment_kind == AstFragmentKind::ForeignItems && !self.cx.ecfg.macros_in_extern() {
+            if let SyntaxExtensionKind::NonMacroAttr { .. } = ext {} else {
                 emit_feature_err(&self.cx.parse_sess, sym::macros_in_extern,
-                                 invoc.span(), GateIssue::Language,
+                                 span, GateIssue::Language,
                                  "macro invocations in `extern {}` blocks are experimental");
             }
         }
 
-        let result = match invoc.kind {
-            InvocationKind::Bang { .. } => self.expand_bang_invoc(invoc, ext)?,
-            InvocationKind::Attr { .. } => self.expand_attr_invoc(invoc, ext)?,
-            InvocationKind::Derive { .. } => self.expand_derive_invoc(invoc, ext)?,
-        };
-
         if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit {
             let info = self.cx.current_expansion.mark.expn_info().unwrap();
             let suggested_limit = self.cx.ecfg.recursion_limit * 2;
             let mut err = self.cx.struct_span_err(info.call_site,
                 &format!("recursion limit reached while expanding the macro `{}`",
-                         info.format.name()));
+                         info.kind.descr()));
             err.help(&format!(
                 "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate",
                 suggested_limit));
@@ -515,61 +506,87 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             FatalError.raise();
         }
 
-        Some(result)
-    }
-
-    fn expand_attr_invoc(&mut self,
-                         invoc: Invocation,
-                         ext: &SyntaxExtension)
-                         -> Option<AstFragment> {
-        let (attr, mut item) = match invoc.kind {
-            InvocationKind::Attr { attr, item, .. } => (attr?, item),
-            _ => unreachable!(),
-        };
-
-        match &ext.kind {
-            SyntaxExtensionKind::NonMacroAttr { mark_used } => {
-                attr::mark_known(&attr);
-                if *mark_used {
-                    attr::mark_used(&attr);
+        match invoc.kind {
+            InvocationKind::Bang { mac, .. } => match ext {
+                SyntaxExtensionKind::Bang(expander) => {
+                    self.gate_proc_macro_expansion_kind(span, fragment_kind);
+                    let tok_result = expander.expand(self.cx, span, mac.node.stream());
+                    let result =
+                        self.parse_ast_fragment(tok_result, fragment_kind, &mac.node.path, span);
+                    self.gate_proc_macro_expansion(span, &result);
+                    result
                 }
-                item.visit_attrs(|attrs| attrs.push(attr));
-                Some(invoc.fragment_kind.expect_from_annotatables(iter::once(item)))
-            }
-            SyntaxExtensionKind::LegacyAttr(expander) => {
-                let meta = attr.parse_meta(self.cx.parse_sess)
-                               .map_err(|mut e| { e.emit(); }).ok()?;
-                let item = expander.expand(self.cx, attr.span, &meta, item);
-                Some(invoc.fragment_kind.expect_from_annotatables(item))
-            }
-            SyntaxExtensionKind::Attr(expander) => {
-                self.gate_proc_macro_attr_item(attr.span, &item);
-                let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item {
-                    Annotatable::Item(item) => token::NtItem(item),
-                    Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
-                    Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
-                    Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
-                    Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
-                    Annotatable::Expr(expr) => token::NtExpr(expr),
-                })), DUMMY_SP).into();
-                let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span);
-                let tok_result = expander.expand(self.cx, attr.span, input, item_tok);
-                let res = self.parse_ast_fragment(tok_result, invoc.fragment_kind,
-                                                  &attr.path, attr.span);
-                self.gate_proc_macro_expansion(attr.span, &res);
-                res
+                SyntaxExtensionKind::LegacyBang(expander) => {
+                    let tok_result = expander.expand(self.cx, span, mac.node.stream());
+                    if let Some(result) = fragment_kind.make_from(tok_result) {
+                        result
+                    } else {
+                        let msg = format!("non-{kind} macro in {kind} position: {path}",
+                                          kind = fragment_kind.name(), path = mac.node.path);
+                        self.cx.span_err(span, &msg);
+                        self.cx.trace_macros_diag();
+                        fragment_kind.dummy(span)
+                    }
+                }
+                _ => unreachable!()
             }
-            SyntaxExtensionKind::Derive(..) | SyntaxExtensionKind::LegacyDerive(..) => {
-                self.cx.span_err(attr.span, &format!("`{}` is a derive macro", attr.path));
-                self.cx.trace_macros_diag();
-                invoc.fragment_kind.dummy(attr.span)
+            InvocationKind::Attr { attr, mut item, .. } => match ext {
+                SyntaxExtensionKind::Attr(expander) => {
+                    self.gate_proc_macro_attr_item(span, &item);
+                    let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item {
+                        Annotatable::Item(item) => token::NtItem(item),
+                        Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()),
+                        Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()),
+                        Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()),
+                        Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()),
+                        Annotatable::Expr(expr) => token::NtExpr(expr),
+                    })), DUMMY_SP).into();
+                    let input = self.extract_proc_macro_attr_input(attr.tokens, span);
+                    let tok_result = expander.expand(self.cx, span, input, item_tok);
+                    let res = self.parse_ast_fragment(tok_result, fragment_kind, &attr.path, span);
+                    self.gate_proc_macro_expansion(span, &res);
+                    res
+                }
+                SyntaxExtensionKind::LegacyAttr(expander) => {
+                    match attr.parse_meta(self.cx.parse_sess) {
+                        Ok(meta) => {
+                            let item = expander.expand(self.cx, span, &meta, item);
+                            fragment_kind.expect_from_annotatables(item)
+                        }
+                        Err(mut err) => {
+                            err.emit();
+                            fragment_kind.dummy(span)
+                        }
+                    }
+                }
+                SyntaxExtensionKind::NonMacroAttr { mark_used } => {
+                    attr::mark_known(&attr);
+                    if *mark_used {
+                        attr::mark_used(&attr);
+                    }
+                    item.visit_attrs(|attrs| attrs.push(attr));
+                    fragment_kind.expect_from_annotatables(iter::once(item))
+                }
+                _ => unreachable!()
             }
-            _ => {
-                let msg = &format!("macro `{}` may not be used in attributes", attr.path);
-                self.cx.span_err(attr.span, msg);
-                self.cx.trace_macros_diag();
-                invoc.fragment_kind.dummy(attr.span)
+            InvocationKind::Derive { path, item, item_with_markers } => match ext {
+                SyntaxExtensionKind::Derive(expander) |
+                SyntaxExtensionKind::LegacyDerive(expander) => {
+                    let (path, item) = match ext {
+                        SyntaxExtensionKind::LegacyDerive(..) => (path, item_with_markers),
+                        _ => (path, item),
+                    };
+                    if !item.derive_allowed() {
+                        return fragment_kind.dummy(span);
+                    }
+                    let meta = ast::MetaItem { node: ast::MetaItemKind::Word, span, path };
+                    let span = span.with_ctxt(self.cx.backtrace());
+                    let items = expander.expand(self.cx, span, &meta, item);
+                    fragment_kind.expect_from_annotatables(items)
+                }
+                _ => unreachable!()
             }
+            InvocationKind::DeriveContainer { .. } => unreachable!()
         }
     }
 
@@ -616,14 +633,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         );
     }
 
-    fn gate_proc_macro_expansion(&self, span: Span, fragment: &Option<AstFragment>) {
+    fn gate_proc_macro_expansion(&self, span: Span, fragment: &AstFragment) {
         if self.cx.ecfg.proc_macro_hygiene() {
             return
         }
-        let fragment = match fragment {
-            Some(fragment) => fragment,
-            None => return,
-        };
 
         fragment.visit_with(&mut DisallowMacros {
             span,
@@ -655,58 +668,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         }
     }
 
-    /// Expand a macro invocation. Returns the resulting expanded AST fragment.
-    fn expand_bang_invoc(&mut self,
-                         invoc: Invocation,
-                         ext: &SyntaxExtension)
-                         -> Option<AstFragment> {
-        let kind = invoc.fragment_kind;
-        let (mac, span) = match invoc.kind {
-            InvocationKind::Bang { mac, span } => (mac, span),
-            _ => unreachable!(),
-        };
-        let path = &mac.node.path;
-
-        let opt_expanded = match &ext.kind {
-            SyntaxExtensionKind::Bang(expander) => {
-                self.gate_proc_macro_expansion_kind(span, kind);
-                let tok_result = expander.expand(self.cx, span, mac.node.stream());
-                let result = self.parse_ast_fragment(tok_result, kind, path, span);
-                self.gate_proc_macro_expansion(span, &result);
-                result
-            }
-            SyntaxExtensionKind::LegacyBang(expander) => {
-                let tok_result = expander.expand(self.cx, span, mac.node.stream(), Some(ext.span));
-                kind.make_from(tok_result)
-            }
-
-            SyntaxExtensionKind::Attr(..) |
-            SyntaxExtensionKind::LegacyAttr(..) |
-            SyntaxExtensionKind::NonMacroAttr { .. } => {
-                self.cx.span_err(path.span,
-                                 &format!("`{}` can only be used in attributes", path));
-                self.cx.trace_macros_diag();
-                kind.dummy(span)
-            }
-
-            SyntaxExtensionKind::Derive(..) | SyntaxExtensionKind::LegacyDerive(..) => {
-                self.cx.span_err(path.span, &format!("`{}` is a derive macro", path));
-                self.cx.trace_macros_diag();
-                kind.dummy(span)
-            }
-        };
-
-        if opt_expanded.is_some() {
-            opt_expanded
-        } else {
-            let msg = format!("non-{kind} macro in {kind} position: {name}",
-                              name = path.segments[0].ident.name, kind = kind.name());
-            self.cx.span_err(path.span, &msg);
-            self.cx.trace_macros_diag();
-            kind.dummy(span)
-        }
-    }
-
     fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) {
         let kind = match kind {
             AstFragmentKind::Expr => "expressions",
@@ -731,47 +692,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         );
     }
 
-    /// Expand a derive invocation. Returns the resulting expanded AST fragment.
-    fn expand_derive_invoc(&mut self,
-                           invoc: Invocation,
-                           ext: &SyntaxExtension)
-                           -> Option<AstFragment> {
-        let (path, item) = match invoc.kind {
-            InvocationKind::Derive { path, item } => (path, item),
-            _ => unreachable!(),
-        };
-        if !item.derive_allowed() {
-            return None;
-        }
-
-        match &ext.kind {
-            SyntaxExtensionKind::Derive(expander) |
-            SyntaxExtensionKind::LegacyDerive(expander) => {
-                let meta = ast::MetaItem { node: ast::MetaItemKind::Word, span: path.span, path };
-                let span = meta.span.with_ctxt(self.cx.backtrace());
-                let items = expander.expand(self.cx, span, &meta, item);
-                Some(invoc.fragment_kind.expect_from_annotatables(items))
-            }
-            _ => {
-                let msg = &format!("macro `{}` may not be used for derive attributes", path);
-                self.cx.span_err(path.span, msg);
-                self.cx.trace_macros_diag();
-                invoc.fragment_kind.dummy(path.span)
-            }
-        }
-    }
-
     fn parse_ast_fragment(&mut self,
                           toks: TokenStream,
                           kind: AstFragmentKind,
                           path: &Path,
                           span: Span)
-                          -> Option<AstFragment> {
+                          -> AstFragment {
         let mut parser = self.cx.new_parser_from_tts(&toks.into_trees().collect::<Vec<_>>());
         match parser.parse_ast_fragment(kind, false) {
             Ok(fragment) => {
                 parser.ensure_complete_parse(path, kind.name(), span);
-                Some(fragment)
+                fragment
             }
             Err(mut err) => {
                 err.set_span(span);
@@ -881,7 +812,17 @@ struct InvocationCollector<'a, 'b> {
 
 impl<'a, 'b> InvocationCollector<'a, 'b> {
     fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment {
-        let mark = Mark::fresh(self.cx.current_expansion.mark);
+        // Expansion info for all the collected invocations is set upon their resolution,
+        // with exception of the derive container case which is not resolved and can get
+        // its expansion info immediately.
+        let expn_info = match &kind {
+            InvocationKind::DeriveContainer { item, .. } => Some(ExpnInfo::default(
+                ExpnKind::Macro(MacroKind::Attr, sym::derive),
+                item.span(), self.cx.parse_sess.edition,
+            )),
+            _ => None,
+        };
+        let mark = Mark::fresh(self.cx.current_expansion.mark, expn_info);
         self.invocations.push(Invocation {
             kind,
             fragment_kind,
@@ -900,12 +841,15 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
 
     fn collect_attr(&mut self,
                     attr: Option<ast::Attribute>,
-                    traits: Vec<Path>,
+                    derives: Vec<Path>,
                     item: Annotatable,
                     kind: AstFragmentKind,
                     after_derive: bool)
                     -> AstFragment {
-        self.collect(kind, InvocationKind::Attr { attr, traits, item, after_derive })
+        self.collect(kind, match attr {
+            Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive },
+            None => InvocationKind::DeriveContainer { derives, item },
+        })
     }
 
     fn find_attr_invoc(&self, attrs: &mut Vec<ast::Attribute>, after_derive: &mut bool)
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 665c794422d..5c6438a7ef5 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -88,6 +88,7 @@ impl<'a> ParserAnyMacro<'a> {
 
 struct MacroRulesMacroExpander {
     name: ast::Ident,
+    span: Span,
     lhses: Vec<quoted::TokenTree>,
     rhses: Vec<quoted::TokenTree>,
     valid: bool,
@@ -99,12 +100,11 @@ impl TTMacroExpander for MacroRulesMacroExpander {
         cx: &'cx mut ExtCtxt<'_>,
         sp: Span,
         input: TokenStream,
-        def_span: Option<Span>,
     ) -> Box<dyn MacResult + 'cx> {
         if !self.valid {
             return DummyResult::any(sp);
         }
-        generic_extension(cx, sp, def_span, self.name, input, &self.lhses, &self.rhses)
+        generic_extension(cx, sp, self.span, self.name, input, &self.lhses, &self.rhses)
     }
 }
 
@@ -117,7 +117,7 @@ fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) {
 fn generic_extension<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
-    def_span: Option<Span>,
+    def_span: Span,
     name: ast::Ident,
     arg: TokenStream,
     lhses: &[quoted::TokenTree],
@@ -199,10 +199,8 @@ fn generic_extension<'cx>(
     let span = token.span.substitute_dummy(sp);
     let mut err = cx.struct_span_err(span, &parse_failure_msg(&token));
     err.span_label(span, label);
-    if let Some(sp) = def_span {
-        if cx.source_map().span_to_filename(sp).is_real() && !sp.is_dummy() {
-            err.span_label(cx.source_map().def_span(sp), "when calling this macro");
-        }
+    if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() {
+        err.span_label(cx.source_map().def_span(def_span), "when calling this macro");
     }
 
     // Check whether there's a missing comma in this macro call, like `println!("{}" a);`
@@ -377,7 +375,7 @@ pub fn compile(
     }
 
     let expander: Box<_> =
-        Box::new(MacroRulesMacroExpander { name: def.ident, lhses, rhses, valid });
+        Box::new(MacroRulesMacroExpander { name: def.ident, span: def.span, lhses, rhses, valid });
 
     let (default_transparency, transparency_error) =
         attr::find_transparency(&def.attrs, body.legacy);
diff --git a/src/libsyntax/json.rs b/src/libsyntax/json.rs
index 767ab74355e..ec0222d90eb 100644
--- a/src/libsyntax/json.rs
+++ b/src/libsyntax/json.rs
@@ -170,7 +170,7 @@ struct DiagnosticSpanMacroExpansion {
     macro_decl_name: String,
 
     /// span where macro was defined (if known)
-    def_site_span: Option<DiagnosticSpan>,
+    def_site_span: DiagnosticSpan,
 }
 
 #[derive(RustcEncodable)]
@@ -300,14 +300,13 @@ impl DiagnosticSpan {
                                      None,
                                      backtrace,
                                      je);
-            let def_site_span = bt.def_site_span.map(|sp| {
-                Self::from_span_full(sp,
+            let def_site_span =
+                Self::from_span_full(bt.def_site_span,
                                      false,
                                      None,
                                      None,
                                      vec![].into_iter(),
-                                     je)
-            });
+                                     je);
             Box::new(DiagnosticSpanMacroExpansion {
                 span: call_site,
                 macro_decl_name: bt.macro_decl_name,
diff --git a/src/libsyntax/source_map.rs b/src/libsyntax/source_map.rs
index ac30cbb471a..bbf62ef1e23 100644
--- a/src/libsyntax/source_map.rs
+++ b/src/libsyntax/source_map.rs
@@ -7,10 +7,8 @@
 //! within the SourceMap, which upon request can be converted to line and column
 //! information, source code snippets, etc.
 
-
 pub use syntax_pos::*;
-pub use syntax_pos::hygiene::{ExpnFormat, ExpnInfo};
-pub use ExpnFormat::*;
+pub use syntax_pos::hygiene::{ExpnKind, ExpnInfo};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::StableHasher;
diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs
index 6630bf90815..d86b76f71ec 100644
--- a/src/libsyntax/std_inject.rs
+++ b/src/libsyntax/std_inject.rs
@@ -1,26 +1,15 @@
 use crate::ast;
 use crate::attr;
 use crate::edition::Edition;
-use crate::ext::hygiene::{Mark, SyntaxContext};
+use crate::ext::hygiene::{Mark, MacroKind};
 use crate::symbol::{Ident, Symbol, kw, sym};
-use crate::source_map::{ExpnInfo, MacroAttribute, dummy_spanned, respan};
+use crate::source_map::{ExpnInfo, ExpnKind, dummy_spanned, respan};
 use crate::ptr::P;
 use crate::tokenstream::TokenStream;
 
 use std::cell::Cell;
 use std::iter;
-use syntax_pos::{DUMMY_SP, Span};
-
-/// Craft a span that will be ignored by the stability lint's
-/// call to source_map's `is_internal` check.
-/// The expanded code uses the unstable `#[prelude_import]` attribute.
-fn ignored_span(sp: Span, edition: Edition) -> Span {
-    let mark = Mark::fresh(Mark::root());
-    mark.set_expn_info(ExpnInfo::with_unstable(
-        MacroAttribute(Symbol::intern("std_inject")), sp, edition, &[sym::prelude_import]
-    ));
-    sp.with_ctxt(SyntaxContext::empty().apply_mark(mark))
-}
+use syntax_pos::DUMMY_SP;
 
 pub fn injected_crate_name() -> Option<&'static str> {
     INJECTED_CRATE_NAME.with(|name| name.get())
@@ -86,7 +75,11 @@ pub fn maybe_inject_crates_ref(
 
     INJECTED_CRATE_NAME.with(|opt_name| opt_name.set(Some(name)));
 
-    let span = ignored_span(DUMMY_SP, edition);
+    let span = DUMMY_SP.fresh_expansion(Mark::root(), ExpnInfo::allow_unstable(
+        ExpnKind::Macro(MacroKind::Attr, sym::std_inject), DUMMY_SP, edition,
+        [sym::prelude_import][..].into(),
+    ));
+
     krate.module.items.insert(0, P(ast::Item {
         attrs: vec![ast::Attribute {
             style: ast::AttrStyle::Outer,
diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs
index c717f140ca3..799d64a9962 100644
--- a/src/libsyntax/test.rs
+++ b/src/libsyntax/test.rs
@@ -15,13 +15,13 @@ use smallvec::{smallvec, SmallVec};
 use syntax_pos::{DUMMY_SP, NO_EXPANSION, Span, SourceFile, BytePos};
 
 use crate::attr::{self, HasAttrs};
-use crate::source_map::{self, SourceMap, ExpnInfo, MacroAttribute, dummy_spanned, respan};
+use crate::source_map::{self, SourceMap, ExpnInfo, ExpnKind, dummy_spanned, respan};
 use crate::config;
 use crate::entry::{self, EntryPointType};
 use crate::ext::base::{ExtCtxt, Resolver};
 use crate::ext::build::AstBuilder;
 use crate::ext::expand::ExpansionConfig;
-use crate::ext::hygiene::{self, Mark, SyntaxContext};
+use crate::ext::hygiene::{self, Mark, SyntaxContext, MacroKind};
 use crate::mut_visit::{*, ExpectOne};
 use crate::feature_gate::Features;
 use crate::util::map_in_place::MapInPlace;
@@ -43,7 +43,6 @@ struct TestCtxt<'a> {
     test_cases: Vec<Test>,
     reexport_test_harness_main: Option<Symbol>,
     is_libtest: bool,
-    ctxt: SyntaxContext,
     features: &'a Features,
     test_runner: Option<ast::Path>,
 
@@ -259,8 +258,6 @@ fn generate_test_harness(sess: &ParseSess,
     let mut cleaner = EntryPointCleaner { depth: 0 };
     cleaner.visit_crate(krate);
 
-    let mark = Mark::fresh(Mark::root());
-
     let mut econfig = ExpansionConfig::default("test".to_string());
     econfig.features = Some(features);
 
@@ -274,16 +271,10 @@ fn generate_test_harness(sess: &ParseSess,
         is_libtest: attr::find_crate_name(&krate.attrs)
             .map(|s| s == sym::test).unwrap_or(false),
         toplevel_reexport: None,
-        ctxt: SyntaxContext::empty().apply_mark(mark),
         features,
         test_runner
     };
 
-    mark.set_expn_info(ExpnInfo::with_unstable(
-        MacroAttribute(sym::test_case), DUMMY_SP, sess.edition,
-        &[sym::main, sym::test, sym::rustc_attrs],
-    ));
-
     TestHarnessGenerator {
         cx,
         tests: Vec::new(),
@@ -291,13 +282,6 @@ fn generate_test_harness(sess: &ParseSess,
     }.visit_crate(krate);
 }
 
-/// Craft a span that will be ignored by the stability lint's
-/// call to source_map's `is_internal` check.
-/// The expanded code calls some unstable functions in the test crate.
-fn ignored_span(cx: &TestCtxt<'_>, sp: Span) -> Span {
-    sp.with_ctxt(cx.ctxt)
-}
-
 enum HasTestSignature {
     Yes,
     No(BadTestSignature),
@@ -314,12 +298,15 @@ enum BadTestSignature {
 /// Creates a function item for use as the main function of a test build.
 /// This function will call the `test_runner` as specified by the crate attribute
 fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
-    // Writing this out by hand with 'ignored_span':
+    // Writing this out by hand:
     //        pub fn main() {
     //            #![main]
     //            test::test_main_static(&[..tests]);
     //        }
-    let sp = ignored_span(cx, DUMMY_SP);
+    let sp = DUMMY_SP.fresh_expansion(Mark::root(), ExpnInfo::allow_unstable(
+        ExpnKind::Macro(MacroKind::Attr, sym::test_case), DUMMY_SP, cx.ext_cx.parse_sess.edition,
+        [sym::main, sym::test, sym::rustc_attrs][..].into(),
+    ));
     let ecx = &cx.ext_cx;
     let test_id = Ident::with_empty_ctxt(sym::test);