Implement token-based handling of attributes during expansion

This PR modifies the macro expansion infrastructure to handle attributes
in a fully token-based manner. As a result:

* Derives macros no longer lose spans when their input is modified
  by eager cfg-expansion. This is accomplished by performing eager
  cfg-expansion on the token stream that we pass to the derive
  proc-macro
* Inner attributes now preserve spans in all cases, including when we
  have multiple inner attributes in a row.

This is accomplished through the following changes:

* New structs `AttrAnnotatedTokenStream` and `AttrAnnotatedTokenTree` are introduced.
  These are very similar to a normal `TokenTree`, but they also track
  the position of attributes and attribute targets within the stream.
  They are built when we collect tokens during parsing.
  An `AttrAnnotatedTokenStream` is converted to a regular `TokenStream` when
  we invoke a macro.
* Token capturing and `LazyTokenStream` are modified to work with
  `AttrAnnotatedTokenStream`. A new `ReplaceRange` type is introduced, which
  is created during the parsing of a nested AST node to make the 'outer'
  AST node aware of the attributes and attribute target stored deeper in the token stream.
* When we need to perform eager cfg-expansion (either due to `#[derive]` or `#[cfg_eval]`),
we tokenize and reparse our target, capturing additional information about the locations of
`#[cfg]` and `#[cfg_attr]` attributes at any depth within the target.
This is a performance optimization, allowing us to perform less work
in the typical case where captured tokens never have eager cfg-expansion run.

1 files changed, 55 insertions, 0 deletions

diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs
index f3d33309124..e9d597d1ba6 100644
--- a/compiler/rustc_session/src/utils.rs
+++ b/compiler/rustc_session/src/utils.rs
@@ -1,7 +1,13 @@
+use crate::parse::ParseSess;
 use crate::session::Session;
+use rustc_ast::token::{self, DelimToken, Nonterminal, Token};
+use rustc_ast::tokenstream::CanSynthesizeMissingTokens;
+use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_data_structures::profiling::VerboseTimingGuard;
 use std::path::{Path, PathBuf};
 
+pub type NtToTokenstream = fn(&Nonterminal, &ParseSess, CanSynthesizeMissingTokens) -> TokenStream;
+
 impl Session {
     pub fn timer<'a>(&'a self, what: &'static str) -> VerboseTimingGuard<'a> {
         self.prof.verbose_generic_activity(what)
@@ -53,3 +59,52 @@ impl CanonicalizedPath {
         &self.original
     }
 }
+
+// FIXME: Find a better spot for this - it needs to be accessible from `rustc_ast_lowering`,
+// and needs to access `ParseSess
+pub struct FlattenNonterminals<'a> {
+    pub parse_sess: &'a ParseSess,
+    pub synthesize_tokens: CanSynthesizeMissingTokens,
+    pub nt_to_tokenstream: NtToTokenstream,
+}
+
+impl<'a> FlattenNonterminals<'a> {
+    pub fn process_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
+        fn can_skip(stream: &TokenStream) -> bool {
+            stream.trees().all(|tree| match tree {
+                TokenTree::Token(token) => !matches!(token.kind, token::Interpolated(_)),
+                TokenTree::Delimited(_, _, inner) => can_skip(&inner),
+            })
+        }
+
+        if can_skip(&tokens) {
+            return tokens;
+        }
+
+        tokens.into_trees().flat_map(|tree| self.process_token_tree(tree).into_trees()).collect()
+    }
+
+    pub fn process_token_tree(&mut self, tree: TokenTree) -> TokenStream {
+        match tree {
+            TokenTree::Token(token) => self.process_token(token),
+            TokenTree::Delimited(span, delim, tts) => {
+                TokenTree::Delimited(span, delim, self.process_token_stream(tts)).into()
+            }
+        }
+    }
+
+    pub fn process_token(&mut self, token: Token) -> TokenStream {
+        match token.kind {
+            token::Interpolated(nt) => {
+                let tts = (self.nt_to_tokenstream)(&nt, self.parse_sess, self.synthesize_tokens);
+                TokenTree::Delimited(
+                    DelimSpan::from_single(token.span),
+                    DelimToken::NoDelim,
+                    self.process_token_stream(tts),
+                )
+                .into()
+            }
+            _ => TokenTree::Token(token).into(),
+        }
+    }
+}