Auto merge of #44601 - alexcrichton:lower-attributes-in-hir, r=nrc

rustc: Forbid interpolated tokens in the HIR

Right now the HIR contains raw `syntax::ast::Attribute` structure but nowadays
these can contain arbitrary tokens. One variant of the `Token` enum is an
"interpolated" token which basically means to shove all the tokens for a
nonterminal in this position. A "nonterminal" in this case is roughly analagous
to a macro argument:

    macro_rules! foo {
        ($a:expr) => {
            // $a is a nonterminal as an expression
        }
    }

Currently nonterminals contain namely items and expressions, and this poses a
problem for incremental compilation! With incremental we want a stable hash of
all HIR items, but this means we may transitively need a stable hash *of the
entire AST*, which is certainly not stable w/ node ids and whatnot. Hence today
there's a "bug" where the "stable hash" of an AST is just the raw hash value of
the AST, and this only arises with interpolated nonterminals. The downside of
this approach, however, is that a bunch of errors get spewed out during
compilation about how this isn't a great idea.

This PR is focused at fixing these warnings, basically deleting them from the
compiler. The implementation here is to alter attributes as they're lowered from
the AST to HIR, expanding all nonterminals in-place as we see them. This code
for expanding a nonterminal to a token stream already exists for the
`proc_macro` crate, so we basically just reuse the same implementation there.

After this PR it's considered a bug to have an `Interpolated` token and hence
the stable hash implementation simply uses `bug!` in this location.

Closes #40946

1 files changed, 85 insertions, 0 deletions

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index d39f11bc3ee..a316733bdb5 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -15,10 +15,15 @@ pub use self::Lit::*;
 pub use self::Token::*;
 
 use ast::{self};
+use parse::ParseSess;
+use print::pprust;
 use ptr::P;
 use serialize::{Decodable, Decoder, Encodable, Encoder};
 use symbol::keywords;
+use syntax::parse::parse_stream_from_source_str;
+use syntax_pos::{self, Span};
 use tokenstream::{TokenStream, TokenTree};
+use tokenstream;
 
 use std::cell::Cell;
 use std::{cmp, fmt};
@@ -421,6 +426,59 @@ impl Token {
     pub fn is_reserved_ident(&self) -> bool {
         self.is_special_ident() || self.is_used_keyword() || self.is_unused_keyword()
     }
+
+    pub fn interpolated_to_tokenstream(&self, sess: &ParseSess, span: Span)
+        -> TokenStream
+    {
+        let nt = match *self {
+            Token::Interpolated(ref nt) => nt,
+            _ => panic!("only works on interpolated tokens"),
+        };
+
+        // An `Interpolated` token means that we have a `Nonterminal`
+        // which is often a parsed AST item. At this point we now need
+        // to convert the parsed AST to an actual token stream, e.g.
+        // un-parse it basically.
+        //
+        // Unfortunately there's not really a great way to do that in a
+        // guaranteed lossless fashion right now. The fallback here is
+        // to just stringify the AST node and reparse it, but this loses
+        // all span information.
+        //
+        // As a result, some AST nodes are annotated with the token
+        // stream they came from. Attempt to extract these lossless
+        // token streams before we fall back to the stringification.
+        let mut tokens = None;
+
+        match nt.0 {
+            Nonterminal::NtItem(ref item) => {
+                tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
+            }
+            Nonterminal::NtTraitItem(ref item) => {
+                tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
+            }
+            Nonterminal::NtImplItem(ref item) => {
+                tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
+            }
+            Nonterminal::NtIdent(ident) => {
+                let token = Token::Ident(ident.node);
+                tokens = Some(TokenTree::Token(ident.span, token).into());
+            }
+            Nonterminal::NtTT(ref tt) => {
+                tokens = Some(tt.clone().into());
+            }
+            _ => {}
+        }
+
+        tokens.unwrap_or_else(|| {
+            nt.1.force(|| {
+                // FIXME(jseyfried): Avoid this pretty-print + reparse hack
+                let name = "<macro expansion>".to_owned();
+                let source = pprust::token_to_string(self);
+                parse_stream_from_source_str(name, source, sess, Some(span))
+            })
+        })
+    }
 }
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
@@ -533,3 +591,30 @@ impl Decodable for LazyTokenStream {
 impl ::std::hash::Hash for LazyTokenStream {
     fn hash<H: ::std::hash::Hasher>(&self, _hasher: &mut H) {}
 }
+
+fn prepend_attrs(sess: &ParseSess,
+                 attrs: &[ast::Attribute],
+                 tokens: Option<&tokenstream::TokenStream>,
+                 span: syntax_pos::Span)
+    -> Option<tokenstream::TokenStream>
+{
+    let tokens = match tokens {
+        Some(tokens) => tokens,
+        None => return None,
+    };
+    if attrs.len() == 0 {
+        return Some(tokens.clone())
+    }
+    let mut builder = tokenstream::TokenStreamBuilder::new();
+    for attr in attrs {
+        assert_eq!(attr.style, ast::AttrStyle::Outer,
+                   "inner attributes should prevent cached tokens from existing");
+        // FIXME: Avoid this pretty-print + reparse hack as bove
+        let name = "<macro expansion>".to_owned();
+        let source = pprust::attr_to_string(attr);
+        let stream = parse_stream_from_source_str(name, source, sess, Some(span));
+        builder.push(stream);
+    }
+    builder.push(tokens.clone());
+    Some(builder.build())
+}