Merge remote-tracking branch 'origin/master' into gen

1 files changed, 71 insertions, 8 deletions

diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs
index 12ed4fba402..1bffffd6c9e 100644
--- a/src/libproc_macro/lib.rs
+++ b/src/libproc_macro/lib.rs
@@ -509,14 +509,49 @@ impl TokenTree {
             Ident(ident) | Lifetime(ident) => TokenNode::Term(Term(ident.name)),
             Literal(..) | DocComment(..) => TokenNode::Literal(self::Literal(token)),
 
-            Interpolated(ref nt) => __internal::with_sess(|(sess, _)| {
-                TokenNode::Group(Delimiter::None, TokenStream(nt.1.force(|| {
-                    // FIXME(jseyfried): Avoid this pretty-print + reparse hack
-                    let name = "<macro expansion>".to_owned();
-                    let source = pprust::token_to_string(&token);
-                    parse_stream_from_source_str(name, source, sess, Some(span))
-                })))
-            }),
+            Interpolated(ref nt) => {
+                // An `Interpolated` token means that we have a `Nonterminal`
+                // which is often a parsed AST item. At this point we now need
+                // to convert the parsed AST to an actual token stream, e.g.
+                // un-parse it basically.
+                //
+                // Unfortunately there's not really a great way to do that in a
+                // guaranteed lossless fashion right now. The fallback here is
+                // to just stringify the AST node and reparse it, but this loses
+                // all span information.
+                //
+                // As a result, some AST nodes are annotated with the token
+                // stream they came from. Attempt to extract these lossless
+                // token streams before we fall back to the stringification.
+                let mut tokens = None;
+
+                match nt.0 {
+                    Nonterminal::NtItem(ref item) => {
+                        tokens = prepend_attrs(&item.attrs, item.tokens.as_ref(), span);
+                    }
+                    Nonterminal::NtTraitItem(ref item) => {
+                        tokens = prepend_attrs(&item.attrs, item.tokens.as_ref(), span);
+                    }
+                    Nonterminal::NtImplItem(ref item) => {
+                        tokens = prepend_attrs(&item.attrs, item.tokens.as_ref(), span);
+                    }
+                    _ => {}
+                }
+
+                tokens.map(|tokens| {
+                    TokenNode::Group(Delimiter::None,
+                                     TokenStream(tokens.clone()))
+                }).unwrap_or_else(|| {
+                    __internal::with_sess(|(sess, _)| {
+                        TokenNode::Group(Delimiter::None, TokenStream(nt.1.force(|| {
+                            // FIXME(jseyfried): Avoid this pretty-print + reparse hack
+                            let name = "<macro expansion>".to_owned();
+                            let source = pprust::token_to_string(&token);
+                            parse_stream_from_source_str(name, source, sess, Some(span))
+                        })))
+                    })
+                })
+            }
 
             OpenDelim(..) | CloseDelim(..) => unreachable!(),
             Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
@@ -580,6 +615,34 @@ impl TokenTree {
     }
 }
 
+fn prepend_attrs(attrs: &[ast::Attribute],
+                 tokens: Option<&tokenstream::TokenStream>,
+                 span: syntax_pos::Span)
+    -> Option<tokenstream::TokenStream>
+{
+    let tokens = match tokens {
+        Some(tokens) => tokens,
+        None => return None,
+    };
+    if attrs.len() == 0 {
+        return Some(tokens.clone())
+    }
+    let mut builder = tokenstream::TokenStreamBuilder::new();
+    for attr in attrs {
+        assert_eq!(attr.style, ast::AttrStyle::Outer,
+                   "inner attributes should prevent cached tokens from existing");
+        let stream = __internal::with_sess(|(sess, _)| {
+            // FIXME: Avoid this pretty-print + reparse hack as bove
+            let name = "<macro expansion>".to_owned();
+            let source = pprust::attr_to_string(attr);
+            parse_stream_from_source_str(name, source, sess, Some(span))
+        });
+        builder.push(stream);
+    }
+    builder.push(tokens.clone());
+    Some(builder.build())
+}
+
 /// Permanently unstable internal implementation details of this crate. This
 /// should not be used.
 ///