Interpolate AST nodes in quasiquote.

This changes the `ToTokens` implementations for expressions, statements,
etc. with almost-trivial ones that produce `Interpolated(*Nt(...))`
pseudo-tokens. In this way, quasiquote now works the same way as macros
do: already-parsed AST fragments are used as-is, not reparsed.

The `ToSource` trait is removed. Quasiquote no longer involves
pretty-printing at all, which removes the need for the
`encode_with_hygiene` hack. All associated machinery is removed.

A new `Nonterminal` is added, NtArm, which the parser now interpolates.
This is just for quasiquote, not macros (although it could be in the
future).

`ToTokens` is no longer implemented for `Arg` (although this could be
added again) and `Generics` (which I don't think makes sense).

This breaks any compiler extensions that relied on the ability of
`ToTokens` to turn AST fragments back into inspectable token trees. For
this reason, this closes #16987.

As such, this is a [breaking-change].

Fixes #16472.
Fixes #15962.
Fixes #17397.
Fixes #16617.

1 files changed, 128 insertions, 235 deletions

diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 5776fa99740..e100b7705d8 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -30,16 +30,16 @@ pub mod rt {
     use ext::base::ExtCtxt;
     use parse::token;
     use parse;
-    use print::pprust;
     use ptr::P;
+    use std::rc::Rc;
 
-    use ast::{TokenTree, Generics, Expr};
+    use ast::{TokenTree, Expr};
 
     pub use parse::new_parser_from_tts;
-    pub use codemap::{BytePos, Span, dummy_spanned};
+    pub use codemap::{BytePos, Span, dummy_spanned, DUMMY_SP};
 
     pub trait ToTokens {
-        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> ;
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree>;
     }
 
     impl ToTokens for TokenTree {
@@ -70,277 +70,189 @@ pub mod rt {
         }
     }
 
-    /* Should be (when bugs in default methods are fixed):
-
-    trait ToSource : ToTokens {
-        // Takes a thing and generates a string containing rust code for it.
-        pub fn to_source() -> String;
-
-        // If you can make source, you can definitely make tokens.
-        pub fn to_tokens(cx: &ExtCtxt) -> ~[TokenTree] {
-            cx.parse_tts(self.to_source())
+    impl ToTokens for ast::Ident {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(DUMMY_SP, token::Ident(*self, token::Plain))]
         }
     }
 
-    */
-
-    // FIXME: Move this trait to pprust and get rid of *_to_str?
-    pub trait ToSource {
-        // Takes a thing and generates a string containing rust code for it.
-        fn to_source(&self) -> String;
+    impl ToTokens for ast::Path {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtPath(Box::new(self.clone()))))]
+        }
     }
 
-    // FIXME (Issue #16472): This should go away after ToToken impls
-    // are revised to go directly to token-trees.
-    trait ToSourceWithHygiene : ToSource {
-        // Takes a thing and generates a string containing rust code
-        // for it, encoding Idents as special byte sequences to
-        // maintain hygiene across serialization and deserialization.
-        fn to_source_with_hygiene(&self) -> String;
+    impl ToTokens for ast::Ty {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtTy(P(self.clone()))))]
+        }
     }
 
-    macro_rules! impl_to_source {
-        (P<$t:ty>, $pp:ident) => (
-            impl ToSource for P<$t> {
-                fn to_source(&self) -> String {
-                    pprust::$pp(&**self)
-                }
-            }
-            impl ToSourceWithHygiene for P<$t> {
-                fn to_source_with_hygiene(&self) -> String {
-                    pprust::with_hygiene::$pp(&**self)
-                }
-            }
-        );
-        ($t:ty, $pp:ident) => (
-            impl ToSource for $t {
-                fn to_source(&self) -> String {
-                    pprust::$pp(self)
-                }
-            }
-            impl ToSourceWithHygiene for $t {
-                fn to_source_with_hygiene(&self) -> String {
-                    pprust::with_hygiene::$pp(self)
-                }
-            }
-        );
+    impl ToTokens for ast::Block {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtBlock(P(self.clone()))))]
+        }
     }
 
-    fn slice_to_source<'a, T: ToSource>(sep: &'static str, xs: &'a [T]) -> String {
-        xs.iter()
-            .map(|i| i.to_source())
-            .collect::<Vec<String>>()
-            .connect(sep)
-            .to_string()
+    impl ToTokens for P<ast::Item> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtItem(self.clone())))]
+        }
     }
 
-    fn slice_to_source_with_hygiene<'a, T: ToSourceWithHygiene>(
-        sep: &'static str, xs: &'a [T]) -> String {
-        xs.iter()
-            .map(|i| i.to_source_with_hygiene())
-            .collect::<Vec<String>>()
-            .connect(sep)
-            .to_string()
+    impl ToTokens for P<ast::ImplItem> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtImplItem(self.clone())))]
+        }
     }
 
-    macro_rules! impl_to_source_slice {
-        ($t:ty, $sep:expr) => (
-            impl ToSource for [$t] {
-                fn to_source(&self) -> String {
-                    slice_to_source($sep, self)
-                }
-            }
-
-            impl ToSourceWithHygiene for [$t] {
-                fn to_source_with_hygiene(&self) -> String {
-                    slice_to_source_with_hygiene($sep, self)
-                }
-            }
-        )
+    impl ToTokens for P<ast::TraitItem> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtTraitItem(self.clone())))]
+        }
     }
 
-    impl ToSource for ast::Ident {
-        fn to_source(&self) -> String {
-            token::get_ident(*self).to_string()
+    impl ToTokens for P<ast::Stmt> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtStmt(self.clone())))]
         }
     }
 
-    impl ToSourceWithHygiene for ast::Ident {
-        fn to_source_with_hygiene(&self) -> String {
-            self.encode_with_hygiene()
+    impl ToTokens for P<ast::Expr> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtExpr(self.clone())))]
         }
     }
 
-    impl_to_source! { ast::Path, path_to_string }
-    impl_to_source! { ast::Ty, ty_to_string }
-    impl_to_source! { ast::Block, block_to_string }
-    impl_to_source! { ast::Arg, arg_to_string }
-    impl_to_source! { Generics, generics_to_string }
-    impl_to_source! { ast::WhereClause, where_clause_to_string }
-    impl_to_source! { P<ast::Item>, item_to_string }
-    impl_to_source! { P<ast::ImplItem>, impl_item_to_string }
-    impl_to_source! { P<ast::TraitItem>, trait_item_to_string }
-    impl_to_source! { P<ast::Stmt>, stmt_to_string }
-    impl_to_source! { P<ast::Expr>, expr_to_string }
-    impl_to_source! { P<ast::Pat>, pat_to_string }
-    impl_to_source! { ast::Arm, arm_to_string }
-    impl_to_source_slice! { ast::Ty, ", " }
-    impl_to_source_slice! { P<ast::Item>, "\n\n" }
-
-    impl ToSource for ast::Attribute_ {
-        fn to_source(&self) -> String {
-            pprust::attribute_to_string(&dummy_spanned(self.clone()))
+    impl ToTokens for P<ast::Pat> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(self.span, token::Interpolated(token::NtPat(self.clone())))]
         }
     }
-    impl ToSourceWithHygiene for ast::Attribute_ {
-        fn to_source_with_hygiene(&self) -> String {
-            self.to_source()
+
+    impl ToTokens for ast::Arm {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtArm(self.clone())))]
         }
     }
 
-    impl ToSource for str {
-        fn to_source(&self) -> String {
-            let lit = dummy_spanned(ast::LitStr(
-                    token::intern_and_get_ident(self), ast::CookedStr));
-            pprust::lit_to_string(&lit)
-        }
+    macro_rules! impl_to_tokens_slice {
+        ($t: ty, $sep: expr) => {
+            impl ToTokens for [$t] {
+                fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
+                    let mut v = vec![];
+                    for (i, x) in self.iter().enumerate() {
+                        if i > 0 {
+                            v.push_all(&$sep);
+                        }
+                        v.extend(x.to_tokens(cx));
+                    }
+                    v
+                }
+            }
+        };
     }
-    impl ToSourceWithHygiene for str {
-        fn to_source_with_hygiene(&self) -> String {
-            self.to_source()
+
+    impl_to_tokens_slice! { ast::Ty, [ast::TtToken(DUMMY_SP, token::Comma)] }
+    impl_to_tokens_slice! { P<ast::Item>, [] }
+
+    impl ToTokens for P<ast::MetaItem> {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtToken(DUMMY_SP, token::Interpolated(token::NtMeta(self.clone())))]
         }
     }
 
-    impl ToSource for () {
-        fn to_source(&self) -> String {
-            "()".to_string()
+    impl ToTokens for ast::Attribute {
+        fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
+            let mut r = vec![];
+            // FIXME: The spans could be better
+            r.push(ast::TtToken(self.span, token::Pound));
+            if self.node.style == ast::AttrInner {
+                r.push(ast::TtToken(self.span, token::Not));
+            }
+            r.push(ast::TtDelimited(self.span, Rc::new(ast::Delimited {
+                delim: token::Bracket,
+                open_span: self.span,
+                tts: self.node.value.to_tokens(cx),
+                close_span: self.span,
+            })));
+            r
         }
     }
-    impl ToSourceWithHygiene for () {
-        fn to_source_with_hygiene(&self) -> String {
-            self.to_source()
+
+    impl ToTokens for str {
+        fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
+            let lit = ast::LitStr(
+                token::intern_and_get_ident(self), ast::CookedStr);
+            dummy_spanned(lit).to_tokens(cx)
         }
     }
 
-    impl ToSource for bool {
-        fn to_source(&self) -> String {
-            let lit = dummy_spanned(ast::LitBool(*self));
-            pprust::lit_to_string(&lit)
+    impl ToTokens for () {
+        fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
+            vec![ast::TtDelimited(DUMMY_SP, Rc::new(ast::Delimited {
+                delim: token::Paren,
+                open_span: DUMMY_SP,
+                tts: vec![],
+                close_span: DUMMY_SP,
+            }))]
         }
     }
-    impl ToSourceWithHygiene for bool {
-        fn to_source_with_hygiene(&self) -> String {
-            self.to_source()
+
+    impl ToTokens for ast::Lit {
+        fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
+            // FIXME: This is wrong
+            P(ast::Expr {
+                id: ast::DUMMY_NODE_ID,
+                node: ast::ExprLit(P(self.clone())),
+                span: DUMMY_SP,
+            }).to_tokens(cx)
         }
     }
 
-    impl ToSource for char {
-        fn to_source(&self) -> String {
-            let lit = dummy_spanned(ast::LitChar(*self));
-            pprust::lit_to_string(&lit)
+    impl ToTokens for bool {
+        fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
+            dummy_spanned(ast::LitBool(*self)).to_tokens(cx)
         }
     }
-    impl ToSourceWithHygiene for char {
-        fn to_source_with_hygiene(&self) -> String {
-            self.to_source()
+
+    impl ToTokens for char {
+        fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
+            dummy_spanned(ast::LitChar(*self)).to_tokens(cx)
         }
     }
 
-    macro_rules! impl_to_source_int {
+    macro_rules! impl_to_tokens_int {
         (signed, $t:ty, $tag:expr) => (
-            impl ToSource for $t {
-                fn to_source(&self) -> String {
+            impl ToTokens for $t {
+                fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
                     let lit = ast::LitInt(*self as u64, ast::SignedIntLit($tag,
                                                                           ast::Sign::new(*self)));
-                    pprust::lit_to_string(&dummy_spanned(lit))
-                }
-            }
-            impl ToSourceWithHygiene for $t {
-                fn to_source_with_hygiene(&self) -> String {
-                    self.to_source()
+                    dummy_spanned(lit).to_tokens(cx)
                 }
             }
         );
         (unsigned, $t:ty, $tag:expr) => (
-            impl ToSource for $t {
-                fn to_source(&self) -> String {
+            impl ToTokens for $t {
+                fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
                     let lit = ast::LitInt(*self as u64, ast::UnsignedIntLit($tag));
-                    pprust::lit_to_string(&dummy_spanned(lit))
-                }
-            }
-            impl ToSourceWithHygiene for $t {
-                fn to_source_with_hygiene(&self) -> String {
-                    self.to_source()
+                    dummy_spanned(lit).to_tokens(cx)
                 }
             }
         );
     }
 
-    impl_to_source_int! { signed, isize, ast::TyIs }
-    impl_to_source_int! { signed, i8,  ast::TyI8 }
-    impl_to_source_int! { signed, i16, ast::TyI16 }
-    impl_to_source_int! { signed, i32, ast::TyI32 }
-    impl_to_source_int! { signed, i64, ast::TyI64 }
-
-    impl_to_source_int! { unsigned, usize, ast::TyUs }
-    impl_to_source_int! { unsigned, u8,   ast::TyU8 }
-    impl_to_source_int! { unsigned, u16,  ast::TyU16 }
-    impl_to_source_int! { unsigned, u32,  ast::TyU32 }
-    impl_to_source_int! { unsigned, u64,  ast::TyU64 }
-
-    // Alas ... we write these out instead. All redundant.
-
-    macro_rules! impl_to_tokens {
-        ($t:ty) => (
-            impl ToTokens for $t {
-                fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
-                    cx.parse_tts_with_hygiene(self.to_source_with_hygiene())
-                }
-            }
-        )
-    }
+    impl_to_tokens_int! { signed, isize, ast::TyIs }
+    impl_to_tokens_int! { signed, i8,  ast::TyI8 }
+    impl_to_tokens_int! { signed, i16, ast::TyI16 }
+    impl_to_tokens_int! { signed, i32, ast::TyI32 }
+    impl_to_tokens_int! { signed, i64, ast::TyI64 }
 
-    macro_rules! impl_to_tokens_lifetime {
-        ($t:ty) => (
-            impl<'a> ToTokens for $t {
-                fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
-                    cx.parse_tts_with_hygiene(self.to_source_with_hygiene())
-                }
-            }
-        )
-    }
-
-    impl_to_tokens! { ast::Ident }
-    impl_to_tokens! { ast::Path }
-    impl_to_tokens! { P<ast::Item> }
-    impl_to_tokens! { P<ast::ImplItem> }
-    impl_to_tokens! { P<ast::TraitItem> }
-    impl_to_tokens! { P<ast::Pat> }
-    impl_to_tokens! { ast::Arm }
-    impl_to_tokens_lifetime! { &'a [P<ast::Item>] }
-    impl_to_tokens! { ast::Ty }
-    impl_to_tokens_lifetime! { &'a [ast::Ty] }
-    impl_to_tokens! { Generics }
-    impl_to_tokens! { ast::WhereClause }
-    impl_to_tokens! { P<ast::Stmt> }
-    impl_to_tokens! { P<ast::Expr> }
-    impl_to_tokens! { ast::Block }
-    impl_to_tokens! { ast::Arg }
-    impl_to_tokens! { ast::Attribute_ }
-    impl_to_tokens_lifetime! { &'a str }
-    impl_to_tokens! { () }
-    impl_to_tokens! { char }
-    impl_to_tokens! { bool }
-    impl_to_tokens! { isize }
-    impl_to_tokens! { i8 }
-    impl_to_tokens! { i16 }
-    impl_to_tokens! { i32 }
-    impl_to_tokens! { i64 }
-    impl_to_tokens! { usize }
-    impl_to_tokens! { u8 }
-    impl_to_tokens! { u16 }
-    impl_to_tokens! { u32 }
-    impl_to_tokens! { u64 }
+    impl_to_tokens_int! { unsigned, usize, ast::TyUs }
+    impl_to_tokens_int! { unsigned, u8,   ast::TyU8 }
+    impl_to_tokens_int! { unsigned, u16,  ast::TyU16 }
+    impl_to_tokens_int! { unsigned, u32,  ast::TyU32 }
+    impl_to_tokens_int! { unsigned, u64,  ast::TyU64 }
 
     pub trait ExtParseUtils {
         fn parse_item(&self, s: String) -> P<ast::Item>;
@@ -349,12 +261,6 @@ pub mod rt {
         fn parse_tts(&self, s: String) -> Vec<ast::TokenTree>;
     }
 
-    trait ExtParseUtilsWithHygiene {
-        // FIXME (Issue #16472): This should go away after ToToken impls
-        // are revised to go directly to token-trees.
-        fn parse_tts_with_hygiene(&self, s: String) -> Vec<ast::TokenTree>;
-    }
-
     impl<'a> ExtParseUtils for ExtCtxt<'a> {
 
         fn parse_item(&self, s: String) -> P<ast::Item> {
@@ -386,19 +292,6 @@ pub mod rt {
                                              self.parse_sess())
         }
     }
-
-    impl<'a> ExtParseUtilsWithHygiene for ExtCtxt<'a> {
-
-        fn parse_tts_with_hygiene(&self, s: String) -> Vec<ast::TokenTree> {
-            use parse::with_hygiene::parse_tts_from_source_str;
-            parse_tts_from_source_str("<quote expansion>".to_string(),
-                                      s,
-                                      self.cfg(),
-                                      self.parse_sess())
-        }
-
-    }
-
 }
 
 pub fn expand_quote_tokens<'cx>(cx: &'cx mut ExtCtxt,