quote_expr macro: embed Ident using special encoding that preserves hygiene.

This adds support to `quote_expr!` and friends for round-trip hygienic
preservation of Ident.

Here are the pieces of the puzzle:

* adding a method for encoding Ident for re-reading into token tree.

* Support for reading such encoded Idents in the lexer.  Note that one
  must peek ahead for MOD_SEP after scan_embedded_hygienic_ident.

* To ensure that encoded Idents are only read when we are in the midst
  of expanding a `quote_expr` or similar, added a
  `read_embedded_ident` flag on `StringReader`.

* pprust support for encoding Ident's as (uint,uint) pairs (for hygiene).

1 files changed, 96 insertions, 3 deletions

diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index cc07b531258..d7d6c20b475 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -97,6 +97,15 @@ pub mod rt {
         fn to_source(&self) -> String;
     }
 
+    // FIXME (Issue #16472): This should go away after ToToken impls
+    // are revised to go directly to token-trees.
+    trait ToSourceWithHygiene : ToSource {
+        // Takes a thing and generates a string containing rust code
+        // for it, encoding Idents as special byte sequences to
+        // maintain hygiene across serialization and deserialization.
+        fn to_source_with_hygiene(&self) -> String;
+    }
+
     macro_rules! impl_to_source(
         (Gc<$t:ty>, $pp:ident) => (
             impl ToSource for Gc<$t> {
@@ -104,6 +113,11 @@ pub mod rt {
                     pprust::$pp(&**self)
                 }
             }
+            impl ToSourceWithHygiene for Gc<$t> {
+                fn to_source_with_hygiene(&self) -> String {
+                    pprust::with_hygiene::$pp(&**self)
+                }
+            }
         );
         ($t:ty, $pp:ident) => (
             impl ToSource for $t {
@@ -111,6 +125,11 @@ pub mod rt {
                     pprust::$pp(self)
                 }
             }
+            impl ToSourceWithHygiene for $t {
+                fn to_source_with_hygiene(&self) -> String {
+                    pprust::with_hygiene::$pp(self)
+                }
+            }
         );
     )
 
@@ -122,6 +141,15 @@ pub mod rt {
             .to_string()
     }
 
+    fn slice_to_source_with_hygiene<'a, T: ToSourceWithHygiene>(
+        sep: &'static str, xs: &'a [T]) -> String {
+        xs.iter()
+            .map(|i| i.to_source_with_hygiene())
+            .collect::<Vec<String>>()
+            .connect(sep)
+            .to_string()
+    }
+
     macro_rules! impl_to_source_slice(
         ($t:ty, $sep:expr) => (
             impl<'a> ToSource for &'a [$t] {
@@ -129,6 +157,12 @@ pub mod rt {
                     slice_to_source($sep, *self)
                 }
             }
+
+            impl<'a> ToSourceWithHygiene for &'a [$t] {
+                fn to_source_with_hygiene(&self) -> String {
+                    slice_to_source_with_hygiene($sep, *self)
+                }
+            }
         )
     )
 
@@ -138,6 +172,12 @@ pub mod rt {
         }
     }
 
+    impl ToSourceWithHygiene for ast::Ident {
+        fn to_source_with_hygiene(&self) -> String {
+            self.encode_with_hygiene()
+        }
+    }
+
     impl_to_source!(ast::Ty, ty_to_string)
     impl_to_source!(ast::Block, block_to_string)
     impl_to_source!(ast::Arg, arg_to_string)
@@ -156,6 +196,11 @@ pub mod rt {
             pprust::attribute_to_string(&dummy_spanned(*self))
         }
     }
+    impl ToSourceWithHygiene for ast::Attribute_ {
+        fn to_source_with_hygiene(&self) -> String {
+            self.to_source()
+        }
+    }
 
     impl<'a> ToSource for &'a str {
         fn to_source(&self) -> String {
@@ -164,12 +209,22 @@ pub mod rt {
             pprust::lit_to_string(&lit)
         }
     }
+    impl<'a> ToSourceWithHygiene for &'a str {
+        fn to_source_with_hygiene(&self) -> String {
+            self.to_source()
+        }
+    }
 
     impl ToSource for () {
         fn to_source(&self) -> String {
             "()".to_string()
         }
     }
+    impl ToSourceWithHygiene for () {
+        fn to_source_with_hygiene(&self) -> String {
+            self.to_source()
+        }
+    }
 
     impl ToSource for bool {
         fn to_source(&self) -> String {
@@ -177,6 +232,11 @@ pub mod rt {
             pprust::lit_to_string(&lit)
         }
     }
+    impl ToSourceWithHygiene for bool {
+        fn to_source_with_hygiene(&self) -> String {
+            self.to_source()
+        }
+    }
 
     impl ToSource for char {
         fn to_source(&self) -> String {
@@ -184,6 +244,11 @@ pub mod rt {
             pprust::lit_to_string(&lit)
         }
     }
+    impl ToSourceWithHygiene for char {
+        fn to_source_with_hygiene(&self) -> String {
+            self.to_source()
+        }
+    }
 
     macro_rules! impl_to_source_int(
         (signed, $t:ty, $tag:ident) => (
@@ -194,6 +259,11 @@ pub mod rt {
                     pprust::lit_to_string(&dummy_spanned(lit))
                 }
             }
+            impl ToSourceWithHygiene for $t {
+                fn to_source_with_hygiene(&self) -> String {
+                    self.to_source()
+                }
+            }
         );
         (unsigned, $t:ty, $tag:ident) => (
             impl ToSource for $t {
@@ -202,6 +272,11 @@ pub mod rt {
                     pprust::lit_to_string(&dummy_spanned(lit))
                 }
             }
+            impl ToSourceWithHygiene for $t {
+                fn to_source_with_hygiene(&self) -> String {
+                    self.to_source()
+                }
+            }
         );
     )
 
@@ -223,7 +298,7 @@ pub mod rt {
         ($t:ty) => (
             impl ToTokens for $t {
                 fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
-                    cx.parse_tts(self.to_source())
+                    cx.parse_tts_with_hygiene(self.to_source_with_hygiene())
                 }
             }
         )
@@ -233,7 +308,7 @@ pub mod rt {
         ($t:ty) => (
             impl<'a> ToTokens for $t {
                 fn to_tokens(&self, cx: &ExtCtxt) -> Vec<TokenTree> {
-                    cx.parse_tts(self.to_source())
+                    cx.parse_tts_with_hygiene(self.to_source_with_hygiene())
                 }
             }
         )
@@ -272,7 +347,13 @@ pub mod rt {
         fn parse_item(&self, s: String) -> Gc<ast::Item>;
         fn parse_expr(&self, s: String) -> Gc<ast::Expr>;
         fn parse_stmt(&self, s: String) -> Gc<ast::Stmt>;
-        fn parse_tts(&self, s: String) -> Vec<ast::TokenTree> ;
+        fn parse_tts(&self, s: String) -> Vec<ast::TokenTree>;
+    }
+
+    trait ExtParseUtilsWithHygiene {
+        // FIXME (Issue #16472): This should go away after ToToken impls
+        // are revised to go directly to token-trees.
+        fn parse_tts_with_hygiene(&self, s: String) -> Vec<ast::TokenTree>;
     }
 
     impl<'a> ExtParseUtils for ExtCtxt<'a> {
@@ -315,6 +396,18 @@ pub mod rt {
         }
     }
 
+    impl<'a> ExtParseUtilsWithHygiene for ExtCtxt<'a> {
+
+        fn parse_tts_with_hygiene(&self, s: String) -> Vec<ast::TokenTree> {
+            use parse::with_hygiene::parse_tts_from_source_str;
+            parse_tts_from_source_str("<quote expansion>".to_string(),
+                                      s,
+                                      self.cfg(),
+                                      self.parse_sess())
+        }
+
+    }
+
 }
 
 pub fn expand_quote_tokens(cx: &mut ExtCtxt,