lexer: lex WS/COMMENT/SHEBANG rather than skipping

Now, the lexer will categorize every byte in its input according to the
grammar. The parser skips over these while parsing, thus avoiding their
presence in the input to syntax extensions.

1 files changed, 15 insertions, 1 deletions

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 83d373d033b..e65f9f208a3 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -97,8 +97,18 @@ pub enum Token {
 
     /* For interpolation */
     INTERPOLATED(Nonterminal),
-
     DOC_COMMENT(Ident),
+
+    // Junk. These carry no data because we don't really care about the data
+    // they *would* carry, and don't really want to allocate a new ident for
+    // them. Instead, users could extract that from the associated span.
+
+    /// Whitespace
+    WS,
+    /// Comment
+    COMMENT,
+    SHEBANG(Ident),
+
     EOF,
 }
 
@@ -231,6 +241,10 @@ pub fn to_string(t: &Token) -> String {
       /* Other */
       DOC_COMMENT(s) => get_ident(s).get().to_string(),
       EOF => "<eof>".to_string(),
+      WS => " ".to_string(),
+      COMMENT => "/* */".to_string(),
+      SHEBANG(s) => format!("/* shebang: {}*/", s.as_str()),
+
       INTERPOLATED(ref nt) => {
         match nt {
             &NtExpr(ref e) => ::print::pprust::expr_to_string(&**e),