auto merge of #15339 : cmr/rust/rewrite-lexer2, r=huonw

Mostly minor things that rebasing is becoming painful.

1 files changed, 93 insertions, 103 deletions

diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 55db3482a61..5839df67022 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -10,7 +10,6 @@
 
 use ast;
 use ast::{P, Ident, Name, Mrk};
-use ast_util;
 use ext::mtwt;
 use parse::token;
 use util::interner::{RcStr, StrInterner};
@@ -79,30 +78,37 @@ pub enum Token {
     QUESTION,
 
     /* Literals */
-    LIT_BYTE(u8),
-    LIT_CHAR(char),
-    LIT_INT(i64, ast::IntTy),
-    LIT_UINT(u64, ast::UintTy),
-    LIT_INT_UNSUFFIXED(i64),
-    LIT_FLOAT(ast::Ident, ast::FloatTy),
-    LIT_FLOAT_UNSUFFIXED(ast::Ident),
-    LIT_STR(ast::Ident),
-    LIT_STR_RAW(ast::Ident, uint), /* raw str delimited by n hash symbols */
-    LIT_BINARY(Rc<Vec<u8>>),
-    LIT_BINARY_RAW(Rc<Vec<u8>>, uint), /* raw binary str delimited by n hash symbols */
+    LIT_BYTE(Name),
+    LIT_CHAR(Name),
+    LIT_INTEGER(Name),
+    LIT_FLOAT(Name),
+    LIT_STR(Name),
+    LIT_STR_RAW(Name, uint), /* raw str delimited by n hash symbols */
+    LIT_BINARY(Name),
+    LIT_BINARY_RAW(Name, uint), /* raw binary str delimited by n hash symbols */
 
     /* Name components */
-    // an identifier contains an "is_mod_name" boolean,
-    // indicating whether :: follows this token with no
-    // whitespace in between.
-    IDENT(ast::Ident, bool),
+    /// An identifier contains an "is_mod_name" boolean,
+    /// indicating whether :: follows this token with no
+    /// whitespace in between.
+    IDENT(Ident, bool),
     UNDERSCORE,
-    LIFETIME(ast::Ident),
+    LIFETIME(Ident),
 
     /* For interpolation */
     INTERPOLATED(Nonterminal),
+    DOC_COMMENT(Name),
+
+    // Junk. These carry no data because we don't really care about the data
+    // they *would* carry, and don't really want to allocate a new ident for
+    // them. Instead, users could extract that from the associated span.
+
+    /// Whitespace
+    WS,
+    /// Comment
+    COMMENT,
+    SHEBANG(Name),
 
-    DOC_COMMENT(ast::Ident),
     EOF,
 }
 
@@ -115,11 +121,12 @@ pub enum Nonterminal {
     NtPat( Gc<ast::Pat>),
     NtExpr(Gc<ast::Expr>),
     NtTy(  P<ast::Ty>),
-    // see IDENT, above, for meaning of bool in NtIdent:
-    NtIdent(Box<ast::Ident>, bool),
-    NtMeta(Gc<ast::MetaItem>), // stuff inside brackets for attributes
+    /// See IDENT, above, for meaning of bool in NtIdent:
+    NtIdent(Box<Ident>, bool),
+    /// Stuff inside brackets for attributes
+    NtMeta(Gc<ast::MetaItem>),
     NtPath(Box<ast::Path>),
-    NtTT(  Gc<ast::TokenTree>), // needs @ed to break a circularity
+    NtTT(  Gc<ast::TokenTree>), // needs Gc'd to break a circularity
     NtMatchers(Vec<ast::Matcher> )
 }
 
@@ -200,54 +207,28 @@ pub fn to_string(t: &Token) -> String {
 
       /* Literals */
       LIT_BYTE(b) => {
-          let mut res = String::from_str("b'");
-          (b as char).escape_default(|c| {
-              res.push_char(c);
-          });
-          res.push_char('\'');
-          res
+          format!("b'{}'", b.as_str())
       }
       LIT_CHAR(c) => {
-          let mut res = String::from_str("'");
-          c.escape_default(|c| {
-              res.push_char(c);
-          });
-          res.push_char('\'');
-          res
-      }
-      LIT_INT(i, t) => ast_util::int_ty_to_string(t, Some(i)),
-      LIT_UINT(u, t) => ast_util::uint_ty_to_string(t, Some(u)),
-      LIT_INT_UNSUFFIXED(i) => { (i as u64).to_string() }
-      LIT_FLOAT(s, t) => {
-        let mut body = String::from_str(get_ident(s).get());
-        if body.as_slice().ends_with(".") {
-            body.push_char('0');  // `10.f` is not a float literal
-        }
-        body.push_str(ast_util::float_ty_to_string(t).as_slice());
-        body
+          format!("'{}'", c.as_str())
       }
-      LIT_FLOAT_UNSUFFIXED(s) => {
-        let mut body = String::from_str(get_ident(s).get());
-        if body.as_slice().ends_with(".") {
-            body.push_char('0');  // `10.f` is not a float literal
-        }
-        body
+      LIT_INTEGER(c) | LIT_FLOAT(c) => {
+          c.as_str().to_string()
       }
+
       LIT_STR(s) => {
-          format!("\"{}\"", get_ident(s).get().escape_default())
+          format!("\"{}\"", s.as_str())
       }
       LIT_STR_RAW(s, n) => {
         format!("r{delim}\"{string}\"{delim}",
-                 delim="#".repeat(n), string=get_ident(s))
+                 delim="#".repeat(n), string=s.as_str())
       }
-      LIT_BINARY(ref v) => {
-          format!(
-            "b\"{}\"",
-            v.iter().map(|&b| b as char).collect::<String>().escape_default())
+      LIT_BINARY(v) => {
+          format!("b\"{}\"", v.as_str())
       }
-      LIT_BINARY_RAW(ref s, n) => {
+      LIT_BINARY_RAW(s, n) => {
         format!("br{delim}\"{string}\"{delim}",
-                 delim="#".repeat(n), string=s.as_slice().to_ascii().as_str_ascii())
+                 delim="#".repeat(n), string=s.as_str())
       }
 
       /* Name components */
@@ -258,8 +239,12 @@ pub fn to_string(t: &Token) -> String {
       UNDERSCORE => "_".to_string(),
 
       /* Other */
-      DOC_COMMENT(s) => get_ident(s).get().to_string(),
+      DOC_COMMENT(s) => s.as_str().to_string(),
       EOF => "<eof>".to_string(),
+      WS => " ".to_string(),
+      COMMENT => "/* */".to_string(),
+      SHEBANG(s) => format!("/* shebang: {}*/", s.as_str()),
+
       INTERPOLATED(ref nt) => {
         match nt {
             &NtExpr(ref e) => ::print::pprust::expr_to_string(&**e),
@@ -296,11 +281,8 @@ pub fn can_begin_expr(t: &Token) -> bool {
       TILDE => true,
       LIT_BYTE(_) => true,
       LIT_CHAR(_) => true,
-      LIT_INT(_, _) => true,
-      LIT_UINT(_, _) => true,
-      LIT_INT_UNSUFFIXED(_) => true,
-      LIT_FLOAT(_, _) => true,
-      LIT_FLOAT_UNSUFFIXED(_) => true,
+      LIT_INTEGER(_) => true,
+      LIT_FLOAT(_) => true,
       LIT_STR(_) => true,
       LIT_STR_RAW(_, _) => true,
       LIT_BINARY(_) => true,
@@ -337,11 +319,8 @@ pub fn is_lit(t: &Token) -> bool {
     match *t {
       LIT_BYTE(_) => true,
       LIT_CHAR(_) => true,
-      LIT_INT(_, _) => true,
-      LIT_UINT(_, _) => true,
-      LIT_INT_UNSUFFIXED(_) => true,
-      LIT_FLOAT(_, _) => true,
-      LIT_FLOAT_UNSUFFIXED(_) => true,
+      LIT_INTEGER(_) => true,
+      LIT_FLOAT(_) => true,
       LIT_STR(_) => true,
       LIT_STR_RAW(_, _) => true,
       LIT_BINARY(_) => true,
@@ -395,19 +374,19 @@ macro_rules! declare_special_idents_and_keywords {(
         $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )*
     }
 ) => {
-    static STRICT_KEYWORD_START: Name = first!($( $sk_name, )*);
-    static STRICT_KEYWORD_FINAL: Name = last!($( $sk_name, )*);
-    static RESERVED_KEYWORD_START: Name = first!($( $rk_name, )*);
-    static RESERVED_KEYWORD_FINAL: Name = last!($( $rk_name, )*);
+    static STRICT_KEYWORD_START: Name = first!($( Name($sk_name), )*);
+    static STRICT_KEYWORD_FINAL: Name = last!($( Name($sk_name), )*);
+    static RESERVED_KEYWORD_START: Name = first!($( Name($rk_name), )*);
+    static RESERVED_KEYWORD_FINAL: Name = last!($( Name($rk_name), )*);
 
     pub mod special_idents {
-        use ast::Ident;
-        $( pub static $si_static: Ident = Ident { name: $si_name, ctxt: 0 }; )*
+        use ast::{Ident, Name};
+        $( pub static $si_static: Ident = Ident { name: Name($si_name), ctxt: 0 }; )*
     }
 
     pub mod special_names {
         use ast::Name;
-        $( pub static $si_static: Name =  $si_name; )*
+        $( pub static $si_static: Name =  Name($si_name); )*
     }
 
     /**
@@ -428,8 +407,8 @@ macro_rules! declare_special_idents_and_keywords {(
         impl Keyword {
             pub fn to_name(&self) -> Name {
                 match *self {
-                    $( $sk_variant => $sk_name, )*
-                    $( $rk_variant => $rk_name, )*
+                    $( $sk_variant => Name($sk_name), )*
+                    $( $rk_variant => Name($rk_name), )*
                 }
             }
         }
@@ -448,8 +427,11 @@ macro_rules! declare_special_idents_and_keywords {(
 }}
 
 // If the special idents get renumbered, remember to modify these two as appropriate
-pub static SELF_KEYWORD_NAME: Name = 1;
-static STATIC_KEYWORD_NAME: Name = 2;
+pub static SELF_KEYWORD_NAME: Name = Name(SELF_KEYWORD_NAME_NUM);
+static STATIC_KEYWORD_NAME: Name = Name(STATIC_KEYWORD_NAME_NUM);
+
+pub static SELF_KEYWORD_NAME_NUM: u32 = 1;
+static STATIC_KEYWORD_NAME_NUM: u32 = 2;
 
 // NB: leaving holes in the ident table is bad! a different ident will get
 // interned with the id from the hole, but it will be between the min and max
@@ -459,8 +441,8 @@ declare_special_idents_and_keywords! {
     pub mod special_idents {
         // These ones are statics
         (0,                          invalid,                "");
-        (super::SELF_KEYWORD_NAME,   self_,                  "self");
-        (super::STATIC_KEYWORD_NAME, statik,                 "static");
+        (super::SELF_KEYWORD_NAME_NUM,   self_,                  "self");
+        (super::STATIC_KEYWORD_NAME_NUM, statik,                 "static");
         (3,                          static_lifetime,        "'static");
 
         // for matcher NTs
@@ -500,8 +482,8 @@ declare_special_idents_and_keywords! {
         (29,                         Ref,        "ref");
         (30,                         Return,     "return");
         // Static and Self are also special idents (prefill de-dupes)
-        (super::STATIC_KEYWORD_NAME, Static,     "static");
-        (super::SELF_KEYWORD_NAME,   Self,       "self");
+        (super::STATIC_KEYWORD_NAME_NUM, Static,     "static");
+        (super::SELF_KEYWORD_NAME_NUM,   Self,       "self");
         (31,                         Struct,     "struct");
         (32,                         Super,      "super");
         (33,                         True,       "true");
@@ -683,20 +665,20 @@ pub fn gensym(s: &str) -> Name {
 
 /// Maps a string to an identifier with an empty syntax context.
 #[inline]
-pub fn str_to_ident(s: &str) -> ast::Ident {
-    ast::Ident::new(intern(s))
+pub fn str_to_ident(s: &str) -> Ident {
+    Ident::new(intern(s))
 }
 
 /// Maps a string to a gensym'ed identifier.
 #[inline]
-pub fn gensym_ident(s: &str) -> ast::Ident {
-    ast::Ident::new(gensym(s))
+pub fn gensym_ident(s: &str) -> Ident {
+    Ident::new(gensym(s))
 }
 
 // create a fresh name that maps to the same string as the old one.
 // note that this guarantees that str_ptr_eq(ident_to_string(src),interner_get(fresh_name(src)));
 // that is, that the new name and the old one are connected to ptr_eq strings.
-pub fn fresh_name(src: &ast::Ident) -> Name {
+pub fn fresh_name(src: &Ident) -> Name {
     let interner = get_ident_interner();
     interner.gensym_copy(src.name)
     // following: debug version. Could work in final except that it's incompatible with
@@ -708,7 +690,7 @@ pub fn fresh_name(src: &ast::Ident) -> Name {
 
 // create a fresh mark.
 pub fn fresh_mark() -> Mrk {
-    gensym("mark")
+    gensym("mark").uint() as u32
 }
 
 // See the macro above about the types of keywords
@@ -722,10 +704,13 @@ pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool {
 
 pub fn is_any_keyword(tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => match sid.name {
-            SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME |
-            STRICT_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true,
-            _ => false,
+        token::IDENT(sid, false) => {
+            let n = sid.name;
+
+               n == SELF_KEYWORD_NAME
+            || n == STATIC_KEYWORD_NAME
+            || STRICT_KEYWORD_START <= n
+            && n <= RESERVED_KEYWORD_FINAL
         },
         _ => false
     }
@@ -733,10 +718,13 @@ pub fn is_any_keyword(tok: &Token) -> bool {
 
 pub fn is_strict_keyword(tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => match sid.name {
-            SELF_KEYWORD_NAME | STATIC_KEYWORD_NAME |
-            STRICT_KEYWORD_START .. STRICT_KEYWORD_FINAL => true,
-            _ => false,
+        token::IDENT(sid, false) => {
+            let n = sid.name;
+
+               n == SELF_KEYWORD_NAME
+            || n == STATIC_KEYWORD_NAME
+            || STRICT_KEYWORD_START <= n
+            && n <= STRICT_KEYWORD_FINAL
         },
         _ => false,
     }
@@ -744,9 +732,11 @@ pub fn is_strict_keyword(tok: &Token) -> bool {
 
 pub fn is_reserved_keyword(tok: &Token) -> bool {
     match *tok {
-        token::IDENT(sid, false) => match sid.name {
-            RESERVED_KEYWORD_START .. RESERVED_KEYWORD_FINAL => true,
-            _ => false,
+        token::IDENT(sid, false) => {
+            let n = sid.name;
+
+               RESERVED_KEYWORD_START <= n
+            && n <= RESERVED_KEYWORD_FINAL
         },
         _ => false,
     }
@@ -768,7 +758,7 @@ mod test {
     use ext::mtwt;
 
     fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
-        ast::Ident{name:id.name,ctxt:mtwt::apply_mark(m,id.ctxt)}
+        ast::Ident { name: id.name, ctxt:mtwt::apply_mark(m, id.ctxt) }
     }
 
     #[test] fn mtwt_token_eq_test() {