auto merge of #18365 : bjz/rust/token, r=alexcrichton

[breaking-change]

(for syntax-extensions)

- Token variant identifiers have been converted to PascalCase for consistency with Rust coding standards
- Some free-functions in `syntax::token` have been converted to methods on `syntax::token::Token`:
    - `can_begin_expr`         -> `Token::can_begin_expr`
    - `close_delimiter_for`    -> `Token::get_close_delimiter`
    - `is_lit`                 -> `Token::is_lit`
    - `is_ident`               -> `Token::is_ident`
    - `is_path`                -> `Token::is_path`
    - `is_plain_ident`         -> `Token::is_plain_ident`
    - `is_lifetime`            -> `Token::is_lifetime`
    - `is_mutability`          -> `Token::is_mutability`
    - `to_binop`               -> `Token::to_binop`
    - `is_keyword`             -> `Token::is_keyword`
    - `is_any_keyword`         -> `Token:is_any_keyword`
    - `is_strict_keyword`      -> `Token::is_strict_keyword`
    - `is_reserved_keyword`    -> `Token::is_reserved_keyword`
    - `mtwt_token_eq`          -> `Token::mtwt_eq`
- `token::Ident` now takes an enum instead of a boolean for clarity
- `token::{to_string, binop_to_string}` were moved to `pprust::{token_to_string, binop_to_string}`

3 files changed, 26 insertions, 24 deletions

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index cea8cab5265..073bebcb3f6 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -85,8 +85,9 @@ use parse::lexer::*; //resolve bug?
 use parse::ParseSess;
 use parse::attr::ParserAttr;
 use parse::parser::{LifetimeAndTypesWithoutColons, Parser};
-use parse::token::{Token, EOF, Nonterminal};
+use parse::token::{Token, Nonterminal};
 use parse::token;
+use print::pprust;
 use ptr::P;
 
 use std::rc::Rc;
@@ -226,8 +227,8 @@ pub fn parse_or_else(sess: &ParseSess,
 /// unhygienic comparison)
 pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
     match (t1,t2) {
-        (&token::IDENT(id1,_),&token::IDENT(id2,_))
-        | (&token::LIFETIME(id1),&token::LIFETIME(id2)) =>
+        (&token::Ident(id1,_),&token::Ident(id2,_))
+        | (&token::Lifetime(id1),&token::Lifetime(id2)) =>
             id1.name == id2.name,
         _ => *t1 == *t2
     }
@@ -354,9 +355,9 @@ pub fn parse(sess: &ParseSess,
                     // Built-in nonterminals never start with these tokens,
                     // so we can eliminate them from consideration.
                     match tok {
-                        token::RPAREN |
-                        token::RBRACE |
-                        token::RBRACKET => {},
+                        token::RParen |
+                        token::RBrace |
+                        token::RBracket => {},
                         _ => bb_eis.push(ei)
                     }
                   }
@@ -372,7 +373,7 @@ pub fn parse(sess: &ParseSess,
         }
 
         /* error messages here could be improved with links to orig. rules */
-        if token_name_eq(&tok, &EOF) {
+        if token_name_eq(&tok, &token::Eof) {
             if eof_eis.len() == 1u {
                 let mut v = Vec::new();
                 for dv in eof_eis.get_mut(0).matches.iter_mut() {
@@ -402,7 +403,7 @@ pub fn parse(sess: &ParseSess,
                     nts, next_eis.len()).to_string());
             } else if bb_eis.len() == 0u && next_eis.len() == 0u {
                 return Failure(sp, format!("no rules expected the token `{}`",
-                            token::to_string(&tok)).to_string());
+                            pprust::token_to_string(&tok)).to_string());
             } else if next_eis.len() > 0u {
                 /* Now process the next token */
                 while next_eis.len() > 0u {
@@ -447,9 +448,9 @@ pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal {
       "ty" => token::NtTy(p.parse_ty(false /* no need to disambiguate*/)),
       // this could be handled like a token, since it is one
       "ident" => match p.token {
-        token::IDENT(sn,b) => { p.bump(); token::NtIdent(box sn,b) }
+        token::Ident(sn,b) => { p.bump(); token::NtIdent(box sn,b) }
         _ => {
-            let token_str = token::to_string(&p.token);
+            let token_str = pprust::token_to_string(&p.token);
             p.fatal((format!("expected ident, found {}",
                              token_str.as_slice())).as_slice())
         }
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 3b51fb380b8..20428e50c7f 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -20,7 +20,7 @@ use parse::lexer::new_tt_reader;
 use parse::parser::Parser;
 use parse::attr::ParserAttr;
 use parse::token::{special_idents, gensym_ident};
-use parse::token::{FAT_ARROW, SEMI, NtMatchers, NtTT, EOF};
+use parse::token::{NtMatchers, NtTT};
 use parse::token;
 use print;
 use ptr::P;
@@ -43,10 +43,10 @@ impl<'a> ParserAnyMacro<'a> {
     /// allowed to be there.
     fn ensure_complete_parse(&self, allow_semi: bool) {
         let mut parser = self.parser.borrow_mut();
-        if allow_semi && parser.token == SEMI {
+        if allow_semi && parser.token == token::Semi {
             parser.bump()
         }
-        if parser.token != EOF {
+        if parser.token != token::Eof {
             let token_str = parser.this_token_to_string();
             let msg = format!("macro expansion ignores token `{}` and any \
                                following",
@@ -89,7 +89,7 @@ impl<'a> MacResult for ParserAnyMacro<'a> {
         loop {
             let mut parser = self.parser.borrow_mut();
             match parser.token {
-                EOF => break,
+                token::Eof => break,
                 _ => {
                     let attrs = parser.parse_outer_attributes();
                     ret.push(parser.parse_method(attrs, ast::Inherited))
@@ -231,12 +231,13 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
     let argument_gram = vec!(
         ms(MatchSeq(vec!(
             ms(MatchNonterminal(lhs_nm, special_idents::matchers, 0u)),
-            ms(MatchTok(FAT_ARROW)),
-            ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), Some(SEMI),
-                                ast::OneOrMore, 0u, 2u)),
+            ms(MatchTok(token::FatArrow)),
+            ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))),
+                                Some(token::Semi), ast::OneOrMore, 0u, 2u)),
         //to phase into semicolon-termination instead of
         //semicolon-separation
-        ms(MatchSeq(vec!(ms(MatchTok(SEMI))), None, ast::ZeroOrMore, 2u, 2u)));
+        ms(MatchSeq(vec!(ms(MatchTok(token::Semi))), None,
+                            ast::ZeroOrMore, 2u, 2u)));
 
 
     // Parse the macro_rules! invocation (`none` is for no interpolations):
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 1bb519f66cd..2c7b583d460 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -13,7 +13,7 @@ use ast::{TokenTree, TtDelimited, TtToken, TtSequence, TtNonterminal, Ident};
 use codemap::{Span, DUMMY_SP};
 use diagnostic::SpanHandler;
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
-use parse::token::{EOF, INTERPOLATED, IDENT, Token, NtIdent};
+use parse::token::{Token, NtIdent};
 use parse::token;
 use parse::lexer::TokenAndSpan;
 
@@ -66,7 +66,7 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler,
         repeat_idx: Vec::new(),
         repeat_len: Vec::new(),
         /* dummy values, never read: */
-        cur_tok: EOF,
+        cur_tok: token::Eof,
         cur_span: DUMMY_SP,
     };
     tt_next_token(&mut r); /* get cur_tok and cur_span set up */
@@ -158,7 +158,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
     loop {
         let should_pop = match r.stack.last() {
             None => {
-                assert_eq!(ret_val.tok, EOF);
+                assert_eq!(ret_val.tok, token::Eof);
                 return ret_val;
             }
             Some(frame) => {
@@ -175,7 +175,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
             let prev = r.stack.pop().unwrap();
             match r.stack.last_mut() {
                 None => {
-                    r.cur_tok = EOF;
+                    r.cur_tok = token::Eof;
                     return ret_val;
                 }
                 Some(frame) => {
@@ -272,13 +272,13 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
                        (b) we actually can, since it's a token. */
                     MatchedNonterminal(NtIdent(box sn, b)) => {
                         r.cur_span = sp;
-                        r.cur_tok = IDENT(sn,b);
+                        r.cur_tok = token::Ident(sn,b);
                         return ret_val;
                     }
                     MatchedNonterminal(ref other_whole_nt) => {
                         // FIXME(pcwalton): Bad copy.
                         r.cur_span = sp;
-                        r.cur_tok = INTERPOLATED((*other_whole_nt).clone());
+                        r.cur_tok = token::Interpolated((*other_whole_nt).clone());
                         return ret_val;
                     }
                     MatchedSeq(..) => {