From d8b1fa0ae0c27e54d3539190683c01e194d36fbd Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Mon, 27 Oct 2014 19:22:52 +1100 Subject: Use PascalCase for token variants --- src/libsyntax/ext/asm.rs | 46 ++++++------ src/libsyntax/ext/base.rs | 10 +-- src/libsyntax/ext/cfg.rs | 2 +- src/libsyntax/ext/concat_idents.rs | 10 +-- src/libsyntax/ext/format.rs | 18 ++--- src/libsyntax/ext/quote.rs | 135 +++++++++++++++++------------------ src/libsyntax/ext/tt/macro_parser.rs | 16 ++--- src/libsyntax/ext/tt/macro_rules.rs | 17 ++--- src/libsyntax/ext/tt/transcribe.rs | 12 ++-- 9 files changed, 133 insertions(+), 133 deletions(-) (limited to 'src/libsyntax/ext') diff --git a/src/libsyntax/ext/asm.rs b/src/libsyntax/ext/asm.rs index 702be0c0eee..2b52b7feacc 100644 --- a/src/libsyntax/ext/asm.rs +++ b/src/libsyntax/ext/asm.rs @@ -72,21 +72,21 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]) asm_str_style = Some(style); } Outputs => { - while p.token != token::EOF && - p.token != token::COLON && - p.token != token::MOD_SEP { + while p.token != token::Eof && + p.token != token::Colon && + p.token != token::ModSep { if outputs.len() != 0 { - p.eat(&token::COMMA); + p.eat(&token::Comma); } let (constraint, _str_style) = p.parse_str(); let span = p.last_span; - p.expect(&token::LPAREN); + p.expect(&token::LParen); let out = p.parse_expr(); - p.expect(&token::RPAREN); + p.expect(&token::RParen); // Expands a read+write operand into two operands. // @@ -113,12 +113,12 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]) } } Inputs => { - while p.token != token::EOF && - p.token != token::COLON && - p.token != token::MOD_SEP { + while p.token != token::Eof && + p.token != token::Colon && + p.token != token::ModSep { if inputs.len() != 0 { - p.eat(&token::COMMA); + p.eat(&token::Comma); } let (constraint, _str_style) = p.parse_str(); @@ -129,21 +129,21 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]) cx.span_err(p.last_span, "input operand constraint contains '+'"); } - p.expect(&token::LPAREN); + p.expect(&token::LParen); let input = p.parse_expr(); - p.expect(&token::RPAREN); + p.expect(&token::RParen); inputs.push((constraint, input)); } } Clobbers => { let mut clobs = Vec::new(); - while p.token != token::EOF && - p.token != token::COLON && - p.token != token::MOD_SEP { + while p.token != token::Eof && + p.token != token::Colon && + p.token != token::ModSep { if clobs.len() != 0 { - p.eat(&token::COMMA); + p.eat(&token::Comma); } let (s, _str_style) = p.parse_str(); @@ -172,8 +172,8 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]) cx.span_warn(p.last_span, "unrecognized option"); } - if p.token == token::COMMA { - p.eat(&token::COMMA); + if p.token == token::Comma { + p.eat(&token::Comma); } } StateNone => () @@ -183,17 +183,17 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[ast::TokenTree]) // MOD_SEP is a double colon '::' without space in between. // When encountered, the state must be advanced twice. match (&p.token, state.next(), state.next().next()) { - (&token::COLON, StateNone, _) | - (&token::MOD_SEP, _, StateNone) => { + (&token::Colon, StateNone, _) | + (&token::ModSep, _, StateNone) => { p.bump(); break 'statement; } - (&token::COLON, st, _) | - (&token::MOD_SEP, _, st) => { + (&token::Colon, st, _) | + (&token::ModSep, _, st) => { p.bump(); state = st; } - (&token::EOF, _, _) => break 'statement, + (&token::Eof, _, _) => break 'statement, _ => break } } diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 64c8068607a..a8326e79ef3 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -684,8 +684,8 @@ pub fn get_single_str_from_tts(cx: &ExtCtxt, cx.span_err(sp, format!("{} takes 1 argument.", name).as_slice()); } else { match tts[0] { - ast::TtToken(_, token::LIT_STR(ident)) => return Some(parse::str_lit(ident.as_str())), - ast::TtToken(_, token::LIT_STR_RAW(ident, _)) => { + ast::TtToken(_, token::LitStr(ident)) => return Some(parse::str_lit(ident.as_str())), + ast::TtToken(_, token::LitStrRaw(ident, _)) => { return Some(parse::raw_str_lit(ident.as_str())) } _ => { @@ -704,12 +704,12 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt, tts: &[ast::TokenTree]) -> Option>> { let mut p = cx.new_parser_from_tts(tts); let mut es = Vec::new(); - while p.token != token::EOF { + while p.token != token::Eof { es.push(cx.expander().fold_expr(p.parse_expr())); - if p.eat(&token::COMMA) { + if p.eat(&token::Comma) { continue; } - if p.token != token::EOF { + if p.token != token::Eof { cx.span_err(sp, "expected token: `,`"); return None; } diff --git a/src/libsyntax/ext/cfg.rs b/src/libsyntax/ext/cfg.rs index f697acb417d..72da60ffe09 100644 --- a/src/libsyntax/ext/cfg.rs +++ b/src/libsyntax/ext/cfg.rs @@ -29,7 +29,7 @@ pub fn expand_cfg<'cx>(cx: &mut ExtCtxt, let mut p = cx.new_parser_from_tts(tts); let cfg = p.parse_meta_item(); - if !p.eat(&token::EOF) { + if !p.eat(&token::Eof) { cx.span_err(sp, "expected 1 cfg-pattern"); return DummyResult::expr(sp); } diff --git a/src/libsyntax/ext/concat_idents.rs b/src/libsyntax/ext/concat_idents.rs index e12f9ee133a..e5e93a7d8b3 100644 --- a/src/libsyntax/ext/concat_idents.rs +++ b/src/libsyntax/ext/concat_idents.rs @@ -23,21 +23,21 @@ pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[ast::TokenTree] for (i, e) in tts.iter().enumerate() { if i & 1 == 1 { match *e { - ast::TtToken(_, token::COMMA) => (), + ast::TtToken(_, token::Comma) => {}, _ => { cx.span_err(sp, "concat_idents! expecting comma."); return DummyResult::expr(sp); - } + }, } } else { match *e { - ast::TtToken(_, token::IDENT(ident,_)) => { + ast::TtToken(_, token::Ident(ident, _)) => { res_str.push_str(token::get_ident(ident).get()) - } + }, _ => { cx.span_err(sp, "concat_idents! requires ident args."); return DummyResult::expr(sp); - } + }, } } } diff --git a/src/libsyntax/ext/format.rs b/src/libsyntax/ext/format.rs index 87cd61c9b22..fdf61d4abd9 100644 --- a/src/libsyntax/ext/format.rs +++ b/src/libsyntax/ext/format.rs @@ -91,7 +91,7 @@ fn parse_args(ecx: &mut ExtCtxt, sp: Span, allow_method: bool, // Parse the leading function expression (maybe a block, maybe a path) let invocation = if allow_method { let e = p.parse_expr(); - if !p.eat(&token::COMMA) { + if !p.eat(&token::Comma) { ecx.span_err(sp, "expected token: `,`"); return (Call(e), None); } @@ -99,28 +99,28 @@ fn parse_args(ecx: &mut ExtCtxt, sp: Span, allow_method: bool, } else { Call(p.parse_expr()) }; - if !p.eat(&token::COMMA) { + if !p.eat(&token::Comma) { ecx.span_err(sp, "expected token: `,`"); return (invocation, None); } - if p.token == token::EOF { + if p.token == token::Eof { ecx.span_err(sp, "requires at least a format string argument"); return (invocation, None); } let fmtstr = p.parse_expr(); let mut named = false; - while p.token != token::EOF { - if !p.eat(&token::COMMA) { + while p.token != token::Eof { + if !p.eat(&token::Comma) { ecx.span_err(sp, "expected token: `,`"); return (invocation, None); } - if p.token == token::EOF { break } // accept trailing commas + if p.token == token::Eof { break } // accept trailing commas if named || (token::is_ident(&p.token) && - p.look_ahead(1, |t| *t == token::EQ)) { + p.look_ahead(1, |t| *t == token::Eq)) { named = true; let ident = match p.token { - token::IDENT(i, _) => { + token::Ident(i, _) => { p.bump(); i } @@ -139,7 +139,7 @@ fn parse_args(ecx: &mut ExtCtxt, sp: Span, allow_method: bool, }; let interned_name = token::get_ident(ident); let name = interned_name.get(); - p.expect(&token::EQ); + p.expect(&token::Eq); let e = p.parse_expr(); match names.find_equiv(&name) { None => {} diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 6f1fd90adfa..39a538f917b 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -515,123 +515,122 @@ fn mk_token_path(cx: &ExtCtxt, sp: Span, name: &str) -> P { cx.expr_path(cx.path_global(sp, idents)) } -fn mk_binop(cx: &ExtCtxt, sp: Span, bop: token::BinOp) -> P { +fn mk_binop(cx: &ExtCtxt, sp: Span, bop: token::BinOpToken) -> P { let name = match bop { - PLUS => "PLUS", - MINUS => "MINUS", - STAR => "STAR", - SLASH => "SLASH", - PERCENT => "PERCENT", - CARET => "CARET", - AND => "AND", - OR => "OR", - SHL => "SHL", - SHR => "SHR" + token::Plus => "Plus", + token::Minus => "Minus", + token::Star => "Star", + token::Slash => "Slash", + token::Percent => "Percent", + token::Caret => "Caret", + token::And => "And", + token::Or => "Or", + token::Shl => "Shl", + token::Shr => "Shr" }; mk_token_path(cx, sp, name) } fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { - match *tok { - BINOP(binop) => { - return cx.expr_call(sp, mk_token_path(cx, sp, "BINOP"), vec!(mk_binop(cx, sp, binop))); + token::BinOp(binop) => { + return cx.expr_call(sp, mk_token_path(cx, sp, "BinOp"), vec!(mk_binop(cx, sp, binop))); } - BINOPEQ(binop) => { - return cx.expr_call(sp, mk_token_path(cx, sp, "BINOPEQ"), + token::BinOpEq(binop) => { + return cx.expr_call(sp, mk_token_path(cx, sp, "BinOpEq"), vec!(mk_binop(cx, sp, binop))); } - LIT_BYTE(i) => { + token::LitByte(i) => { let e_byte = mk_name(cx, sp, i.ident()); - return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_BYTE"), vec!(e_byte)); + return cx.expr_call(sp, mk_token_path(cx, sp, "LitByte"), vec!(e_byte)); } - LIT_CHAR(i) => { + token::LitChar(i) => { let e_char = mk_name(cx, sp, i.ident()); - return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_CHAR"), vec!(e_char)); + return cx.expr_call(sp, mk_token_path(cx, sp, "LitChar"), vec!(e_char)); } - LIT_INTEGER(i) => { + token::LitInteger(i) => { let e_int = mk_name(cx, sp, i.ident()); - return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_INTEGER"), vec!(e_int)); + return cx.expr_call(sp, mk_token_path(cx, sp, "LitInteger"), vec!(e_int)); } - LIT_FLOAT(fident) => { + token::LitFloat(fident) => { let e_fident = mk_name(cx, sp, fident.ident()); - return cx.expr_call(sp, mk_token_path(cx, sp, "LIT_FLOAT"), vec!(e_fident)); + return cx.expr_call(sp, mk_token_path(cx, sp, "LitFloat"), vec!(e_fident)); } - LIT_STR(ident) => { + token::LitStr(ident) => { return cx.expr_call(sp, - mk_token_path(cx, sp, "LIT_STR"), + mk_token_path(cx, sp, "LitStr"), vec!(mk_name(cx, sp, ident.ident()))); } - LIT_STR_RAW(ident, n) => { + token::LitStrRaw(ident, n) => { return cx.expr_call(sp, - mk_token_path(cx, sp, "LIT_STR_RAW"), + mk_token_path(cx, sp, "LitStrRaw"), vec!(mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))); } - IDENT(ident, b) => { + token::Ident(ident, b) => { return cx.expr_call(sp, - mk_token_path(cx, sp, "IDENT"), + mk_token_path(cx, sp, "Ident"), vec!(mk_ident(cx, sp, ident), cx.expr_bool(sp, b))); } - LIFETIME(ident) => { + token::Lifetime(ident) => { return cx.expr_call(sp, - mk_token_path(cx, sp, "LIFETIME"), + mk_token_path(cx, sp, "Lifetime"), vec!(mk_ident(cx, sp, ident))); } - DOC_COMMENT(ident) => { + token::DocComment(ident) => { return cx.expr_call(sp, - mk_token_path(cx, sp, "DOC_COMMENT"), + mk_token_path(cx, sp, "DocComment"), vec!(mk_name(cx, sp, ident.ident()))); } - INTERPOLATED(_) => fail!("quote! with interpolated token"), + token::Interpolated(_) => fail!("quote! with interpolated token"), _ => () } let name = match *tok { - EQ => "EQ", - LT => "LT", - LE => "LE", - EQEQ => "EQEQ", - NE => "NE", - GE => "GE", - GT => "GT", - ANDAND => "ANDAND", - OROR => "OROR", - NOT => "NOT", - TILDE => "TILDE", - AT => "AT", - DOT => "DOT", - DOTDOT => "DOTDOT", - COMMA => "COMMA", - SEMI => "SEMI", - COLON => "COLON", - MOD_SEP => "MOD_SEP", - RARROW => "RARROW", - LARROW => "LARROW", - FAT_ARROW => "FAT_ARROW", - LPAREN => "LPAREN", - RPAREN => "RPAREN", - LBRACKET => "LBRACKET", - RBRACKET => "RBRACKET", - LBRACE => "LBRACE", - RBRACE => "RBRACE", - POUND => "POUND", - DOLLAR => "DOLLAR", - UNDERSCORE => "UNDERSCORE", - EOF => "EOF", - _ => fail!() + token::Eq => "Eq", + token::Lt => "Lt", + token::Le => "Le", + token::EqEq => "EqEq", + token::Ne => "Ne", + token::Ge => "Ge", + token::Gt => "Gt", + token::AndAnd => "AndAnd", + token::OrOr => "OrOr", + token::Not => "Not", + token::Tilde => "Tilde", + token::At => "At", + token::Dot => "Dot", + token::DotDot => "DotDot", + token::Comma => "Comma", + token::Semi => "Semi", + token::Colon => "Colon", + token::ModSep => "ModSep", + token::RArrow => "RArrow", + token::LArrow => "LArrow", + token::FatArrow => "FatArrow", + token::LParen => "LParen", + token::RParen => "RParen", + token::LBracket => "LBracket", + token::RBracket => "RBracket", + token::LBrace => "LBrace", + token::RBrace => "RBrace", + token::Pound => "Pound", + token::Dollar => "Dollar", + token::Underscore => "Underscore", + token::Eof => "Eof", + _ => fail!(), }; mk_token_path(cx, sp, name) } @@ -702,7 +701,7 @@ fn expand_tts(cx: &ExtCtxt, sp: Span, tts: &[ast::TokenTree]) p.quote_depth += 1u; let cx_expr = p.parse_expr(); - if !p.eat(&token::COMMA) { + if !p.eat(&token::Comma) { p.fatal("expected token `,`"); } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index cea8cab5265..6d30de96a3c 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -85,7 +85,7 @@ use parse::lexer::*; //resolve bug? use parse::ParseSess; use parse::attr::ParserAttr; use parse::parser::{LifetimeAndTypesWithoutColons, Parser}; -use parse::token::{Token, EOF, Nonterminal}; +use parse::token::{Token, Nonterminal}; use parse::token; use ptr::P; @@ -226,8 +226,8 @@ pub fn parse_or_else(sess: &ParseSess, /// unhygienic comparison) pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { match (t1,t2) { - (&token::IDENT(id1,_),&token::IDENT(id2,_)) - | (&token::LIFETIME(id1),&token::LIFETIME(id2)) => + (&token::Ident(id1,_),&token::Ident(id2,_)) + | (&token::Lifetime(id1),&token::Lifetime(id2)) => id1.name == id2.name, _ => *t1 == *t2 } @@ -354,9 +354,9 @@ pub fn parse(sess: &ParseSess, // Built-in nonterminals never start with these tokens, // so we can eliminate them from consideration. match tok { - token::RPAREN | - token::RBRACE | - token::RBRACKET => {}, + token::RParen | + token::RBrace | + token::RBracket => {}, _ => bb_eis.push(ei) } } @@ -372,7 +372,7 @@ pub fn parse(sess: &ParseSess, } /* error messages here could be improved with links to orig. rules */ - if token_name_eq(&tok, &EOF) { + if token_name_eq(&tok, &token::Eof) { if eof_eis.len() == 1u { let mut v = Vec::new(); for dv in eof_eis.get_mut(0).matches.iter_mut() { @@ -447,7 +447,7 @@ pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal { "ty" => token::NtTy(p.parse_ty(false /* no need to disambiguate*/)), // this could be handled like a token, since it is one "ident" => match p.token { - token::IDENT(sn,b) => { p.bump(); token::NtIdent(box sn,b) } + token::Ident(sn,b) => { p.bump(); token::NtIdent(box sn,b) } _ => { let token_str = token::to_string(&p.token); p.fatal((format!("expected ident, found {}", diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 3b51fb380b8..20428e50c7f 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -20,7 +20,7 @@ use parse::lexer::new_tt_reader; use parse::parser::Parser; use parse::attr::ParserAttr; use parse::token::{special_idents, gensym_ident}; -use parse::token::{FAT_ARROW, SEMI, NtMatchers, NtTT, EOF}; +use parse::token::{NtMatchers, NtTT}; use parse::token; use print; use ptr::P; @@ -43,10 +43,10 @@ impl<'a> ParserAnyMacro<'a> { /// allowed to be there. fn ensure_complete_parse(&self, allow_semi: bool) { let mut parser = self.parser.borrow_mut(); - if allow_semi && parser.token == SEMI { + if allow_semi && parser.token == token::Semi { parser.bump() } - if parser.token != EOF { + if parser.token != token::Eof { let token_str = parser.this_token_to_string(); let msg = format!("macro expansion ignores token `{}` and any \ following", @@ -89,7 +89,7 @@ impl<'a> MacResult for ParserAnyMacro<'a> { loop { let mut parser = self.parser.borrow_mut(); match parser.token { - EOF => break, + token::Eof => break, _ => { let attrs = parser.parse_outer_attributes(); ret.push(parser.parse_method(attrs, ast::Inherited)) @@ -231,12 +231,13 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt, let argument_gram = vec!( ms(MatchSeq(vec!( ms(MatchNonterminal(lhs_nm, special_idents::matchers, 0u)), - ms(MatchTok(FAT_ARROW)), - ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), Some(SEMI), - ast::OneOrMore, 0u, 2u)), + ms(MatchTok(token::FatArrow)), + ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), + Some(token::Semi), ast::OneOrMore, 0u, 2u)), //to phase into semicolon-termination instead of //semicolon-separation - ms(MatchSeq(vec!(ms(MatchTok(SEMI))), None, ast::ZeroOrMore, 2u, 2u))); + ms(MatchSeq(vec!(ms(MatchTok(token::Semi))), None, + ast::ZeroOrMore, 2u, 2u))); // Parse the macro_rules! invocation (`none` is for no interpolations): diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 1bb519f66cd..2c7b583d460 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -13,7 +13,7 @@ use ast::{TokenTree, TtDelimited, TtToken, TtSequence, TtNonterminal, Ident}; use codemap::{Span, DUMMY_SP}; use diagnostic::SpanHandler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; -use parse::token::{EOF, INTERPOLATED, IDENT, Token, NtIdent}; +use parse::token::{Token, NtIdent}; use parse::token; use parse::lexer::TokenAndSpan; @@ -66,7 +66,7 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, repeat_idx: Vec::new(), repeat_len: Vec::new(), /* dummy values, never read: */ - cur_tok: EOF, + cur_tok: token::Eof, cur_span: DUMMY_SP, }; tt_next_token(&mut r); /* get cur_tok and cur_span set up */ @@ -158,7 +158,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { loop { let should_pop = match r.stack.last() { None => { - assert_eq!(ret_val.tok, EOF); + assert_eq!(ret_val.tok, token::Eof); return ret_val; } Some(frame) => { @@ -175,7 +175,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { let prev = r.stack.pop().unwrap(); match r.stack.last_mut() { None => { - r.cur_tok = EOF; + r.cur_tok = token::Eof; return ret_val; } Some(frame) => { @@ -272,13 +272,13 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { (b) we actually can, since it's a token. */ MatchedNonterminal(NtIdent(box sn, b)) => { r.cur_span = sp; - r.cur_tok = IDENT(sn,b); + r.cur_tok = token::Ident(sn,b); return ret_val; } MatchedNonterminal(ref other_whole_nt) => { // FIXME(pcwalton): Bad copy. r.cur_span = sp; - r.cur_tok = INTERPOLATED((*other_whole_nt).clone()); + r.cur_tok = token::Interpolated((*other_whole_nt).clone()); return ret_val; } MatchedSeq(..) => { -- cgit 1.4.1-3-g733a5 From fcb78d65f2a3b553b9aaca762910daf10fbb1dce Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Mon, 27 Oct 2014 23:33:30 +1100 Subject: Convert some token functions into methods --- src/librustc/middle/save/span_utils.rs | 20 +- src/librustdoc/html/highlight.rs | 4 +- src/libsyntax/ast.rs | 2 +- src/libsyntax/ext/format.rs | 3 +- src/libsyntax/ext/trace_macros.rs | 6 +- src/libsyntax/parse/lexer/comments.rs | 2 +- src/libsyntax/parse/lexer/mod.rs | 9 +- src/libsyntax/parse/parser.rs | 190 +++++++---------- src/libsyntax/parse/token.rs | 380 ++++++++++++++++++--------------- 9 files changed, 305 insertions(+), 311 deletions(-) (limited to 'src/libsyntax/ext') diff --git a/src/librustc/middle/save/span_utils.rs b/src/librustc/middle/save/span_utils.rs index 08567dba3a4..511d8aa5bac 100644 --- a/src/librustc/middle/save/span_utils.rs +++ b/src/librustc/middle/save/span_utils.rs @@ -19,7 +19,7 @@ use syntax::codemap::*; use syntax::parse::lexer; use syntax::parse::lexer::{Reader,StringReader}; use syntax::parse::token; -use syntax::parse::token::{is_keyword,keywords,is_ident,Token}; +use syntax::parse::token::{keywords, Token}; pub struct SpanUtils<'a> { pub sess: &'a Session, @@ -97,7 +97,7 @@ impl<'a> SpanUtils<'a> { return self.make_sub_span(span, result) } if bracket_count == 0 && - (is_ident(&ts.tok) || is_keyword(keywords::Self, &ts.tok)) { + (ts.tok.is_ident() || ts.tok.is_keyword(keywords::Self)) { result = Some(ts.sp); } @@ -120,7 +120,7 @@ impl<'a> SpanUtils<'a> { return None; } if bracket_count == 0 && - (is_ident(&ts.tok) || is_keyword(keywords::Self, &ts.tok)) { + (ts.tok.is_ident() || ts.tok.is_keyword(keywords::Self)) { return self.make_sub_span(span, Some(ts.sp)); } @@ -148,7 +148,7 @@ impl<'a> SpanUtils<'a> { if (next.tok == token::LParen || next.tok == token::Lt) && bracket_count == 0 && - is_ident(&prev.tok) { + prev.tok.is_ident() { result = Some(prev.sp); } @@ -158,7 +158,7 @@ impl<'a> SpanUtils<'a> { prev = next; next = toks.next_token(); if next.tok == token::Lt && - is_ident(&old.tok) { + old.tok.is_ident() { result = Some(old.sp); } } @@ -170,7 +170,7 @@ impl<'a> SpanUtils<'a> { _ => 0 }; - if is_ident(&prev.tok) && bracket_count == 0 { + if prev.tok.is_ident() && bracket_count == 0 { last_span = Some(prev.sp); } prev = next; @@ -194,7 +194,7 @@ impl<'a> SpanUtils<'a> { if (next.tok == token::Lt || next.tok == token::Colon) && bracket_count == 0 && - is_ident(&prev.tok) { + prev.tok.is_ident() { result = Some(prev.sp); } @@ -216,7 +216,7 @@ impl<'a> SpanUtils<'a> { format!("Mis-counted brackets when breaking path? Parsing '{}' in {}, line {}", self.snippet(span), loc.file.name, loc.line).as_slice()); } - if result.is_none() && is_ident(&prev.tok) && bracket_count == 0 { + if result.is_none() && prev.tok.is_ident() && bracket_count == 0 { return self.make_sub_span(span, Some(prev.sp)); } self.make_sub_span(span, result) @@ -254,7 +254,7 @@ impl<'a> SpanUtils<'a> { token::BinOp(token::Shr) => -2, _ => 0 }; - if is_ident(&ts.tok) && + if ts.tok.is_ident() && bracket_count == nesting { result.push(self.make_sub_span(span, Some(ts.sp)).unwrap()); } @@ -285,7 +285,7 @@ impl<'a> SpanUtils<'a> { if ts.tok == token::Eof { return None; } - if is_keyword(keyword, &ts.tok) { + if ts.tok.is_keyword(keyword) { let ts = toks.next_token(); if ts.tok == token::Eof { return None diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 481cd197394..0441e6b791f 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -101,7 +101,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader, token::RParen | token::LBracket | token::LBrace | token::RBrace | token::Question => "", token::Dollar => { - if token::is_ident(&lexer.peek().tok) { + if lexer.peek().tok.is_ident() { is_macro_nonterminal = true; "macro-nonterminal" } else { @@ -146,7 +146,7 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader, "Option" | "Result" => "prelude-ty", "Some" | "None" | "Ok" | "Err" => "prelude-val", - _ if token::is_any_keyword(&next.tok) => "kw", + _ if next.tok.is_any_keyword() => "kw", _ => { if is_macro_nonterminal { is_macro_nonterminal = false; diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index eaa370d95a8..a4f060cd9fc 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -82,7 +82,7 @@ impl PartialEq for Ident { // // On the other hand, if the comparison does need to be hygienic, // one example and its non-hygienic counterpart would be: - // syntax::parse::token::mtwt_token_eq + // syntax::parse::token::Token::mtwt_eq // syntax::ext::tt::macro_parser::token_name_eq fail!("not allowed to compare these idents: {}, {}. \ Probably related to issue \\#6993", self, other); diff --git a/src/libsyntax/ext/format.rs b/src/libsyntax/ext/format.rs index fdf61d4abd9..1b12ae67ee5 100644 --- a/src/libsyntax/ext/format.rs +++ b/src/libsyntax/ext/format.rs @@ -116,8 +116,7 @@ fn parse_args(ecx: &mut ExtCtxt, sp: Span, allow_method: bool, return (invocation, None); } if p.token == token::Eof { break } // accept trailing commas - if named || (token::is_ident(&p.token) && - p.look_ahead(1, |t| *t == token::Eq)) { + if named || (p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq)) { named = true; let ident = match p.token { token::Ident(i, _) => { diff --git a/src/libsyntax/ext/trace_macros.rs b/src/libsyntax/ext/trace_macros.rs index abf798ddacb..76f7b7b0d7b 100644 --- a/src/libsyntax/ext/trace_macros.rs +++ b/src/libsyntax/ext/trace_macros.rs @@ -12,7 +12,7 @@ use ast; use codemap::Span; use ext::base::ExtCtxt; use ext::base; -use parse::token::{keywords, is_keyword}; +use parse::token::keywords; pub fn expand_trace_macros(cx: &mut ExtCtxt, @@ -20,10 +20,10 @@ pub fn expand_trace_macros(cx: &mut ExtCtxt, tt: &[ast::TokenTree]) -> Box { match tt { - [ast::TtToken(_, ref tok)] if is_keyword(keywords::True, tok) => { + [ast::TtToken(_, ref tok)] if tok.is_keyword(keywords::True) => { cx.set_trace_macros(true); } - [ast::TtToken(_, ref tok)] if is_keyword(keywords::False, tok) => { + [ast::TtToken(_, ref tok)] if tok.is_keyword(keywords::False) => { cx.set_trace_macros(false); } _ => cx.span_err(sp, "trace_macros! accepts only `true` or `false`"), diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 3298eae125a..66b21aed552 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -367,7 +367,7 @@ pub fn gather_comments_and_literals(span_diagnostic: &diagnostic::SpanHandler, rdr.next_token(); //discard, and look ahead; we're working with internal state let TokenAndSpan { tok, sp } = rdr.peek(); - if token::is_lit(&tok) { + if tok.is_lit() { rdr.with_str_from(bstart, |s| { debug!("tok lit: {}", s); literals.push(Literal {lit: s.to_string(), pos: sp.lo}); diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 4226c3ce3a4..5c642f4096a 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1058,15 +1058,14 @@ impl<'a> StringReader<'a> { let keyword_checking_token = &token::Ident(keyword_checking_ident, false); let last_bpos = self.last_pos; - if token::is_keyword(token::keywords::Self, - keyword_checking_token) { + if keyword_checking_token.is_keyword(token::keywords::Self) { self.err_span_(start, last_bpos, "invalid lifetime name: 'self \ is no longer a special lifetime"); - } else if token::is_any_keyword(keyword_checking_token) && - !token::is_keyword(token::keywords::Static, - keyword_checking_token) { + } else if keyword_checking_token.is_any_keyword() && + !keyword_checking_token.is_keyword(token::keywords::Static) + { self.err_span_(start, last_bpos, "invalid lifetime name"); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index bd977962e91..bcea1449139 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -74,9 +74,8 @@ use parse::common::{seq_sep_trailing_allowed}; use parse::lexer::Reader; use parse::lexer::TokenAndSpan; use parse::obsolete::*; -use parse::token::{InternedString, can_begin_expr}; -use parse::token::{is_ident, is_ident_or_path, is_plain_ident}; -use parse::token::{keywords, special_idents, token_to_binop}; +use parse::token::InternedString; +use parse::token::{keywords, special_idents}; use parse::token; use parse::{new_sub_parser_from_file, ParseSess}; use ptr::P; @@ -335,7 +334,7 @@ pub struct Parser<'a> { } fn is_plain_ident_or_underscore(t: &token::Token) -> bool { - is_plain_ident(t) || *t == token::Underscore + t.is_plain_ident() || *t == token::Underscore } /// Get a token the parser cares about @@ -517,7 +516,7 @@ impl<'a> Parser<'a> { pub fn commit_stmt(&mut self, edible: &[token::Token], inedible: &[token::Token]) { if self.last_token .as_ref() - .map_or(false, |t| is_ident_or_path(&**t)) { + .map_or(false, |t| t.is_ident() || t.is_path()) { let mut expected = edible.iter().map(|x| x.clone()).collect::>(); expected.push_all(inedible.as_slice()); self.check_for_erroneous_unit_struct_expecting( @@ -569,14 +568,10 @@ impl<'a> Parser<'a> { is_present } - pub fn is_keyword(&mut self, kw: keywords::Keyword) -> bool { - token::is_keyword(kw, &self.token) - } - /// If the next token is the given keyword, eat it and return /// true. Otherwise, return false. pub fn eat_keyword(&mut self, kw: keywords::Keyword) -> bool { - if self.is_keyword(kw) { + if self.token.is_keyword(kw) { self.bump(); true } else { @@ -598,7 +593,7 @@ impl<'a> Parser<'a> { /// Signal an error if the given string is a strict keyword pub fn check_strict_keywords(&mut self) { - if token::is_strict_keyword(&self.token) { + if self.token.is_strict_keyword() { let token_str = self.this_token_to_string(); let span = self.span; self.span_err(span, @@ -609,7 +604,7 @@ impl<'a> Parser<'a> { /// Signal an error if the current token is a reserved keyword pub fn check_reserved_keywords(&mut self) { - if token::is_reserved_keyword(&self.token) { + if self.token.is_reserved_keyword() { let token_str = self.this_token_to_string(); self.fatal(format!("`{}` is a reserved keyword", token_str).as_slice()) @@ -896,7 +891,7 @@ impl<'a> Parser<'a> { pub fn bump(&mut self) { self.last_span = self.span; // Stash token for error recovery (sometimes; clone is not necessarily cheap). - self.last_token = if is_ident_or_path(&self.token) { + self.last_token = if self.token.is_ident() || self.token.is_path() { Some(box self.token.clone()) } else { None @@ -986,13 +981,13 @@ impl<'a> Parser<'a> { /// Is the current token one of the keywords that signals a bare function /// type? pub fn token_is_bare_fn_keyword(&mut self) -> bool { - if token::is_keyword(keywords::Fn, &self.token) { + if self.token.is_keyword(keywords::Fn) { return true } - if token::is_keyword(keywords::Unsafe, &self.token) || - token::is_keyword(keywords::Once, &self.token) { - return self.look_ahead(1, |t| token::is_keyword(keywords::Fn, t)) + if self.token.is_keyword(keywords::Unsafe) || + self.token.is_keyword(keywords::Once) { + return self.look_ahead(1, |t| t.is_keyword(keywords::Fn)) } false @@ -1000,23 +995,16 @@ impl<'a> Parser<'a> { /// Is the current token one of the keywords that signals a closure type? pub fn token_is_closure_keyword(&mut self) -> bool { - token::is_keyword(keywords::Unsafe, &self.token) || - token::is_keyword(keywords::Once, &self.token) + self.token.is_keyword(keywords::Unsafe) || + self.token.is_keyword(keywords::Once) } /// Is the current token one of the keywords that signals an old-style /// closure type (with explicit sigil)? pub fn token_is_old_style_closure_keyword(&mut self) -> bool { - token::is_keyword(keywords::Unsafe, &self.token) || - token::is_keyword(keywords::Once, &self.token) || - token::is_keyword(keywords::Fn, &self.token) - } - - pub fn token_is_lifetime(tok: &token::Token) -> bool { - match *tok { - token::Lifetime(..) => true, - _ => false, - } + self.token.is_keyword(keywords::Unsafe) || + self.token.is_keyword(keywords::Once) || + self.token.is_keyword(keywords::Fn) } pub fn get_lifetime(&mut self) -> ast::Ident { @@ -1103,10 +1091,8 @@ impl<'a> Parser<'a> { pub fn parse_optional_unboxed_closure_kind(&mut self) -> Option { if self.token == token::BinOp(token::And) && - self.look_ahead(1, |t| { - token::is_keyword(keywords::Mut, t) - }) && - self.look_ahead(2, |t| *t == token::Colon) { + self.look_ahead(1, |t| t.is_keyword(keywords::Mut)) && + self.look_ahead(2, |t| *t == token::Colon) { self.bump(); self.bump(); self.bump(); @@ -1485,8 +1471,8 @@ impl<'a> Parser<'a> { // BORROWED POINTER self.expect_and(); self.parse_borrowed_pointee() - } else if self.is_keyword(keywords::Extern) || - self.is_keyword(keywords::Unsafe) || + } else if self.token.is_keyword(keywords::Extern) || + self.token.is_keyword(keywords::Unsafe) || self.token_is_bare_fn_keyword() { // BARE FUNCTION self.parse_ty_bare_fn() @@ -1495,7 +1481,7 @@ impl<'a> Parser<'a> { self.token == token::OrOr || (self.token == token::Lt && self.look_ahead(1, |t| { - *t == token::Gt || Parser::token_is_lifetime(t) + *t == token::Gt || t.is_lifetime() })) { // CLOSURE @@ -1524,7 +1510,8 @@ impl<'a> Parser<'a> { item_name: item_name, })) } else if self.token == token::ModSep - || is_ident_or_path(&self.token) { + || self.token.is_ident() + || self.token.is_path() { // NAMED TYPE let mode = if plus_allowed { LifetimeAndTypesAndBounds @@ -1577,7 +1564,7 @@ impl<'a> Parser<'a> { let offset = match self.token { token::BinOp(token::And) => 1, token::AndAnd => 1, - _ if token::is_keyword(keywords::Mut, &self.token) => 1, + _ if self.token.is_keyword(keywords::Mut) => 1, _ => 0 }; @@ -1925,11 +1912,6 @@ impl<'a> Parser<'a> { } } - pub fn token_is_mutability(tok: &token::Token) -> bool { - token::is_keyword(keywords::Mut, tok) || - token::is_keyword(keywords::Const, tok) - } - /// Parse mutability declaration (mut/const/imm) pub fn parse_mutability(&mut self) -> Mutability { if self.eat_keyword(keywords::Mut) { @@ -2157,7 +2139,7 @@ impl<'a> Parser<'a> { if self.eat_keyword(keywords::While) { return self.parse_while_expr(None); } - if Parser::token_is_lifetime(&self.token) { + if self.token.is_lifetime() { let lifetime = self.get_lifetime(); self.bump(); self.expect(&token::Colon); @@ -2177,7 +2159,7 @@ impl<'a> Parser<'a> { } if self.eat_keyword(keywords::Continue) { let lo = self.span.lo; - let ex = if Parser::token_is_lifetime(&self.token) { + let ex = if self.token.is_lifetime() { let lifetime = self.get_lifetime(); self.bump(); ExprAgain(Some(lifetime)) @@ -2197,7 +2179,7 @@ impl<'a> Parser<'a> { } if self.eat_keyword(keywords::Return) { // RETURN expression - if can_begin_expr(&self.token) { + if self.token.can_begin_expr() { let e = self.parse_expr(); hi = e.span.hi; ex = ExprRet(Some(e)); @@ -2206,7 +2188,7 @@ impl<'a> Parser<'a> { } } else if self.eat_keyword(keywords::Break) { // BREAK expression - if Parser::token_is_lifetime(&self.token) { + if self.token.is_lifetime() { let lifetime = self.get_lifetime(); self.bump(); ex = ExprBreak(Some(lifetime)); @@ -2215,9 +2197,9 @@ impl<'a> Parser<'a> { } hi = self.span.hi; } else if self.token == token::ModSep || - is_ident(&self.token) && - !self.is_keyword(keywords::True) && - !self.is_keyword(keywords::False) { + self.token.is_ident() && + !self.token.is_keyword(keywords::True) && + !self.token.is_keyword(keywords::False) { let pth = self.parse_path(LifetimeAndTypesWithColons).path; @@ -2226,7 +2208,7 @@ impl<'a> Parser<'a> { // MACRO INVOCATION expression self.bump(); - let ket = token::close_delimiter_for(&self.token) + let ket = self.token.get_close_delimiter() .unwrap_or_else(|| { self.fatal("expected open delimiter") }); @@ -2581,7 +2563,7 @@ impl<'a> Parser<'a> { } } - match (&self.token, token::close_delimiter_for(&self.token)) { + match (&self.token, self.token.get_close_delimiter()) { (&token::Eof, _) => { let open_braces = self.open_braces.clone(); for sp in open_braces.iter() { @@ -2638,7 +2620,7 @@ impl<'a> Parser<'a> { // the interpolation of Matcher's maybe_whole!(self, NtMatchers); let mut name_idx = 0u; - match token::close_delimiter_for(&self.token) { + match self.token.get_close_delimiter() { Some(other_delimiter) => { self.bump(); self.parse_matcher_subseq_upto(&mut name_idx, &other_delimiter) @@ -2743,7 +2725,7 @@ impl<'a> Parser<'a> { ex = self.mk_unary(UnUniq, e); } token::Ident(_, _) => { - if !self.is_keyword(keywords::Box) { + if !self.token.is_keyword(keywords::Box) { return self.parse_dot_or_call_expr(); } @@ -2789,7 +2771,7 @@ impl<'a> Parser<'a> { return lhs; } - let cur_opt = token_to_binop(&self.token); + let cur_opt = self.token.to_binop(); match cur_opt { Some(cur_op) => { let cur_prec = operator_prec(cur_op); @@ -2860,7 +2842,7 @@ impl<'a> Parser<'a> { /// Parse an 'if' or 'if let' expression ('if' token already eaten) pub fn parse_if_expr(&mut self) -> P { - if self.is_keyword(keywords::Let) { + if self.token.is_keyword(keywords::Let) { return self.parse_if_let_expr(); } let lo = self.last_span.lo; @@ -2951,7 +2933,7 @@ impl<'a> Parser<'a> { /// Parse a 'while' or 'while let' expression ('while' token already eaten) pub fn parse_while_expr(&mut self, opt_ident: Option) -> P { - if self.is_keyword(keywords::Let) { + if self.token.is_keyword(keywords::Let) { return self.parse_while_let_expr(opt_ident); } let lo = self.last_span.lo; @@ -3272,9 +3254,10 @@ impl<'a> Parser<'a> { } // at this point, token != _, ~, &, &&, (, [ - if (!is_ident_or_path(&self.token) && self.token != token::ModSep) - || self.is_keyword(keywords::True) - || self.is_keyword(keywords::False) { + if (!(self.token.is_ident() || self.token.is_path()) + && self.token != token::ModSep) + || self.token.is_keyword(keywords::True) + || self.token.is_keyword(keywords::False) { // Parse an expression pattern or exp .. exp. // // These expressions are limited to literals (possibly @@ -3285,7 +3268,7 @@ impl<'a> Parser<'a> { *t != token::Comma && *t != token::RBracket }) { self.bump(); - let end = if is_ident_or_path(&self.token) { + let end = if self.token.is_ident() || self.token.is_path() { let path = self.parse_path(LifetimeAndTypesWithColons) .path; let hi = self.span.hi; @@ -3333,13 +3316,13 @@ impl<'a> Parser<'a> { self.eat(&token::DotDotDot); let end = self.parse_expr_res(RESTRICTION_NO_BAR_OP); pat = PatRange(start, end); - } else if is_plain_ident(&self.token) && !can_be_enum_or_struct { + } else if self.token.is_plain_ident() && !can_be_enum_or_struct { let id = self.parse_ident(); let id_span = self.last_span; let pth1 = codemap::Spanned{span:id_span, node: id}; if self.eat(&token::Not) { // macro invocation - let ket = token::close_delimiter_for(&self.token) + let ket = self.token.get_close_delimiter() .unwrap_or_else(|| self.fatal("expected open delimiter")); self.bump(); @@ -3438,7 +3421,7 @@ impl<'a> Parser<'a> { fn parse_pat_ident(&mut self, binding_mode: ast::BindingMode) -> ast::Pat_ { - if !is_plain_ident(&self.token) { + if !self.token.is_plain_ident() { let span = self.span; let tok_str = self.this_token_to_string(); self.span_fatal(span, @@ -3504,7 +3487,7 @@ impl<'a> Parser<'a> { fn parse_name_and_ty(&mut self, pr: Visibility, attrs: Vec ) -> StructField { let lo = self.span.lo; - if !is_plain_ident(&self.token) { + if !self.token.is_plain_ident() { self.fatal("expected ident"); } let name = self.parse_ident(); @@ -3542,13 +3525,13 @@ impl<'a> Parser<'a> { } let lo = self.span.lo; - if self.is_keyword(keywords::Let) { + if self.token.is_keyword(keywords::Let) { check_expected_item(self, item_attrs.as_slice()); self.expect_keyword(keywords::Let); let decl = self.parse_let(); P(spanned(lo, decl.span.hi, StmtDecl(decl, ast::DUMMY_NODE_ID))) - } else if is_ident(&self.token) - && !token::is_any_keyword(&self.token) + } else if self.token.is_ident() + && !self.token.is_any_keyword() && self.look_ahead(1, |t| *t == token::Not) { // it's a macro invocation: @@ -3559,7 +3542,7 @@ impl<'a> Parser<'a> { let pth = self.parse_path(NoTypesAllowed).path; self.bump(); - let id = if token::close_delimiter_for(&self.token).is_some() { + let id = if self.token.get_close_delimiter().is_some() { token::special_idents::invalid // no special identifier } else { self.parse_ident() @@ -3568,7 +3551,7 @@ impl<'a> Parser<'a> { // check that we're pointing at delimiters (need to check // again after the `if`, because of `parse_ident` // consuming more tokens). - let (bra, ket) = match token::close_delimiter_for(&self.token) { + let (bra, ket) = match self.token.get_close_delimiter() { Some(ket) => (self.token.clone(), ket), None => { // we only expect an ident if we didn't parse one @@ -3993,7 +3976,7 @@ impl<'a> Parser<'a> { } fn forbid_lifetime(&mut self) { - if Parser::token_is_lifetime(&self.token) { + if self.token.is_lifetime() { let span = self.span; self.span_fatal(span, "lifetime parameters must be declared \ prior to type parameters"); @@ -4145,29 +4128,22 @@ impl<'a> Parser<'a> { // // We already know that the current token is `&`. - if this.look_ahead(1, |t| token::is_keyword(keywords::Self, t)) { + if this.look_ahead(1, |t| t.is_keyword(keywords::Self)) { this.bump(); SelfRegion(None, MutImmutable, this.expect_self_ident()) - } else if this.look_ahead(1, |t| Parser::token_is_mutability(t)) && - this.look_ahead(2, - |t| token::is_keyword(keywords::Self, - t)) { + } else if this.look_ahead(1, |t| t.is_mutability()) && + this.look_ahead(2, |t| t.is_keyword(keywords::Self)) { this.bump(); let mutability = this.parse_mutability(); SelfRegion(None, mutability, this.expect_self_ident()) - } else if this.look_ahead(1, |t| Parser::token_is_lifetime(t)) && - this.look_ahead(2, - |t| token::is_keyword(keywords::Self, - t)) { + } else if this.look_ahead(1, |t| t.is_lifetime()) && + this.look_ahead(2, |t| t.is_keyword(keywords::Self)) { this.bump(); let lifetime = this.parse_lifetime(); SelfRegion(Some(lifetime), MutImmutable, this.expect_self_ident()) - } else if this.look_ahead(1, |t| Parser::token_is_lifetime(t)) && - this.look_ahead(2, |t| { - Parser::token_is_mutability(t) - }) && - this.look_ahead(3, |t| token::is_keyword(keywords::Self, - t)) { + } else if this.look_ahead(1, |t| t.is_lifetime()) && + this.look_ahead(2, |t| t.is_mutability()) && + this.look_ahead(3, |t| t.is_keyword(keywords::Self)) { this.bump(); let lifetime = this.parse_lifetime(); let mutability = this.parse_mutability(); @@ -4195,7 +4171,7 @@ impl<'a> Parser<'a> { } token::Tilde => { // We need to make sure it isn't a type - if self.look_ahead(1, |t| token::is_keyword(keywords::Self, t)) { + if self.look_ahead(1, |t| t.is_keyword(keywords::Self)) { self.bump(); drop(self.expect_self_ident()); let last_span = self.last_span; @@ -4207,7 +4183,7 @@ impl<'a> Parser<'a> { // Possibly "*self" or "*mut self" -- not supported. Try to avoid // emitting cryptic "unexpected token" errors. self.bump(); - let _mutability = if Parser::token_is_mutability(&self.token) { + let _mutability = if self.token.is_mutability() { self.parse_mutability() } else { MutImmutable @@ -4231,10 +4207,8 @@ impl<'a> Parser<'a> { } else { SelfValue(self_ident) } - } else if Parser::token_is_mutability(&self.token) && - self.look_ahead(1, |t| { - token::is_keyword(keywords::Self, t) - }) { + } else if self.token.is_mutability() && + self.look_ahead(1, |t| t.is_keyword(keywords::Self)) { mutbl_self = self.parse_mutability(); let self_ident = self.expect_self_ident(); @@ -4245,11 +4219,9 @@ impl<'a> Parser<'a> { } else { SelfValue(self_ident) } - } else if Parser::token_is_mutability(&self.token) && + } else if self.token.is_mutability() && self.look_ahead(1, |t| *t == token::Tilde) && - self.look_ahead(2, |t| { - token::is_keyword(keywords::Self, t) - }) { + self.look_ahead(2, |t| t.is_keyword(keywords::Self)) { mutbl_self = self.parse_mutability(); self.bump(); drop(self.expect_self_ident()); @@ -4430,7 +4402,7 @@ impl<'a> Parser<'a> { // code copied from parse_macro_use_or_failure... abstraction! let (method_, hi, new_attrs) = { - if !token::is_any_keyword(&self.token) + if !self.token.is_any_keyword() && self.look_ahead(1, |t| *t == token::Not) && (self.look_ahead(2, |t| *t == token::LParen) || self.look_ahead(2, |t| *t == token::LBrace)) { @@ -4439,7 +4411,7 @@ impl<'a> Parser<'a> { self.expect(&token::Not); // eat a matched-delimiter token tree: - let tts = match token::close_delimiter_for(&self.token) { + let tts = match self.token.get_close_delimiter() { Some(ket) => { self.bump(); self.parse_seq_to_end(&ket, @@ -5037,7 +5009,7 @@ impl<'a> Parser<'a> { Some(path) } else if self.eat_keyword(keywords::As) { // skip the ident if there is one - if is_ident(&self.token) { self.bump(); } + if self.token.is_ident() { self.bump(); } self.span_err(span, format!("expected `;`, found `as`; perhaps you meant \ @@ -5335,7 +5307,7 @@ impl<'a> Parser<'a> { } // the rest are all guaranteed to be items: - if self.is_keyword(keywords::Static) { + if self.token.is_keyword(keywords::Static) { // STATIC ITEM self.bump(); let m = if self.eat_keyword(keywords::Mut) {MutMutable} else {MutImmutable}; @@ -5349,7 +5321,7 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return IoviItem(item); } - if self.is_keyword(keywords::Const) { + if self.token.is_keyword(keywords::Const) { // CONST ITEM self.bump(); if self.eat_keyword(keywords::Mut) { @@ -5367,7 +5339,7 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return IoviItem(item); } - if self.is_keyword(keywords::Fn) && + if self.token.is_keyword(keywords::Fn) && self.look_ahead(1, |f| !Parser::fn_expr_lookahead(f)) { // FUNCTION ITEM self.bump(); @@ -5382,7 +5354,7 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return IoviItem(item); } - if self.is_keyword(keywords::Unsafe) + if self.token.is_keyword(keywords::Unsafe) && self.look_ahead(1u, |t| *t != token::LBrace) { // UNSAFE FUNCTION ITEM self.bump(); @@ -5489,12 +5461,12 @@ impl<'a> Parser<'a> { let visibility = self.parse_visibility(); - if self.is_keyword(keywords::Static) { + if self.token.is_keyword(keywords::Static) { // FOREIGN STATIC ITEM let item = self.parse_item_foreign_static(visibility, attrs); return IoviForeignItem(item); } - if self.is_keyword(keywords::Fn) || self.is_keyword(keywords::Unsafe) { + if self.token.is_keyword(keywords::Fn) || self.token.is_keyword(keywords::Unsafe) { // FOREIGN FUNCTION ITEM let item = self.parse_item_foreign_fn(visibility, attrs); return IoviForeignItem(item); @@ -5510,9 +5482,9 @@ impl<'a> Parser<'a> { lo: BytePos, visibility: Visibility ) -> ItemOrViewItem { - if macros_allowed && !token::is_any_keyword(&self.token) + if macros_allowed && !self.token.is_any_keyword() && self.look_ahead(1, |t| *t == token::Not) - && (self.look_ahead(2, |t| is_plain_ident(t)) + && (self.look_ahead(2, |t| t.is_plain_ident()) || self.look_ahead(2, |t| *t == token::LParen) || self.look_ahead(2, |t| *t == token::LBrace)) { // MACRO INVOCATION ITEM @@ -5524,13 +5496,13 @@ impl<'a> Parser<'a> { // a 'special' identifier (like what `macro_rules!` uses) // is optional. We should eventually unify invoc syntax // and remove this. - let id = if is_plain_ident(&self.token) { + let id = if self.token.is_plain_ident() { self.parse_ident() } else { token::special_idents::invalid // no special identifier }; // eat a matched-delimiter token tree: - let tts = match token::close_delimiter_for(&self.token) { + let tts = match self.token.get_close_delimiter() { Some(ket) => { self.bump(); self.parse_seq_to_end(&ket, diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 0cc56b2ab2b..6ffe766684d 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -173,6 +173,206 @@ pub enum Token { Eof, } +impl Token { + /// Returns `true` if the token can appear at the start of an expression. + pub fn can_begin_expr(&self) -> bool { + match *self { + LParen => true, + LBrace => true, + LBracket => true, + Ident(_, _) => true, + Underscore => true, + Tilde => true, + LitByte(_) => true, + LitChar(_) => true, + LitInteger(_) => true, + LitFloat(_) => true, + LitStr(_) => true, + LitStrRaw(_, _) => true, + LitBinary(_) => true, + LitBinaryRaw(_, _) => true, + Pound => true, + At => true, + Not => true, + BinOp(Minus) => true, + BinOp(Star) => true, + BinOp(And) => true, + BinOp(Or) => true, // in lambda syntax + OrOr => true, // in lambda syntax + ModSep => true, + Interpolated(NtExpr(..)) => true, + Interpolated(NtIdent(..)) => true, + Interpolated(NtBlock(..)) => true, + Interpolated(NtPath(..)) => true, + _ => false, + } + } + + /// Returns the matching close delimiter if this is an open delimiter, + /// otherwise `None`. + pub fn get_close_delimiter(&self) -> Option { + match *self { + LParen => Some(RParen), + LBrace => Some(RBrace), + LBracket => Some(RBracket), + _ => None, + } + } + + /// Returns `true` if the token is any literal + pub fn is_lit(&self) -> bool { + match *self { + LitByte(_) => true, + LitChar(_) => true, + LitInteger(_) => true, + LitFloat(_) => true, + LitStr(_) => true, + LitStrRaw(_, _) => true, + LitBinary(_) => true, + LitBinaryRaw(_, _) => true, + _ => false, + } + } + + /// Returns `true` if the token is an identifier. + pub fn is_ident(&self) -> bool { + match *self { + Ident(_, _) => true, + _ => false, + } + } + + /// Returns `true` if the token is an interpolated path. + pub fn is_path(&self) -> bool { + match *self { + Interpolated(NtPath(..)) => true, + _ => false, + } + } + + /// Returns `true` if the token is a path that is not followed by a `::` + /// token. + pub fn is_plain_ident(&self) -> bool { + match *self { + Ident(_, false) => true, + _ => false, + } + } + + /// Returns `true` if the token is a lifetime. + pub fn is_lifetime(&self) -> bool { + match *self { + Lifetime(..) => true, + _ => false, + } + } + + /// Returns `true` if the token is either the `mut` or `const` keyword. + pub fn is_mutability(&self) -> bool { + self.is_keyword(keywords::Mut) || + self.is_keyword(keywords::Const) + } + + /// Maps a token to its corresponding binary operator. + pub fn to_binop(&self) -> Option { + match *self { + BinOp(Star) => Some(ast::BiMul), + BinOp(Slash) => Some(ast::BiDiv), + BinOp(Percent) => Some(ast::BiRem), + BinOp(Plus) => Some(ast::BiAdd), + BinOp(Minus) => Some(ast::BiSub), + BinOp(Shl) => Some(ast::BiShl), + BinOp(Shr) => Some(ast::BiShr), + BinOp(And) => Some(ast::BiBitAnd), + BinOp(Caret) => Some(ast::BiBitXor), + BinOp(Or) => Some(ast::BiBitOr), + Lt => Some(ast::BiLt), + Le => Some(ast::BiLe), + Ge => Some(ast::BiGe), + Gt => Some(ast::BiGt), + EqEq => Some(ast::BiEq), + Ne => Some(ast::BiNe), + AndAnd => Some(ast::BiAnd), + OrOr => Some(ast::BiOr), + _ => None, + } + } + + /// Returns `true` if the token is a given keyword, `kw`. + pub fn is_keyword(&self, kw: keywords::Keyword) -> bool { + match *self { + Ident(sid, false) => kw.to_name() == sid.name, + _ => false, + } + } + + /// Returns `true` if the token is either a special identifier, or a strict + /// or reserved keyword. + pub fn is_any_keyword(&self) -> bool { + match *self { + Ident(sid, false) => { + let n = sid.name; + + n == SELF_KEYWORD_NAME + || n == STATIC_KEYWORD_NAME + || n == SUPER_KEYWORD_NAME + || STRICT_KEYWORD_START <= n + && n <= RESERVED_KEYWORD_FINAL + }, + _ => false + } + } + + /// Returns `true` if the token may not appear as an identifier. + pub fn is_strict_keyword(&self) -> bool { + match *self { + Ident(sid, false) => { + let n = sid.name; + + n == SELF_KEYWORD_NAME + || n == STATIC_KEYWORD_NAME + || n == SUPER_KEYWORD_NAME + || STRICT_KEYWORD_START <= n + && n <= STRICT_KEYWORD_FINAL + }, + Ident(sid, true) => { + let n = sid.name; + + n != SELF_KEYWORD_NAME + && n != SUPER_KEYWORD_NAME + && STRICT_KEYWORD_START <= n + && n <= STRICT_KEYWORD_FINAL + } + _ => false, + } + } + + /// Returns `true` if the token is a keyword that has been reserved for + /// possible future use. + pub fn is_reserved_keyword(&self) -> bool { + match *self { + Ident(sid, false) => { + let n = sid.name; + + RESERVED_KEYWORD_START <= n + && n <= RESERVED_KEYWORD_FINAL + }, + _ => false, + } + } + + /// Hygienic identifier equality comparison. + /// + /// See `styntax::ext::mtwt`. + pub fn mtwt_eq(&self, other : &Token) -> bool { + match (self, other) { + (&Ident(id1,_), &Ident(id2,_)) | (&Lifetime(id1), &Lifetime(id2)) => + mtwt::resolve(id1) == mtwt::resolve(id2), + _ => *self == *other + } + } +} + #[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash)] /// For interpolation during macro expansion. pub enum Nonterminal { @@ -304,86 +504,6 @@ pub fn to_string(t: &Token) -> String { } } -pub fn can_begin_expr(t: &Token) -> bool { - match *t { - LParen => true, - LBrace => true, - LBracket => true, - Ident(_, _) => true, - Underscore => true, - Tilde => true, - LitByte(_) => true, - LitChar(_) => true, - LitInteger(_) => true, - LitFloat(_) => true, - LitStr(_) => true, - LitStrRaw(_, _) => true, - LitBinary(_) => true, - LitBinaryRaw(_, _) => true, - Pound => true, - At => true, - Not => true, - BinOp(Minus) => true, - BinOp(Star) => true, - BinOp(And) => true, - BinOp(Or) => true, // in lambda syntax - OrOr => true, // in lambda syntax - ModSep => true, - Interpolated(NtExpr(..)) => true, - Interpolated(NtIdent(..)) => true, - Interpolated(NtBlock(..)) => true, - Interpolated(NtPath(..)) => true, - _ => false, - } -} - -/// Returns the matching close delimiter if this is an open delimiter, -/// otherwise `None`. -pub fn close_delimiter_for(t: &Token) -> Option { - match *t { - LParen => Some(RParen), - LBrace => Some(RBrace), - LBracket => Some(RBracket), - _ => None, - } -} - -pub fn is_lit(t: &Token) -> bool { - match *t { - LitByte(_) => true, - LitChar(_) => true, - LitInteger(_) => true, - LitFloat(_) => true, - LitStr(_) => true, - LitStrRaw(_, _) => true, - LitBinary(_) => true, - LitBinaryRaw(_, _) => true, - _ => false, - } -} - -pub fn is_ident(t: &Token) -> bool { - match *t { - Ident(_, _) => true, - _ => false, - } -} - -pub fn is_ident_or_path(t: &Token) -> bool { - match *t { - Ident(_, _) => true, - Interpolated(NtPath(..)) => true, - _ => false, - } -} - -pub fn is_plain_ident(t: &Token) -> bool { - match *t { - Ident(_, false) => true, - _ => false, - } -} - // Get the first "argument" macro_rules! first { ( $first:expr, $( $remainder:expr, )* ) => ( $first ) @@ -570,34 +690,6 @@ declare_special_idents_and_keywords! { } } -/** - * Maps a token to a record specifying the corresponding binary - * operator - */ -pub fn token_to_binop(tok: &Token) -> Option { - match *tok { - BinOp(Star) => Some(ast::BiMul), - BinOp(Slash) => Some(ast::BiDiv), - BinOp(Percent) => Some(ast::BiRem), - BinOp(Plus) => Some(ast::BiAdd), - BinOp(Minus) => Some(ast::BiSub), - BinOp(Shl) => Some(ast::BiShl), - BinOp(Shr) => Some(ast::BiShr), - BinOp(And) => Some(ast::BiBitAnd), - BinOp(Caret) => Some(ast::BiBitXor), - BinOp(Or) => Some(ast::BiBitOr), - Lt => Some(ast::BiLt), - Le => Some(ast::BiLe), - Ge => Some(ast::BiGe), - Gt => Some(ast::BiGt), - EqEq => Some(ast::BiEq), - Ne => Some(ast::BiNe), - AndAnd => Some(ast::BiAnd), - OrOr => Some(ast::BiOr), - _ => None - } -} - // looks like we can get rid of this completely... pub type IdentInterner = StrInterner; @@ -751,74 +843,6 @@ pub fn fresh_mark() -> ast::Mrk { gensym("mark").uint() as u32 } -// See the macro above about the types of keywords - -pub fn is_keyword(kw: keywords::Keyword, tok: &Token) -> bool { - match *tok { - Ident(sid, false) => { kw.to_name() == sid.name } - _ => { false } - } -} - -pub fn is_any_keyword(tok: &Token) -> bool { - match *tok { - Ident(sid, false) => { - let n = sid.name; - - n == SELF_KEYWORD_NAME - || n == STATIC_KEYWORD_NAME - || n == SUPER_KEYWORD_NAME - || STRICT_KEYWORD_START <= n - && n <= RESERVED_KEYWORD_FINAL - }, - _ => false - } -} - -pub fn is_strict_keyword(tok: &Token) -> bool { - match *tok { - Ident(sid, false) => { - let n = sid.name; - - n == SELF_KEYWORD_NAME - || n == STATIC_KEYWORD_NAME - || n == SUPER_KEYWORD_NAME - || STRICT_KEYWORD_START <= n - && n <= STRICT_KEYWORD_FINAL - }, - Ident(sid, true) => { - let n = sid.name; - - n != SELF_KEYWORD_NAME - && n != SUPER_KEYWORD_NAME - && STRICT_KEYWORD_START <= n - && n <= STRICT_KEYWORD_FINAL - } - _ => false, - } -} - -pub fn is_reserved_keyword(tok: &Token) -> bool { - match *tok { - Ident(sid, false) => { - let n = sid.name; - - RESERVED_KEYWORD_START <= n - && n <= RESERVED_KEYWORD_FINAL - }, - _ => false, - } -} - -pub fn mtwt_token_eq(t1 : &Token, t2 : &Token) -> bool { - match (t1,t2) { - (&Ident(id1,_),&Ident(id2,_)) | (&Lifetime(id1),&Lifetime(id2)) => - mtwt::resolve(id1) == mtwt::resolve(id2), - _ => *t1 == *t2 - } -} - - #[cfg(test)] mod test { use super::*; @@ -830,9 +854,9 @@ mod test { } #[test] fn mtwt_token_eq_test() { - assert!(mtwt_token_eq(&Gt,&Gt)); + assert!(Gt.mtwt_eq(&Gt)); let a = str_to_ident("bac"); let a1 = mark_ident(a,92); - assert!(mtwt_token_eq(&Ident(a,true),&Ident(a1,false))); + assert!(Ident(a,true).mtwt_eq(&Ident(a1,false))); } } -- cgit 1.4.1-3-g733a5 From cd049591a25973cd41ca5b69e7a151ae5fa0b71f Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Tue, 28 Oct 2014 02:01:44 +1100 Subject: Use an enum rather than a bool in token::Ident --- src/grammar/verify.rs | 5 ++-- src/libsyntax/ext/quote.rs | 9 ++++-- src/libsyntax/parse/lexer/mod.rs | 61 ++++++++++++++++++++++++---------------- src/libsyntax/parse/mod.rs | 18 ++++++------ src/libsyntax/parse/parser.rs | 12 ++++---- src/libsyntax/parse/token.rs | 48 ++++++++++++++++++++++--------- 6 files changed, 96 insertions(+), 57 deletions(-) (limited to 'src/libsyntax/ext') diff --git a/src/grammar/verify.rs b/src/grammar/verify.rs index 16abf5160fa..fef7d3510e7 100644 --- a/src/grammar/verify.rs +++ b/src/grammar/verify.rs @@ -35,7 +35,7 @@ use syntax::parse::lexer::TokenAndSpan; fn parse_token_list(file: &str) -> HashMap { fn id() -> Token { - token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, false) + token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain) } let mut res = HashMap::new(); @@ -198,7 +198,8 @@ fn parse_antlr_token(s: &str, tokens: &HashMap) -> TokenAndSpan { token::LitFloat(..) => token::LitFloat(nm), token::LitBinary(..) => token::LitBinary(nm), token::LitBinaryRaw(..) => token::LitBinaryRaw(fix(content), count(content)), - token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 }, true), + token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 }, + token::ModName), token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }), ref t => t.clone() }; diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 39a538f917b..dc7a495523f 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -531,6 +531,7 @@ fn mk_binop(cx: &ExtCtxt, sp: Span, bop: token::BinOpToken) -> P { mk_token_path(cx, sp, name) } +#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { match *tok { token::BinOp(binop) => { @@ -575,10 +576,14 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { vec!(mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n))); } - token::Ident(ident, b) => { + token::Ident(ident, style) => { return cx.expr_call(sp, mk_token_path(cx, sp, "Ident"), - vec!(mk_ident(cx, sp, ident), cx.expr_bool(sp, b))); + vec![mk_ident(cx, sp, ident), + match style { + ModName => mk_token_path(cx, sp, "ModName"), + Plain => mk_token_path(cx, sp, "Plain"), + }]); } token::Lifetime(ident) => { diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 5c642f4096a..b439353ad95 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -921,12 +921,14 @@ impl<'a> StringReader<'a> { if string == "_" { token::Underscore } else { - let is_mod_name = self.curr_is(':') && self.nextch_is(':'); - // FIXME: perform NFKC normalization here. (Issue #2253) - token::Ident(str_to_ident(string), is_mod_name) + if self.curr_is(':') && self.nextch_is(':') { + token::Ident(str_to_ident(string), token::ModName) + } else { + token::Ident(str_to_ident(string), token::Plain) + } } - }) + }); } if is_dec_digit(c) { @@ -937,8 +939,11 @@ impl<'a> StringReader<'a> { match (c.unwrap(), self.nextch(), self.nextnextch()) { ('\x00', Some('n'), Some('a')) => { let ast_ident = self.scan_embedded_hygienic_ident(); - let is_mod_name = self.curr_is(':') && self.nextch_is(':'); - return token::Ident(ast_ident, is_mod_name); + return if self.curr_is(':') && self.nextch_is(':') { + token::Ident(ast_ident, token::ModName) + } else { + token::Ident(ast_ident, token::Plain) + }; } _ => {} } @@ -1056,7 +1061,7 @@ impl<'a> StringReader<'a> { str_to_ident(lifetime_name) }); let keyword_checking_token = - &token::Ident(keyword_checking_ident, false); + &token::Ident(keyword_checking_ident, token::Plain); let last_bpos = self.last_pos; if keyword_checking_token.is_keyword(token::keywords::Self) { self.err_span_(start, @@ -1434,7 +1439,7 @@ mod test { assert_eq!(string_reader.next_token().tok, token::Whitespace); let tok1 = string_reader.next_token(); let tok2 = TokenAndSpan{ - tok:token::Ident(id, false), + tok:token::Ident(id, token::Plain), sp:Span {lo:BytePos(21),hi:BytePos(23),expn_id: NO_EXPANSION}}; assert_eq!(tok1,tok2); assert_eq!(string_reader.next_token().tok, token::Whitespace); @@ -1443,7 +1448,7 @@ mod test { // read another token: let tok3 = string_reader.next_token(); let tok4 = TokenAndSpan{ - tok:token::Ident(str_to_ident("main"), false), + tok:token::Ident(str_to_ident("main"), token::Plain), sp:Span {lo:BytePos(24),hi:BytePos(28),expn_id: NO_EXPANSION}}; assert_eq!(tok3,tok4); // the lparen is already read: @@ -1458,39 +1463,45 @@ mod test { } } - // make the identifier by looking up the string in the interner + #[cfg(stage0)] fn mk_ident (id: &str, is_mod_name: bool) -> token::Token { - token::Ident (str_to_ident(id),is_mod_name) + token::Ident(str_to_ident(id), is_mod_name) + } + + // make the identifier by looking up the string in the interner + #[cfg(not(stage0))] + fn mk_ident(id: &str, style: token::IdentStyle) -> token::Token { + token::Ident(str_to_ident(id), style) } #[test] fn doublecolonparsing () { check_tokenization(setup(&mk_sh(), "a b".to_string()), - vec!(mk_ident("a",false), - token::Whitespace, - mk_ident("b",false))); + vec![mk_ident("a", token::Plain), + token::Whitespace, + mk_ident("b", token::Plain)]); } #[test] fn dcparsing_2 () { check_tokenization(setup(&mk_sh(), "a::b".to_string()), - vec!(mk_ident("a",true), - token::ModSep, - mk_ident("b",false))); + vec![mk_ident("a",token::ModName), + token::ModSep, + mk_ident("b", token::Plain)]); } #[test] fn dcparsing_3 () { check_tokenization(setup(&mk_sh(), "a ::b".to_string()), - vec!(mk_ident("a",false), - token::Whitespace, - token::ModSep, - mk_ident("b",false))); + vec![mk_ident("a", token::Plain), + token::Whitespace, + token::ModSep, + mk_ident("b", token::Plain)]); } #[test] fn dcparsing_4 () { check_tokenization(setup(&mk_sh(), "a:: b".to_string()), - vec!(mk_ident("a",true), - token::ModSep, - token::Whitespace, - mk_ident("b",false))); + vec![mk_ident("a",token::ModName), + token::ModSep, + token::Whitespace, + mk_ident("b", token::Plain)]); } #[test] fn character_a() { diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 6c0df39daeb..e60da0867f7 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -793,9 +793,9 @@ mod test { let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string()); let tts: &[ast::TokenTree] = tts.as_slice(); match tts { - [ast::TtToken(_, token::Ident(name_macro_rules, false)), + [ast::TtToken(_, token::Ident(name_macro_rules, token::Plain)), ast::TtToken(_, token::Not), - ast::TtToken(_, token::Ident(name_zip, false)), + ast::TtToken(_, token::Ident(name_zip, token::Plain)), ast::TtDelimited(_, ref macro_delimed)] if name_macro_rules.as_str() == "macro_rules" && name_zip.as_str() == "zip" => { @@ -810,7 +810,7 @@ mod test { match (first_open, first_tts.as_slice(), first_close) { (&ast::Delimiter { token: token::LParen, .. }, [ast::TtToken(_, token::Dollar), - ast::TtToken(_, token::Ident(name, false))], + ast::TtToken(_, token::Ident(name, token::Plain))], &ast::Delimiter { token: token::RParen, .. }) if name.as_str() == "a" => {}, _ => fail!("value 3: {}", **first_delimed), @@ -819,7 +819,7 @@ mod test { match (second_open, second_tts.as_slice(), second_close) { (&ast::Delimiter { token: token::LParen, .. }, [ast::TtToken(_, token::Dollar), - ast::TtToken(_, token::Ident(name, false))], + ast::TtToken(_, token::Ident(name, token::Plain))], &ast::Delimiter { token: token::RParen, .. }) if name.as_str() == "a" => {}, _ => fail!("value 4: {}", **second_delimed), @@ -845,7 +845,7 @@ mod test { \"variant\":\"Ident\",\ \"fields\":[\ \"fn\",\ - false\ + \"Plain\"\ ]\ }\ ]\ @@ -858,7 +858,7 @@ mod test { \"variant\":\"Ident\",\ \"fields\":[\ \"a\",\ - false\ + \"Plain\"\ ]\ }\ ]\ @@ -881,7 +881,7 @@ mod test { \"variant\":\"Ident\",\ \"fields\":[\ \"b\",\ - false\ + \"Plain\"\ ]\ }\ ]\ @@ -901,7 +901,7 @@ mod test { \"variant\":\"Ident\",\ \"fields\":[\ \"int\",\ - false\ + \"Plain\"\ ]\ }\ ]\ @@ -932,7 +932,7 @@ mod test { \"variant\":\"Ident\",\ \"fields\":[\ \"b\",\ - false\ + \"Plain\"\ ]\ }\ ]\ diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index bcea1449139..54b1cc2dbad 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2067,10 +2067,10 @@ impl<'a> Parser<'a> { }, // FIXME #13626: Should be able to stick in // token::SELF_KEYWORD_NAME - token::Ident(id @ ast::Ident{ - name: ast::Name(token::SELF_KEYWORD_NAME_NUM), - ctxt: _ - } ,false) => { + token::Ident(id @ ast::Ident { + name: ast::Name(token::SELF_KEYWORD_NAME_NUM), + ctxt: _ + }, token::Plain) => { self.bump(); let path = ast_util::ident_to_path(mk_sp(lo, hi), id); ex = ExprPath(path); @@ -4094,14 +4094,14 @@ impl<'a> Parser<'a> { fn is_self_ident(&mut self) -> bool { match self.token { - token::Ident(id, false) => id.name == special_idents::self_.name, + token::Ident(id, token::Plain) => id.name == special_idents::self_.name, _ => false } } fn expect_self_ident(&mut self) -> ast::Ident { match self.token { - token::Ident(id, false) if id.name == special_idents::self_.name => { + token::Ident(id, token::Plain) if id.name == special_idents::self_.name => { self.bump(); id }, diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 6ffe766684d..1a69944bffa 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -98,6 +98,21 @@ pub enum BinOpToken { Shr, } +#[cfg(stage0)] +#[allow(non_uppercase_statics)] +pub const ModName: bool = true; +#[cfg(stage0)] +#[allow(non_uppercase_statics)] +pub const Plain: bool = false; + +#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)] +#[cfg(not(stage0))] +pub enum IdentStyle { + /// `::` follows the identifier with no whitespace in-between. + ModName, + Plain, +} + #[allow(non_camel_case_types)] #[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)] pub enum Token { @@ -149,10 +164,10 @@ pub enum Token { LitBinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */ /* Name components */ - /// An identifier contains an "is_mod_name" boolean, - /// indicating whether :: follows this token with no - /// whitespace in between. + #[cfg(stage0)] Ident(ast::Ident, bool), + #[cfg(not(stage0))] + Ident(ast::Ident, IdentStyle), Underscore, Lifetime(ast::Ident), @@ -252,10 +267,11 @@ impl Token { /// Returns `true` if the token is a path that is not followed by a `::` /// token. + #[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot pub fn is_plain_ident(&self) -> bool { match *self { - Ident(_, false) => true, - _ => false, + Ident(_, Plain) => true, + _ => false, } } @@ -299,18 +315,20 @@ impl Token { } /// Returns `true` if the token is a given keyword, `kw`. + #[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot pub fn is_keyword(&self, kw: keywords::Keyword) -> bool { match *self { - Ident(sid, false) => kw.to_name() == sid.name, - _ => false, + Ident(sid, Plain) => kw.to_name() == sid.name, + _ => false, } } /// Returns `true` if the token is either a special identifier, or a strict /// or reserved keyword. + #[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot pub fn is_any_keyword(&self) -> bool { match *self { - Ident(sid, false) => { + Ident(sid, Plain) => { let n = sid.name; n == SELF_KEYWORD_NAME @@ -324,9 +342,10 @@ impl Token { } /// Returns `true` if the token may not appear as an identifier. + #[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot pub fn is_strict_keyword(&self) -> bool { match *self { - Ident(sid, false) => { + Ident(sid, Plain) => { let n = sid.name; n == SELF_KEYWORD_NAME @@ -335,7 +354,7 @@ impl Token { || STRICT_KEYWORD_START <= n && n <= STRICT_KEYWORD_FINAL }, - Ident(sid, true) => { + Ident(sid, ModName) => { let n = sid.name; n != SELF_KEYWORD_NAME @@ -349,9 +368,10 @@ impl Token { /// Returns `true` if the token is a keyword that has been reserved for /// possible future use. + #[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot pub fn is_reserved_keyword(&self) -> bool { match *self { - Ident(sid, false) => { + Ident(sid, Plain) => { let n = sid.name; RESERVED_KEYWORD_START <= n @@ -382,8 +402,10 @@ pub enum Nonterminal { NtPat( P), NtExpr( P), NtTy( P), - /// See IDENT, above, for meaning of bool in NtIdent: + #[cfg(stage0)] NtIdent(Box, bool), + #[cfg(not(stage0))] + NtIdent(Box, IdentStyle), /// Stuff inside brackets for attributes NtMeta( P), NtPath(Box), @@ -857,6 +879,6 @@ mod test { assert!(Gt.mtwt_eq(&Gt)); let a = str_to_ident("bac"); let a1 = mark_ident(a,92); - assert!(Ident(a,true).mtwt_eq(&Ident(a1,false))); + assert!(Ident(a, ModName).mtwt_eq(&Ident(a1, Plain))); } } -- cgit 1.4.1-3-g733a5 From 665ad9c175f746b78c7eae81432b543d2e16c3c9 Mon Sep 17 00:00:00 2001 From: Brendan Zabarauskas Date: Tue, 28 Oct 2014 11:05:28 +1100 Subject: Move token-to-string functions into print::pprust --- src/libsyntax/ext/tt/macro_parser.rs | 5 +- src/libsyntax/parse/lexer/comments.rs | 4 +- src/libsyntax/parse/parser.rs | 3 +- src/libsyntax/parse/token.rs | 95 ------------------------------ src/libsyntax/print/pprust.rs | 107 ++++++++++++++++++++++++++++++++-- 5 files changed, 108 insertions(+), 106 deletions(-) (limited to 'src/libsyntax/ext') diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 6d30de96a3c..073bebcb3f6 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -87,6 +87,7 @@ use parse::attr::ParserAttr; use parse::parser::{LifetimeAndTypesWithoutColons, Parser}; use parse::token::{Token, Nonterminal}; use parse::token; +use print::pprust; use ptr::P; use std::rc::Rc; @@ -402,7 +403,7 @@ pub fn parse(sess: &ParseSess, nts, next_eis.len()).to_string()); } else if bb_eis.len() == 0u && next_eis.len() == 0u { return Failure(sp, format!("no rules expected the token `{}`", - token::to_string(&tok)).to_string()); + pprust::token_to_string(&tok)).to_string()); } else if next_eis.len() > 0u { /* Now process the next token */ while next_eis.len() > 0u { @@ -449,7 +450,7 @@ pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal { "ident" => match p.token { token::Ident(sn,b) => { p.bump(); token::NtIdent(box sn,b) } _ => { - let token_str = token::to_string(&p.token); + let token_str = pprust::token_to_string(&p.token); p.fatal((format!("expected ident, found {}", token_str.as_slice())).as_slice()) } diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 66b21aed552..3814ecfbe5b 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -15,7 +15,7 @@ use parse::lexer::{is_whitespace, Reader}; use parse::lexer::{StringReader, TokenAndSpan}; use parse::lexer::is_block_doc_comment; use parse::lexer; -use parse::token; +use print::pprust; use std::io; use std::str; @@ -373,7 +373,7 @@ pub fn gather_comments_and_literals(span_diagnostic: &diagnostic::SpanHandler, literals.push(Literal {lit: s.to_string(), pos: sp.lo}); }) } else { - debug!("tok: {}", token::to_string(&tok)); + debug!("tok: {}", pprust::token_to_string(&tok)); } first_read = false; } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 54b1cc2dbad..654de709566 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -78,6 +78,7 @@ use parse::token::InternedString; use parse::token::{keywords, special_idents}; use parse::token; use parse::{new_sub_parser_from_file, ParseSess}; +use print::pprust; use ptr::P; use owned_slice::OwnedSlice; @@ -394,7 +395,7 @@ impl<'a> Parser<'a> { /// Convert a token to a string using self's reader pub fn token_to_string(token: &token::Token) -> String { - token::to_string(token) + pprust::token_to_string(token) } /// Convert the current token to a string using self's reader diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 1a69944bffa..9ed8e4bc3a7 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -431,101 +431,6 @@ impl fmt::Show for Nonterminal { } } -pub fn binop_to_string(o: BinOpToken) -> &'static str { - match o { - Plus => "+", - Minus => "-", - Star => "*", - Slash => "/", - Percent => "%", - Caret => "^", - And => "&", - Or => "|", - Shl => "<<", - Shr => ">>", - } -} - -pub fn to_string(t: &Token) -> String { - match *t { - Eq => "=".into_string(), - Lt => "<".into_string(), - Le => "<=".into_string(), - EqEq => "==".into_string(), - Ne => "!=".into_string(), - Ge => ">=".into_string(), - Gt => ">".into_string(), - Not => "!".into_string(), - Tilde => "~".into_string(), - OrOr => "||".into_string(), - AndAnd => "&&".into_string(), - BinOp(op) => binop_to_string(op).into_string(), - BinOpEq(op) => format!("{}=", binop_to_string(op)), - - /* Structural symbols */ - At => "@".into_string(), - Dot => ".".into_string(), - DotDot => "..".into_string(), - DotDotDot => "...".into_string(), - Comma => ",".into_string(), - Semi => ";".into_string(), - Colon => ":".into_string(), - ModSep => "::".into_string(), - RArrow => "->".into_string(), - LArrow => "<-".into_string(), - FatArrow => "=>".into_string(), - LParen => "(".into_string(), - RParen => ")".into_string(), - LBracket => "[".into_string(), - RBracket => "]".into_string(), - LBrace => "{".into_string(), - RBrace => "}".into_string(), - Pound => "#".into_string(), - Dollar => "$".into_string(), - Question => "?".into_string(), - - /* Literals */ - LitByte(b) => format!("b'{}'", b.as_str()), - LitChar(c) => format!("'{}'", c.as_str()), - LitFloat(c) => c.as_str().into_string(), - LitInteger(c) => c.as_str().into_string(), - LitStr(s) => format!("\"{}\"", s.as_str()), - LitStrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", - delim="#".repeat(n), - string=s.as_str()), - LitBinary(v) => format!("b\"{}\"", v.as_str()), - LitBinaryRaw(s, n) => format!("br{delim}\"{string}\"{delim}", - delim="#".repeat(n), - string=s.as_str()), - - /* Name components */ - Ident(s, _) => get_ident(s).get().into_string(), - Lifetime(s) => format!("{}", get_ident(s)), - Underscore => "_".into_string(), - - /* Other */ - DocComment(s) => s.as_str().into_string(), - Eof => "".into_string(), - Whitespace => " ".into_string(), - Comment => "/* */".into_string(), - Shebang(s) => format!("/* shebang: {}*/", s.as_str()), - - Interpolated(ref nt) => match *nt { - NtExpr(ref e) => ::print::pprust::expr_to_string(&**e), - NtMeta(ref e) => ::print::pprust::meta_item_to_string(&**e), - NtTy(ref e) => ::print::pprust::ty_to_string(&**e), - NtPath(ref e) => ::print::pprust::path_to_string(&**e), - NtItem(..) => "an interpolated item".into_string(), - NtBlock(..) => "an interpolated block".into_string(), - NtStmt(..) => "an interpolated statement".into_string(), - NtPat(..) => "an interpolated pattern".into_string(), - NtIdent(..) => "an interpolated identifier".into_string(), - NtTT(..) => "an interpolated tt".into_string(), - NtMatchers(..) => "an interpolated matcher sequence".into_string(), - } - } -} - // Get the first "argument" macro_rules! first { ( $first:expr, $( $remainder:expr, )* ) => ( $first ) diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 25ef8700ed0..a52987f5bd1 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -21,6 +21,7 @@ use attr::{AttrMetaMethods, AttributeMethods}; use codemap::{CodeMap, BytePos}; use codemap; use diagnostic; +use parse::token::{BinOpToken, Token}; use parse::token; use parse::lexer::comments; use parse; @@ -181,6 +182,101 @@ pub fn to_string(f: |&mut State| -> IoResult<()>) -> String { } } +pub fn binop_to_string(op: BinOpToken) -> &'static str { + match op { + token::Plus => "+", + token::Minus => "-", + token::Star => "*", + token::Slash => "/", + token::Percent => "%", + token::Caret => "^", + token::And => "&", + token::Or => "|", + token::Shl => "<<", + token::Shr => ">>", + } +} + +pub fn token_to_string(tok: &Token) -> String { + match *tok { + token::Eq => "=".into_string(), + token::Lt => "<".into_string(), + token::Le => "<=".into_string(), + token::EqEq => "==".into_string(), + token::Ne => "!=".into_string(), + token::Ge => ">=".into_string(), + token::Gt => ">".into_string(), + token::Not => "!".into_string(), + token::Tilde => "~".into_string(), + token::OrOr => "||".into_string(), + token::AndAnd => "&&".into_string(), + token::BinOp(op) => binop_to_string(op).into_string(), + token::BinOpEq(op) => format!("{}=", binop_to_string(op)), + + /* Structural symbols */ + token::At => "@".into_string(), + token::Dot => ".".into_string(), + token::DotDot => "..".into_string(), + token::DotDotDot => "...".into_string(), + token::Comma => ",".into_string(), + token::Semi => ";".into_string(), + token::Colon => ":".into_string(), + token::ModSep => "::".into_string(), + token::RArrow => "->".into_string(), + token::LArrow => "<-".into_string(), + token::FatArrow => "=>".into_string(), + token::LParen => "(".into_string(), + token::RParen => ")".into_string(), + token::LBracket => "[".into_string(), + token::RBracket => "]".into_string(), + token::LBrace => "{".into_string(), + token::RBrace => "}".into_string(), + token::Pound => "#".into_string(), + token::Dollar => "$".into_string(), + token::Question => "?".into_string(), + + /* Literals */ + token::LitByte(b) => format!("b'{}'", b.as_str()), + token::LitChar(c) => format!("'{}'", c.as_str()), + token::LitFloat(c) => c.as_str().into_string(), + token::LitInteger(c) => c.as_str().into_string(), + token::LitStr(s) => format!("\"{}\"", s.as_str()), + token::LitStrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", + delim="#".repeat(n), + string=s.as_str()), + token::LitBinary(v) => format!("b\"{}\"", v.as_str()), + token::LitBinaryRaw(s, n) => format!("br{delim}\"{string}\"{delim}", + delim="#".repeat(n), + string=s.as_str()), + + /* Name components */ + token::Ident(s, _) => token::get_ident(s).get().into_string(), + token::Lifetime(s) => format!("{}", token::get_ident(s)), + token::Underscore => "_".into_string(), + + /* Other */ + token::DocComment(s) => s.as_str().into_string(), + token::Eof => "".into_string(), + token::Whitespace => " ".into_string(), + token::Comment => "/* */".into_string(), + token::Shebang(s) => format!("/* shebang: {}*/", s.as_str()), + + token::Interpolated(ref nt) => match *nt { + token::NtExpr(ref e) => expr_to_string(&**e), + token::NtMeta(ref e) => meta_item_to_string(&**e), + token::NtTy(ref e) => ty_to_string(&**e), + token::NtPath(ref e) => path_to_string(&**e), + token::NtItem(..) => "an interpolated item".into_string(), + token::NtBlock(..) => "an interpolated block".into_string(), + token::NtStmt(..) => "an interpolated statement".into_string(), + token::NtPat(..) => "an interpolated pattern".into_string(), + token::NtIdent(..) => "an interpolated identifier".into_string(), + token::NtTT(..) => "an interpolated tt".into_string(), + token::NtMatchers(..) => "an interpolated matcher sequence".into_string(), + } + } +} + // FIXME (Issue #16472): the thing_to_string_impls macro should go away // after we revise the syntax::ext::quote::ToToken impls to go directly // to token-trees instead of thing -> string -> token-trees. @@ -1026,14 +1122,14 @@ impl<'a> State<'a> { match *tt { ast::TtDelimited(_, ref delimed) => { let (ref open, ref tts, ref close) = **delimed; - try!(word(&mut self.s, parse::token::to_string(&open.token).as_slice())); + try!(word(&mut self.s, token_to_string(&open.token).as_slice())); try!(space(&mut self.s)); try!(self.print_tts(tts.as_slice())); try!(space(&mut self.s)); - word(&mut self.s, parse::token::to_string(&close.token).as_slice()) + word(&mut self.s, token_to_string(&close.token).as_slice()) }, ast::TtToken(_, ref tk) => { - try!(word(&mut self.s, parse::token::to_string(tk).as_slice())); + try!(word(&mut self.s, token_to_string(tk).as_slice())); match *tk { parse::token::DocComment(..) => { hardbreak(&mut self.s) @@ -1049,10 +1145,9 @@ impl<'a> State<'a> { try!(word(&mut self.s, ")")); match *separator { Some(ref tk) => { - try!(word(&mut self.s, - parse::token::to_string(tk).as_slice())); + try!(word(&mut self.s, token_to_string(tk).as_slice())); } - None => () + None => {}, } match kleene_op { ast::ZeroOrMore => word(&mut self.s, "*"), -- cgit 1.4.1-3-g733a5