diff options
Diffstat (limited to 'src/libsyntax/parse/token.rs')
| -rw-r--r-- | src/libsyntax/parse/token.rs | 358 |
1 files changed, 110 insertions, 248 deletions
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 16417ac0044..fcb6c3539db 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -11,7 +11,6 @@ pub use self::BinOpToken::*; pub use self::Nonterminal::*; pub use self::DelimToken::*; -pub use self::IdentStyle::*; pub use self::Lit::*; pub use self::Token::*; @@ -26,7 +25,6 @@ use std::fmt; use std::ops::Deref; use std::rc::Rc; -#[allow(non_camel_case_types)] #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] pub enum BinOpToken { Plus, @@ -53,13 +51,6 @@ pub enum DelimToken { } #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] -pub enum IdentStyle { - /// `::` follows the identifier with no whitespace in-between. - ModName, - Plain, -} - -#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)] pub enum SpecialMacroVar { /// `$crate` will be filled in with the name of the crate a macro was /// imported from, if any. @@ -99,7 +90,6 @@ impl Lit { } } -#[allow(non_camel_case_types)] #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)] pub enum Token { /* Expression-operator symbols. */ @@ -141,7 +131,7 @@ pub enum Token { Literal(Lit, Option<ast::Name>), /* Name components */ - Ident(ast::Ident, IdentStyle), + Ident(ast::Ident), Underscore, Lifetime(ast::Ident), @@ -151,11 +141,11 @@ pub enum Token { /// Doc comment DocComment(ast::Name), // In left-hand-sides of MBE macros: - /// Parse a nonterminal (name to bind, name of NT, styles of their idents) - MatchNt(ast::Ident, ast::Ident, IdentStyle, IdentStyle), + /// Parse a nonterminal (name to bind, name of NT) + MatchNt(ast::Ident, ast::Ident), // In right-hand-sides of MBE macros: /// A syntactic variable that will be filled in by macro expansion. - SubstNt(ast::Ident, IdentStyle), + SubstNt(ast::Ident), /// A macro variable with special meaning. SpecialVarNt(SpecialMacroVar), @@ -185,7 +175,7 @@ impl Token { pub fn can_begin_expr(&self) -> bool { match *self { OpenDelim(_) => true, - Ident(_, _) => true, + Ident(..) => true, Underscore => true, Tilde => true, Literal(_, _) => true, @@ -218,7 +208,7 @@ impl Token { /// Returns `true` if the token is an identifier. pub fn is_ident(&self) -> bool { match *self { - Ident(_, _) => true, + Ident(..) => true, _ => false, } } @@ -239,16 +229,6 @@ impl Token { } } - /// Returns `true` if the token is a path that is not followed by a `::` - /// token. - #[allow(non_upper_case_globals)] - pub fn is_plain_ident(&self) -> bool { - match *self { - Ident(_, Plain) => true, - _ => false, - } - } - /// Returns `true` if the token is a lifetime. pub fn is_lifetime(&self) -> bool { match *self { @@ -263,6 +243,11 @@ impl Token { self.is_keyword(keywords::Const) } + pub fn is_path_start(&self) -> bool { + self == &ModSep || self == &Lt || self.is_path() || + self.is_path_segment_keyword() || self.is_ident() && !self.is_any_keyword() + } + /// Maps a token to its corresponding binary operator. pub fn to_binop(&self) -> Option<BinOpKind> { match *self { @@ -289,77 +274,41 @@ impl Token { } /// Returns `true` if the token is a given keyword, `kw`. - #[allow(non_upper_case_globals)] pub fn is_keyword(&self, kw: keywords::Keyword) -> bool { match *self { - Ident(sid, Plain) => kw.to_name() == sid.name, - _ => false, + Ident(id) => id.name == kw.name(), + _ => false, } } - pub fn is_keyword_allow_following_colon(&self, kw: keywords::Keyword) -> bool { + pub fn is_path_segment_keyword(&self) -> bool { match *self { - Ident(sid, _) => { kw.to_name() == sid.name } - _ => { false } + Ident(id) => id.name == keywords::Super.name() || + id.name == keywords::SelfValue.name() || + id.name == keywords::SelfType.name(), + _ => false, } } - /// Returns `true` if the token is either a special identifier, or a strict - /// or reserved keyword. - #[allow(non_upper_case_globals)] + /// Returns `true` if the token is either a strict or reserved keyword. pub fn is_any_keyword(&self) -> bool { - match *self { - Ident(sid, Plain) => { - let n = sid.name; - - n == SELF_KEYWORD_NAME - || n == STATIC_KEYWORD_NAME - || n == SUPER_KEYWORD_NAME - || n == SELF_TYPE_KEYWORD_NAME - || STRICT_KEYWORD_START <= n - && n <= RESERVED_KEYWORD_FINAL - }, - _ => false - } + self.is_strict_keyword() || self.is_reserved_keyword() } - /// Returns `true` if the token may not appear as an identifier. - #[allow(non_upper_case_globals)] + /// Returns `true` if the token is a strict keyword. pub fn is_strict_keyword(&self) -> bool { match *self { - Ident(sid, Plain) => { - let n = sid.name; - - n == SELF_KEYWORD_NAME - || n == STATIC_KEYWORD_NAME - || n == SUPER_KEYWORD_NAME - || n == SELF_TYPE_KEYWORD_NAME - || STRICT_KEYWORD_START <= n - && n <= STRICT_KEYWORD_FINAL - }, - Ident(sid, ModName) => { - let n = sid.name; - - n != SELF_KEYWORD_NAME - && n != SUPER_KEYWORD_NAME - && STRICT_KEYWORD_START <= n - && n <= STRICT_KEYWORD_FINAL - } + Ident(id) => id.name >= keywords::As.name() && + id.name <= keywords::While.name(), _ => false, } } - /// Returns `true` if the token is a keyword that has been reserved for - /// possible future use. - #[allow(non_upper_case_globals)] + /// Returns `true` if the token is a keyword reserved for possible future use. pub fn is_reserved_keyword(&self) -> bool { match *self { - Ident(sid, Plain) => { - let n = sid.name; - - RESERVED_KEYWORD_START <= n - && n <= RESERVED_KEYWORD_FINAL - }, + Ident(id) => id.name >= keywords::Abstract.name() && + id.name <= keywords::Yield.name(), _ => false, } } @@ -369,7 +318,7 @@ impl Token { /// See `styntax::ext::mtwt`. pub fn mtwt_eq(&self, other : &Token) -> bool { match (self, other) { - (&Ident(id1,_), &Ident(id2,_)) | (&Lifetime(id1), &Lifetime(id2)) => + (&Ident(id1), &Ident(id2)) | (&Lifetime(id1), &Lifetime(id2)) => mtwt::resolve(id1) == mtwt::resolve(id2), _ => *self == *other } @@ -385,7 +334,7 @@ pub enum Nonterminal { NtPat(P<ast::Pat>), NtExpr(P<ast::Expr>), NtTy(P<ast::Ty>), - NtIdent(Box<ast::SpannedIdent>, IdentStyle), + NtIdent(Box<ast::SpannedIdent>), /// Stuff inside brackets for attributes NtMeta(P<ast::MetaItem>), NtPath(Box<ast::Path>), @@ -422,191 +371,104 @@ impl fmt::Debug for Nonterminal { } } - -// Get the first "argument" -macro_rules! first { - ( $first:expr, $( $remainder:expr, )* ) => ( $first ) -} - -// Get the last "argument" (has to be done recursively to avoid phoney local ambiguity error) -macro_rules! last { - ( $first:expr, $( $remainder:expr, )+ ) => ( last!( $( $remainder, )+ ) ); - ( $first:expr, ) => ( $first ) -} - // In this macro, there is the requirement that the name (the number) must be monotonically // increasing by one in the special identifiers, starting at 0; the same holds for the keywords, -// except starting from the next number instead of zero, and with the additional exception that -// special identifiers are *also* allowed (they are deduplicated in the important place, the -// interner), an exception which is demonstrated by "static" and "self". -macro_rules! declare_special_idents_and_keywords {( - // So now, in these rules, why is each definition parenthesised? - // Answer: otherwise we get a spurious local ambiguity bug on the "}" - pub mod special_idents { - $( ($si_name:expr, $si_static:ident, $si_str:expr); )* - } - - pub mod keywords { - 'strict: - $( ($sk_name:expr, $sk_variant:ident, $sk_str:expr); )* - 'reserved: - $( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )* - } +// except starting from the next number instead of zero. +macro_rules! declare_keywords {( + $( ($index: expr, $konst: ident, $string: expr) )* ) => { - const STRICT_KEYWORD_START: ast::Name = first!($( ast::Name($sk_name), )*); - const STRICT_KEYWORD_FINAL: ast::Name = last!($( ast::Name($sk_name), )*); - const RESERVED_KEYWORD_START: ast::Name = first!($( ast::Name($rk_name), )*); - const RESERVED_KEYWORD_FINAL: ast::Name = last!($( ast::Name($rk_name), )*); - - pub mod special_idents { - use ast; - $( - #[allow(non_upper_case_globals)] - pub const $si_static: ast::Ident = ast::Ident { - name: ast::Name($si_name), - ctxt: ast::EMPTY_CTXT, - }; - )* - } - - pub mod special_names { - use ast; - $( - #[allow(non_upper_case_globals)] - pub const $si_static: ast::Name = ast::Name($si_name); - )* - } - - /// All the valid words that have meaning in the Rust language. - /// - /// Rust keywords are either 'strict' or 'reserved'. Strict keywords may not - /// appear as identifiers at all. Reserved keywords are not used anywhere in - /// the language and may not appear as identifiers. pub mod keywords { - pub use self::Keyword::*; use ast; - - #[derive(Copy, Clone, PartialEq, Eq)] - pub enum Keyword { - $( $sk_variant, )* - $( $rk_variant, )* + #[derive(Clone, Copy, PartialEq, Eq)] + pub struct Keyword { + ident: ast::Ident, } - impl Keyword { - pub fn to_name(&self) -> ast::Name { - match *self { - $( $sk_variant => ast::Name($sk_name), )* - $( $rk_variant => ast::Name($rk_name), )* - } - } + #[inline] pub fn ident(self) -> ast::Ident { self.ident } + #[inline] pub fn name(self) -> ast::Name { self.ident.name } } + $( + #[allow(non_upper_case_globals)] + pub const $konst: Keyword = Keyword { + ident: ast::Ident::with_empty_ctxt(ast::Name($index)) + }; + )* } fn mk_fresh_ident_interner() -> IdentInterner { - let mut init_vec = Vec::new(); - $(init_vec.push($si_str);)* - $(init_vec.push($sk_str);)* - $(init_vec.push($rk_str);)* - interner::StrInterner::prefill(&init_vec[..]) + interner::StrInterner::prefill(&[$($string,)*]) } }} -// If the special idents get renumbered, remember to modify these two as appropriate -pub const SELF_KEYWORD_NAME: ast::Name = ast::Name(SELF_KEYWORD_NAME_NUM); -const STATIC_KEYWORD_NAME: ast::Name = ast::Name(STATIC_KEYWORD_NAME_NUM); -pub const SUPER_KEYWORD_NAME: ast::Name = ast::Name(SUPER_KEYWORD_NAME_NUM); -const SELF_TYPE_KEYWORD_NAME: ast::Name = ast::Name(SELF_TYPE_KEYWORD_NAME_NUM); - -pub const SELF_KEYWORD_NAME_NUM: u32 = 1; -const STATIC_KEYWORD_NAME_NUM: u32 = 2; -const SUPER_KEYWORD_NAME_NUM: u32 = 3; -const SELF_TYPE_KEYWORD_NAME_NUM: u32 = 10; - // NB: leaving holes in the ident table is bad! a different ident will get // interned with the id from the hole, but it will be between the min and max // of the reserved words, and thus tagged as "reserved". - -declare_special_idents_and_keywords! { - pub mod special_idents { - // These ones are statics - (0, invalid, ""); - (super::SELF_KEYWORD_NAME_NUM, self_, "self"); - (super::STATIC_KEYWORD_NAME_NUM, statik, "static"); - (super::SUPER_KEYWORD_NAME_NUM, super_, "super"); - (4, static_lifetime, "'static"); - - // for matcher NTs - (5, tt, "tt"); - (6, matchers, "matchers"); - - // outside of libsyntax - (7, clownshoe_abi, "__rust_abi"); - (8, opaque, "<opaque>"); - (9, __unused1, "<__unused1>"); - (super::SELF_TYPE_KEYWORD_NAME_NUM, type_self, "Self"); - (11, prelude_import, "prelude_import"); - (12, DEFAULT, "default"); - } - - pub mod keywords { - // These ones are variants of the Keyword enum - - 'strict: - (13, As, "as"); - (14, Break, "break"); - (15, Crate, "crate"); - (16, Else, "else"); - (17, Enum, "enum"); - (18, Extern, "extern"); - (19, False, "false"); - (20, Fn, "fn"); - (21, For, "for"); - (22, If, "if"); - (23, Impl, "impl"); - (24, In, "in"); - (25, Let, "let"); - (26, Loop, "loop"); - (27, Match, "match"); - (28, Mod, "mod"); - (29, Move, "move"); - (30, Mut, "mut"); - (31, Pub, "pub"); - (32, Ref, "ref"); - (33, Return, "return"); - // Static and Self are also special idents (prefill de-dupes) - (super::STATIC_KEYWORD_NAME_NUM, Static, "static"); - (super::SELF_KEYWORD_NAME_NUM, SelfValue, "self"); - (super::SELF_TYPE_KEYWORD_NAME_NUM, SelfType, "Self"); - (34, Struct, "struct"); - (super::SUPER_KEYWORD_NAME_NUM, Super, "super"); - (35, True, "true"); - (36, Trait, "trait"); - (37, Type, "type"); - (38, Unsafe, "unsafe"); - (39, Use, "use"); - (40, While, "while"); - (41, Continue, "continue"); - (42, Box, "box"); - (43, Const, "const"); - (44, Where, "where"); - 'reserved: - (45, Virtual, "virtual"); - (46, Proc, "proc"); - (47, Alignof, "alignof"); - (48, Become, "become"); - (49, Offsetof, "offsetof"); - (50, Priv, "priv"); - (51, Pure, "pure"); - (52, Sizeof, "sizeof"); - (53, Typeof, "typeof"); - (54, Unsized, "unsized"); - (55, Yield, "yield"); - (56, Do, "do"); - (57, Abstract, "abstract"); - (58, Final, "final"); - (59, Override, "override"); - (60, Macro, "macro"); - } +// After modifying this list adjust `is_strict_keyword`/`is_reserved_keyword`, +// this should be rarely necessary though if the keywords are kept in alphabetic order. +declare_keywords! { + // Invalid identifier + (0, Invalid, "") + + // Strict keywords used in the language. + (1, As, "as") + (2, Box, "box") + (3, Break, "break") + (4, Const, "const") + (5, Continue, "continue") + (6, Crate, "crate") + (7, Else, "else") + (8, Enum, "enum") + (9, Extern, "extern") + (10, False, "false") + (11, Fn, "fn") + (12, For, "for") + (13, If, "if") + (14, Impl, "impl") + (15, In, "in") + (16, Let, "let") + (17, Loop, "loop") + (18, Match, "match") + (19, Mod, "mod") + (20, Move, "move") + (21, Mut, "mut") + (22, Pub, "pub") + (23, Ref, "ref") + (24, Return, "return") + (25, SelfValue, "self") + (26, SelfType, "Self") + (27, Static, "static") + (28, Struct, "struct") + (29, Super, "super") + (30, Trait, "trait") + (31, True, "true") + (32, Type, "type") + (33, Unsafe, "unsafe") + (34, Use, "use") + (35, Where, "where") + (36, While, "while") + + // Keywords reserved for future use. + (37, Abstract, "abstract") + (38, Alignof, "alignof") + (39, Become, "become") + (40, Do, "do") + (41, Final, "final") + (42, Macro, "macro") + (43, Offsetof, "offsetof") + (44, Override, "override") + (45, Priv, "priv") + (46, Proc, "proc") + (47, Pure, "pure") + (48, Sizeof, "sizeof") + (49, Typeof, "typeof") + (50, Unsized, "unsized") + (51, Virtual, "virtual") + (52, Yield, "yield") + + // Weak keywords, have special meaning only in specific contexts. + (53, Default, "default") + (54, StaticLifetime, "'static") + (55, Union, "union") } // looks like we can get rid of this completely... @@ -779,6 +641,6 @@ mod tests { assert!(Gt.mtwt_eq(&Gt)); let a = str_to_ident("bac"); let a1 = mark_ident(a,92); - assert!(Ident(a, ModName).mtwt_eq(&Ident(a1, Plain))); + assert!(Ident(a).mtwt_eq(&Ident(a1))); } } |