From 9d6768a478b8a6afa1e16dfebe9d79bd3f508cdf Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 12:46:32 +0200 Subject: syntax::parser::token -> syntax::token --- src/libsyntax/ast.rs | 2 +- src/libsyntax/attr/mod.rs | 2 +- src/libsyntax/feature_gate/check.rs | 2 +- src/libsyntax/lib.rs | 1 + src/libsyntax/mut_visit.rs | 2 +- src/libsyntax/parse/lexer/mod.rs | 2 +- src/libsyntax/parse/lexer/tests.rs | 2 +- src/libsyntax/parse/lexer/tokentrees.rs | 2 +- src/libsyntax/parse/lexer/unicode_chars.rs | 2 +- src/libsyntax/parse/literal.rs | 2 +- src/libsyntax/parse/mod.rs | 3 +- src/libsyntax/parse/parser/attr.rs | 2 +- src/libsyntax/parse/parser/diagnostics.rs | 2 +- src/libsyntax/parse/parser/expr.rs | 2 +- src/libsyntax/parse/parser/generics.rs | 2 +- src/libsyntax/parse/parser/mod.rs | 2 +- src/libsyntax/parse/parser/module.rs | 2 +- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/parser/path.rs | 2 +- src/libsyntax/parse/parser/stmt.rs | 2 +- src/libsyntax/parse/parser/ty.rs | 2 +- src/libsyntax/parse/tests.rs | 2 +- src/libsyntax/parse/token.rs | 728 ----------------------------- src/libsyntax/print/pprust.rs | 2 +- src/libsyntax/token.rs | 728 +++++++++++++++++++++++++++++ src/libsyntax/tokenstream.rs | 2 +- src/libsyntax/util/parser.rs | 2 +- src/libsyntax/visit.rs | 2 +- 28 files changed, 754 insertions(+), 754 deletions(-) delete mode 100644 src/libsyntax/parse/token.rs create mode 100644 src/libsyntax/token.rs (limited to 'src/libsyntax') diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 67d1acbccfb..18151a1586c 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -24,9 +24,9 @@ pub use crate::util::parser::ExprPrecedence; pub use syntax_pos::symbol::{Ident, Symbol as Name}; -use crate::parse::token::{self, DelimToken}; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; +use crate::token::{self, DelimToken}; use crate::tokenstream::TokenStream; use syntax_pos::symbol::{kw, sym, Symbol}; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index c663995eb8f..1534d2723c8 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -17,7 +17,7 @@ use crate::source_map::{BytePos, Spanned}; use crate::parse::lexer::comments::doc_comment_style; use crate::parse; use crate::parse::PResult; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::ptr::P; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs index 5b1493ebc9b..ecff89ad59b 100644 --- a/src/libsyntax/feature_gate/check.rs +++ b/src/libsyntax/feature_gate/check.rs @@ -11,7 +11,7 @@ use crate::attr::{self, check_builtin_attribute}; use crate::source_map::Spanned; use crate::edition::{ALL_EDITIONS, Edition}; use crate::visit::{self, FnKind, Visitor}; -use crate::parse::token; +use crate::token; use crate::sess::ParseSess; use crate::symbol::{Symbol, sym}; use crate::tokenstream::TokenTree; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 7be6e6c7e18..4c134430f30 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -107,6 +107,7 @@ pub mod show_span; pub use syntax_pos::edition; pub use syntax_pos::symbol; pub mod sess; +pub mod token; pub mod tokenstream; pub mod visit; diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 7261601e144..0c90652526d 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -9,7 +9,7 @@ use crate::ast::*; use crate::source_map::{Spanned, respan}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::ptr::P; use crate::ThinVec; use crate::tokenstream::*; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index a6dfed542d6..5499a3cae5f 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, TokenKind}; +use crate::token::{self, Token, TokenKind}; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs index de301b1fc49..b9b82b2bcef 100644 --- a/src/libsyntax/parse/lexer/tests.rs +++ b/src/libsyntax/parse/lexer/tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::symbol::Symbol; use crate::source_map::{SourceMap, FilePathMapping}; -use crate::parse::token; +use crate::token; use crate::with_default_globals; use errors::{Handler, emitter::EmitterWriter}; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index de8ac2c71e8..3c15a5e1dae 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -4,8 +4,8 @@ use syntax_pos::Span; use super::{StringReader, UnmatchedBrace}; use crate::print::pprust::token_to_string; -use crate::parse::token::{self, Token}; use crate::parse::PResult; +use crate::token::{self, Token}; use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint}; impl<'a> StringReader<'a> { diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index 525b4215aff..6eb995b61d3 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -4,7 +4,7 @@ use super::StringReader; use errors::{Applicability, DiagnosticBuilder}; use syntax_pos::{BytePos, Pos, Span, symbol::kw}; -use crate::parse::token; +use crate::token; #[rustfmt::skip] // for line breaks const UNICODE_ARRAY: &[(char, &str, char)] = &[ diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index a8eeac59954..d4c9b7850c5 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -1,8 +1,8 @@ //! Code related to parsing literals. use crate::ast::{self, Lit, LitKind}; -use crate::parse::token::{self, Token}; use crate::symbol::{kw, sym, Symbol}; +use crate::token::{self, Token}; use crate::tokenstream::TokenTree; use log::debug; diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 5b020d3d288..7db0ca0b78d 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -2,7 +2,7 @@ use crate::ast; use crate::parse::parser::{Parser, emit_unclosed_delims, make_unclosed_delims_error}; -use crate::parse::token::Nonterminal; +use crate::token::{self, Nonterminal}; use crate::tokenstream::{self, TokenStream, TokenTree}; use crate::print::pprust; use crate::sess::ParseSess; @@ -25,7 +25,6 @@ mod tests; #[macro_use] pub mod parser; pub mod lexer; -pub mod token; crate mod classify; crate mod literal; diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 1c292661f24..aa0542a09a9 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,7 +1,7 @@ use super::{SeqSep, PResult, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; -use crate::parse::token::{self, Nonterminal, DelimToken}; +use crate::token::{self, Nonterminal, DelimToken}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; diff --git a/src/libsyntax/parse/parser/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index 49a517a5c44..09f0b9b74b2 100644 --- a/src/libsyntax/parse/parser/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -6,7 +6,7 @@ use crate::ast::{ self, Param, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, }; -use crate::parse::token::{self, TokenKind, token_can_begin_expr}; +use crate::token::{self, TokenKind, token_can_begin_expr}; use crate::print::pprust; use crate::ptr::P; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index 509e6482dcc..4b962201e85 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -12,7 +12,7 @@ use crate::ast::{ }; use crate::maybe_recover_from_interpolated_ty_qpath; use crate::parse::classify; -use crate::parse::token::{self, Token, TokenKind}; +use crate::token::{self, Token, TokenKind}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, Span}; diff --git a/src/libsyntax/parse/parser/generics.rs b/src/libsyntax/parse/parser/generics.rs index 3c094750b4d..cfc3768cac5 100644 --- a/src/libsyntax/parse/parser/generics.rs +++ b/src/libsyntax/parse/parser/generics.rs @@ -1,7 +1,7 @@ use super::{Parser, PResult}; use crate::ast::{self, WhereClause, GenericParam, GenericParamKind, GenericBounds, Attribute}; -use crate::parse::token; +use crate::token; use crate::source_map::DUMMY_SP; use syntax_pos::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/mod.rs b/src/libsyntax/parse/parser/mod.rs index 1284e89f195..6498b674a3b 100644 --- a/src/libsyntax/parse/parser/mod.rs +++ b/src/libsyntax/parse/parser/mod.rs @@ -18,7 +18,7 @@ use crate::ast::{ use crate::parse::{PResult, Directory, DirectoryOwnership}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use crate::parse::token::{self, Token, TokenKind, DelimToken}; +use crate::token::{self, Token, TokenKind, DelimToken}; use crate::print::pprust; use crate::ptr::P; use crate::sess::ParseSess; diff --git a/src/libsyntax/parse/parser/module.rs b/src/libsyntax/parse/parser/module.rs index 242a17659a0..70e0dfc3e88 100644 --- a/src/libsyntax/parse/parser/module.rs +++ b/src/libsyntax/parse/parser/module.rs @@ -5,7 +5,7 @@ use super::diagnostics::Error; use crate::attr; use crate::ast::{self, Ident, Attribute, ItemKind, Mod, Crate}; use crate::parse::{new_sub_parser_from_file, DirectoryOwnership}; -use crate::parse::token::{self, TokenKind}; +use crate::token::{self, TokenKind}; use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName}; use crate::symbol::sym; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index fcb7c4d1323..c4a983188fa 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -5,7 +5,7 @@ use crate::ptr::P; use crate::ast::{self, Attribute, Pat, PatKind, FieldPat, RangeEnd, RangeSyntax, Mac}; use crate::ast::{BindingMode, Ident, Mutability, Path, QSelf, Expr, ExprKind}; use crate::mut_visit::{noop_visit_pat, noop_visit_mac, MutVisitor}; -use crate::parse::token; +use crate::token; use crate::print::pprust; use crate::source_map::{respan, Span, Spanned}; use crate::ThinVec; diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index 4438d61d9ee..2659e0c680e 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -3,7 +3,7 @@ use super::{Parser, PResult, TokenType}; use crate::{maybe_whole, ThinVec}; use crate::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs}; use crate::ast::{AnonConst, GenericArg, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::source_map::{Span, BytePos}; use syntax_pos::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index 12c530f3cbb..010fd9c37fd 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -9,7 +9,7 @@ use crate::{maybe_whole, ThinVec}; use crate::ast::{self, DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind}; use crate::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter}; use crate::parse::{classify, DirectoryOwnership}; -use crate::parse::token; +use crate::token; use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index b770b90705c..4183416e628 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -6,7 +6,7 @@ use crate::ptr::P; use crate::ast::{self, Ty, TyKind, MutTy, BareFnTy, FunctionRetTy, GenericParam, Lifetime, Ident}; use crate::ast::{TraitBoundModifier, TraitObjectSyntax, GenericBound, GenericBounds, PolyTraitRef}; use crate::ast::{Mutability, AnonConst, Mac}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::source_map::Span; use crate::symbol::{kw}; diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs index 201eaca947b..27ca2b6472f 100644 --- a/src/libsyntax/parse/tests.rs +++ b/src/libsyntax/parse/tests.rs @@ -4,7 +4,7 @@ use crate::ast::{self, Name, PatKind}; use crate::attr::first_attr_value_str_by_name; use crate::sess::ParseSess; use crate::parse::{PResult, new_parser_from_source_str}; -use crate::parse::token::Token; +use crate::token::Token; use crate::print::pprust::item_to_string; use crate::ptr::P; use crate::source_map::FilePathMapping; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs deleted file mode 100644 index 6f3da344ccf..00000000000 --- a/src/libsyntax/parse/token.rs +++ /dev/null @@ -1,728 +0,0 @@ -pub use BinOpToken::*; -pub use Nonterminal::*; -pub use DelimToken::*; -pub use LitKind::*; -pub use TokenKind::*; - -use crate::ast; -use crate::ptr::P; -use crate::symbol::kw; -use crate::tokenstream::TokenTree; - -use syntax_pos::symbol::Symbol; -use syntax_pos::{self, Span, DUMMY_SP}; - -use std::fmt; -use std::mem; -#[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert_size; -use rustc_data_structures::sync::Lrc; - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum BinOpToken { - Plus, - Minus, - Star, - Slash, - Percent, - Caret, - And, - Or, - Shl, - Shr, -} - -/// A delimiter token. -#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum DelimToken { - /// A round parenthesis (i.e., `(` or `)`). - Paren, - /// A square bracket (i.e., `[` or `]`). - Bracket, - /// A curly brace (i.e., `{` or `}`). - Brace, - /// An empty delimiter. - NoDelim, -} - -impl DelimToken { - pub fn len(self) -> usize { - if self == NoDelim { 0 } else { 1 } - } - - pub fn is_empty(self) -> bool { - self == NoDelim - } -} - -#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub enum LitKind { - Bool, // AST only, must never appear in a `Token` - Byte, - Char, - Integer, - Float, - Str, - StrRaw(u16), // raw string delimited by `n` hash symbols - ByteStr, - ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols - Err, -} - -/// A literal token. -#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Lit { - pub kind: LitKind, - pub symbol: Symbol, - pub suffix: Option, -} - -impl fmt::Display for Lit { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Lit { kind, symbol, suffix } = *self; - match kind { - Byte => write!(f, "b'{}'", symbol)?, - Char => write!(f, "'{}'", symbol)?, - Str => write!(f, "\"{}\"", symbol)?, - StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=symbol)?, - ByteStr => write!(f, "b\"{}\"", symbol)?, - ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=symbol)?, - Integer | - Float | - Bool | - Err => write!(f, "{}", symbol)?, - } - - if let Some(suffix) = suffix { - write!(f, "{}", suffix)?; - } - - Ok(()) - } -} - -impl LitKind { - /// An English article for the literal token kind. - crate fn article(self) -> &'static str { - match self { - Integer | Err => "an", - _ => "a", - } - } - - crate fn descr(self) -> &'static str { - match self { - Bool => panic!("literal token contains `Lit::Bool`"), - Byte => "byte", - Char => "char", - Integer => "integer", - Float => "float", - Str | StrRaw(..) => "string", - ByteStr | ByteStrRaw(..) => "byte string", - Err => "error", - } - } - - crate fn may_have_suffix(self) -> bool { - match self { - Integer | Float | Err => true, - _ => false, - } - } -} - -impl Lit { - pub fn new(kind: LitKind, symbol: Symbol, suffix: Option) -> Lit { - Lit { kind, symbol, suffix } - } -} - -pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { - let ident_token = Token::new(Ident(name, is_raw), span); - token_can_begin_expr(&ident_token) -} - -pub(crate) fn token_can_begin_expr(ident_token: &Token) -> bool { - !ident_token.is_reserved_ident() || - ident_token.is_path_segment_keyword() || - match ident_token.kind { - TokenKind::Ident(ident, _) => [ - kw::Async, - kw::Do, - kw::Box, - kw::Break, - kw::Continue, - kw::False, - kw::For, - kw::If, - kw::Let, - kw::Loop, - kw::Match, - kw::Move, - kw::Return, - kw::True, - kw::Unsafe, - kw::While, - kw::Yield, - kw::Static, - ].contains(&ident), - _=> false, - } -} - -fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { - let ident_token = Token::new(Ident(name, is_raw), span); - - !ident_token.is_reserved_ident() || - ident_token.is_path_segment_keyword() || - [ - kw::Underscore, - kw::For, - kw::Impl, - kw::Fn, - kw::Unsafe, - kw::Extern, - kw::Typeof, - kw::Dyn, - ].contains(&name) -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub enum TokenKind { - /* Expression-operator symbols. */ - Eq, - Lt, - Le, - EqEq, - Ne, - Ge, - Gt, - AndAnd, - OrOr, - Not, - Tilde, - BinOp(BinOpToken), - BinOpEq(BinOpToken), - - /* Structural symbols */ - At, - Dot, - DotDot, - DotDotDot, - DotDotEq, - Comma, - Semi, - Colon, - ModSep, - RArrow, - LArrow, - FatArrow, - Pound, - Dollar, - Question, - /// Used by proc macros for representing lifetimes, not generated by lexer right now. - SingleQuote, - /// An opening delimiter (e.g., `{`). - OpenDelim(DelimToken), - /// A closing delimiter (e.g., `}`). - CloseDelim(DelimToken), - - /* Literals */ - Literal(Lit), - - /* Name components */ - Ident(ast::Name, /* is_raw */ bool), - Lifetime(ast::Name), - - Interpolated(Lrc), - - // Can be expanded into several tokens. - /// A doc comment. - DocComment(ast::Name), - - // Junk. These carry no data because we don't really care about the data - // they *would* carry, and don't really want to allocate a new ident for - // them. Instead, users could extract that from the associated span. - - /// Whitespace. - Whitespace, - /// A comment. - Comment, - Shebang(ast::Name), - /// A completely invalid token which should be skipped. - Unknown(ast::Name), - - Eof, -} - -// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(target_arch = "x86_64")] -static_assert_size!(TokenKind, 16); - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Token { - pub kind: TokenKind, - pub span: Span, -} - -impl TokenKind { - pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { - Literal(Lit::new(kind, symbol, suffix)) - } - - /// Returns tokens that are likely to be typed accidentally instead of the current token. - /// Enables better error recovery when the wrong token is found. - crate fn similar_tokens(&self) -> Option> { - match *self { - Comma => Some(vec![Dot, Lt, Semi]), - Semi => Some(vec![Colon, Comma]), - _ => None - } - } -} - -impl Token { - pub fn new(kind: TokenKind, span: Span) -> Self { - Token { kind, span } - } - - /// Some token that will be thrown away later. - crate fn dummy() -> Self { - Token::new(TokenKind::Whitespace, DUMMY_SP) - } - - /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: ast::Ident) -> Self { - Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) - } - - /// Return this token by value and leave a dummy token in its place. - pub fn take(&mut self) -> Self { - mem::replace(self, Token::dummy()) - } - - crate fn is_op(&self) -> bool { - match self.kind { - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | - Ident(..) | Lifetime(..) | Interpolated(..) | - Whitespace | Comment | Shebang(..) | Eof => false, - _ => true, - } - } - - crate fn is_like_plus(&self) -> bool { - match self.kind { - BinOp(Plus) | BinOpEq(Plus) => true, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of an expression. - pub fn can_begin_expr(&self) -> bool { - match self.kind { - Ident(name, is_raw) => - ident_can_begin_expr(name, self.span, is_raw), // value name or keyword - OpenDelim(..) | // tuple, array or block - Literal(..) | // literal - Not | // operator not - BinOp(Minus) | // unary minus - BinOp(Star) | // dereference - BinOp(Or) | OrOr | // closure - BinOp(And) | // reference - AndAnd | // double reference - // DotDotDot is no longer supported, but we need some way to display the error - DotDot | DotDotDot | DotDotEq | // range notation - Lt | BinOp(Shl) | // associated path - ModSep | // global path - Lifetime(..) | // labeled loop - Pound => true, // expression attributes - Interpolated(ref nt) => match **nt { - NtLiteral(..) | - NtIdent(..) | - NtExpr(..) | - NtBlock(..) | - NtPath(..) | - NtLifetime(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of a type. - pub fn can_begin_type(&self) -> bool { - match self.kind { - Ident(name, is_raw) => - ident_can_begin_type(name, self.span, is_raw), // type name or keyword - OpenDelim(Paren) | // tuple - OpenDelim(Bracket) | // array - Not | // never - BinOp(Star) | // raw pointer - BinOp(And) | // reference - AndAnd | // double reference - Question | // maybe bound in trait object - Lifetime(..) | // lifetime bound in trait object - Lt | BinOp(Shl) | // associated path - ModSep => true, // global path - Interpolated(ref nt) => match **nt { - NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of a const param. - crate fn can_begin_const_arg(&self) -> bool { - match self.kind { - OpenDelim(Brace) => true, - Interpolated(ref nt) => match **nt { - NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, - _ => false, - } - _ => self.can_begin_literal_or_bool(), - } - } - - /// Returns `true` if the token can appear at the start of a generic bound. - crate fn can_begin_bound(&self) -> bool { - self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || - self == &Question || self == &OpenDelim(Paren) - } - - /// Returns `true` if the token is any literal - pub fn is_lit(&self) -> bool { - match self.kind { - Literal(..) => true, - _ => false, - } - } - - /// Returns `true` if the token is any literal, a minus (which can prefix a literal, - /// for example a '-42', or one of the boolean idents). - pub fn can_begin_literal_or_bool(&self) -> bool { - match self.kind { - Literal(..) | BinOp(Minus) => true, - Ident(name, false) if name.is_bool_lit() => true, - Interpolated(ref nt) => match **nt { - NtLiteral(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns an identifier if this token is an identifier. - pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { - match self.kind { - Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), - Interpolated(ref nt) => match **nt { - NtIdent(ident, is_raw) => Some((ident, is_raw)), - _ => None, - }, - _ => None, - } - } - - /// Returns a lifetime identifier if this token is a lifetime. - pub fn lifetime(&self) -> Option { - match self.kind { - Lifetime(name) => Some(ast::Ident::new(name, self.span)), - Interpolated(ref nt) => match **nt { - NtLifetime(ident) => Some(ident), - _ => None, - }, - _ => None, - } - } - - /// Returns `true` if the token is an identifier. - pub fn is_ident(&self) -> bool { - self.ident().is_some() - } - - /// Returns `true` if the token is a lifetime. - crate fn is_lifetime(&self) -> bool { - self.lifetime().is_some() - } - - /// Returns `true` if the token is a identifier whose name is the given - /// string slice. - crate fn is_ident_named(&self, name: Symbol) -> bool { - self.ident().map_or(false, |(ident, _)| ident.name == name) - } - - /// Returns `true` if the token is an interpolated path. - fn is_path(&self) -> bool { - if let Interpolated(ref nt) = self.kind { - if let NtPath(..) = **nt { - return true; - } - } - false - } - - /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? - /// That is, is this a pre-parsed expression dropped into the token stream - /// (which happens while parsing the result of macro expansion)? - crate fn is_whole_expr(&self) -> bool { - if let Interpolated(ref nt) = self.kind { - if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { - return true; - } - } - - false - } - - /// Returns `true` if the token is either the `mut` or `const` keyword. - crate fn is_mutability(&self) -> bool { - self.is_keyword(kw::Mut) || - self.is_keyword(kw::Const) - } - - crate fn is_qpath_start(&self) -> bool { - self == &Lt || self == &BinOp(Shl) - } - - crate fn is_path_start(&self) -> bool { - self == &ModSep || self.is_qpath_start() || self.is_path() || - self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() - } - - /// Returns `true` if the token is a given keyword, `kw`. - pub fn is_keyword(&self, kw: Symbol) -> bool { - self.is_non_raw_ident_where(|id| id.name == kw) - } - - crate fn is_path_segment_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) - } - - // Returns true for reserved identifiers used internally for elided lifetimes, - // unnamed method parameters, crate root module, error recovery etc. - crate fn is_special_ident(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_special) - } - - /// Returns `true` if the token is a keyword used in the language. - crate fn is_used_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_used_keyword) - } - - /// Returns `true` if the token is a keyword reserved for possible future use. - crate fn is_unused_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) - } - - /// Returns `true` if the token is either a special identifier or a keyword. - pub fn is_reserved_ident(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_reserved) - } - - /// Returns `true` if the token is the identifier `true` or `false`. - crate fn is_bool_lit(&self) -> bool { - self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) - } - - /// Returns `true` if the token is a non-raw identifier for which `pred` holds. - fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { - match self.ident() { - Some((id, false)) => pred(id), - _ => false, - } - } - - crate fn glue(&self, joint: &Token) -> Option { - let kind = match self.kind { - Eq => match joint.kind { - Eq => EqEq, - Gt => FatArrow, - _ => return None, - }, - Lt => match joint.kind { - Eq => Le, - Lt => BinOp(Shl), - Le => BinOpEq(Shl), - BinOp(Minus) => LArrow, - _ => return None, - }, - Gt => match joint.kind { - Eq => Ge, - Gt => BinOp(Shr), - Ge => BinOpEq(Shr), - _ => return None, - }, - Not => match joint.kind { - Eq => Ne, - _ => return None, - }, - BinOp(op) => match joint.kind { - Eq => BinOpEq(op), - BinOp(And) if op == And => AndAnd, - BinOp(Or) if op == Or => OrOr, - Gt if op == Minus => RArrow, - _ => return None, - }, - Dot => match joint.kind { - Dot => DotDot, - DotDot => DotDotDot, - _ => return None, - }, - DotDot => match joint.kind { - Dot => DotDotDot, - Eq => DotDotEq, - _ => return None, - }, - Colon => match joint.kind { - Colon => ModSep, - _ => return None, - }, - SingleQuote => match joint.kind { - Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), - _ => return None, - }, - - Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | - DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | - Question | OpenDelim(..) | CloseDelim(..) | - Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, - }; - - Some(Token::new(kind, self.span.to(joint.span))) - } - - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { - if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { - return false - } - match (&self.kind, &other.kind) { - (&Eq, &Eq) | - (&Lt, &Lt) | - (&Le, &Le) | - (&EqEq, &EqEq) | - (&Ne, &Ne) | - (&Ge, &Ge) | - (&Gt, &Gt) | - (&AndAnd, &AndAnd) | - (&OrOr, &OrOr) | - (&Not, &Not) | - (&Tilde, &Tilde) | - (&At, &At) | - (&Dot, &Dot) | - (&DotDot, &DotDot) | - (&DotDotDot, &DotDotDot) | - (&DotDotEq, &DotDotEq) | - (&Comma, &Comma) | - (&Semi, &Semi) | - (&Colon, &Colon) | - (&ModSep, &ModSep) | - (&RArrow, &RArrow) | - (&LArrow, &LArrow) | - (&FatArrow, &FatArrow) | - (&Pound, &Pound) | - (&Dollar, &Dollar) | - (&Question, &Question) | - (&Whitespace, &Whitespace) | - (&Comment, &Comment) | - (&Eof, &Eof) => true, - - (&BinOp(a), &BinOp(b)) | - (&BinOpEq(a), &BinOpEq(b)) => a == b, - - (&OpenDelim(a), &OpenDelim(b)) | - (&CloseDelim(a), &CloseDelim(b)) => a == b, - - (&DocComment(a), &DocComment(b)) | - (&Shebang(a), &Shebang(b)) => a == b, - - (&Literal(a), &Literal(b)) => a == b, - - (&Lifetime(a), &Lifetime(b)) => a == b, - (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || - a == kw::DollarCrate || - c == kw::DollarCrate), - - (&Interpolated(_), &Interpolated(_)) => false, - - _ => panic!("forgot to add a token?"), - } - } -} - -impl PartialEq for Token { - fn eq(&self, rhs: &TokenKind) -> bool { - self.kind == *rhs - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable)] -/// For interpolation during macro expansion. -pub enum Nonterminal { - NtItem(P), - NtBlock(P), - NtStmt(ast::Stmt), - NtPat(P), - NtExpr(P), - NtTy(P), - NtIdent(ast::Ident, /* is_raw */ bool), - NtLifetime(ast::Ident), - NtLiteral(P), - /// Stuff inside brackets for attributes - NtMeta(ast::AttrItem), - NtPath(ast::Path), - NtVis(ast::Visibility), - NtTT(TokenTree), - // Used only for passing items to proc macro attributes (they are not - // strictly necessary for that, `Annotatable` can be converted into - // tokens directly, but doing that naively regresses pretty-printing). - NtTraitItem(ast::TraitItem), - NtImplItem(ast::ImplItem), - NtForeignItem(ast::ForeignItem), -} - -impl PartialEq for Nonterminal { - fn eq(&self, rhs: &Self) -> bool { - match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs, - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them - // correctly based on data from AST. This will prevent them from matching each other - // in macros. The comparison will become possible only when each nonterminal has an - // attached token stream from which it was parsed. - _ => false, - } - } -} - -impl fmt::Debug for Nonterminal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - NtItem(..) => f.pad("NtItem(..)"), - NtBlock(..) => f.pad("NtBlock(..)"), - NtStmt(..) => f.pad("NtStmt(..)"), - NtPat(..) => f.pad("NtPat(..)"), - NtExpr(..) => f.pad("NtExpr(..)"), - NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), - NtLiteral(..) => f.pad("NtLiteral(..)"), - NtMeta(..) => f.pad("NtMeta(..)"), - NtPath(..) => f.pad("NtPath(..)"), - NtTT(..) => f.pad("NtTT(..)"), - NtImplItem(..) => f.pad("NtImplItem(..)"), - NtTraitItem(..) => f.pad("NtTraitItem(..)"), - NtForeignItem(..) => f.pad("NtForeignItem(..)"), - NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), - } - } -} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 1d59c13a9d0..049f34efc26 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -4,7 +4,7 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg}; use crate::util::parser::{self, AssocOp, Fixity}; use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; -use crate::parse::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; +use crate::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; use crate::parse::lexer::comments; use crate::parse; use crate::print::pp::{self, Breaks}; diff --git a/src/libsyntax/token.rs b/src/libsyntax/token.rs new file mode 100644 index 00000000000..6f3da344ccf --- /dev/null +++ b/src/libsyntax/token.rs @@ -0,0 +1,728 @@ +pub use BinOpToken::*; +pub use Nonterminal::*; +pub use DelimToken::*; +pub use LitKind::*; +pub use TokenKind::*; + +use crate::ast; +use crate::ptr::P; +use crate::symbol::kw; +use crate::tokenstream::TokenTree; + +use syntax_pos::symbol::Symbol; +use syntax_pos::{self, Span, DUMMY_SP}; + +use std::fmt; +use std::mem; +#[cfg(target_arch = "x86_64")] +use rustc_data_structures::static_assert_size; +use rustc_data_structures::sync::Lrc; + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +pub enum BinOpToken { + Plus, + Minus, + Star, + Slash, + Percent, + Caret, + And, + Or, + Shl, + Shr, +} + +/// A delimiter token. +#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +pub enum DelimToken { + /// A round parenthesis (i.e., `(` or `)`). + Paren, + /// A square bracket (i.e., `[` or `]`). + Bracket, + /// A curly brace (i.e., `{` or `}`). + Brace, + /// An empty delimiter. + NoDelim, +} + +impl DelimToken { + pub fn len(self) -> usize { + if self == NoDelim { 0 } else { 1 } + } + + pub fn is_empty(self) -> bool { + self == NoDelim + } +} + +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum LitKind { + Bool, // AST only, must never appear in a `Token` + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u16), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols + Err, +} + +/// A literal token. +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Lit { + pub kind: LitKind, + pub symbol: Symbol, + pub suffix: Option, +} + +impl fmt::Display for Lit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Lit { kind, symbol, suffix } = *self; + match kind { + Byte => write!(f, "b'{}'", symbol)?, + Char => write!(f, "'{}'", symbol)?, + Str => write!(f, "\"{}\"", symbol)?, + StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + ByteStr => write!(f, "b\"{}\"", symbol)?, + ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + Integer | + Float | + Bool | + Err => write!(f, "{}", symbol)?, + } + + if let Some(suffix) = suffix { + write!(f, "{}", suffix)?; + } + + Ok(()) + } +} + +impl LitKind { + /// An English article for the literal token kind. + crate fn article(self) -> &'static str { + match self { + Integer | Err => "an", + _ => "a", + } + } + + crate fn descr(self) -> &'static str { + match self { + Bool => panic!("literal token contains `Lit::Bool`"), + Byte => "byte", + Char => "char", + Integer => "integer", + Float => "float", + Str | StrRaw(..) => "string", + ByteStr | ByteStrRaw(..) => "byte string", + Err => "error", + } + } + + crate fn may_have_suffix(self) -> bool { + match self { + Integer | Float | Err => true, + _ => false, + } + } +} + +impl Lit { + pub fn new(kind: LitKind, symbol: Symbol, suffix: Option) -> Lit { + Lit { kind, symbol, suffix } + } +} + +pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); + token_can_begin_expr(&ident_token) +} + +pub(crate) fn token_can_begin_expr(ident_token: &Token) -> bool { + !ident_token.is_reserved_ident() || + ident_token.is_path_segment_keyword() || + match ident_token.kind { + TokenKind::Ident(ident, _) => [ + kw::Async, + kw::Do, + kw::Box, + kw::Break, + kw::Continue, + kw::False, + kw::For, + kw::If, + kw::Let, + kw::Loop, + kw::Match, + kw::Move, + kw::Return, + kw::True, + kw::Unsafe, + kw::While, + kw::Yield, + kw::Static, + ].contains(&ident), + _=> false, + } +} + +fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); + + !ident_token.is_reserved_ident() || + ident_token.is_path_segment_keyword() || + [ + kw::Underscore, + kw::For, + kw::Impl, + kw::Fn, + kw::Unsafe, + kw::Extern, + kw::Typeof, + kw::Dyn, + ].contains(&name) +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum TokenKind { + /* Expression-operator symbols. */ + Eq, + Lt, + Le, + EqEq, + Ne, + Ge, + Gt, + AndAnd, + OrOr, + Not, + Tilde, + BinOp(BinOpToken), + BinOpEq(BinOpToken), + + /* Structural symbols */ + At, + Dot, + DotDot, + DotDotDot, + DotDotEq, + Comma, + Semi, + Colon, + ModSep, + RArrow, + LArrow, + FatArrow, + Pound, + Dollar, + Question, + /// Used by proc macros for representing lifetimes, not generated by lexer right now. + SingleQuote, + /// An opening delimiter (e.g., `{`). + OpenDelim(DelimToken), + /// A closing delimiter (e.g., `}`). + CloseDelim(DelimToken), + + /* Literals */ + Literal(Lit), + + /* Name components */ + Ident(ast::Name, /* is_raw */ bool), + Lifetime(ast::Name), + + Interpolated(Lrc), + + // Can be expanded into several tokens. + /// A doc comment. + DocComment(ast::Name), + + // Junk. These carry no data because we don't really care about the data + // they *would* carry, and don't really want to allocate a new ident for + // them. Instead, users could extract that from the associated span. + + /// Whitespace. + Whitespace, + /// A comment. + Comment, + Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), + + Eof, +} + +// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(target_arch = "x86_64")] +static_assert_size!(TokenKind, 16); + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Token { + pub kind: TokenKind, + pub span: Span, +} + +impl TokenKind { + pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { + Literal(Lit::new(kind, symbol, suffix)) + } + + /// Returns tokens that are likely to be typed accidentally instead of the current token. + /// Enables better error recovery when the wrong token is found. + crate fn similar_tokens(&self) -> Option> { + match *self { + Comma => Some(vec![Dot, Lt, Semi]), + Semi => Some(vec![Colon, Comma]), + _ => None + } + } +} + +impl Token { + pub fn new(kind: TokenKind, span: Span) -> Self { + Token { kind, span } + } + + /// Some token that will be thrown away later. + crate fn dummy() -> Self { + Token::new(TokenKind::Whitespace, DUMMY_SP) + } + + /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. + pub fn from_ast_ident(ident: ast::Ident) -> Self { + Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) + } + + /// Return this token by value and leave a dummy token in its place. + pub fn take(&mut self) -> Self { + mem::replace(self, Token::dummy()) + } + + crate fn is_op(&self) -> bool { + match self.kind { + OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | + Ident(..) | Lifetime(..) | Interpolated(..) | + Whitespace | Comment | Shebang(..) | Eof => false, + _ => true, + } + } + + crate fn is_like_plus(&self) -> bool { + match self.kind { + BinOp(Plus) | BinOpEq(Plus) => true, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of an expression. + pub fn can_begin_expr(&self) -> bool { + match self.kind { + Ident(name, is_raw) => + ident_can_begin_expr(name, self.span, is_raw), // value name or keyword + OpenDelim(..) | // tuple, array or block + Literal(..) | // literal + Not | // operator not + BinOp(Minus) | // unary minus + BinOp(Star) | // dereference + BinOp(Or) | OrOr | // closure + BinOp(And) | // reference + AndAnd | // double reference + // DotDotDot is no longer supported, but we need some way to display the error + DotDot | DotDotDot | DotDotEq | // range notation + Lt | BinOp(Shl) | // associated path + ModSep | // global path + Lifetime(..) | // labeled loop + Pound => true, // expression attributes + Interpolated(ref nt) => match **nt { + NtLiteral(..) | + NtIdent(..) | + NtExpr(..) | + NtBlock(..) | + NtPath(..) | + NtLifetime(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of a type. + pub fn can_begin_type(&self) -> bool { + match self.kind { + Ident(name, is_raw) => + ident_can_begin_type(name, self.span, is_raw), // type name or keyword + OpenDelim(Paren) | // tuple + OpenDelim(Bracket) | // array + Not | // never + BinOp(Star) | // raw pointer + BinOp(And) | // reference + AndAnd | // double reference + Question | // maybe bound in trait object + Lifetime(..) | // lifetime bound in trait object + Lt | BinOp(Shl) | // associated path + ModSep => true, // global path + Interpolated(ref nt) => match **nt { + NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of a const param. + crate fn can_begin_const_arg(&self) -> bool { + match self.kind { + OpenDelim(Brace) => true, + Interpolated(ref nt) => match **nt { + NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, + _ => false, + } + _ => self.can_begin_literal_or_bool(), + } + } + + /// Returns `true` if the token can appear at the start of a generic bound. + crate fn can_begin_bound(&self) -> bool { + self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || + self == &Question || self == &OpenDelim(Paren) + } + + /// Returns `true` if the token is any literal + pub fn is_lit(&self) -> bool { + match self.kind { + Literal(..) => true, + _ => false, + } + } + + /// Returns `true` if the token is any literal, a minus (which can prefix a literal, + /// for example a '-42', or one of the boolean idents). + pub fn can_begin_literal_or_bool(&self) -> bool { + match self.kind { + Literal(..) | BinOp(Minus) => true, + Ident(name, false) if name.is_bool_lit() => true, + Interpolated(ref nt) => match **nt { + NtLiteral(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns an identifier if this token is an identifier. + pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { + match self.kind { + Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), + Interpolated(ref nt) => match **nt { + NtIdent(ident, is_raw) => Some((ident, is_raw)), + _ => None, + }, + _ => None, + } + } + + /// Returns a lifetime identifier if this token is a lifetime. + pub fn lifetime(&self) -> Option { + match self.kind { + Lifetime(name) => Some(ast::Ident::new(name, self.span)), + Interpolated(ref nt) => match **nt { + NtLifetime(ident) => Some(ident), + _ => None, + }, + _ => None, + } + } + + /// Returns `true` if the token is an identifier. + pub fn is_ident(&self) -> bool { + self.ident().is_some() + } + + /// Returns `true` if the token is a lifetime. + crate fn is_lifetime(&self) -> bool { + self.lifetime().is_some() + } + + /// Returns `true` if the token is a identifier whose name is the given + /// string slice. + crate fn is_ident_named(&self, name: Symbol) -> bool { + self.ident().map_or(false, |(ident, _)| ident.name == name) + } + + /// Returns `true` if the token is an interpolated path. + fn is_path(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtPath(..) = **nt { + return true; + } + } + false + } + + /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? + /// That is, is this a pre-parsed expression dropped into the token stream + /// (which happens while parsing the result of macro expansion)? + crate fn is_whole_expr(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { + return true; + } + } + + false + } + + /// Returns `true` if the token is either the `mut` or `const` keyword. + crate fn is_mutability(&self) -> bool { + self.is_keyword(kw::Mut) || + self.is_keyword(kw::Const) + } + + crate fn is_qpath_start(&self) -> bool { + self == &Lt || self == &BinOp(Shl) + } + + crate fn is_path_start(&self) -> bool { + self == &ModSep || self.is_qpath_start() || self.is_path() || + self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() + } + + /// Returns `true` if the token is a given keyword, `kw`. + pub fn is_keyword(&self, kw: Symbol) -> bool { + self.is_non_raw_ident_where(|id| id.name == kw) + } + + crate fn is_path_segment_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) + } + + // Returns true for reserved identifiers used internally for elided lifetimes, + // unnamed method parameters, crate root module, error recovery etc. + crate fn is_special_ident(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_special) + } + + /// Returns `true` if the token is a keyword used in the language. + crate fn is_used_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_used_keyword) + } + + /// Returns `true` if the token is a keyword reserved for possible future use. + crate fn is_unused_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) + } + + /// Returns `true` if the token is either a special identifier or a keyword. + pub fn is_reserved_ident(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_reserved) + } + + /// Returns `true` if the token is the identifier `true` or `false`. + crate fn is_bool_lit(&self) -> bool { + self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) + } + + /// Returns `true` if the token is a non-raw identifier for which `pred` holds. + fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { + match self.ident() { + Some((id, false)) => pred(id), + _ => false, + } + } + + crate fn glue(&self, joint: &Token) -> Option { + let kind = match self.kind { + Eq => match joint.kind { + Eq => EqEq, + Gt => FatArrow, + _ => return None, + }, + Lt => match joint.kind { + Eq => Le, + Lt => BinOp(Shl), + Le => BinOpEq(Shl), + BinOp(Minus) => LArrow, + _ => return None, + }, + Gt => match joint.kind { + Eq => Ge, + Gt => BinOp(Shr), + Ge => BinOpEq(Shr), + _ => return None, + }, + Not => match joint.kind { + Eq => Ne, + _ => return None, + }, + BinOp(op) => match joint.kind { + Eq => BinOpEq(op), + BinOp(And) if op == And => AndAnd, + BinOp(Or) if op == Or => OrOr, + Gt if op == Minus => RArrow, + _ => return None, + }, + Dot => match joint.kind { + Dot => DotDot, + DotDot => DotDotDot, + _ => return None, + }, + DotDot => match joint.kind { + Dot => DotDotDot, + Eq => DotDotEq, + _ => return None, + }, + Colon => match joint.kind { + Colon => ModSep, + _ => return None, + }, + SingleQuote => match joint.kind { + Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), + _ => return None, + }, + + Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | + DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | + Question | OpenDelim(..) | CloseDelim(..) | + Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, + }; + + Some(Token::new(kind, self.span.to(joint.span))) + } + + // See comments in `Nonterminal::to_tokenstream` for why we care about + // *probably* equal here rather than actual equality + crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { + if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { + return false + } + match (&self.kind, &other.kind) { + (&Eq, &Eq) | + (&Lt, &Lt) | + (&Le, &Le) | + (&EqEq, &EqEq) | + (&Ne, &Ne) | + (&Ge, &Ge) | + (&Gt, &Gt) | + (&AndAnd, &AndAnd) | + (&OrOr, &OrOr) | + (&Not, &Not) | + (&Tilde, &Tilde) | + (&At, &At) | + (&Dot, &Dot) | + (&DotDot, &DotDot) | + (&DotDotDot, &DotDotDot) | + (&DotDotEq, &DotDotEq) | + (&Comma, &Comma) | + (&Semi, &Semi) | + (&Colon, &Colon) | + (&ModSep, &ModSep) | + (&RArrow, &RArrow) | + (&LArrow, &LArrow) | + (&FatArrow, &FatArrow) | + (&Pound, &Pound) | + (&Dollar, &Dollar) | + (&Question, &Question) | + (&Whitespace, &Whitespace) | + (&Comment, &Comment) | + (&Eof, &Eof) => true, + + (&BinOp(a), &BinOp(b)) | + (&BinOpEq(a), &BinOpEq(b)) => a == b, + + (&OpenDelim(a), &OpenDelim(b)) | + (&CloseDelim(a), &CloseDelim(b)) => a == b, + + (&DocComment(a), &DocComment(b)) | + (&Shebang(a), &Shebang(b)) => a == b, + + (&Literal(a), &Literal(b)) => a == b, + + (&Lifetime(a), &Lifetime(b)) => a == b, + (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || + a == kw::DollarCrate || + c == kw::DollarCrate), + + (&Interpolated(_), &Interpolated(_)) => false, + + _ => panic!("forgot to add a token?"), + } + } +} + +impl PartialEq for Token { + fn eq(&self, rhs: &TokenKind) -> bool { + self.kind == *rhs + } +} + +#[derive(Clone, RustcEncodable, RustcDecodable)] +/// For interpolation during macro expansion. +pub enum Nonterminal { + NtItem(P), + NtBlock(P), + NtStmt(ast::Stmt), + NtPat(P), + NtExpr(P), + NtTy(P), + NtIdent(ast::Ident, /* is_raw */ bool), + NtLifetime(ast::Ident), + NtLiteral(P), + /// Stuff inside brackets for attributes + NtMeta(ast::AttrItem), + NtPath(ast::Path), + NtVis(ast::Visibility), + NtTT(TokenTree), + // Used only for passing items to proc macro attributes (they are not + // strictly necessary for that, `Annotatable` can be converted into + // tokens directly, but doing that naively regresses pretty-printing). + NtTraitItem(ast::TraitItem), + NtImplItem(ast::ImplItem), + NtForeignItem(ast::ForeignItem), +} + +impl PartialEq for Nonterminal { + fn eq(&self, rhs: &Self) -> bool { + match (self, rhs) { + (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => + ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs, + (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, + (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, + // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them + // correctly based on data from AST. This will prevent them from matching each other + // in macros. The comparison will become possible only when each nonterminal has an + // attached token stream from which it was parsed. + _ => false, + } + } +} + +impl fmt::Debug for Nonterminal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + NtItem(..) => f.pad("NtItem(..)"), + NtBlock(..) => f.pad("NtBlock(..)"), + NtStmt(..) => f.pad("NtStmt(..)"), + NtPat(..) => f.pad("NtPat(..)"), + NtExpr(..) => f.pad("NtExpr(..)"), + NtTy(..) => f.pad("NtTy(..)"), + NtIdent(..) => f.pad("NtIdent(..)"), + NtLiteral(..) => f.pad("NtLiteral(..)"), + NtMeta(..) => f.pad("NtMeta(..)"), + NtPath(..) => f.pad("NtPath(..)"), + NtTT(..) => f.pad("NtTT(..)"), + NtImplItem(..) => f.pad("NtImplItem(..)"), + NtTraitItem(..) => f.pad("NtTraitItem(..)"), + NtForeignItem(..) => f.pad("NtForeignItem(..)"), + NtVis(..) => f.pad("NtVis(..)"), + NtLifetime(..) => f.pad("NtLifetime(..)"), + } + } +} diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index a51d208704a..6e1bb85ce1a 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -13,7 +13,7 @@ //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking //! ownership of the original. -use crate::parse::token::{self, DelimToken, Token, TokenKind}; +use crate::token::{self, DelimToken, Token, TokenKind}; use syntax_pos::{Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index 982755e8680..edb708d7e97 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, BinOpToken}; +use crate::token::{self, Token, BinOpToken}; use crate::symbol::kw; use crate::ast::{self, BinOpKind}; diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 117787d08c7..cfd160fd577 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -14,7 +14,7 @@ //! those that are created by the expansion of a macro. use crate::ast::*; -use crate::parse::token::Token; +use crate::token::Token; use crate::tokenstream::{TokenTree, TokenStream}; use syntax_pos::Span; -- cgit 1.4.1-3-g733a5