diff options
24 files changed, 119 insertions, 119 deletions
diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 089e5de01a2..919f682fc4f 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -67,7 +67,7 @@ use syntax::source_map::CompilerDesugaringKind::IfTemporary; use syntax::std_inject; use syntax::symbol::{kw, sym, Symbol}; use syntax::tokenstream::{TokenStream, TokenTree}; -use syntax::parse::token::{self, Token}; +use syntax::parse::token::{self, TokenKind}; use syntax::visit::{self, Visitor}; use syntax_pos::{DUMMY_SP, edition, Span}; @@ -1337,7 +1337,7 @@ impl<'a> LoweringContext<'a> { } } - fn lower_token(&mut self, token: Token, span: Span) -> TokenStream { + fn lower_token(&mut self, token: TokenKind, span: Span) -> TokenStream { match token { token::Interpolated(nt) => { let tts = nt.to_tokenstream(&self.sess.parse_sess, span); diff --git a/src/librustc/hir/map/def_collector.rs b/src/librustc/hir/map/def_collector.rs index 0fa0d1ea00c..b9a80ebb78f 100644 --- a/src/librustc/hir/map/def_collector.rs +++ b/src/librustc/hir/map/def_collector.rs @@ -6,7 +6,7 @@ use syntax::ast::*; use syntax::ext::hygiene::Mark; use syntax::visit; use syntax::symbol::{kw, sym}; -use syntax::parse::token::{self, Token}; +use syntax::parse::token::{self, TokenKind}; use syntax_pos::Span; /// Creates `DefId`s for nodes in the AST. @@ -325,7 +325,7 @@ impl<'a> visit::Visitor<'a> for DefCollector<'a> { } } - fn visit_token(&mut self, t: Token) { + fn visit_token(&mut self, t: TokenKind) { if let token::Interpolated(nt) = t { if let token::NtExpr(ref expr) = *nt { if let ExprKind::Mac(..) = expr.node { diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 6e1eba0af56..8e2550d3c45 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -307,7 +307,7 @@ impl_stable_hash_for!(struct token::Lit { }); fn hash_token<'a, 'gcx, W: StableHasherResult>( - token: &token::Token, + token: &token::TokenKind, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher<W>, ) { diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index 76279cc0283..a7a78a69952 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -34,7 +34,7 @@ use syntax::ext::base::Determinacy::Undetermined; use syntax::ext::hygiene::Mark; use syntax::ext::tt::macro_rules; use syntax::feature_gate::is_builtin_attr; -use syntax::parse::token::{self, Token}; +use syntax::parse::token::{self, TokenKind}; use syntax::span_err; use syntax::std_inject::injected_crate_name; use syntax::symbol::{kw, sym}; @@ -1052,7 +1052,7 @@ impl<'a, 'b> Visitor<'a> for BuildReducedGraphVisitor<'a, 'b> { self.resolver.current_module = parent; } - fn visit_token(&mut self, t: Token) { + fn visit_token(&mut self, t: TokenKind) { if let token::Interpolated(nt) = t { if let token::NtExpr(ref expr) = *nt { if let ast::ExprKind::Mac(..) = expr.node { diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs index e2c93b6d331..5527fcb923b 100644 --- a/src/librustc_save_analysis/span_utils.rs +++ b/src/librustc_save_analysis/span_utils.rs @@ -5,7 +5,7 @@ use crate::generated_code; use std::cell::Cell; use syntax::parse::lexer::{self, StringReader}; -use syntax::parse::token::{self, Token}; +use syntax::parse::token::{self, TokenKind}; use syntax_pos::*; #[derive(Clone)] @@ -56,7 +56,7 @@ impl<'a> SpanUtils<'a> { lexer::StringReader::retokenize(&self.sess.parse_sess, span) } - pub fn sub_span_of_token(&self, span: Span, tok: Token) -> Option<Span> { + pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> { let mut toks = self.retokenise_span(span); loop { let next = toks.real_token(); diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index c57510ab1a0..ade15f024a6 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned}; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::parse::parser::Parser; use crate::parse::{self, ParseSess, PResult}; -use crate::parse::token::{self, Token}; +use crate::parse::token::{self, TokenKind}; use crate::ptr::P; use crate::symbol::{sym, Symbol}; use crate::ThinVec; @@ -468,7 +468,7 @@ impl MetaItem { idents.push(TokenTree::Token(mod_sep_span, token::ModSep).into()); } idents.push(TokenTree::Token(segment.ident.span, - Token::from_ast_ident(segment.ident)).into()); + TokenKind::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 473a5f414df..c22952ed750 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -78,7 +78,7 @@ use crate::ast::Ident; use crate::ext::tt::quoted::{self, TokenTree}; use crate::parse::{Directory, ParseSess}; use crate::parse::parser::{Parser, PathStyle}; -use crate::parse::token::{self, DocComment, Nonterminal, Token}; +use crate::parse::token::{self, DocComment, Nonterminal, TokenKind}; use crate::print::pprust; use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::{DelimSpan, TokenStream}; @@ -199,7 +199,7 @@ struct MatcherPos<'root, 'tt: 'root> { seq_op: Option<quoted::KleeneOp>, /// The separator if we are in a repetition. - sep: Option<Token>, + sep: Option<TokenKind>, /// The "parent" matcher position if we are in a repetition. That is, the matcher position just /// before we enter the sequence. @@ -273,7 +273,7 @@ pub enum ParseResult<T> { Success(T), /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. - Failure(syntax_pos::Span, Token, &'static str), + Failure(syntax_pos::Span, TokenKind, &'static str), /// Fatal error (malformed macro?). Abort compilation. Error(syntax_pos::Span, String), } @@ -417,7 +417,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>( /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For /// other tokens, this is "unexpected token...". -pub fn parse_failure_msg(tok: Token) -> String { +pub fn parse_failure_msg(tok: TokenKind) -> String { match tok { token::Eof => "unexpected end of macro invocation".to_string(), _ => format!( @@ -428,7 +428,7 @@ pub fn parse_failure_msg(tok: Token) -> String { } /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1: &Token, t2: &Token) -> bool { +fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool { if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) { id1.name == id2.name && is_raw1 == is_raw2 } else if let (Some(id1), Some(id2)) = (t1.lifetime(), t2.lifetime()) { @@ -466,7 +466,7 @@ fn inner_parse_loop<'root, 'tt>( next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>, eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - token: &Token, + token: &TokenKind, span: syntax_pos::Span, ) -> ParseResult<()> { // Pop items from `cur_items` until it is empty. @@ -807,7 +807,7 @@ pub fn parse( /// The token is an identifier, but not `_`. /// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> { +fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> { match *token { token::Ident(ident, is_raw) if ident.name != kw::Underscore => Some((ident, is_raw)), @@ -819,7 +819,7 @@ fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> { /// /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that /// token. Be conservative (return true) if not sure. -fn may_begin_with(name: Symbol, token: &Token) -> bool { +fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { /// Checks whether the non-terminal may contain a single (non-keyword) identifier. fn may_be_ident(nt: &token::Nonterminal) -> bool { match *nt { diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 285c88357a6..9d3ea4d8645 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -12,7 +12,7 @@ use crate::feature_gate::Features; use crate::parse::{Directory, ParseSess}; use crate::parse::parser::Parser; use crate::parse::token::{self, NtTT}; -use crate::parse::token::Token::*; +use crate::parse::token::TokenKind::*; use crate::symbol::{Symbol, kw, sym}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index a029c654659..fe0cb56b29e 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -23,12 +23,12 @@ pub struct Delimited { impl Delimited { /// Returns the opening delimiter (possibly `NoDelim`). - pub fn open_token(&self) -> token::Token { + pub fn open_token(&self) -> token::TokenKind { token::OpenDelim(self.delim) } /// Returns the closing delimiter (possibly `NoDelim`). - pub fn close_token(&self) -> token::Token { + pub fn close_token(&self) -> token::TokenKind { token::CloseDelim(self.delim) } @@ -58,7 +58,7 @@ pub struct SequenceRepetition { /// The sequence of token trees pub tts: Vec<TokenTree>, /// The optional separator - pub separator: Option<token::Token>, + pub separator: Option<token::TokenKind>, /// Whether the sequence can be repeated zero (*), or one or more times (+) pub op: KleeneOp, /// The number of `Match`s that appear in the sequence (and subsequences) @@ -81,7 +81,7 @@ pub enum KleeneOp { /// are "first-class" token trees. Useful for parsing macros. #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum TokenTree { - Token(Span, token::Token), + Token(Span, token::TokenKind), Delimited(DelimSpan, Lrc<Delimited>), /// A kleene-style repetition sequence Sequence(DelimSpan, Lrc<SequenceRepetition>), @@ -366,7 +366,7 @@ where /// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return /// `None`. -fn kleene_op(token: &token::Token) -> Option<KleeneOp> { +fn kleene_op(token: &token::TokenKind) -> Option<KleeneOp> { match *token { token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), @@ -383,7 +383,7 @@ fn kleene_op(token: &token::Token) -> Option<KleeneOp> { fn parse_kleene_op<I>( input: &mut I, span: Span, -) -> Result<Result<(KleeneOp, Span), (token::Token, Span)>, Span> +) -> Result<Result<(KleeneOp, Span), (token::TokenKind, Span)>, Span> where I: Iterator<Item = tokenstream::TokenTree>, { @@ -422,7 +422,7 @@ fn parse_sep_and_kleene_op<I>( attrs: &[ast::Attribute], edition: Edition, macro_node_id: NodeId, -) -> (Option<token::Token>, KleeneOp) +) -> (Option<token::TokenKind>, KleeneOp) where I: Iterator<Item = tokenstream::TokenTree>, { @@ -447,7 +447,7 @@ fn parse_sep_and_kleene_op_2015<I>( _features: &Features, _attrs: &[ast::Attribute], macro_node_id: NodeId, -) -> (Option<token::Token>, KleeneOp) +) -> (Option<token::TokenKind>, KleeneOp) where I: Iterator<Item = tokenstream::TokenTree>, { @@ -565,7 +565,7 @@ fn parse_sep_and_kleene_op_2018<I>( sess: &ParseSess, _features: &Features, _attrs: &[ast::Attribute], -) -> (Option<token::Token>, KleeneOp) +) -> (Option<token::TokenKind>, KleeneOp) where I: Iterator<Item = tokenstream::TokenTree>, { diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index c2a1866b03a..1b169d7696a 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -4,7 +4,7 @@ use crate::ext::expand::Marker; use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use crate::ext::tt::quoted; use crate::mut_visit::noop_visit_tt; -use crate::parse::token::{self, NtTT, Token}; +use crate::parse::token::{self, NtTT, TokenKind}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use smallvec::{smallvec, SmallVec}; @@ -18,7 +18,7 @@ use std::rc::Rc; /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). enum Frame { Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan }, - Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> }, + Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<TokenKind> }, } impl Frame { @@ -242,7 +242,7 @@ pub fn transcribe( Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark)); sp = sp.apply_mark(cx.current_expansion.mark); result.push(TokenTree::Token(sp, token::Dollar).into()); - result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into()); + result.push(TokenTree::Token(sp, token::TokenKind::from_ast_ident(ident)).into()); } } diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index fb1a7a680ba..289f2c0ce48 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -9,7 +9,7 @@ use crate::ast::*; use crate::source_map::{Spanned, respan}; -use crate::parse::token::{self, Token}; +use crate::parse::token::{self, TokenKind}; use crate::ptr::P; use crate::ThinVec; use crate::tokenstream::*; @@ -262,7 +262,7 @@ pub trait MutVisitor: Sized { noop_visit_tts(tts, self); } - fn visit_token(&mut self, t: &mut Token) { + fn visit_token(&mut self, t: &mut TokenKind) { noop_visit_token(t, self); } @@ -596,7 +596,7 @@ pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &m } // apply ident visitor if it's an ident, apply other visits to interpolated nodes -pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) { +pub fn noop_visit_token<T: MutVisitor>(t: &mut TokenKind, vis: &mut T) { match t { token::Ident(id, _is_raw) => vis.visit_ident(id), token::Lifetime(id) => vis.visit_ident(id), diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/diagnostics.rs index fc09943d4f5..b391f7ca327 100644 --- a/src/libsyntax/parse/diagnostics.rs +++ b/src/libsyntax/parse/diagnostics.rs @@ -229,8 +229,8 @@ impl<'a> Parser<'a> { pub fn expected_one_of_not_found( &mut self, - edible: &[token::Token], - inedible: &[token::Token], + edible: &[token::TokenKind], + inedible: &[token::TokenKind], ) -> PResult<'a, bool /* recovered */> { fn tokens_to_string(tokens: &[TokenType]) -> String { let mut i = tokens.iter(); @@ -368,7 +368,7 @@ impl<'a> Parser<'a> { /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. - crate fn eat_to_tokens(&mut self, kets: &[&token::Token]) { + crate fn eat_to_tokens(&mut self, kets: &[&token::TokenKind]) { let handler = self.diagnostic(); if let Err(ref mut err) = self.parse_seq_to_before_tokens( @@ -388,7 +388,7 @@ impl<'a> Parser<'a> { /// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>(); /// ^^ help: remove extra angle brackets /// ``` - crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::Token) { + crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::TokenKind) { // This function is intended to be invoked after parsing a path segment where there are two // cases: // @@ -726,7 +726,7 @@ impl<'a> Parser<'a> { /// closing delimiter. pub fn unexpected_try_recover( &mut self, - t: &token::Token, + t: &token::TokenKind, ) -> PResult<'a, bool /* recovered */> { let token_str = pprust::token_to_string(t); let this_token_str = self.this_token_descr(); @@ -903,7 +903,7 @@ impl<'a> Parser<'a> { crate fn recover_closing_delimiter( &mut self, - tokens: &[token::Token], + tokens: &[token::TokenKind], mut err: DiagnosticBuilder<'a>, ) -> PResult<'a, bool> { let mut pos = None; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index a06a84f162a..ca9199975bb 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,6 +1,6 @@ use crate::ast::{self, Ident}; use crate::parse::ParseSess; -use crate::parse::token::{self, Token}; +use crate::parse::token::{self, TokenKind}; use crate::symbol::{sym, Symbol}; use crate::parse::unescape; use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char}; @@ -22,7 +22,7 @@ mod unicode_chars; #[derive(Clone, Debug)] pub struct TokenAndSpan { - pub tok: Token, + pub tok: TokenKind, pub sp: Span, } @@ -56,7 +56,7 @@ pub struct StringReader<'a> { /// Stop reading src at this index. crate end_src_index: usize, // cached: - peek_tok: Token, + peek_tok: TokenKind, peek_span: Span, peek_span_src_raw: Span, fatal_errs: Vec<DiagnosticBuilder<'a>>, @@ -847,7 +847,7 @@ impl<'a> StringReader<'a> { } } - fn binop(&mut self, op: token::BinOpToken) -> Token { + fn binop(&mut self, op: token::BinOpToken) -> TokenKind { self.bump(); if self.ch_is('=') { self.bump(); @@ -859,7 +859,7 @@ impl<'a> StringReader<'a> { /// Returns the next token from the string, advances the input past that /// token, and updates the interner - fn next_token_inner(&mut self) -> Result<Token, ()> { + fn next_token_inner(&mut self) -> Result<TokenKind, ()> { let c = self.ch; if ident_start(c) { @@ -916,7 +916,7 @@ impl<'a> StringReader<'a> { let (kind, symbol) = self.scan_number(c.unwrap()); let suffix = self.scan_optional_raw_name(); debug!("next_token_inner: scanned number {:?}, {:?}, {:?}", kind, symbol, suffix); - return Ok(Token::lit(kind, symbol, suffix)); + return Ok(TokenKind::lit(kind, symbol, suffix)); } match c.expect("next_token_inner called at EOF") { @@ -1077,7 +1077,7 @@ impl<'a> StringReader<'a> { let symbol = self.name_from(start); self.bump(); self.validate_char_escape(start_with_quote); - return Ok(Token::lit(token::Char, symbol, None)); + return Ok(TokenKind::lit(token::Char, symbol, None)); } // Include the leading `'` in the real identifier, for macro @@ -1102,7 +1102,7 @@ impl<'a> StringReader<'a> { let symbol = self.scan_single_quoted_string(start_with_quote, msg); self.validate_char_escape(start_with_quote); let suffix = self.scan_optional_raw_name(); - Ok(Token::lit(token::Char, symbol, suffix)) + Ok(TokenKind::lit(token::Char, symbol, suffix)) } 'b' => { self.bump(); @@ -1127,7 +1127,7 @@ impl<'a> StringReader<'a> { }; let suffix = self.scan_optional_raw_name(); - Ok(Token::lit(kind, symbol, suffix)) + Ok(TokenKind::lit(kind, symbol, suffix)) } '"' => { let start_with_quote = self.pos; @@ -1135,7 +1135,7 @@ impl<'a> StringReader<'a> { let symbol = self.scan_double_quoted_string(msg); self.validate_str_escape(start_with_quote); let suffix = self.scan_optional_raw_name(); - Ok(Token::lit(token::Str, symbol, suffix)) + Ok(TokenKind::lit(token::Str, symbol, suffix)) } 'r' => { let start_bpos = self.pos; @@ -1213,7 +1213,7 @@ impl<'a> StringReader<'a> { }; let suffix = self.scan_optional_raw_name(); - Ok(Token::lit(token::StrRaw(hash_count), symbol, suffix)) + Ok(TokenKind::lit(token::StrRaw(hash_count), symbol, suffix)) } '-' => { if self.nextch_is('>') { @@ -1638,19 +1638,19 @@ mod tests { // check that the given reader produces the desired stream // of tokens (stop checking after exhausting the expected vec) - fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<Token>) { + fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) { for expected_tok in &expected { assert_eq!(&string_reader.next_token().tok, expected_tok); } } // make the identifier by looking up the string in the interner - fn mk_ident(id: &str) -> Token { - Token::from_ast_ident(Ident::from_str(id)) + fn mk_ident(id: &str) -> TokenKind { + TokenKind::from_ast_ident(Ident::from_str(id)) } - fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> Token { - Token::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) + fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { + TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) } #[test] diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 4bfc5bb16c0..b8cd32883b8 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -23,7 +23,7 @@ impl<'a> StringReader<'a> { struct TokenTreesReader<'a> { string_reader: StringReader<'a>, - token: token::Token, + token: token::TokenKind, span: Span, /// Stack of open delimiters and their spans. Used for error message. open_braces: Vec<(token::DelimToken, Span)>, diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 18019a89130..945475ff981 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -3,7 +3,7 @@ use crate::ast::{self, Ident, Lit, LitKind}; use crate::parse::parser::Parser; use crate::parse::PResult; -use crate::parse::token::{self, Token}; +use crate::parse::token::{self, TokenKind}; use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte}; use crate::print::pprust; use crate::symbol::{kw, sym, Symbol}; @@ -228,7 +228,7 @@ impl Lit { } /// Converts arbitrary token into an AST literal. - crate fn from_token(token: &Token, span: Span) -> Result<Lit, LitError> { + crate fn from_token(token: &TokenKind, span: Span) -> Result<Lit, LitError> { let lit = match *token { token::Ident(ident, false) if ident.name == kw::True || ident.name == kw::False => token::Lit::new(token::Bool, ident.name, None), @@ -276,7 +276,7 @@ impl<'a> Parser<'a> { let next_span = self.look_ahead_span(1); if self.span.hi() == next_span.lo() { let s = String::from("0.") + &symbol.as_str(); - let token = Token::lit(token::Float, Symbol::intern(&s), suffix); + let token = TokenKind::lit(token::Float, Symbol::intern(&s), suffix); return Some((token, self.span.to(next_span))); } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 60d04ae9d94..7f8b96508bd 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -358,13 +358,13 @@ pub fn stream_to_parser_with_base_dir<'a>( /// A sequence separator. pub struct SeqSep { /// The seperator token. - pub sep: Option<token::Token>, + pub sep: Option<token::TokenKind>, /// `true` if a trailing separator is allowed. pub trailing_sep_allowed: bool, } impl SeqSep { - pub fn trailing_allowed(t: token::Token) -> SeqSep { + pub fn trailing_allowed(t: token::TokenKind) -> SeqSep { SeqSep { sep: Some(t), trailing_sep_allowed: true, diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 8409e300fc9..8fc02dd9259 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -196,9 +196,9 @@ enum PrevTokenKind { #[derive(Clone)] pub struct Parser<'a> { pub sess: &'a ParseSess, - /// The current token. - pub token: token::Token, - /// The span of the current token. + /// the current token: + pub token: token::TokenKind, + /// the span of the current token: pub span: Span, meta_var_span: Option<Span>, /// The span of the previous token. @@ -355,7 +355,7 @@ impl TokenCursor { [ TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)), TokenTree::Token(sp, token::Eq), - TokenTree::Token(sp, token::Token::lit( + TokenTree::Token(sp, token::TokenKind::lit( token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None )), ] @@ -380,7 +380,7 @@ impl TokenCursor { #[derive(Clone, PartialEq)] crate enum TokenType { - Token(token::Token), + Token(token::TokenKind), Keyword(Symbol), Operator, Lifetime, @@ -410,7 +410,7 @@ impl TokenType { /// /// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes /// that `IDENT` is not the ident of a fn trait. -fn can_continue_type_after_non_fn_ident(t: &token::Token) -> bool { +fn can_continue_type_after_non_fn_ident(t: &token::TokenKind) -> bool { t == &token::ModSep || t == &token::Lt || t == &token::BinOp(token::Shl) } @@ -559,7 +559,7 @@ impl<'a> Parser<'a> { } /// Expects and consumes the token `t`. Signals an error if the next token is not `t`. - pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> { + pub fn expect(&mut self, t: &token::TokenKind) -> PResult<'a, bool /* recovered */> { if self.expected_tokens.is_empty() { if self.token == *t { self.bump(); @@ -577,8 +577,8 @@ impl<'a> Parser<'a> { /// anything. Signal a fatal error if next token is unexpected. pub fn expect_one_of( &mut self, - edible: &[token::Token], - inedible: &[token::Token], + edible: &[token::TokenKind], + inedible: &[token::TokenKind], ) -> PResult<'a, bool /* recovered */> { if edible.contains(&self.token) { self.bump(); @@ -640,14 +640,14 @@ impl<'a> Parser<'a> { /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not /// encountered. - crate fn check(&mut self, tok: &token::Token) -> bool { + crate fn check(&mut self, tok: &token::TokenKind) -> bool { let is_present = self.token == *tok; if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); } is_present } /// Consumes a token 'tok' if it exists. Returns whether the given token was present. - pub fn eat(&mut self, tok: &token::Token) -> bool { + pub fn eat(&mut self, tok: &token::TokenKind) -> bool { let is_present = self.check(tok); if is_present { self.bump() } is_present @@ -883,7 +883,7 @@ impl<'a> Parser<'a> { /// `f` must consume tokens until reaching the next separator or /// closing bracket. pub fn parse_seq_to_end<T, F>(&mut self, - ket: &token::Token, + ket: &token::TokenKind, sep: SeqSep, f: F) -> PResult<'a, Vec<T>> where @@ -901,7 +901,7 @@ impl<'a> Parser<'a> { /// closing bracket. pub fn parse_seq_to_before_end<T, F>( &mut self, - ket: &token::Token, + ket: &token::TokenKind, sep: SeqSep, f: F, ) -> PResult<'a, (Vec<T>, bool)> @@ -912,7 +912,7 @@ impl<'a> Parser<'a> { crate fn parse_seq_to_before_tokens<T, F>( &mut self, - kets: &[&token::Token], + kets: &[&token::TokenKind], sep: SeqSep, expect: TokenExpectType, mut f: F, @@ -986,8 +986,8 @@ impl<'a> Parser<'a> { /// closing bracket. fn parse_unspanned_seq<T, F>( &mut self, - bra: &token::Token, - ket: &token::Token, + bra: &token::TokenKind, + ket: &token::TokenKind, sep: SeqSep, f: F, ) -> PResult<'a, Vec<T>> where @@ -1032,7 +1032,7 @@ impl<'a> Parser<'a> { /// Advance the parser using provided token as a next one. Use this when /// consuming a part of a token. For example a single `<` from `<<`. - fn bump_with(&mut self, next: token::Token, span: Span) { + fn bump_with(&mut self, next: token::TokenKind, span: Span) { self.prev_span = self.span.with_hi(span.lo()); // It would be incorrect to record the kind of the current token, but // fortunately for tokens currently using `bump_with`, the @@ -1044,7 +1044,7 @@ impl<'a> Parser<'a> { } pub fn look_ahead<R, F>(&self, dist: usize, f: F) -> R where - F: FnOnce(&token::Token) -> R, + F: FnOnce(&token::TokenKind) -> R, { if dist == 0 { return f(&self.token) @@ -1763,7 +1763,7 @@ impl<'a> Parser<'a> { fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> { let ident = self.parse_path_segment_ident()?; - let is_args_start = |token: &token::Token| match *token { + let is_args_start = |token: &token::TokenKind| match *token { token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren) | token::LArrow => true, _ => false, @@ -1992,7 +1992,7 @@ impl<'a> Parser<'a> { let ex: ExprKind; - // Note: when adding new syntax here, don't forget to adjust Token::can_begin_expr(). + // Note: when adding new syntax here, don't forget to adjust TokenKind::can_begin_expr(). match self.token { token::OpenDelim(token::Paren) => { self.bump(); @@ -2706,7 +2706,7 @@ impl<'a> Parser<'a> { -> PResult<'a, P<Expr>> { let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?; let lo = self.span; - // Note: when adding new unary operators, don't forget to adjust Token::can_begin_expr() + // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() let (hi, ex) = match self.token { token::Not => { self.bump(); @@ -2760,7 +2760,7 @@ impl<'a> Parser<'a> { // `not` is just an ordinary identifier in Rust-the-language, // but as `rustc`-the-compiler, we can issue clever diagnostics // for confused users who really want to say `!` - let token_cannot_continue_expr = |t: &token::Token| match *t { + let token_cannot_continue_expr = |t: &token::TokenKind| match *t { // These tokens can start an expression after `!`, but // can't continue an expression after an ident token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw), @@ -4779,7 +4779,7 @@ impl<'a> Parser<'a> { let mut last_plus_span = None; let mut was_negative = false; loop { - // This needs to be synchronized with `Token::can_begin_bound`. + // This needs to be synchronized with `TokenKind::can_begin_bound`. let is_bound_start = self.check_path() || self.check_lifetime() || self.check(&token::Not) || // used for error reporting only self.check(&token::Question) || @@ -6413,7 +6413,7 @@ impl<'a> Parser<'a> { } /// Given a termination token, parses all of the items in a module. - fn parse_mod_items(&mut self, term: &token::Token, inner_lo: Span) -> PResult<'a, Mod> { + fn parse_mod_items(&mut self, term: &token::TokenKind, inner_lo: Span) -> PResult<'a, Mod> { let mut items = vec![]; while let Some(item) = self.parse_item()? { items.push(item); diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index d54d12698bb..aa1e8fd060f 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -2,7 +2,7 @@ pub use BinOpToken::*; pub use Nonterminal::*; pub use DelimToken::*; pub use LitKind::*; -pub use Token::*; +pub use TokenKind::*; use crate::ast::{self}; use crate::parse::ParseSess; @@ -118,7 +118,7 @@ impl Lit { } pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool { - let ident_token: Token = Ident(ident, is_raw); + let ident_token: TokenKind = Ident(ident, is_raw); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || @@ -149,7 +149,7 @@ pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool { } fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool { - let ident_token: Token = Ident(ident, is_raw); + let ident_token: TokenKind = Ident(ident, is_raw); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || @@ -166,7 +166,7 @@ fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool { } #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)] -pub enum Token { +pub enum TokenKind { /* Expression-operator symbols. */ Eq, Lt, @@ -231,13 +231,13 @@ pub enum Token { Eof, } -// `Token` is used a lot. Make sure it doesn't unintentionally get bigger. +// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert_size!(Token, 16); +static_assert_size!(TokenKind, 16); -impl Token { - /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: ast::Ident) -> Token { +impl TokenKind { + /// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary. + pub fn from_ast_ident(ident: ast::Ident) -> TokenKind { Ident(ident, ident.is_raw_guess()) } @@ -323,7 +323,7 @@ impl Token { self == &Question || self == &OpenDelim(Paren) } - pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Token { + pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { Literal(Lit::new(kind, symbol, suffix)) } @@ -468,7 +468,7 @@ impl Token { } } - crate fn glue(self, joint: Token) -> Option<Token> { + crate fn glue(self, joint: TokenKind) -> Option<TokenKind> { Some(match self { Eq => match joint { Eq => EqEq, @@ -534,7 +534,7 @@ impl Token { /// Returns tokens that are likely to be typed accidentally instead of the current token. /// Enables better error recovery when the wrong token is found. - crate fn similar_tokens(&self) -> Option<Vec<Token>> { + crate fn similar_tokens(&self) -> Option<Vec<TokenKind>> { match *self { Comma => Some(vec![Dot, Lt, Semi]), Semi => Some(vec![Colon, Comma]), @@ -544,7 +544,7 @@ impl Token { // See comments in `Nonterminal::to_tokenstream` for why we care about // *probably* equal here rather than actual equality - crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { + crate fn probably_equal_for_proc_macro(&self, other: &TokenKind) -> bool { if mem::discriminant(self) != mem::discriminant(other) { return false } @@ -743,7 +743,7 @@ impl Nonterminal { } } -crate fn is_op(tok: &Token) -> bool { +crate fn is_op(tok: &TokenKind) -> bool { match *tok { OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) | Lifetime(..) | Interpolated(..) | diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 57c01e9e3ef..cd7106191be 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -6,7 +6,7 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg}; use crate::util::parser::{self, AssocOp, Fixity}; use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; -use crate::parse::token::{self, BinOpToken, Nonterminal, Token}; +use crate::parse::token::{self, BinOpToken, Nonterminal, TokenKind}; use crate::parse::lexer::comments; use crate::parse::{self, ParseSess}; use crate::print::pp::{self, Breaks}; @@ -189,7 +189,7 @@ pub fn literal_to_string(lit: token::Lit) -> String { out } -pub fn token_to_string(tok: &Token) -> String { +pub fn token_to_string(tok: &TokenKind) -> String { match *tok { token::Eq => "=".to_string(), token::Lt => "<".to_string(), diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 5a934cd9f08..0f50f51f5d3 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -16,7 +16,7 @@ use crate::ext::base; use crate::ext::tt::{macro_parser, quoted}; use crate::parse::Directory; -use crate::parse::token::{self, DelimToken, Token}; +use crate::parse::token::{self, DelimToken, TokenKind}; use crate::print::pprust; use syntax_pos::{BytePos, Mark, Span, DUMMY_SP}; @@ -44,7 +44,7 @@ use std::{fmt, iter, mem}; #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum TokenTree { /// A single token - Token(Span, token::Token), + Token(Span, token::TokenKind), /// A delimited sequence of token trees Delimited(DelimSpan, DelimToken, TokenStream), } @@ -54,7 +54,7 @@ pub enum TokenTree { fn _dummy() where Span: Send + Sync, - token::Token: Send + Sync, + token::TokenKind: Send + Sync, DelimSpan: Send + Sync, DelimToken: Send + Sync, TokenStream: Send + Sync, @@ -130,7 +130,7 @@ impl TokenTree { } /// Indicates if the stream is a token that is equal to the provided token. - pub fn eq_token(&self, t: Token) -> bool { + pub fn eq_token(&self, t: TokenKind) -> bool { match *self { TokenTree::Token(_, ref tk) => *tk == t, _ => false, @@ -241,8 +241,8 @@ impl From<TokenTree> for TreeAndJoint { } } -impl From<Token> for TokenStream { - fn from(token: Token) -> TokenStream { +impl From<TokenKind> for TokenStream { + fn from(token: TokenKind) -> TokenStream { TokenTree::Token(DUMMY_SP, token).into() } } @@ -580,7 +580,7 @@ mod tests { use super::*; use crate::syntax::ast::Ident; use crate::with_default_globals; - use crate::parse::token::Token; + use crate::parse::token::TokenKind; use crate::util::parser_testing::string_to_stream; use syntax_pos::{Span, BytePos, NO_EXPANSION}; diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index 8dc9ce39915..9e26f1bf7d3 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, BinOpToken}; +use crate::parse::token::{self, TokenKind, BinOpToken}; use crate::symbol::kw; use crate::ast::{self, BinOpKind}; @@ -69,7 +69,7 @@ pub enum Fixity { impl AssocOp { /// Creates a new AssocOP from a token - pub fn from_token(t: &Token) -> Option<AssocOp> { + pub fn from_token(t: &TokenKind) -> Option<AssocOp> { use AssocOp::*; match *t { token::BinOpEq(k) => Some(AssignOp(k)), diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 334709b1521..35f70092be4 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -14,7 +14,7 @@ //! those that are created by the expansion of a macro. use crate::ast::*; -use crate::parse::token::Token; +use crate::parse::token::TokenKind; use crate::tokenstream::{TokenTree, TokenStream}; use syntax_pos::Span; @@ -151,7 +151,7 @@ pub trait Visitor<'ast>: Sized { fn visit_tts(&mut self, tts: TokenStream) { walk_tts(self, tts) } - fn visit_token(&mut self, _t: Token) {} + fn visit_token(&mut self, _t: TokenKind) {} // FIXME: add `visit_interpolated` and `walk_interpolated` fn visit_vis(&mut self, vis: &'ast Visibility) { walk_vis(self, vis) diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index 13342c8e28e..29dd445e751 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -4,7 +4,7 @@ use syntax::ast::{self, *}; use syntax::source_map::Spanned; use syntax::ext::base::*; use syntax::ext::build::AstBuilder; -use syntax::parse::token::{self, Token}; +use syntax::parse::token::{self, TokenKind}; use syntax::parse::parser::Parser; use syntax::print::pprust; use syntax::ptr::P; @@ -31,7 +31,7 @@ pub fn expand_assert<'cx>( tts: custom_message.unwrap_or_else(|| { TokenStream::from(TokenTree::Token( DUMMY_SP, - Token::lit(token::Str, Symbol::intern(&format!( + TokenKind::lit(token::Str, Symbol::intern(&format!( "assertion failed: {}", pprust::expr_to_string(&cond_expr).escape_debug() )), None), diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs index cc05ecf8df5..119b83b7527 100644 --- a/src/libsyntax_ext/proc_macro_server.rs +++ b/src/libsyntax_ext/proc_macro_server.rs @@ -161,7 +161,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)> let stream = vec![ Ident(ast::Ident::new(sym::doc, span), false), Eq, - Token::lit(token::Str, Symbol::intern(&escaped), None), + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), ] .into_iter() .map(|token| tokenstream::TokenTree::Token(span, token)) @@ -220,7 +220,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> { }) if symbol.as_str().starts_with("-") => { let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); - let integer = Token::lit(token::Integer, symbol, suffix); + let integer = TokenKind::lit(token::Integer, symbol, suffix); let a = tokenstream::TokenTree::Token(span, minus); let b = tokenstream::TokenTree::Token(span, integer); return vec![a, b].into_iter().collect(); @@ -231,7 +231,7 @@ impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> { }) if symbol.as_str().starts_with("-") => { let minus = BinOp(BinOpToken::Minus); let symbol = Symbol::intern(&symbol.as_str()[1..]); - let float = Token::lit(token::Float, symbol, suffix); + let float = TokenKind::lit(token::Float, symbol, suffix); let a = tokenstream::TokenTree::Token(span, minus); let b = tokenstream::TokenTree::Token(span, float); return vec![a, b].into_iter().collect(); |