From e0127dbf8135b766a332ce21c4eee48998b59bef Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Tue, 4 Jun 2019 20:42:43 +0300 Subject: syntax: Use `Token` in `TokenTree::Token` --- src/libsyntax/diagnostics/plugin.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src/libsyntax/diagnostics') diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 0c57c23b2b5..b342e4bc472 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -5,7 +5,7 @@ use crate::ast::{self, Ident, Name}; use crate::source_map; use crate::ext::base::{ExtCtxt, MacEager, MacResult}; use crate::ext::build::AstBuilder; -use crate::parse::token; +use crate::parse::token::{self, Token}; use crate::ptr::P; use crate::symbol::kw; use crate::tokenstream::{TokenTree}; @@ -34,7 +34,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>, token_tree: &[TokenTree]) -> Box { let code = match (token_tree.len(), token_tree.get(0)) { - (1, Some(&TokenTree::Token(_, token::Ident(code, _)))) => code, + (1, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. }))) => code, _ => unreachable!() }; @@ -72,12 +72,12 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>, token_tree.get(1), token_tree.get(2) ) { - (1, Some(&TokenTree::Token(_, token::Ident(ref code, _))), None, None) => { + (1, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })), None, None) => { (code, None) }, - (3, Some(&TokenTree::Token(_, token::Ident(ref code, _))), - Some(&TokenTree::Token(_, token::Comma)), - Some(&TokenTree::Token(_, token::Literal(token::Lit { symbol, .. })))) => { + (3, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })), + Some(&TokenTree::Token(Token { kind: token::Comma, .. })), + Some(&TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), .. }))) => { (code, Some(symbol)) } _ => unreachable!() @@ -143,9 +143,9 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, let (crate_name, name) = match (&token_tree[0], &token_tree[2]) { ( // Crate name. - &TokenTree::Token(_, token::Ident(ref crate_name, _)), + &TokenTree::Token(Token { kind: token::Ident(ref crate_name, _), .. }), // DIAGNOSTICS ident. - &TokenTree::Token(_, token::Ident(ref name, _)) + &TokenTree::Token(Token { kind: token::Ident(ref name, _), .. }) ) => (*&crate_name, name), _ => unreachable!() }; -- cgit 1.4.1-3-g733a5 From f745e5f9b676be02cc1dfbab0bfb338dc72b4dd3 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 5 Jun 2019 11:56:06 +0300 Subject: syntax: Remove duplicate span from `token::Ident` --- src/librustc/ich/impls_syntax.rs | 4 +- src/librustdoc/html/highlight.rs | 4 +- src/libsyntax/attr/mod.rs | 10 ++-- src/libsyntax/diagnostics/plugin.rs | 18 +++---- src/libsyntax/ext/base.rs | 2 +- src/libsyntax/ext/tt/macro_parser.rs | 39 +++++++------- src/libsyntax/ext/tt/macro_rules.rs | 12 ++--- src/libsyntax/ext/tt/quoted.rs | 5 +- src/libsyntax/mut_visit.rs | 1 - src/libsyntax/parse/diagnostics.rs | 6 +-- src/libsyntax/parse/lexer/mod.rs | 22 +++----- src/libsyntax/parse/literal.rs | 8 +-- src/libsyntax/parse/mod.rs | 31 +++++------ src/libsyntax/parse/parser.rs | 48 ++++++++--------- src/libsyntax/parse/token.rs | 94 +++++++++++++++++++++------------- src/libsyntax/tokenstream.rs | 4 +- src/libsyntax_ext/concat_idents.rs | 4 +- src/libsyntax_ext/format.rs | 6 +-- src/libsyntax_ext/proc_macro_decls.rs | 4 +- src/libsyntax_ext/proc_macro_server.rs | 13 +++-- src/libsyntax_pos/symbol.rs | 30 ++++++----- 21 files changed, 181 insertions(+), 184 deletions(-) (limited to 'src/libsyntax/diagnostics') diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 20d308e5fe8..abe4196abd1 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -353,8 +353,8 @@ impl<'a> HashStable> for token::TokenKind { } token::Literal(lit) => lit.hash_stable(hcx, hasher), - token::Ident(ident, is_raw) => { - ident.name.hash_stable(hcx, hasher); + token::Ident(name, is_raw) => { + name.hash_stable(hcx, hasher); is_raw.hash_stable(hcx, hasher); } token::Lifetime(name) => name.hash_stable(hcx, hasher), diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index bc6eaaaa8b9..281bd72deeb 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -325,8 +325,8 @@ impl<'a> Classifier<'a> { } // Keywords are also included in the identifier set. - token::Ident(ident, is_raw) => { - match ident.name { + token::Ident(name, is_raw) => { + match name { kw::Ref | kw::Mut if !is_raw => Class::RefKeyWord, kw::SelfLower | kw::SelfUpper => Class::Self_, diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 56afc8728b4..39ffabaa4a9 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -482,19 +482,19 @@ impl MetaItem { let path = match tokens.next() { Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) | Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: { - let mut segments = if let token::Ident(ident, _) = kind { + let mut segments = if let token::Ident(name, _) = kind { if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() { tokens.next(); - vec![PathSegment::from_ident(ident.with_span_pos(span))] + vec![PathSegment::from_ident(Ident::new(name, span))] } else { - break 'arm Path::from_ident(ident.with_span_pos(span)); + break 'arm Path::from_ident(Ident::new(name, span)); } } else { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(Token { kind: token::Ident(ident, _), span })) = tokens.next() { - segments.push(PathSegment::from_ident(ident.with_span_pos(span))); + if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = tokens.next() { + segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { return None; } diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index b342e4bc472..8d9848d98fb 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -39,7 +39,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>, }; ecx.parse_sess.registered_diagnostics.with_lock(|diagnostics| { - match diagnostics.get_mut(&code.name) { + match diagnostics.get_mut(&code) { // Previously used errors. Some(&mut ErrorInfo { description: _, use_site: Some(previous_span) }) => { ecx.struct_span_warn(span, &format!( @@ -72,10 +72,10 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>, token_tree.get(1), token_tree.get(2) ) { - (1, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })), None, None) => { + (1, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. })), None, None) => { (code, None) }, - (3, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })), + (3, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. })), Some(&TokenTree::Token(Token { kind: token::Comma, .. })), Some(&TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), .. }))) => { (code, Some(symbol)) @@ -112,7 +112,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>, description, use_site: None }; - if diagnostics.insert(code.name, info).is_some() { + if diagnostics.insert(code, info).is_some() { ecx.span_err(span, &format!( "diagnostic code {} already registered", code )); @@ -140,13 +140,13 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, token_tree: &[TokenTree]) -> Box { assert_eq!(token_tree.len(), 3); - let (crate_name, name) = match (&token_tree[0], &token_tree[2]) { + let (crate_name, ident) = match (&token_tree[0], &token_tree[2]) { ( // Crate name. - &TokenTree::Token(Token { kind: token::Ident(ref crate_name, _), .. }), + &TokenTree::Token(Token { kind: token::Ident(crate_name, _), .. }), // DIAGNOSTICS ident. - &TokenTree::Token(Token { kind: token::Ident(ref name, _), .. }) - ) => (*&crate_name, name), + &TokenTree::Token(Token { kind: token::Ident(name, _), span }) + ) => (crate_name, Ident::new(name, span)), _ => unreachable!() }; @@ -209,7 +209,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, MacEager::items(smallvec![ P(ast::Item { - ident: *name, + ident, attrs: Vec::new(), id: ast::DUMMY_NODE_ID, node: ast::ItemKind::Const( diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 0c2ab672407..3b24837e365 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -269,7 +269,7 @@ impl TTMacroExpander for F if let token::Interpolated(nt) = &token.kind { if let token::NtIdent(ident, is_raw) = **nt { *tt = tokenstream::TokenTree::token(ident.span, - token::Ident(ident, is_raw)); + token::Ident(ident.name, is_raw)); } } } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index f93b548c501..82cc9e8ac22 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -74,7 +74,7 @@ pub use NamedMatch::*; pub use ParseResult::*; use TokenTreeOrTokenTreeSlice::*; -use crate::ast::Ident; +use crate::ast::{Ident, Name}; use crate::ext::tt::quoted::{self, TokenTree}; use crate::parse::{Directory, ParseSess}; use crate::parse::parser::{Parser, PathStyle}; @@ -429,8 +429,8 @@ pub fn parse_failure_msg(tok: TokenKind) -> String { /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool { - if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) { - id1.name == id2.name && is_raw1 == is_raw2 + if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) { + name1 == name2 && is_raw1 == is_raw2 } else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) { name1 == name2 } else { @@ -466,8 +466,7 @@ fn inner_parse_loop<'root, 'tt>( next_items: &mut Vec>, eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, - token: &TokenKind, - span: syntax_pos::Span, + token: &Token, ) -> ParseResult<()> { // Pop items from `cur_items` until it is empty. while let Some(mut item) = cur_items.pop() { @@ -510,7 +509,7 @@ fn inner_parse_loop<'root, 'tt>( // Add matches from this repetition to the `matches` of `up` for idx in item.match_lo..item.match_hi { let sub = item.matches[idx].clone(); - let span = DelimSpan::from_pair(item.sp_open, span); + let span = DelimSpan::from_pair(item.sp_open, token.span); new_pos.push_match(idx, MatchedSeq(sub, span)); } @@ -598,7 +597,7 @@ fn inner_parse_loop<'root, 'tt>( TokenTree::MetaVarDecl(_, _, id) => { // Built-in nonterminals never start with these tokens, // so we can eliminate them from consideration. - if may_begin_with(id.name, token) { + if may_begin_with(token, id.name) { bb_items.push(item); } } @@ -698,7 +697,6 @@ pub fn parse( &mut eof_items, &mut bb_items, &parser.token, - parser.span, ) { Success(_) => {} Failure(token, msg) => return Failure(token, msg), @@ -806,10 +804,9 @@ pub fn parse( /// The token is an identifier, but not `_`. /// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> { +fn get_macro_name(token: &TokenKind) -> Option<(Name, bool)> { match *token { - token::Ident(ident, is_raw) if ident.name != kw::Underscore => - Some((ident, is_raw)), + token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), _ => None, } } @@ -818,7 +815,7 @@ fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> { /// /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that /// token. Be conservative (return true) if not sure. -fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { +fn may_begin_with(token: &Token, name: Name) -> bool { /// Checks whether the non-terminal may contain a single (non-keyword) identifier. fn may_be_ident(nt: &token::Nonterminal) -> bool { match *nt { @@ -830,14 +827,14 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { match name { sym::expr => token.can_begin_expr(), sym::ty => token.can_begin_type(), - sym::ident => get_macro_ident(token).is_some(), + sym::ident => get_macro_name(token).is_some(), sym::literal => token.can_begin_literal_or_bool(), - sym::vis => match *token { + sym::vis => match token.kind { // The follow-set of :vis + "priv" keyword + interpolated token::Comma | token::Ident(..) | token::Interpolated(_) => true, _ => token.can_begin_type(), }, - sym::block => match *token { + sym::block => match token.kind { token::OpenDelim(token::Brace) => true, token::Interpolated(ref nt) => match **nt { token::NtItem(_) @@ -851,7 +848,7 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { }, _ => false, }, - sym::path | sym::meta => match *token { + sym::path | sym::meta => match token.kind { token::ModSep | token::Ident(..) => true, token::Interpolated(ref nt) => match **nt { token::NtPath(_) | token::NtMeta(_) => true, @@ -859,7 +856,7 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { }, _ => false, }, - sym::pat => match *token { + sym::pat => match token.kind { token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) token::OpenDelim(token::Paren) | // tuple pattern token::OpenDelim(token::Bracket) | // slice pattern @@ -875,7 +872,7 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { token::Interpolated(ref nt) => may_be_ident(nt), _ => false, }, - sym::lifetime => match *token { + sym::lifetime => match token.kind { token::Lifetime(_) => true, token::Interpolated(ref nt) => match **nt { token::NtLifetime(_) | token::NtTT(_) => true, @@ -883,7 +880,7 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool { }, _ => false, }, - _ => match *token { + _ => match token.kind { token::CloseDelim(_) => false, _ => true, }, @@ -929,10 +926,10 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> Nonterminal { sym::literal => token::NtLiteral(panictry!(p.parse_literal_maybe_minus())), sym::ty => token::NtTy(panictry!(p.parse_ty())), // this could be handled like a token, since it is one - sym::ident => if let Some((ident, is_raw)) = get_macro_ident(&p.token) { + sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { let span = p.span; p.bump(); - token::NtIdent(Ident::new(ident.name, span), is_raw) + token::NtIdent(Ident::new(name, span), is_raw) } else { let token_str = pprust::token_to_string(&p.token); p.fatal(&format!("expected ident, found {}", &token_str)).emit(); diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 05e921b1bfd..77f53c35b0b 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -1046,8 +1046,7 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> IsInFollow { match tok { TokenTree::Token(token) => match token.kind { FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, - Ident(i, false) if i.name == kw::If || - i.name == kw::In => IsInFollow::Yes, + Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, _ => IsInFollow::No(tokens), }, _ => IsInFollow::No(tokens), @@ -1064,8 +1063,8 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> IsInFollow { OpenDelim(token::DelimToken::Bracket) | Comma | FatArrow | Colon | Eq | Gt | BinOp(token::Shr) | Semi | BinOp(token::Or) => IsInFollow::Yes, - Ident(i, false) if i.name == kw::As || - i.name == kw::Where => IsInFollow::Yes, + Ident(name, false) if name == kw::As || + name == kw::Where => IsInFollow::Yes, _ => IsInFollow::No(tokens), }, TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => @@ -1092,9 +1091,8 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> IsInFollow { match tok { TokenTree::Token(token) => match token.kind { Comma => IsInFollow::Yes, - Ident(i, is_raw) if is_raw || i.name != kw::Priv => - IsInFollow::Yes, - ref tok => if tok.can_begin_type() { + Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, + _ => if token.can_begin_type() { IsInFollow::Yes } else { IsInFollow::No(tokens) diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 558b07af611..582d87b911d 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -323,10 +323,9 @@ where // metavariable that names the crate of the invocation. Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { let (ident, is_raw) = token.ident().unwrap(); - let span = token.span.with_lo(span.lo()); + let span = ident.span.with_lo(span.lo()); if ident.name == kw::Crate && !is_raw { - let ident = ast::Ident::new(kw::DollarCrate, ident.span); - TokenTree::token(span, token::Ident(ident, is_raw)) + TokenTree::token(span, token::Ident(kw::DollarCrate, is_raw)) } else { TokenTree::MetaVar(span, ident) } diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 3bb36605299..7eb88de2281 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -598,7 +598,6 @@ pub fn noop_visit_tts(TokenStream(tts): &mut TokenStream, vis: &m // apply ident visitor if it's an ident, apply other visits to interpolated nodes pub fn noop_visit_token(t: &mut TokenKind, vis: &mut T) { match t { - token::Ident(id, _is_raw) => vis.visit_ident(id), token::Interpolated(nt) => { let mut nt = Lrc::make_mut(nt); vis.visit_interpolated(&mut nt); diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/diagnostics.rs index 1759a229cf4..7830b2ce880 100644 --- a/src/libsyntax/parse/diagnostics.rs +++ b/src/libsyntax/parse/diagnostics.rs @@ -201,12 +201,12 @@ impl<'a> Parser<'a> { self.span, &format!("expected identifier, found {}", self.this_token_descr()), ); - if let token::Ident(ident, false) = &self.token.kind { - if ident.is_raw_guess() { + if let token::Ident(name, false) = self.token.kind { + if Ident::new(name, self.span).is_raw_guess() { err.span_suggestion( self.span, "you can escape reserved keywords to use them as identifiers", - format!("r#{}", ident), + format!("r#{}", name), Applicability::MaybeIncorrect, ); } diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index da8c6f5ac22..e3d959c2c54 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,4 +1,4 @@ -use crate::ast::{self, Ident}; +use crate::ast; use crate::parse::ParseSess; use crate::parse::token::{self, Token, TokenKind}; use crate::symbol::{sym, Symbol}; @@ -61,15 +61,6 @@ impl<'a> StringReader<'a> { (real, raw) } - fn mk_ident(&self, string: &str) -> Ident { - let mut ident = Ident::from_str(string); - if let Some(span) = self.override_span { - ident.span = span; - } - - ident - } - fn unwrap_or_abort(&mut self, res: Result) -> Token { match res { Ok(tok) => tok, @@ -858,17 +849,17 @@ impl<'a> StringReader<'a> { return Ok(self.with_str_from(start, |string| { // FIXME: perform NFKC normalization here. (Issue #2253) - let ident = self.mk_ident(string); + let name = ast::Name::intern(string); if is_raw_ident { let span = self.mk_sp(raw_start, self.pos); - if !ident.can_be_raw() { - self.err_span(span, &format!("`{}` cannot be a raw identifier", ident)); + if !name.can_be_raw() { + self.err_span(span, &format!("`{}` cannot be a raw identifier", name)); } self.sess.raw_identifier_spans.borrow_mut().push(span); } - token::Ident(ident, is_raw_ident) + token::Ident(name, is_raw_ident) })); } } @@ -1567,12 +1558,11 @@ mod tests { &sh, "/* my source file */ fn main() { println!(\"zebra\"); }\n" .to_string()); - let id = Ident::from_str("fn"); assert_eq!(string_reader.next_token(), token::Comment); assert_eq!(string_reader.next_token(), token::Whitespace); let tok1 = string_reader.next_token(); let tok2 = Token::new( - token::Ident(id, false), + token::Ident(Symbol::intern("fn"), false), Span::new(BytePos(21), BytePos(23), NO_EXPANSION), ); assert_eq!(tok1.kind, tok2.kind); diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 978fd205ea4..7b27304071c 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -1,6 +1,6 @@ //! Code related to parsing literals. -use crate::ast::{self, Ident, Lit, LitKind}; +use crate::ast::{self, Lit, LitKind}; use crate::parse::parser::Parser; use crate::parse::PResult; use crate::parse::token::{self, Token, TokenKind}; @@ -230,8 +230,8 @@ impl Lit { /// Converts arbitrary token into an AST literal. crate fn from_token(token: &TokenKind, span: Span) -> Result { let lit = match *token { - token::Ident(ident, false) if ident.name == kw::True || ident.name == kw::False => - token::Lit::new(token::Bool, ident.name, None), + token::Ident(name, false) if name == kw::True || name == kw::False => + token::Lit::new(token::Bool, name, None), token::Literal(lit) => lit, token::Interpolated(ref nt) => { @@ -258,7 +258,7 @@ impl Lit { /// Losslessly convert an AST literal into a token stream. crate fn tokens(&self) -> TokenStream { let token = match self.token.kind { - token::Bool => token::Ident(Ident::new(self.token.symbol, self.span), false), + token::Bool => token::Ident(self.token.symbol, false), _ => token::Literal(self.token), }; TokenTree::token(self.span, token).into() diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 5187621258d..2b82767d7e9 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -382,11 +382,12 @@ impl SeqSep { #[cfg(test)] mod tests { use super::*; - use crate::ast::{self, Ident, PatKind}; + use crate::ast::{self, Name, PatKind}; use crate::attr::first_attr_value_str_by_name; use crate::ptr::P; use crate::parse::token::Token; use crate::print::pprust::item_to_string; + use crate::symbol::{kw, sym}; use crate::tokenstream::{DelimSpan, TokenTree}; use crate::util::parser_testing::string_to_stream; use crate::util::parser_testing::{string_to_expr, string_to_item}; @@ -418,8 +419,6 @@ mod tests { #[test] fn string_to_tts_macro () { with_default_globals(|| { - use crate::symbol::sym; - let tts: Vec<_> = string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); let tts: &[TokenTree] = &tts[..]; @@ -432,8 +431,7 @@ mod tests { Some(&TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. })), Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)), ) - if name_macro_rules.name == sym::macro_rules - && name_zip.name.as_str() == "zip" => { + if name_macro_rules == sym::macro_rules && name_zip.as_str() == "zip" => { let tts = ¯o_tts.trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) { ( @@ -448,9 +446,9 @@ mod tests { ( 2, Some(&TokenTree::Token(Token { kind: token::Dollar, .. })), - Some(&TokenTree::Token(Token { kind: token::Ident(ident, false), .. })), + Some(&TokenTree::Token(Token { kind: token::Ident(name, false), .. })), ) - if first_delim == token::Paren && ident.name.as_str() == "a" => {}, + if first_delim == token::Paren && name.as_str() == "a" => {}, _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), } let tts = &second_tts.trees().collect::>(); @@ -458,9 +456,9 @@ mod tests { ( 2, Some(&TokenTree::Token(Token { kind: token::Dollar, .. })), - Some(&TokenTree::Token(Token { kind: token::Ident(ident, false), .. })), + Some(&TokenTree::Token(Token { kind: token::Ident(name, false), .. })), ) - if second_delim == token::Paren && ident.name.as_str() == "a" => {}, + if second_delim == token::Paren && name.as_str() == "a" => {}, _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), } }, @@ -478,25 +476,22 @@ mod tests { let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); let expected = TokenStream::new(vec![ - TokenTree::token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(), - TokenTree::token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), + TokenTree::token(sp(0, 2), token::Ident(kw::Fn, false)).into(), + TokenTree::token(sp(3, 4), token::Ident(Name::intern("a"), false)).into(), TokenTree::Delimited( DelimSpan::from_pair(sp(5, 6), sp(13, 14)), token::DelimToken::Paren, TokenStream::new(vec![ - TokenTree::token(sp(6, 7), - token::Ident(Ident::from_str("b"), false)).into(), + TokenTree::token(sp(6, 7), token::Ident(Name::intern("b"), false)).into(), TokenTree::token(sp(8, 9), token::Colon).into(), - TokenTree::token(sp(10, 13), - token::Ident(Ident::from_str("i32"), false)).into(), + TokenTree::token(sp(10, 13), token::Ident(sym::i32, false)).into(), ]).into(), ).into(), TokenTree::Delimited( DelimSpan::from_pair(sp(15, 16), sp(20, 21)), token::DelimToken::Brace, TokenStream::new(vec![ - TokenTree::token(sp(17, 18), - token::Ident(Ident::from_str("b"), false)).into(), + TokenTree::token(sp(17, 18), token::Ident(Name::intern("b"), false)).into(), TokenTree::token(sp(18, 19), token::Semi).into(), ]).into(), ).into() @@ -604,8 +599,6 @@ mod tests { #[test] fn crlf_doc_comments() { with_default_globals(|| { - use crate::symbol::sym; - let sess = ParseSess::new(FilePathMapping::empty()); let name_1 = FileName::Custom("crlf_source_1".to_string()); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 362f81d02a0..57a49d1524d 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -362,7 +362,7 @@ impl TokenCursor { delim_span, token::Bracket, [ - TokenTree::token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)), + TokenTree::token(sp, token::Ident(sym::doc, false)), TokenTree::token(sp, token::Eq), TokenTree::token(sp, token::TokenKind::lit( token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None @@ -541,9 +541,9 @@ impl<'a> Parser<'a> { crate fn token_descr(&self) -> Option<&'static str> { Some(match &self.token.kind { - t if t.is_special_ident() => "reserved identifier", - t if t.is_used_keyword() => "keyword", - t if t.is_unused_keyword() => "reserved keyword", + _ if self.token.is_special_ident() => "reserved identifier", + _ if self.token.is_used_keyword() => "keyword", + _ if self.token.is_unused_keyword() => "reserved keyword", token::DocComment(..) => "doc comment", _ => return None, }) @@ -619,7 +619,7 @@ impl<'a> Parser<'a> { fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> { match self.token.kind { - token::Ident(ident, _) => { + token::Ident(name, _) => { if self.token.is_reserved_ident() { let mut err = self.expected_ident_found(); if recover { @@ -630,7 +630,7 @@ impl<'a> Parser<'a> { } let span = self.span; self.bump(); - Ok(Ident::new(ident.name, span)) + Ok(Ident::new(name, span)) } _ => { Err(if self.prev_token_kind == PrevTokenKind::DocComment { @@ -1618,10 +1618,10 @@ impl<'a> Parser<'a> { fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> { match self.token.kind { - token::Ident(ident, _) if self.token.is_path_segment_keyword() => { + token::Ident(name, _) if name.is_path_segment_keyword() => { let span = self.span; self.bump(); - Ok(Ident::new(ident.name, span)) + Ok(Ident::new(name, span)) } _ => self.parse_ident(), } @@ -1629,10 +1629,10 @@ impl<'a> Parser<'a> { fn parse_ident_or_underscore(&mut self) -> PResult<'a, ast::Ident> { match self.token.kind { - token::Ident(ident, false) if ident.name == kw::Underscore => { + token::Ident(name, false) if name == kw::Underscore => { let span = self.span; self.bump(); - Ok(Ident::new(ident.name, span)) + Ok(Ident::new(name, span)) } _ => self.parse_ident(), } @@ -2368,13 +2368,11 @@ impl<'a> Parser<'a> { } let mut recovery_field = None; - if let token::Ident(ident, _) = self.token.kind { + if let token::Ident(name, _) = self.token.kind { if !self.token.is_reserved_ident() && self.look_ahead(1, |t| *t == token::Colon) { // Use in case of error after field-looking code: `S { foo: () with a }` - let mut ident = ident.clone(); - ident.span = self.span; recovery_field = Some(ast::Field { - ident, + ident: Ident::new(name, self.span), span: self.span, expr: self.mk_expr(self.span, ExprKind::Err, ThinVec::new()), is_shorthand: false, @@ -2637,7 +2635,7 @@ impl<'a> Parser<'a> { self.look_ahead(1, |t| t.is_ident()) => { self.bump(); let name = match self.token.kind { - token::Ident(ident, _) => ident, + token::Ident(name, _) => name, _ => unreachable!() }; let mut err = self.fatal(&format!("unknown macro variable `{}`", name)); @@ -2651,7 +2649,7 @@ impl<'a> Parser<'a> { // Interpolated identifier and lifetime tokens are replaced with usual identifier // and lifetime tokens, so the former are never encountered during normal parsing. match **nt { - token::NtIdent(ident, is_raw) => Token::new(token::Ident(ident, is_raw), ident.span), + token::NtIdent(ident, is_raw) => Token::new(token::Ident(ident.name, is_raw), ident.span), token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span), _ => return, } @@ -2766,7 +2764,7 @@ impl<'a> Parser<'a> { let token_cannot_continue_expr = |t: &Token| match t.kind { // These tokens can start an expression after `!`, but // can't continue an expression after an ident - token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw), + token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw), token::Literal(..) | token::Pound => true, token::Interpolated(ref nt) => match **nt { token::NtIdent(..) | token::NtExpr(..) | @@ -4328,7 +4326,7 @@ impl<'a> Parser<'a> { -> PResult<'a, Option>> { let token_lo = self.span; let (ident, def) = match self.token.kind { - token::Ident(ident, false) if ident.name == kw::Macro => { + token::Ident(name, false) if name == kw::Macro => { self.bump(); let ident = self.parse_ident()?; let tokens = if self.check(&token::OpenDelim(token::Brace)) { @@ -4356,8 +4354,8 @@ impl<'a> Parser<'a> { (ident, ast::MacroDef { tokens: tokens.into(), legacy: false }) } - token::Ident(ident, _) if ident.name == sym::macro_rules && - self.look_ahead(1, |t| *t == token::Not) => { + token::Ident(name, _) if name == sym::macro_rules && + self.look_ahead(1, |t| *t == token::Not) => { let prev_span = self.prev_span; self.complain_if_pub_macro(&vis.node, prev_span); self.bump(); @@ -5481,8 +5479,8 @@ impl<'a> Parser<'a> { fn parse_self_arg(&mut self) -> PResult<'a, Option> { let expect_ident = |this: &mut Self| match this.token.kind { // Preserve hygienic context. - token::Ident(ident, _) => - { let span = this.span; this.bump(); Ident::new(ident.name, span) } + token::Ident(name, _) => + { let span = this.span; this.bump(); Ident::new(name, span) } _ => unreachable!() }; let isolated_self = |this: &mut Self, n| { @@ -5805,11 +5803,7 @@ impl<'a> Parser<'a> { match *vis { VisibilityKind::Inherited => {} _ => { - let is_macro_rules: bool = match self.token.kind { - token::Ident(sid, _) => sid.name == sym::macro_rules, - _ => false, - }; - let mut err = if is_macro_rules { + let mut err = if self.token.is_keyword(sym::macro_rules) { let mut err = self.diagnostic() .struct_span_err(sp, "can't qualify macro_rules invocation with `pub`"); err.span_suggestion( diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 81c93a4179e..ba7c88e7000 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -118,8 +118,8 @@ impl Lit { } } -pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool { - let ident_token: TokenKind = Ident(ident, is_raw); +pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || @@ -146,11 +146,11 @@ pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool { kw::While, kw::Yield, kw::Static, - ].contains(&ident.name) + ].contains(&name) } -fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool { - let ident_token: TokenKind = Ident(ident, is_raw); +fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || @@ -163,7 +163,7 @@ fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool { kw::Extern, kw::Typeof, kw::Dyn, - ].contains(&ident.name) + ].contains(&name) } #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] @@ -210,7 +210,7 @@ pub enum TokenKind { Literal(Lit), /* Name components */ - Ident(ast::Ident, /* is_raw */ bool), + Ident(ast::Name, /* is_raw */ bool), Lifetime(ast::Name), Interpolated(Lrc), @@ -245,7 +245,7 @@ pub struct Token { impl TokenKind { /// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary. pub fn from_ast_ident(ident: ast::Ident) -> TokenKind { - Ident(ident, ident.is_raw_guess()) + Ident(ident.name, ident.is_raw_guess()) } crate fn is_like_plus(&self) -> bool { @@ -254,12 +254,14 @@ impl TokenKind { _ => false, } } +} +impl Token { /// Returns `true` if the token can appear at the start of an expression. crate fn can_begin_expr(&self) -> bool { - match *self { - Ident(ident, is_raw) => - ident_can_begin_expr(ident, is_raw), // value name or keyword + match self.kind { + Ident(name, is_raw) => + ident_can_begin_expr(name, self.span, is_raw), // value name or keyword OpenDelim(..) | // tuple, array or block Literal(..) | // literal Not | // operator not @@ -289,9 +291,9 @@ impl TokenKind { /// Returns `true` if the token can appear at the start of a type. crate fn can_begin_type(&self) -> bool { - match *self { - Ident(ident, is_raw) => - ident_can_begin_type(ident, is_raw), // type name or keyword + match self.kind { + Ident(name, is_raw) => + ident_can_begin_type(name, self.span, is_raw), // type name or keyword OpenDelim(Paren) | // tuple OpenDelim(Bracket) | // array Not | // never @@ -309,7 +311,9 @@ impl TokenKind { _ => false, } } +} +impl TokenKind { /// Returns `true` if the token can appear at the start of a const param. pub fn can_begin_const_arg(&self) -> bool { match self { @@ -323,13 +327,17 @@ impl TokenKind { _ => self.can_begin_literal_or_bool(), } } +} +impl Token { /// Returns `true` if the token can appear at the start of a generic bound. crate fn can_begin_bound(&self) -> bool { self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || self == &Question || self == &OpenDelim(Paren) } +} +impl TokenKind { pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { Literal(Lit::new(kind, symbol, suffix)) } @@ -355,8 +363,8 @@ impl TokenKind { match *self { Literal(..) => true, BinOp(Minus) => true, - Ident(ident, false) if ident.name == kw::True => true, - Ident(ident, false) if ident.name == kw::False => true, + Ident(name, false) if name == kw::True => true, + Ident(name, false) if name == kw::False => true, Interpolated(ref nt) => match **nt { NtLiteral(..) => true, _ => false, @@ -367,6 +375,18 @@ impl TokenKind { } impl Token { + /// Returns an identifier if this token is an identifier. + pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { + match self.kind { + Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), + Interpolated(ref nt) => match **nt { + NtIdent(ident, is_raw) => Some((ident, is_raw)), + _ => None, + }, + _ => None, + } + } + /// Returns a lifetime identifier if this token is a lifetime. pub fn lifetime(&self) -> Option { match self.kind { @@ -381,12 +401,12 @@ impl Token { } impl TokenKind { - /// Returns an identifier if this token is an identifier. - pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { + /// Returns an identifier name if this token is an identifier. + pub fn ident_name(&self) -> Option<(ast::Name, /* is_raw */ bool)> { match *self { - Ident(ident, is_raw) => Some((ident, is_raw)), + Ident(name, is_raw) => Some((name, is_raw)), Interpolated(ref nt) => match **nt { - NtIdent(ident, is_raw) => Some((ident, is_raw)), + NtIdent(ident, is_raw) => Some((ident.name, is_raw)), _ => None, }, _ => None, @@ -405,7 +425,7 @@ impl TokenKind { } /// Returns `true` if the token is an identifier. pub fn is_ident(&self) -> bool { - self.ident().is_some() + self.ident_name().is_some() } /// Returns `true` if the token is a lifetime. crate fn is_lifetime(&self) -> bool { @@ -415,10 +435,7 @@ impl TokenKind { /// Returns `true` if the token is a identifier whose name is the given /// string slice. crate fn is_ident_named(&self, name: Symbol) -> bool { - match self.ident() { - Some((ident, _)) => ident.name == name, - None => false - } + self.ident_name().map_or(false, |(ident_name, _)| ident_name == name) } /// Returns `true` if the token is an interpolated path. @@ -440,24 +457,30 @@ impl TokenKind { crate fn is_qpath_start(&self) -> bool { self == &Lt || self == &BinOp(Shl) } +} +impl Token { crate fn is_path_start(&self) -> bool { self == &ModSep || self.is_qpath_start() || self.is_path() || self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() } +} +impl TokenKind { /// Returns `true` if the token is a given keyword, `kw`. pub fn is_keyword(&self, kw: Symbol) -> bool { - self.ident().map(|(ident, is_raw)| ident.name == kw && !is_raw).unwrap_or(false) + self.ident_name().map(|(name, is_raw)| name == kw && !is_raw).unwrap_or(false) } pub fn is_path_segment_keyword(&self) -> bool { - match self.ident() { - Some((id, false)) => id.is_path_segment_keyword(), + match self.ident_name() { + Some((name, false)) => name.is_path_segment_keyword(), _ => false, } } +} +impl Token { // Returns true for reserved identifiers used internally for elided lifetimes, // unnamed method parameters, crate root module, error recovery etc. pub fn is_special_ident(&self) -> bool { @@ -490,7 +513,9 @@ impl TokenKind { _ => false, } } +} +impl TokenKind { crate fn glue(self, joint: TokenKind) -> Option { Some(match self { Eq => match joint { @@ -537,7 +562,7 @@ impl TokenKind { _ => return None, }, SingleQuote => match joint { - Ident(ident, false) => Lifetime(Symbol::intern(&format!("'{}", ident))), + Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), _ => return None, }, @@ -608,9 +633,9 @@ impl TokenKind { (&Literal(a), &Literal(b)) => a == b, (&Lifetime(a), &Lifetime(b)) => a == b, - (&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name || - a.name == kw::DollarCrate || - c.name == kw::DollarCrate), + (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || + a == kw::DollarCrate || + c == kw::DollarCrate), (&Interpolated(_), &Interpolated(_)) => false, @@ -738,8 +763,7 @@ impl Nonterminal { prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) } Nonterminal::NtIdent(ident, is_raw) => { - let token = Ident(ident, is_raw); - Some(TokenTree::token(ident.span, token).into()) + Some(TokenTree::token(ident.span, Ident(ident.name, is_raw)).into()) } Nonterminal::NtLifetime(ident) => { Some(TokenTree::token(ident.span, Lifetime(ident.name)).into()) @@ -827,7 +851,7 @@ fn prepend_attrs(sess: &ParseSess, // For simple paths, push the identifier directly if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { let ident = attr.path.segments[0].ident; - let token = Ident(ident, ident.as_str().starts_with("r#")); + let token = Ident(ident.name, ident.as_str().starts_with("r#")); brackets.push(tokenstream::TokenTree::token(ident.span, token)); // ... and for more complicated paths, fall back to a reparse hack that diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 140b77b6b5f..bb80c1a1b3f 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -575,7 +575,7 @@ impl DelimSpan { #[cfg(test)] mod tests { use super::*; - use crate::syntax::ast::Ident; + use crate::syntax::ast::Name; use crate::with_default_globals; use crate::util::parser_testing::string_to_stream; use syntax_pos::{Span, BytePos, NO_EXPANSION}; @@ -660,7 +660,7 @@ mod tests { with_default_globals(|| { let test0: TokenStream = Vec::::new().into_iter().collect(); let test1: TokenStream = - TokenTree::token(sp(0, 1), token::Ident(Ident::from_str("a"), false)).into(); + TokenTree::token(sp(0, 1), token::Ident(Name::intern("a"), false)).into(); let test2 = string_to_ts("foo(bar::baz)"); assert_eq!(test0.is_empty(), true); diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs index 59f25af3742..8f061abc77b 100644 --- a/src/libsyntax_ext/concat_idents.rs +++ b/src/libsyntax_ext/concat_idents.rs @@ -38,8 +38,8 @@ pub fn expand_syntax_ext<'cx>(cx: &'cx mut ExtCtxt<'_>, } } else { match *e { - TokenTree::Token(Token { kind: token::Ident(ident, _), .. }) => - res_str.push_str(&ident.as_str()), + TokenTree::Token(Token { kind: token::Ident(name, _), .. }) => + res_str.push_str(&name.as_str()), _ => { cx.span_err(sp, "concat_idents! requires ident args."); return DummyResult::any(sp); diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs index 0eaac544e33..c78215b77a9 100644 --- a/src/libsyntax_ext/format.rs +++ b/src/libsyntax_ext/format.rs @@ -149,16 +149,16 @@ fn parse_args<'a>( } // accept trailing commas if named || (p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq)) { named = true; - let ident = if let token::Ident(i, _) = p.token.kind { + let name = if let token::Ident(name, _) = p.token.kind { p.bump(); - i + name } else { return Err(ecx.struct_span_err( p.span, "expected ident, positional arguments cannot follow named arguments", )); }; - let name: &str = &ident.as_str(); + let name: &str = &name.as_str(); p.expect(&token::Eq)?; let e = p.parse_expr()?; diff --git a/src/libsyntax_ext/proc_macro_decls.rs b/src/libsyntax_ext/proc_macro_decls.rs index de8b689396f..29297aa913e 100644 --- a/src/libsyntax_ext/proc_macro_decls.rs +++ b/src/libsyntax_ext/proc_macro_decls.rs @@ -132,7 +132,7 @@ impl<'a> CollectProcMacros<'a> { } }; - if !trait_ident.can_be_raw() { + if !trait_ident.name.can_be_raw() { self.handler.span_err(trait_attr.span, &format!("`{}` cannot be a name of derive macro", trait_ident)); } @@ -166,7 +166,7 @@ impl<'a> CollectProcMacros<'a> { return None; } }; - if !ident.can_be_raw() { + if !ident.name.can_be_raw() { self.handler.span_err( attr.span, &format!("`{}` cannot be a name of derive helper attribute", ident), diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs index 6ab613d2abd..ff2835c70f7 100644 --- a/src/libsyntax_ext/proc_macro_server.rs +++ b/src/libsyntax_ext/proc_macro_server.rs @@ -142,9 +142,8 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> Question => op!('?'), SingleQuote => op!('\''), - Ident(ident, false) if ident.name == kw::DollarCrate => - tt!(Ident::dollar_crate()), - Ident(ident, is_raw) => tt!(Ident::new(ident.name, is_raw)), + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), Lifetime(name) => { let ident = ast::Ident::new(name, span).without_first_quote(); stack.push(tt!(Ident::new(ident.name, false))); @@ -159,7 +158,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> escaped.extend(ch.escape_debug()); } let stream = vec![ - Ident(ast::Ident::new(sym::doc, span), false), + Ident(sym::doc, false), Eq, TokenKind::lit(token::Str, Symbol::intern(&escaped), None), ] @@ -211,8 +210,7 @@ impl ToInternal for TokenTree { .into(); } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { - let token = Ident(ast::Ident::new(sym, span), is_raw); - return tokenstream::TokenTree::token(span, token).into(); + return tokenstream::TokenTree::token(span, Ident(sym, is_raw)).into(); } TokenTree::Literal(self::Literal { lit: token::Lit { kind: token::Integer, symbol, suffix }, @@ -338,7 +336,8 @@ impl Ident { if !Self::is_valid(&string) { panic!("`{:?}` is not a valid identifier", string) } - if is_raw && !ast::Ident::from_interned_str(sym.as_interned_str()).can_be_raw() { + // Get rid of gensyms to conservatively check rawness on the string contents only. + if is_raw && !sym.as_interned_str().as_symbol().can_be_raw() { panic!("`{}` cannot be a raw identifier", string); } Ident { sym, is_raw, span } diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs index 4e080d115d2..c37aae0bf31 100644 --- a/src/libsyntax_pos/symbol.rs +++ b/src/libsyntax_pos/symbol.rs @@ -1019,6 +1019,21 @@ impl Symbol { pub fn is_doc_keyword(self) -> bool { self <= kw::Union } + + /// A keyword or reserved identifier that can be used as a path segment. + pub fn is_path_segment_keyword(self) -> bool { + self == kw::Super || + self == kw::SelfLower || + self == kw::SelfUpper || + self == kw::Crate || + self == kw::PathRoot || + self == kw::DollarCrate + } + + /// This symbol can be a raw identifier. + pub fn can_be_raw(self) -> bool { + self != kw::Invalid && self != kw::Underscore && !self.is_path_segment_keyword() + } } impl Ident { @@ -1049,24 +1064,13 @@ impl Ident { /// A keyword or reserved identifier that can be used as a path segment. pub fn is_path_segment_keyword(self) -> bool { - self.name == kw::Super || - self.name == kw::SelfLower || - self.name == kw::SelfUpper || - self.name == kw::Crate || - self.name == kw::PathRoot || - self.name == kw::DollarCrate - } - - /// This identifier can be a raw identifier. - pub fn can_be_raw(self) -> bool { - self.name != kw::Invalid && self.name != kw::Underscore && - !self.is_path_segment_keyword() + self.name.is_path_segment_keyword() } /// We see this identifier in a normal identifier position, like variable name or a type. /// How was it written originally? Did it use the raw form? Let's try to guess. pub fn is_raw_guess(self) -> bool { - self.can_be_raw() && self.is_reserved() + self.name.can_be_raw() && self.is_reserved() } } -- cgit 1.4.1-3-g733a5 From ff40e37b98fb44366a329d1b0d9642d462cc6ab6 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 5 Jun 2019 14:17:56 +0300 Subject: Some code cleanup and tidy/test fixes --- .../unstable-book/src/language-features/plugin.md | 6 +-- src/librustc_lint/builtin.rs | 12 ++--- src/libsyntax/attr/mod.rs | 9 ++-- src/libsyntax/diagnostics/plugin.rs | 4 +- src/libsyntax/early_buffered_lints.rs | 2 +- src/libsyntax/ext/tt/quoted.rs | 41 ++++++++-------- src/libsyntax/ext/tt/transcribe.rs | 2 +- src/libsyntax/lib.rs | 6 --- src/libsyntax/parse/diagnostics.rs | 15 +++--- src/libsyntax/parse/literal.rs | 3 +- src/libsyntax/parse/mod.rs | 19 +++++--- src/libsyntax/parse/parser.rs | 55 ++++++++++++---------- src/libsyntax/parse/token.rs | 3 +- src/libsyntax/tokenstream.rs | 5 +- src/libsyntax_ext/assert.rs | 3 +- src/libsyntax_pos/symbol.rs | 5 +- .../run-pass-fulldeps/auxiliary/roman-numerals.rs | 20 ++++---- 17 files changed, 109 insertions(+), 101 deletions(-) (limited to 'src/libsyntax/diagnostics') diff --git a/src/doc/unstable-book/src/language-features/plugin.md b/src/doc/unstable-book/src/language-features/plugin.md index 43fffd68037..1994cf49188 100644 --- a/src/doc/unstable-book/src/language-features/plugin.md +++ b/src/doc/unstable-book/src/language-features/plugin.md @@ -56,7 +56,7 @@ extern crate syntax_pos; extern crate rustc; extern crate rustc_plugin; -use syntax::parse::token; +use syntax::parse::token::{self, Token}; use syntax::tokenstream::TokenTree; use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager}; use syntax::ext::build::AstBuilder; // A trait for expr_usize. @@ -64,7 +64,7 @@ use syntax_pos::Span; use rustc_plugin::Registry; fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) - -> Box { + -> Box { static NUMERALS: &'static [(&'static str, usize)] = &[ ("M", 1000), ("CM", 900), ("D", 500), ("CD", 400), @@ -80,7 +80,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) } let text = match args[0] { - TokenTree::Token(_, token::Ident(s)) => s.to_string(), + TokenTree::Token(Token { kind: token::Ident(s, _), .. }) => s.to_string(), _ => { cx.span_err(sp, "argument should be a single identifier"); return DummyResult::any(sp); diff --git a/src/librustc_lint/builtin.rs b/src/librustc_lint/builtin.rs index a3da97bd5db..6e4d0e881f7 100644 --- a/src/librustc_lint/builtin.rs +++ b/src/librustc_lint/builtin.rs @@ -1414,15 +1414,9 @@ impl KeywordIdents { fn check_tokens(&mut self, cx: &EarlyContext<'_>, tokens: TokenStream) { for tt in tokens.into_trees() { match tt { - TokenTree::Token(token) => match token.ident() { - // only report non-raw idents - Some((ident, false)) => { - self.check_ident_token(cx, UnderMacro(true), ast::Ident { - span: token.span.substitute_dummy(ident.span), - ..ident - }); - } - _ => {}, + // Only report non-raw idents. + TokenTree::Token(token) => if let Some((ident, false)) = token.ident() { + self.check_ident_token(cx, UnderMacro(true), ident); } TokenTree::Delimited(_, _, tts) => { self.check_tokens(cx, tts) diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 8c9bed57bfd..edfe097c72f 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -483,7 +483,8 @@ impl MetaItem { Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) | Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: { let mut segments = if let token::Ident(name, _) = kind { - if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() { + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) + = tokens.peek() { tokens.next(); vec![PathSegment::from_ident(Ident::new(name, span))] } else { @@ -493,12 +494,14 @@ impl MetaItem { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = tokens.next() { + if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) + = tokens.next() { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { return None; } - if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() { + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) + = tokens.peek() { tokens.next(); } else { break; diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 8d9848d98fb..9f01b9b9f9b 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -77,7 +77,9 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>, }, (3, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. })), Some(&TokenTree::Token(Token { kind: token::Comma, .. })), - Some(&TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), .. }))) => { + Some(&TokenTree::Token(Token { + kind: token::Literal(token::Lit { symbol, .. }), .. + }))) => { (code, Some(symbol)) } _ => unreachable!() diff --git a/src/libsyntax/early_buffered_lints.rs b/src/libsyntax/early_buffered_lints.rs index 977e6d45877..598c8459d15 100644 --- a/src/libsyntax/early_buffered_lints.rs +++ b/src/libsyntax/early_buffered_lints.rs @@ -3,7 +3,7 @@ //! Since we cannot have a dependency on `librustc`, we implement some types here that are somewhat //! redundant. Later, these types can be converted to types for use by the rest of the compiler. -use crate::syntax::ast::NodeId; +use crate::ast::NodeId; use syntax_pos::MultiSpan; /// Since we cannot import `LintId`s from `rustc::lint`, we define some Ids here which can later be diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index b4672fb4a58..ec7d7f705d8 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -24,12 +24,12 @@ pub struct Delimited { impl Delimited { /// Returns the opening delimiter (possibly `NoDelim`). - pub fn open_token(&self) -> token::TokenKind { + pub fn open_token(&self) -> TokenKind { token::OpenDelim(self.delim) } /// Returns the closing delimiter (possibly `NoDelim`). - pub fn close_token(&self) -> token::TokenKind { + pub fn close_token(&self) -> TokenKind { token::CloseDelim(self.delim) } @@ -59,7 +59,7 @@ pub struct SequenceRepetition { /// The sequence of token trees pub tts: Vec, /// The optional separator - pub separator: Option, + pub separator: Option, /// Whether the sequence can be repeated zero (*), or one or more times (+) pub op: KleeneOp, /// The number of `Match`s that appear in the sequence (and subsequences) @@ -210,20 +210,21 @@ pub fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => match trees.next() { - Some(tokenstream::TokenTree::Token(token)) => match token.ident() { - Some((kind, _)) => { - let span = token.span.with_lo(start_sp.lo()); - result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue; - } - _ => token.span, + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => + match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((kind, _)) => { + let span = token.span.with_lo(start_sp.lo()); + result.push(TokenTree::MetaVarDecl(span, ident, kind)); + continue; + } + _ => token.span, + }, + tree => tree + .as_ref() + .map(tokenstream::TokenTree::span) + .unwrap_or(span), }, - tree => tree - .as_ref() - .map(tokenstream::TokenTree::span) - .unwrap_or(span), - }, tree => tree .as_ref() .map(tokenstream::TokenTree::span) @@ -370,7 +371,7 @@ where /// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return /// `None`. -fn kleene_op(token: &token::TokenKind) -> Option { +fn kleene_op(token: &TokenKind) -> Option { match *token { token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), @@ -423,7 +424,7 @@ fn parse_sep_and_kleene_op( attrs: &[ast::Attribute], edition: Edition, macro_node_id: NodeId, -) -> (Option, KleeneOp) +) -> (Option, KleeneOp) where I: Iterator, { @@ -448,7 +449,7 @@ fn parse_sep_and_kleene_op_2015( _features: &Features, _attrs: &[ast::Attribute], macro_node_id: NodeId, -) -> (Option, KleeneOp) +) -> (Option, KleeneOp) where I: Iterator, { @@ -566,7 +567,7 @@ fn parse_sep_and_kleene_op_2018( sess: &ParseSess, _features: &Features, _attrs: &[ast::Attribute], -) -> (Option, KleeneOp) +) -> (Option, KleeneOp) where I: Iterator, { diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index b382893ce4e..90a9cc8f34d 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -242,7 +242,7 @@ pub fn transcribe( Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark)); sp = sp.apply_mark(cx.current_expansion.mark); result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::token(token::TokenKind::from_ast_ident(ident), sp).into()); + result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into()); } } diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 6882586ed2c..c69364d4e19 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -137,12 +137,6 @@ pub mod util { pub mod json; -pub mod syntax { - pub use crate::ext; - pub use crate::parse; - pub use crate::ast; -} - pub mod ast; pub mod attr; pub mod source_map; diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/diagnostics.rs index 7830b2ce880..7f0bf4a9050 100644 --- a/src/libsyntax/parse/diagnostics.rs +++ b/src/libsyntax/parse/diagnostics.rs @@ -2,8 +2,9 @@ use crate::ast::{ self, Arg, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, VariantData, }; -use crate::parse::{SeqSep, token, PResult, Parser}; +use crate::parse::{SeqSep, PResult, Parser}; use crate::parse::parser::{BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType}; +use crate::parse::token::{self, TokenKind}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::Spanned; @@ -229,8 +230,8 @@ impl<'a> Parser<'a> { pub fn expected_one_of_not_found( &mut self, - edible: &[token::TokenKind], - inedible: &[token::TokenKind], + edible: &[TokenKind], + inedible: &[TokenKind], ) -> PResult<'a, bool /* recovered */> { fn tokens_to_string(tokens: &[TokenType]) -> String { let mut i = tokens.iter(); @@ -368,7 +369,7 @@ impl<'a> Parser<'a> { /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. - crate fn eat_to_tokens(&mut self, kets: &[&token::TokenKind]) { + crate fn eat_to_tokens(&mut self, kets: &[&TokenKind]) { let handler = self.diagnostic(); if let Err(ref mut err) = self.parse_seq_to_before_tokens( @@ -388,7 +389,7 @@ impl<'a> Parser<'a> { /// let _ = vec![1, 2, 3].into_iter().collect::>>>(); /// ^^ help: remove extra angle brackets /// ``` - crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::TokenKind) { + crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: TokenKind) { // This function is intended to be invoked after parsing a path segment where there are two // cases: // @@ -726,7 +727,7 @@ impl<'a> Parser<'a> { /// closing delimiter. pub fn unexpected_try_recover( &mut self, - t: &token::TokenKind, + t: &TokenKind, ) -> PResult<'a, bool /* recovered */> { let token_str = pprust::token_to_string(t); let this_token_str = self.this_token_descr(); @@ -903,7 +904,7 @@ impl<'a> Parser<'a> { crate fn recover_closing_delimiter( &mut self, - tokens: &[token::TokenKind], + tokens: &[TokenKind], mut err: DiagnosticBuilder<'a>, ) -> PResult<'a, bool> { let mut pos = None; diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 4979a4dd27f..7d5356ffe4d 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -272,7 +272,8 @@ impl<'a> Parser<'a> { if self.token == token::Dot { // Attempt to recover `.4` as `0.4`. recovered = self.look_ahead(1, |t| { - if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = t.kind { + if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) + = t.kind { let next_span = self.look_ahead_span(1); if self.span.hi() == next_span.lo() { let s = String::from("0.") + &symbol.as_str(); diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 8d3518d0373..063823bbf4d 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -5,7 +5,8 @@ use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; use crate::source_map::{SourceMap, FilePathMapping}; use crate::feature_gate::UnstableFeatures; use crate::parse::parser::Parser; -use crate::syntax::parse::parser::emit_unclosed_delims; +use crate::parse::parser::emit_unclosed_delims; +use crate::parse::token::TokenKind; use crate::tokenstream::{TokenStream, TokenTree}; use crate::diagnostics::plugin::ErrorMap; use crate::print::pprust::token_to_string; @@ -358,13 +359,13 @@ pub fn stream_to_parser_with_base_dir<'a>( /// A sequence separator. pub struct SeqSep { /// The seperator token. - pub sep: Option, + pub sep: Option, /// `true` if a trailing separator is allowed. pub trailing_sep_allowed: bool, } impl SeqSep { - pub fn trailing_allowed(t: token::TokenKind) -> SeqSep { + pub fn trailing_allowed(t: TokenKind) -> SeqSep { SeqSep { sep: Some(t), trailing_sep_allowed: true, @@ -426,7 +427,9 @@ mod tests { match (tts.len(), tts.get(0), tts.get(1), tts.get(2), tts.get(3)) { ( 4, - Some(&TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. })), + Some(&TokenTree::Token(Token { + kind: token::Ident(name_macro_rules, false), .. + })), Some(&TokenTree::Token(Token { kind: token::Not, .. })), Some(&TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. })), Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)), @@ -446,7 +449,9 @@ mod tests { ( 2, Some(&TokenTree::Token(Token { kind: token::Dollar, .. })), - Some(&TokenTree::Token(Token { kind: token::Ident(name, false), .. })), + Some(&TokenTree::Token(Token { + kind: token::Ident(name, false), .. + })), ) if first_delim == token::Paren && name.as_str() == "a" => {}, _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), @@ -456,7 +461,9 @@ mod tests { ( 2, Some(&TokenTree::Token(Token { kind: token::Dollar, .. })), - Some(&TokenTree::Token(Token { kind: token::Ident(name, false), .. })), + Some(&TokenTree::Token(Token { + kind: token::Ident(name, false), .. + })), ) if second_delim == token::Paren && name.as_str() == "a" => {}, _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index e9e908eb858..51bfe3527cf 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -38,7 +38,7 @@ use crate::source_map::{self, SourceMap, Spanned, respan}; use crate::parse::{SeqSep, classify, literal, token}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use crate::parse::token::{Token, DelimToken}; +use crate::parse::token::{Token, TokenKind, DelimToken}; use crate::parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership}; use crate::util::parser::{AssocOp, Fixity}; use crate::print::pprust; @@ -337,8 +337,8 @@ impl TokenCursor { } fn next_desugared(&mut self) -> Token { - let (sp, name) = match self.next() { - Token { span, kind: token::DocComment(name) } => (span, name), + let (name, sp) = match self.next() { + Token { kind: token::DocComment(name), span } => (name, span), tok => return tok, }; @@ -364,7 +364,7 @@ impl TokenCursor { [ TokenTree::token(token::Ident(sym::doc, false), sp), TokenTree::token(token::Eq, sp), - TokenTree::token(token::TokenKind::lit( + TokenTree::token(TokenKind::lit( token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None ), sp), ] @@ -389,7 +389,7 @@ impl TokenCursor { #[derive(Clone, PartialEq)] crate enum TokenType { - Token(token::TokenKind), + Token(TokenKind), Keyword(Symbol), Operator, Lifetime, @@ -419,7 +419,7 @@ impl TokenType { /// /// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes /// that `IDENT` is not the ident of a fn trait. -fn can_continue_type_after_non_fn_ident(t: &token::TokenKind) -> bool { +fn can_continue_type_after_non_fn_ident(t: &TokenKind) -> bool { t == &token::ModSep || t == &token::Lt || t == &token::BinOp(token::Shl) } @@ -565,7 +565,7 @@ impl<'a> Parser<'a> { } /// Expects and consumes the token `t`. Signals an error if the next token is not `t`. - pub fn expect(&mut self, t: &token::TokenKind) -> PResult<'a, bool /* recovered */> { + pub fn expect(&mut self, t: &TokenKind) -> PResult<'a, bool /* recovered */> { if self.expected_tokens.is_empty() { if self.token == *t { self.bump(); @@ -583,8 +583,8 @@ impl<'a> Parser<'a> { /// anything. Signal a fatal error if next token is unexpected. pub fn expect_one_of( &mut self, - edible: &[token::TokenKind], - inedible: &[token::TokenKind], + edible: &[TokenKind], + inedible: &[TokenKind], ) -> PResult<'a, bool /* recovered */> { if edible.contains(&self.token) { self.bump(); @@ -646,14 +646,14 @@ impl<'a> Parser<'a> { /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not /// encountered. - crate fn check(&mut self, tok: &token::TokenKind) -> bool { + crate fn check(&mut self, tok: &TokenKind) -> bool { let is_present = self.token == *tok; if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); } is_present } /// Consumes a token 'tok' if it exists. Returns whether the given token was present. - pub fn eat(&mut self, tok: &token::TokenKind) -> bool { + pub fn eat(&mut self, tok: &TokenKind) -> bool { let is_present = self.check(tok); if is_present { self.bump() } is_present @@ -889,7 +889,7 @@ impl<'a> Parser<'a> { /// `f` must consume tokens until reaching the next separator or /// closing bracket. pub fn parse_seq_to_end(&mut self, - ket: &token::TokenKind, + ket: &TokenKind, sep: SeqSep, f: F) -> PResult<'a, Vec> where @@ -907,7 +907,7 @@ impl<'a> Parser<'a> { /// closing bracket. pub fn parse_seq_to_before_end( &mut self, - ket: &token::TokenKind, + ket: &TokenKind, sep: SeqSep, f: F, ) -> PResult<'a, (Vec, bool)> @@ -918,7 +918,7 @@ impl<'a> Parser<'a> { crate fn parse_seq_to_before_tokens( &mut self, - kets: &[&token::TokenKind], + kets: &[&TokenKind], sep: SeqSep, expect: TokenExpectType, mut f: F, @@ -992,8 +992,8 @@ impl<'a> Parser<'a> { /// closing bracket. fn parse_unspanned_seq( &mut self, - bra: &token::TokenKind, - ket: &token::TokenKind, + bra: &TokenKind, + ket: &TokenKind, sep: SeqSep, f: F, ) -> PResult<'a, Vec> where @@ -1036,7 +1036,7 @@ impl<'a> Parser<'a> { /// Advance the parser using provided token as a next one. Use this when /// consuming a part of a token. For example a single `<` from `<<`. - fn bump_with(&mut self, next: token::TokenKind, span: Span) { + fn bump_with(&mut self, next: TokenKind, span: Span) { self.prev_span = self.span.with_hi(span.lo()); // It would be incorrect to record the kind of the current token, but // fortunately for tokens currently using `bump_with`, the @@ -1050,7 +1050,6 @@ impl<'a> Parser<'a> { F: FnOnce(&Token) -> R, { if dist == 0 { - // FIXME: Avoid cloning here. return f(&self.token); } @@ -1058,7 +1057,8 @@ impl<'a> Parser<'a> { f(&match frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { TokenTree::Token(token) => token, - TokenTree::Delimited(dspan, delim, _) => Token::new(token::OpenDelim(delim), dspan.open), + TokenTree::Delimited(dspan, delim, _) => + Token::new(token::OpenDelim(delim), dspan.open), } None => Token::new(token::CloseDelim(frame.delim), frame.span.close) }) @@ -1768,7 +1768,7 @@ impl<'a> Parser<'a> { fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> { let ident = self.parse_path_segment_ident()?; - let is_args_start = |token: &token::TokenKind| match *token { + let is_args_start = |token: &TokenKind| match *token { token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren) | token::LArrow => true, _ => false, @@ -1864,7 +1864,8 @@ impl<'a> Parser<'a> { } fn parse_field_name(&mut self) -> PResult<'a, Ident> { - if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = self.token.kind { + if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = + self.token.kind { self.expect_no_suffix(self.span, "a tuple index", suffix); self.bump(); Ok(Ident::new(symbol, self.prev_span)) @@ -2649,8 +2650,10 @@ impl<'a> Parser<'a> { // Interpolated identifier and lifetime tokens are replaced with usual identifier // and lifetime tokens, so the former are never encountered during normal parsing. match **nt { - token::NtIdent(ident, is_raw) => Token::new(token::Ident(ident.name, is_raw), ident.span), - token::NtLifetime(ident) => Token::new(token::Lifetime(ident.name), ident.span), + token::NtIdent(ident, is_raw) => + Token::new(token::Ident(ident.name, is_raw), ident.span), + token::NtLifetime(ident) => + Token::new(token::Lifetime(ident.name), ident.span), _ => return, } } @@ -4481,7 +4484,9 @@ impl<'a> Parser<'a> { // We used to incorrectly stop parsing macro-expanded statements here. // If the next token will be an error anyway but could have parsed with the // earlier behavior, stop parsing here and emit a warning to avoid breakage. - else if macro_legacy_warnings && self.token.can_begin_expr() && match self.token.kind { + else if macro_legacy_warnings && + self.token.can_begin_expr() && + match self.token.kind { // These can continue an expression, so we can't stop parsing and warn. token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | token::BinOp(token::Minus) | token::BinOp(token::Star) | @@ -6409,7 +6414,7 @@ impl<'a> Parser<'a> { } /// Given a termination token, parses all of the items in a module. - fn parse_mod_items(&mut self, term: &token::TokenKind, inner_lo: Span) -> PResult<'a, Mod> { + fn parse_mod_items(&mut self, term: &TokenKind, inner_lo: Span) -> PResult<'a, Mod> { let mut items = vec![]; while let Some(item) = self.parse_item()? { items.push(item); diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 58c30a07e3e..28a733728bf 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -5,11 +5,10 @@ pub use LitKind::*; pub use TokenKind::*; use crate::ast::{self}; -use crate::parse::ParseSess; +use crate::parse::{parse_stream_from_source_str, ParseSess}; use crate::print::pprust; use crate::ptr::P; use crate::symbol::kw; -use crate::syntax::parse::parse_stream_from_source_str; use crate::tokenstream::{self, DelimSpan, TokenStream, TokenTree}; use syntax_pos::symbol::Symbol; diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index b4643229285..9dea3a4dcc1 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -203,7 +203,8 @@ impl TokenStream { if let Some((_, next)) = iter.peek() { let sp = match (&ts, &next) { (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, - ((TokenTree::Token(token_left), NonJoint), (TokenTree::Token(token_right), _)) + ((TokenTree::Token(token_left), NonJoint), + (TokenTree::Token(token_right), _)) if ((token_left.is_ident() && !token_left.is_reserved_ident()) || token_left.is_lit()) && ((token_right.is_ident() && !token_right.is_reserved_ident()) @@ -575,7 +576,7 @@ impl DelimSpan { #[cfg(test)] mod tests { use super::*; - use crate::syntax::ast::Name; + use crate::ast::Name; use crate::with_default_globals; use crate::util::parser_testing::string_to_stream; use syntax_pos::{Span, BytePos, NO_EXPANSION}; diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index ce1e3276af3..3886528c74c 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -103,7 +103,8 @@ fn parse_assert<'a>( // // Parse this as an actual message, and suggest inserting a comma. Eventually, this should be // turned into an error. - let custom_message = if let token::Literal(token::Lit { kind: token::Str, .. }) = parser.token.kind { + let custom_message = if let token::Literal(token::Lit { kind: token::Str, .. }) + = parser.token.kind { let mut err = cx.struct_span_warn(parser.span, "unexpected string literal"); let comma_span = cx.source_map().next_point(parser.prev_span); err.span_suggestion_short( diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs index c37aae0bf31..5dd4d6566ed 100644 --- a/src/libsyntax_pos/symbol.rs +++ b/src/libsyntax_pos/symbol.rs @@ -921,10 +921,9 @@ pub struct Interner { impl Interner { fn prefill(init: &[&'static str]) -> Self { - let symbols = (0 .. init.len() as u32).map(Symbol::new); Interner { - strings: init.to_vec(), - names: init.iter().copied().zip(symbols).collect(), + strings: init.into(), + names: init.iter().copied().zip((0..).map(Symbol::new)).collect(), ..Default::default() } } diff --git a/src/test/run-pass-fulldeps/auxiliary/roman-numerals.rs b/src/test/run-pass-fulldeps/auxiliary/roman-numerals.rs index 216c81ca34c..4d9e0129e54 100644 --- a/src/test/run-pass-fulldeps/auxiliary/roman-numerals.rs +++ b/src/test/run-pass-fulldeps/auxiliary/roman-numerals.rs @@ -1,3 +1,9 @@ +// WARNING WARNING WARNING WARNING WARNING +// ======================================= +// +// This code also appears in src/doc/unstable-book/src/language-features/plugin.md. +// Please keep the two copies in sync! FIXME: have rustdoc read this file + // force-host #![crate_type="dylib"] @@ -8,21 +14,15 @@ extern crate syntax_pos; extern crate rustc; extern crate rustc_plugin; -use syntax::parse::token; +use syntax::parse::token::{self, Token}; use syntax::tokenstream::TokenTree; use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager}; -use syntax::ext::build::AstBuilder; // trait for expr_usize +use syntax::ext::build::AstBuilder; // A trait for expr_usize. use syntax_pos::Span; use rustc_plugin::Registry; -// WARNING WARNING WARNING WARNING WARNING -// ======================================= -// -// This code also appears in src/doc/unstable-book/src/language-features/plugin.md. -// Please keep the two copies in sync! FIXME: have rustdoc read this file - fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) - -> Box { + -> Box { static NUMERALS: &'static [(&'static str, usize)] = &[ ("M", 1000), ("CM", 900), ("D", 500), ("CD", 400), @@ -38,7 +38,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree]) } let text = match args[0] { - TokenTree::Token(_, token::Ident(s, _)) => s.to_string(), + TokenTree::Token(Token { kind: token::Ident(s, _), .. }) => s.to_string(), _ => { cx.span_err(sp, "argument should be a single identifier"); return DummyResult::any(sp); -- cgit 1.4.1-3-g733a5