From da116223c709392db3741d966cb49ea0407de19e Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 25 Oct 2019 04:57:40 +0200 Subject: move parse/parser.rs -> parse/parser/mod.rs --- src/libsyntax/parse/parser/mod.rs | 1391 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1391 insertions(+) create mode 100644 src/libsyntax/parse/parser/mod.rs (limited to 'src/libsyntax/parse/parser') diff --git a/src/libsyntax/parse/parser/mod.rs b/src/libsyntax/parse/parser/mod.rs new file mode 100644 index 00000000000..1284e89f195 --- /dev/null +++ b/src/libsyntax/parse/parser/mod.rs @@ -0,0 +1,1391 @@ +pub mod attr; +mod expr; +mod pat; +mod item; +mod module; +mod ty; +mod path; +pub use path::PathStyle; +mod stmt; +mod generics; +mod diagnostics; +use diagnostics::Error; + +use crate::ast::{ + self, Abi, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Ident, + IsAsync, MacDelimiter, Mutability, StrStyle, Visibility, VisibilityKind, Unsafety, +}; +use crate::parse::{PResult, Directory, DirectoryOwnership}; +use crate::parse::lexer::UnmatchedBrace; +use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; +use crate::parse::token::{self, Token, TokenKind, DelimToken}; +use crate::print::pprust; +use crate::ptr::P; +use crate::sess::ParseSess; +use crate::source_map::respan; +use crate::symbol::{kw, sym, Symbol}; +use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; +use crate::ThinVec; + +use errors::{Applicability, DiagnosticBuilder, DiagnosticId, FatalError}; +use syntax_pos::{Span, BytePos, DUMMY_SP, FileName}; +use log::debug; + +use std::borrow::Cow; +use std::{cmp, mem, slice}; +use std::path::PathBuf; + +bitflags::bitflags! { + struct Restrictions: u8 { + const STMT_EXPR = 1 << 0; + const NO_STRUCT_LITERAL = 1 << 1; + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +enum SemiColonMode { + Break, + Ignore, + Comma, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +enum BlockMode { + Break, + Ignore, +} + +/// Like `maybe_whole_expr`, but for things other than expressions. +#[macro_export] +macro_rules! maybe_whole { + ($p:expr, $constructor:ident, |$x:ident| $e:expr) => { + if let token::Interpolated(nt) = &$p.token.kind { + if let token::$constructor(x) = &**nt { + let $x = x.clone(); + $p.bump(); + return Ok($e); + } + } + }; +} + +/// If the next tokens are ill-formed `$ty::` recover them as `<$ty>::`. +#[macro_export] +macro_rules! maybe_recover_from_interpolated_ty_qpath { + ($self: expr, $allow_qpath_recovery: expr) => { + if $allow_qpath_recovery && $self.look_ahead(1, |t| t == &token::ModSep) { + if let token::Interpolated(nt) = &$self.token.kind { + if let token::NtTy(ty) = &**nt { + let ty = ty.clone(); + $self.bump(); + return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_span, ty); + } + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum PrevTokenKind { + DocComment, + Comma, + Plus, + Interpolated, + Eof, + Ident, + BitOr, + Other, +} + +// NOTE: `Ident`s are handled by `common.rs`. + +#[derive(Clone)] +pub struct Parser<'a> { + pub sess: &'a ParseSess, + /// The current normalized token. + /// "Normalized" means that some interpolated tokens + /// (`$i: ident` and `$l: lifetime` meta-variables) are replaced + /// with non-interpolated identifier and lifetime tokens they refer to. + /// Perhaps the normalized / non-normalized setup can be simplified somehow. + pub token: Token, + /// The span of the current non-normalized token. + meta_var_span: Option, + /// The span of the previous non-normalized token. + pub prev_span: Span, + /// The kind of the previous normalized token (in simplified form). + prev_token_kind: PrevTokenKind, + restrictions: Restrictions, + /// Used to determine the path to externally loaded source files. + pub(super) directory: Directory<'a>, + /// `true` to parse sub-modules in other files. + pub(super) recurse_into_file_modules: bool, + /// Name of the root module this parser originated from. If `None`, then the + /// name is not known. This does not change while the parser is descending + /// into modules, and sub-parsers have new values for this name. + pub root_module_name: Option, + expected_tokens: Vec, + token_cursor: TokenCursor, + desugar_doc_comments: bool, + /// `true` we should configure out of line modules as we parse. + cfg_mods: bool, + /// This field is used to keep track of how many left angle brackets we have seen. This is + /// required in order to detect extra leading left angle brackets (`<` characters) and error + /// appropriately. + /// + /// See the comments in the `parse_path_segment` function for more details. + unmatched_angle_bracket_count: u32, + max_angle_bracket_count: u32, + /// A list of all unclosed delimiters found by the lexer. If an entry is used for error recovery + /// it gets removed from here. Every entry left at the end gets emitted as an independent + /// error. + pub(super) unclosed_delims: Vec, + last_unexpected_token_span: Option, + pub last_type_ascription: Option<(Span, bool /* likely path typo */)>, + /// If present, this `Parser` is not parsing Rust code but rather a macro call. + subparser_name: Option<&'static str>, +} + +impl<'a> Drop for Parser<'a> { + fn drop(&mut self) { + emit_unclosed_delims(&mut self.unclosed_delims, &self.sess); + } +} + +#[derive(Clone)] +struct TokenCursor { + frame: TokenCursorFrame, + stack: Vec, +} + +#[derive(Clone)] +struct TokenCursorFrame { + delim: token::DelimToken, + span: DelimSpan, + open_delim: bool, + tree_cursor: tokenstream::Cursor, + close_delim: bool, + last_token: LastToken, +} + +/// This is used in `TokenCursorFrame` above to track tokens that are consumed +/// by the parser, and then that's transitively used to record the tokens that +/// each parse AST item is created with. +/// +/// Right now this has two states, either collecting tokens or not collecting +/// tokens. If we're collecting tokens we just save everything off into a local +/// `Vec`. This should eventually though likely save tokens from the original +/// token stream and just use slicing of token streams to avoid creation of a +/// whole new vector. +/// +/// The second state is where we're passively not recording tokens, but the last +/// token is still tracked for when we want to start recording tokens. This +/// "last token" means that when we start recording tokens we'll want to ensure +/// that this, the first token, is included in the output. +/// +/// You can find some more example usage of this in the `collect_tokens` method +/// on the parser. +#[derive(Clone)] +enum LastToken { + Collecting(Vec), + Was(Option), +} + +impl TokenCursorFrame { + fn new(span: DelimSpan, delim: DelimToken, tts: &TokenStream) -> Self { + TokenCursorFrame { + delim, + span, + open_delim: delim == token::NoDelim, + tree_cursor: tts.clone().into_trees(), + close_delim: delim == token::NoDelim, + last_token: LastToken::Was(None), + } + } +} + +impl TokenCursor { + fn next(&mut self) -> Token { + loop { + let tree = if !self.frame.open_delim { + self.frame.open_delim = true; + TokenTree::open_tt(self.frame.span, self.frame.delim) + } else if let Some(tree) = self.frame.tree_cursor.next() { + tree + } else if !self.frame.close_delim { + self.frame.close_delim = true; + TokenTree::close_tt(self.frame.span, self.frame.delim) + } else if let Some(frame) = self.stack.pop() { + self.frame = frame; + continue + } else { + return Token::new(token::Eof, DUMMY_SP); + }; + + match self.frame.last_token { + LastToken::Collecting(ref mut v) => v.push(tree.clone().into()), + LastToken::Was(ref mut t) => *t = Some(tree.clone().into()), + } + + match tree { + TokenTree::Token(token) => return token, + TokenTree::Delimited(sp, delim, tts) => { + let frame = TokenCursorFrame::new(sp, delim, &tts); + self.stack.push(mem::replace(&mut self.frame, frame)); + } + } + } + } + + fn next_desugared(&mut self) -> Token { + let (name, sp) = match self.next() { + Token { kind: token::DocComment(name), span } => (name, span), + tok => return tok, + }; + + let stripped = strip_doc_comment_decoration(&name.as_str()); + + // Searches for the occurrences of `"#*` and returns the minimum number of `#`s + // required to wrap the text. + let mut num_of_hashes = 0; + let mut count = 0; + for ch in stripped.chars() { + count = match ch { + '"' => 1, + '#' if count > 0 => count + 1, + _ => 0, + }; + num_of_hashes = cmp::max(num_of_hashes, count); + } + + let delim_span = DelimSpan::from_single(sp); + let body = TokenTree::Delimited( + delim_span, + token::Bracket, + [ + TokenTree::token(token::Ident(sym::doc, false), sp), + TokenTree::token(token::Eq, sp), + TokenTree::token(TokenKind::lit( + token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None + ), sp), + ] + .iter().cloned().collect::().into(), + ); + + self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new( + delim_span, + token::NoDelim, + &if doc_comment_style(&name.as_str()) == AttrStyle::Inner { + [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] + .iter().cloned().collect::() + } else { + [TokenTree::token(token::Pound, sp), body] + .iter().cloned().collect::() + }, + ))); + + self.next() + } +} + +#[derive(Clone, PartialEq)] +enum TokenType { + Token(TokenKind), + Keyword(Symbol), + Operator, + Lifetime, + Ident, + Path, + Type, + Const, +} + +impl TokenType { + fn to_string(&self) -> String { + match *self { + TokenType::Token(ref t) => format!("`{}`", pprust::token_kind_to_string(t)), + TokenType::Keyword(kw) => format!("`{}`", kw), + TokenType::Operator => "an operator".to_string(), + TokenType::Lifetime => "lifetime".to_string(), + TokenType::Ident => "identifier".to_string(), + TokenType::Path => "path".to_string(), + TokenType::Type => "type".to_string(), + TokenType::Const => "const".to_string(), + } + } +} + +#[derive(Copy, Clone, Debug)] +enum TokenExpectType { + Expect, + NoExpect, +} + +/// A sequence separator. +struct SeqSep { + /// The separator token. + sep: Option, + /// `true` if a trailing separator is allowed. + trailing_sep_allowed: bool, +} + +impl SeqSep { + fn trailing_allowed(t: TokenKind) -> SeqSep { + SeqSep { + sep: Some(t), + trailing_sep_allowed: true, + } + } + + fn none() -> SeqSep { + SeqSep { + sep: None, + trailing_sep_allowed: false, + } + } +} + +impl<'a> Parser<'a> { + pub fn new( + sess: &'a ParseSess, + tokens: TokenStream, + directory: Option>, + recurse_into_file_modules: bool, + desugar_doc_comments: bool, + subparser_name: Option<&'static str>, + ) -> Self { + let mut parser = Parser { + sess, + token: Token::dummy(), + prev_span: DUMMY_SP, + meta_var_span: None, + prev_token_kind: PrevTokenKind::Other, + restrictions: Restrictions::empty(), + recurse_into_file_modules, + directory: Directory { + path: Cow::from(PathBuf::new()), + ownership: DirectoryOwnership::Owned { relative: None } + }, + root_module_name: None, + expected_tokens: Vec::new(), + token_cursor: TokenCursor { + frame: TokenCursorFrame::new( + DelimSpan::dummy(), + token::NoDelim, + &tokens.into(), + ), + stack: Vec::new(), + }, + desugar_doc_comments, + cfg_mods: true, + unmatched_angle_bracket_count: 0, + max_angle_bracket_count: 0, + unclosed_delims: Vec::new(), + last_unexpected_token_span: None, + last_type_ascription: None, + subparser_name, + }; + + parser.token = parser.next_tok(); + + if let Some(directory) = directory { + parser.directory = directory; + } else if !parser.token.span.is_dummy() { + if let Some(FileName::Real(path)) = + &sess.source_map().lookup_char_pos(parser.token.span.lo()).file.unmapped_path { + if let Some(directory_path) = path.parent() { + parser.directory.path = Cow::from(directory_path.to_path_buf()); + } + } + } + + parser.process_potential_macro_variable(); + parser + } + + fn next_tok(&mut self) -> Token { + let mut next = if self.desugar_doc_comments { + self.token_cursor.next_desugared() + } else { + self.token_cursor.next() + }; + if next.span.is_dummy() { + // Tweak the location for better diagnostics, but keep syntactic context intact. + next.span = self.prev_span.with_ctxt(next.span.ctxt()); + } + next + } + + /// Converts the current token to a string using `self`'s reader. + pub fn this_token_to_string(&self) -> String { + pprust::token_to_string(&self.token) + } + + fn token_descr(&self) -> Option<&'static str> { + Some(match &self.token.kind { + _ if self.token.is_special_ident() => "reserved identifier", + _ if self.token.is_used_keyword() => "keyword", + _ if self.token.is_unused_keyword() => "reserved keyword", + token::DocComment(..) => "doc comment", + _ => return None, + }) + } + + pub(super) fn this_token_descr(&self) -> String { + if let Some(prefix) = self.token_descr() { + format!("{} `{}`", prefix, self.this_token_to_string()) + } else { + format!("`{}`", self.this_token_to_string()) + } + } + + crate fn unexpected(&mut self) -> PResult<'a, T> { + match self.expect_one_of(&[], &[]) { + Err(e) => Err(e), + Ok(_) => unreachable!(), + } + } + + /// Expects and consumes the token `t`. Signals an error if the next token is not `t`. + pub fn expect(&mut self, t: &TokenKind) -> PResult<'a, bool /* recovered */> { + if self.expected_tokens.is_empty() { + if self.token == *t { + self.bump(); + Ok(false) + } else { + self.unexpected_try_recover(t) + } + } else { + self.expect_one_of(slice::from_ref(t), &[]) + } + } + + /// Expect next token to be edible or inedible token. If edible, + /// then consume it; if inedible, then return without consuming + /// anything. Signal a fatal error if next token is unexpected. + pub fn expect_one_of( + &mut self, + edible: &[TokenKind], + inedible: &[TokenKind], + ) -> PResult<'a, bool /* recovered */> { + if edible.contains(&self.token.kind) { + self.bump(); + Ok(false) + } else if inedible.contains(&self.token.kind) { + // leave it in the input + Ok(false) + } else if self.last_unexpected_token_span == Some(self.token.span) { + FatalError.raise(); + } else { + self.expected_one_of_not_found(edible, inedible) + } + } + + fn parse_ident(&mut self) -> PResult<'a, ast::Ident> { + self.parse_ident_common(true) + } + + fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> { + match self.token.kind { + token::Ident(name, _) => { + if self.token.is_reserved_ident() { + let mut err = self.expected_ident_found(); + if recover { + err.emit(); + } else { + return Err(err); + } + } + let span = self.token.span; + self.bump(); + Ok(Ident::new(name, span)) + } + _ => { + Err(if self.prev_token_kind == PrevTokenKind::DocComment { + self.span_fatal_err(self.prev_span, Error::UselessDocComment) + } else { + self.expected_ident_found() + }) + } + } + } + + /// Checks if the next token is `tok`, and returns `true` if so. + /// + /// This method will automatically add `tok` to `expected_tokens` if `tok` is not + /// encountered. + fn check(&mut self, tok: &TokenKind) -> bool { + let is_present = self.token == *tok; + if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); } + is_present + } + + /// Consumes a token 'tok' if it exists. Returns whether the given token was present. + pub fn eat(&mut self, tok: &TokenKind) -> bool { + let is_present = self.check(tok); + if is_present { self.bump() } + is_present + } + + /// If the next token is the given keyword, returns `true` without eating it. + /// An expectation is also added for diagnostics purposes. + fn check_keyword(&mut self, kw: Symbol) -> bool { + self.expected_tokens.push(TokenType::Keyword(kw)); + self.token.is_keyword(kw) + } + + /// If the next token is the given keyword, eats it and returns `true`. + /// Otherwise, returns `false`. An expectation is also added for diagnostics purposes. + fn eat_keyword(&mut self, kw: Symbol) -> bool { + if self.check_keyword(kw) { + self.bump(); + true + } else { + false + } + } + + fn eat_keyword_noexpect(&mut self, kw: Symbol) -> bool { + if self.token.is_keyword(kw) { + self.bump(); + true + } else { + false + } + } + + /// If the given word is not a keyword, signals an error. + /// If the next token is not the given word, signals an error. + /// Otherwise, eats it. + fn expect_keyword(&mut self, kw: Symbol) -> PResult<'a, ()> { + if !self.eat_keyword(kw) { + self.unexpected() + } else { + Ok(()) + } + } + + fn check_or_expected(&mut self, ok: bool, typ: TokenType) -> bool { + if ok { + true + } else { + self.expected_tokens.push(typ); + false + } + } + + fn check_ident(&mut self) -> bool { + self.check_or_expected(self.token.is_ident(), TokenType::Ident) + } + + fn check_path(&mut self) -> bool { + self.check_or_expected(self.token.is_path_start(), TokenType::Path) + } + + fn check_type(&mut self) -> bool { + self.check_or_expected(self.token.can_begin_type(), TokenType::Type) + } + + fn check_const_arg(&mut self) -> bool { + self.check_or_expected(self.token.can_begin_const_arg(), TokenType::Const) + } + + /// Checks to see if the next token is either `+` or `+=`. + /// Otherwise returns `false`. + fn check_plus(&mut self) -> bool { + self.check_or_expected( + self.token.is_like_plus(), + TokenType::Token(token::BinOp(token::Plus)), + ) + } + + /// Expects and consumes a `+`. if `+=` is seen, replaces it with a `=` + /// and continues. If a `+` is not seen, returns `false`. + /// + /// This is used when token-splitting `+=` into `+`. + /// See issue #47856 for an example of when this may occur. + fn eat_plus(&mut self) -> bool { + self.expected_tokens.push(TokenType::Token(token::BinOp(token::Plus))); + match self.token.kind { + token::BinOp(token::Plus) => { + self.bump(); + true + } + token::BinOpEq(token::Plus) => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + self.bump_with(token::Eq, span); + true + } + _ => false, + } + } + + /// Expects and consumes an `&`. If `&&` is seen, replaces it with a single + /// `&` and continues. If an `&` is not seen, signals an error. + fn expect_and(&mut self) -> PResult<'a, ()> { + self.expected_tokens.push(TokenType::Token(token::BinOp(token::And))); + match self.token.kind { + token::BinOp(token::And) => { + self.bump(); + Ok(()) + } + token::AndAnd => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + Ok(self.bump_with(token::BinOp(token::And), span)) + } + _ => self.unexpected() + } + } + + /// Expects and consumes an `|`. If `||` is seen, replaces it with a single + /// `|` and continues. If an `|` is not seen, signals an error. + fn expect_or(&mut self) -> PResult<'a, ()> { + self.expected_tokens.push(TokenType::Token(token::BinOp(token::Or))); + match self.token.kind { + token::BinOp(token::Or) => { + self.bump(); + Ok(()) + } + token::OrOr => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + Ok(self.bump_with(token::BinOp(token::Or), span)) + } + _ => self.unexpected() + } + } + + /// Attempts to consume a `<`. If `<<` is seen, replaces it with a single + /// `<` and continue. If `<-` is seen, replaces it with a single `<` + /// and continue. If a `<` is not seen, returns false. + /// + /// This is meant to be used when parsing generics on a path to get the + /// starting token. + fn eat_lt(&mut self) -> bool { + self.expected_tokens.push(TokenType::Token(token::Lt)); + let ate = match self.token.kind { + token::Lt => { + self.bump(); + true + } + token::BinOp(token::Shl) => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + self.bump_with(token::Lt, span); + true + } + token::LArrow => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + self.bump_with(token::BinOp(token::Minus), span); + true + } + _ => false, + }; + + if ate { + // See doc comment for `unmatched_angle_bracket_count`. + self.unmatched_angle_bracket_count += 1; + self.max_angle_bracket_count += 1; + debug!("eat_lt: (increment) count={:?}", self.unmatched_angle_bracket_count); + } + + ate + } + + fn expect_lt(&mut self) -> PResult<'a, ()> { + if !self.eat_lt() { + self.unexpected() + } else { + Ok(()) + } + } + + /// Expects and consumes a single `>` token. if a `>>` is seen, replaces it + /// with a single `>` and continues. If a `>` is not seen, signals an error. + fn expect_gt(&mut self) -> PResult<'a, ()> { + self.expected_tokens.push(TokenType::Token(token::Gt)); + let ate = match self.token.kind { + token::Gt => { + self.bump(); + Some(()) + } + token::BinOp(token::Shr) => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + Some(self.bump_with(token::Gt, span)) + } + token::BinOpEq(token::Shr) => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + Some(self.bump_with(token::Ge, span)) + } + token::Ge => { + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); + Some(self.bump_with(token::Eq, span)) + } + _ => None, + }; + + match ate { + Some(_) => { + // See doc comment for `unmatched_angle_bracket_count`. + if self.unmatched_angle_bracket_count > 0 { + self.unmatched_angle_bracket_count -= 1; + debug!("expect_gt: (decrement) count={:?}", self.unmatched_angle_bracket_count); + } + + Ok(()) + }, + None => self.unexpected(), + } + } + + /// Parses a sequence, including the closing delimiter. The function + /// `f` must consume tokens until reaching the next separator or + /// closing bracket. + fn parse_seq_to_end( + &mut self, + ket: &TokenKind, + sep: SeqSep, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, Vec> { + let (val, _, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; + if !recovered { + self.bump(); + } + Ok(val) + } + + /// Parses a sequence, not including the closing delimiter. The function + /// `f` must consume tokens until reaching the next separator or + /// closing bracket. + fn parse_seq_to_before_end( + &mut self, + ket: &TokenKind, + sep: SeqSep, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec, bool, bool)> { + self.parse_seq_to_before_tokens(&[ket], sep, TokenExpectType::Expect, f) + } + + fn expect_any_with_type(&mut self, kets: &[&TokenKind], expect: TokenExpectType) -> bool { + kets.iter().any(|k| { + match expect { + TokenExpectType::Expect => self.check(k), + TokenExpectType::NoExpect => self.token == **k, + } + }) + } + + fn parse_seq_to_before_tokens( + &mut self, + kets: &[&TokenKind], + sep: SeqSep, + expect: TokenExpectType, + mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec, bool /* trailing */, bool /* recovered */)> { + let mut first = true; + let mut recovered = false; + let mut trailing = false; + let mut v = vec![]; + while !self.expect_any_with_type(kets, expect) { + if let token::CloseDelim(..) | token::Eof = self.token.kind { + break + } + if let Some(ref t) = sep.sep { + if first { + first = false; + } else { + match self.expect(t) { + Ok(false) => {} + Ok(true) => { + recovered = true; + break; + } + Err(mut e) => { + // Attempt to keep parsing if it was a similar separator. + if let Some(ref tokens) = t.similar_tokens() { + if tokens.contains(&self.token.kind) { + self.bump(); + } + } + e.emit(); + // Attempt to keep parsing if it was an omitted separator. + match f(self) { + Ok(t) => { + v.push(t); + continue; + }, + Err(mut e) => { + e.cancel(); + break; + } + } + } + } + } + } + if sep.trailing_sep_allowed && self.expect_any_with_type(kets, expect) { + trailing = true; + break; + } + + let t = f(self)?; + v.push(t); + } + + Ok((v, trailing, recovered)) + } + + /// Parses a sequence, including the closing delimiter. The function + /// `f` must consume tokens until reaching the next separator or + /// closing bracket. + fn parse_unspanned_seq( + &mut self, + bra: &TokenKind, + ket: &TokenKind, + sep: SeqSep, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec, bool)> { + self.expect(bra)?; + let (result, trailing, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; + if !recovered { + self.eat(ket); + } + Ok((result, trailing)) + } + + fn parse_delim_comma_seq( + &mut self, + delim: DelimToken, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec, bool)> { + self.parse_unspanned_seq( + &token::OpenDelim(delim), + &token::CloseDelim(delim), + SeqSep::trailing_allowed(token::Comma), + f, + ) + } + + fn parse_paren_comma_seq( + &mut self, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec, bool)> { + self.parse_delim_comma_seq(token::Paren, f) + } + + /// Advance the parser by one token. + pub fn bump(&mut self) { + if self.prev_token_kind == PrevTokenKind::Eof { + // Bumping after EOF is a bad sign, usually an infinite loop. + self.bug("attempted to bump the parser past EOF (may be stuck in a loop)"); + } + + self.prev_span = self.meta_var_span.take().unwrap_or(self.token.span); + + // Record last token kind for possible error recovery. + self.prev_token_kind = match self.token.kind { + token::DocComment(..) => PrevTokenKind::DocComment, + token::Comma => PrevTokenKind::Comma, + token::BinOp(token::Plus) => PrevTokenKind::Plus, + token::BinOp(token::Or) => PrevTokenKind::BitOr, + token::Interpolated(..) => PrevTokenKind::Interpolated, + token::Eof => PrevTokenKind::Eof, + token::Ident(..) => PrevTokenKind::Ident, + _ => PrevTokenKind::Other, + }; + + self.token = self.next_tok(); + self.expected_tokens.clear(); + // Check after each token. + self.process_potential_macro_variable(); + } + + /// Advances the parser using provided token as a next one. Use this when + /// consuming a part of a token. For example a single `<` from `<<`. + fn bump_with(&mut self, next: TokenKind, span: Span) { + self.prev_span = self.token.span.with_hi(span.lo()); + // It would be incorrect to record the kind of the current token, but + // fortunately for tokens currently using `bump_with`, the + // `prev_token_kind` will be of no use anyway. + self.prev_token_kind = PrevTokenKind::Other; + self.token = Token::new(next, span); + self.expected_tokens.clear(); + } + + /// Look-ahead `dist` tokens of `self.token` and get access to that token there. + /// When `dist == 0` then the current token is looked at. + pub fn look_ahead(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R { + if dist == 0 { + return looker(&self.token); + } + + let frame = &self.token_cursor.frame; + looker(&match frame.tree_cursor.look_ahead(dist - 1) { + Some(tree) => match tree { + TokenTree::Token(token) => token, + TokenTree::Delimited(dspan, delim, _) => + Token::new(token::OpenDelim(delim), dspan.open), + } + None => Token::new(token::CloseDelim(frame.delim), frame.span.close) + }) + } + + /// Returns whether any of the given keywords are `dist` tokens ahead of the current one. + fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool { + self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw))) + } + + /// Parses asyncness: `async` or nothing. + fn parse_asyncness(&mut self) -> IsAsync { + if self.eat_keyword(kw::Async) { + IsAsync::Async { + closure_id: DUMMY_NODE_ID, + return_impl_trait_id: DUMMY_NODE_ID, + } + } else { + IsAsync::NotAsync + } + } + + /// Parses unsafety: `unsafe` or nothing. + fn parse_unsafety(&mut self) -> Unsafety { + if self.eat_keyword(kw::Unsafe) { + Unsafety::Unsafe + } else { + Unsafety::Normal + } + } + + /// Parses mutability (`mut` or nothing). + fn parse_mutability(&mut self) -> Mutability { + if self.eat_keyword(kw::Mut) { + Mutability::Mutable + } else { + Mutability::Immutable + } + } + + /// Possibly parses mutability (`const` or `mut`). + fn parse_const_or_mut(&mut self) -> Option { + if self.eat_keyword(kw::Mut) { + Some(Mutability::Mutable) + } else if self.eat_keyword(kw::Const) { + Some(Mutability::Immutable) + } else { + None + } + } + + fn parse_field_name(&mut self) -> PResult<'a, Ident> { + if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = + self.token.kind { + self.expect_no_suffix(self.token.span, "a tuple index", suffix); + self.bump(); + Ok(Ident::new(symbol, self.prev_span)) + } else { + self.parse_ident_common(false) + } + } + + fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> { + let delim = match self.token.kind { + token::OpenDelim(delim) => delim, + _ => { + let msg = "expected open delimiter"; + let mut err = self.fatal(msg); + err.span_label(self.token.span, msg); + return Err(err) + } + }; + let tts = match self.parse_token_tree() { + TokenTree::Delimited(_, _, tts) => tts, + _ => unreachable!(), + }; + let delim = match delim { + token::Paren => MacDelimiter::Parenthesis, + token::Bracket => MacDelimiter::Bracket, + token::Brace => MacDelimiter::Brace, + token::NoDelim => self.bug("unexpected no delimiter"), + }; + Ok((delim, tts.into())) + } + + fn parse_or_use_outer_attributes( + &mut self, + already_parsed_attrs: Option>, + ) -> PResult<'a, ThinVec> { + if let Some(attrs) = already_parsed_attrs { + Ok(attrs) + } else { + self.parse_outer_attributes().map(|a| a.into()) + } + } + + pub fn process_potential_macro_variable(&mut self) { + self.token = match self.token.kind { + token::Dollar if self.token.span.from_expansion() && + self.look_ahead(1, |t| t.is_ident()) => { + self.bump(); + let name = match self.token.kind { + token::Ident(name, _) => name, + _ => unreachable!() + }; + let span = self.prev_span.to(self.token.span); + self.diagnostic() + .struct_span_fatal(span, &format!("unknown macro variable `{}`", name)) + .span_label(span, "unknown macro variable") + .emit(); + self.bump(); + return + } + token::Interpolated(ref nt) => { + self.meta_var_span = Some(self.token.span); + // Interpolated identifier and lifetime tokens are replaced with usual identifier + // and lifetime tokens, so the former are never encountered during normal parsing. + match **nt { + token::NtIdent(ident, is_raw) => + Token::new(token::Ident(ident.name, is_raw), ident.span), + token::NtLifetime(ident) => + Token::new(token::Lifetime(ident.name), ident.span), + _ => return, + } + } + _ => return, + }; + } + + /// Parses a single token tree from the input. + pub fn parse_token_tree(&mut self) -> TokenTree { + match self.token.kind { + token::OpenDelim(..) => { + let frame = mem::replace(&mut self.token_cursor.frame, + self.token_cursor.stack.pop().unwrap()); + self.token.span = frame.span.entire(); + self.bump(); + TokenTree::Delimited( + frame.span, + frame.delim, + frame.tree_cursor.stream.into(), + ) + }, + token::CloseDelim(_) | token::Eof => unreachable!(), + _ => { + let token = self.token.take(); + self.bump(); + TokenTree::Token(token) + } + } + } + + /// Parses a stream of tokens into a list of `TokenTree`s, up to EOF. + pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec> { + let mut tts = Vec::new(); + while self.token != token::Eof { + tts.push(self.parse_token_tree()); + } + Ok(tts) + } + + pub fn parse_tokens(&mut self) -> TokenStream { + let mut result = Vec::new(); + loop { + match self.token.kind { + token::Eof | token::CloseDelim(..) => break, + _ => result.push(self.parse_token_tree().into()), + } + } + TokenStream::new(result) + } + + /// Evaluates the closure with restrictions in place. + /// + /// Afters the closure is evaluated, restrictions are reset. + fn with_res(&mut self, res: Restrictions, f: impl FnOnce(&mut Self) -> T) -> T { + let old = self.restrictions; + self.restrictions = res; + let res = f(self); + self.restrictions = old; + res + } + + fn is_crate_vis(&self) -> bool { + self.token.is_keyword(kw::Crate) && self.look_ahead(1, |t| t != &token::ModSep) + } + + /// Parses `pub`, `pub(crate)` and `pub(in path)` plus shortcuts `crate` for `pub(crate)`, + /// `pub(self)` for `pub(in self)` and `pub(super)` for `pub(in super)`. + /// If the following element can't be a tuple (i.e., it's a function definition), then + /// it's not a tuple struct field), and the contents within the parentheses isn't valid, + /// so emit a proper diagnostic. + pub fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { + maybe_whole!(self, NtVis, |x| x); + + self.expected_tokens.push(TokenType::Keyword(kw::Crate)); + if self.is_crate_vis() { + self.bump(); // `crate` + self.sess.gated_spans.gate(sym::crate_visibility_modifier, self.prev_span); + return Ok(respan(self.prev_span, VisibilityKind::Crate(CrateSugar::JustCrate))); + } + + if !self.eat_keyword(kw::Pub) { + // We need a span for our `Spanned`, but there's inherently no + // keyword to grab a span from for inherited visibility; an empty span at the + // beginning of the current token would seem to be the "Schelling span". + return Ok(respan(self.token.span.shrink_to_lo(), VisibilityKind::Inherited)) + } + let lo = self.prev_span; + + if self.check(&token::OpenDelim(token::Paren)) { + // We don't `self.bump()` the `(` yet because this might be a struct definition where + // `()` or a tuple might be allowed. For example, `struct Struct(pub (), pub (usize));`. + // Because of this, we only `bump` the `(` if we're assured it is appropriate to do so + // by the following tokens. + if self.is_keyword_ahead(1, &[kw::Crate]) + && self.look_ahead(2, |t| t != &token::ModSep) // account for `pub(crate::foo)` + { + // Parse `pub(crate)`. + self.bump(); // `(` + self.bump(); // `crate` + self.expect(&token::CloseDelim(token::Paren))?; // `)` + let vis = VisibilityKind::Crate(CrateSugar::PubCrate); + return Ok(respan(lo.to(self.prev_span), vis)); + } else if self.is_keyword_ahead(1, &[kw::In]) { + // Parse `pub(in path)`. + self.bump(); // `(` + self.bump(); // `in` + let path = self.parse_path(PathStyle::Mod)?; // `path` + self.expect(&token::CloseDelim(token::Paren))?; // `)` + let vis = VisibilityKind::Restricted { + path: P(path), + id: ast::DUMMY_NODE_ID, + }; + return Ok(respan(lo.to(self.prev_span), vis)); + } else if self.look_ahead(2, |t| t == &token::CloseDelim(token::Paren)) + && self.is_keyword_ahead(1, &[kw::Super, kw::SelfLower]) + { + // Parse `pub(self)` or `pub(super)`. + self.bump(); // `(` + let path = self.parse_path(PathStyle::Mod)?; // `super`/`self` + self.expect(&token::CloseDelim(token::Paren))?; // `)` + let vis = VisibilityKind::Restricted { + path: P(path), + id: ast::DUMMY_NODE_ID, + }; + return Ok(respan(lo.to(self.prev_span), vis)); + } else if !can_take_tuple { // Provide this diagnostic if this is not a tuple struct. + self.recover_incorrect_vis_restriction()?; + // Emit diagnostic, but continue with public visibility. + } + } + + Ok(respan(lo, VisibilityKind::Public)) + } + + /// Recovery for e.g. `pub(something) fn ...` or `struct X { pub(something) y: Z }` + fn recover_incorrect_vis_restriction(&mut self) -> PResult<'a, ()> { + self.bump(); // `(` + let path = self.parse_path(PathStyle::Mod)?; + self.expect(&token::CloseDelim(token::Paren))?; // `)` + + let msg = "incorrect visibility restriction"; + let suggestion = r##"some possible visibility restrictions are: +`pub(crate)`: visible only on the current crate +`pub(super)`: visible only in the current module's parent +`pub(in path::to::module)`: visible only on the specified path"##; + + let path_str = pprust::path_to_string(&path); + + struct_span_err!(self.sess.span_diagnostic, path.span, E0704, "{}", msg) + .help(suggestion) + .span_suggestion( + path.span, + &format!("make this visible only to module `{}` with `in`", path_str), + format!("in {}", path_str), + Applicability::MachineApplicable, + ) + .emit(); + + Ok(()) + } + + /// Parses `extern string_literal?`. + /// If `extern` is not found, the Rust ABI is used. + /// If `extern` is found and a `string_literal` does not follow, the C ABI is used. + fn parse_extern_abi(&mut self) -> PResult<'a, Abi> { + Ok(if self.eat_keyword(kw::Extern) { + self.parse_opt_abi()? + } else { + Abi::default() + }) + } + + /// Parses a string literal as an ABI spec. + /// If one is not found, the "C" ABI is used. + fn parse_opt_abi(&mut self) -> PResult<'a, Abi> { + let span = if self.token.can_begin_literal_or_bool() { + let ast::Lit { span, kind, .. } = self.parse_lit()?; + match kind { + ast::LitKind::Str(symbol, _) => return Ok(Abi::new(symbol, span)), + ast::LitKind::Err(_) => {} + _ => { + self.struct_span_err(span, "non-string ABI literal") + .span_suggestion( + span, + "specify the ABI with a string literal", + "\"C\"".to_string(), + Applicability::MaybeIncorrect, + ) + .emit(); + } + } + span + } else { + self.prev_span + }; + Ok(Abi::new(sym::C, span)) + } + + /// We are parsing `async fn`. If we are on Rust 2015, emit an error. + fn ban_async_in_2015(&self, async_span: Span) { + if async_span.rust_2015() { + self.diagnostic() + .struct_span_err_with_code( + async_span, + "`async fn` is not permitted in the 2015 edition", + DiagnosticId::Error("E0670".into()) + ) + .emit(); + } + } + + fn collect_tokens( + &mut self, + f: impl FnOnce(&mut Self) -> PResult<'a, R>, + ) -> PResult<'a, (R, TokenStream)> { + // Record all tokens we parse when parsing this item. + let mut tokens = Vec::new(); + let prev_collecting = match self.token_cursor.frame.last_token { + LastToken::Collecting(ref mut list) => { + Some(mem::take(list)) + } + LastToken::Was(ref mut last) => { + tokens.extend(last.take()); + None + } + }; + self.token_cursor.frame.last_token = LastToken::Collecting(tokens); + let prev = self.token_cursor.stack.len(); + let ret = f(self); + let last_token = if self.token_cursor.stack.len() == prev { + &mut self.token_cursor.frame.last_token + } else if self.token_cursor.stack.get(prev).is_none() { + // This can happen due to a bad interaction of two unrelated recovery mechanisms with + // mismatched delimiters *and* recovery lookahead on the likely typo `pub ident(` + // (#62881). + return Ok((ret?, TokenStream::default())); + } else { + &mut self.token_cursor.stack[prev].last_token + }; + + // Pull out the tokens that we've collected from the call to `f` above. + let mut collected_tokens = match *last_token { + LastToken::Collecting(ref mut v) => mem::take(v), + LastToken::Was(ref was) => { + let msg = format!("our vector went away? - found Was({:?})", was); + debug!("collect_tokens: {}", msg); + self.sess.span_diagnostic.delay_span_bug(self.token.span, &msg); + // This can happen due to a bad interaction of two unrelated recovery mechanisms + // with mismatched delimiters *and* recovery lookahead on the likely typo + // `pub ident(` (#62895, different but similar to the case above). + return Ok((ret?, TokenStream::default())); + } + }; + + // If we're not at EOF our current token wasn't actually consumed by + // `f`, but it'll still be in our list that we pulled out. In that case + // put it back. + let extra_token = if self.token != token::Eof { + collected_tokens.pop() + } else { + None + }; + + // If we were previously collecting tokens, then this was a recursive + // call. In that case we need to record all the tokens we collected in + // our parent list as well. To do that we push a clone of our stream + // onto the previous list. + match prev_collecting { + Some(mut list) => { + list.extend(collected_tokens.iter().cloned()); + list.extend(extra_token); + *last_token = LastToken::Collecting(list); + } + None => { + *last_token = LastToken::Was(extra_token); + } + } + + Ok((ret?, TokenStream::new(collected_tokens))) + } + + /// `::{` or `::*` + fn is_import_coupler(&mut self) -> bool { + self.check(&token::ModSep) && + self.look_ahead(1, |t| *t == token::OpenDelim(token::Brace) || + *t == token::BinOp(token::Star)) + } + + fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option)> { + let ret = match self.token.kind { + token::Literal(token::Lit { kind: token::Str, symbol, suffix }) => + (symbol, ast::StrStyle::Cooked, suffix), + token::Literal(token::Lit { kind: token::StrRaw(n), symbol, suffix }) => + (symbol, ast::StrStyle::Raw(n), suffix), + _ => return None + }; + self.bump(); + Some(ret) + } + + pub fn parse_str(&mut self) -> PResult<'a, (Symbol, StrStyle)> { + match self.parse_optional_str() { + Some((s, style, suf)) => { + let sp = self.prev_span; + self.expect_no_suffix(sp, "a string literal", suf); + Ok((s, style)) + } + _ => { + let msg = "expected string literal"; + let mut err = self.fatal(msg); + err.span_label(self.token.span, msg); + Err(err) + } + } + } +} + +crate fn make_unclosed_delims_error( + unmatched: UnmatchedBrace, + sess: &ParseSess, +) -> Option> { + // `None` here means an `Eof` was found. We already emit those errors elsewhere, we add them to + // `unmatched_braces` only for error recovery in the `Parser`. + let found_delim = unmatched.found_delim?; + let mut err = sess.span_diagnostic.struct_span_err(unmatched.found_span, &format!( + "incorrect close delimiter: `{}`", + pprust::token_kind_to_string(&token::CloseDelim(found_delim)), + )); + err.span_label(unmatched.found_span, "incorrect close delimiter"); + if let Some(sp) = unmatched.candidate_span { + err.span_label(sp, "close delimiter possibly meant for this"); + } + if let Some(sp) = unmatched.unclosed_span { + err.span_label(sp, "un-closed delimiter"); + } + Some(err) +} + +pub fn emit_unclosed_delims(unclosed_delims: &mut Vec, sess: &ParseSess) { + *sess.reached_eof.borrow_mut() |= unclosed_delims.iter() + .any(|unmatched_delim| unmatched_delim.found_delim.is_none()); + for unmatched in unclosed_delims.drain(..) { + make_unclosed_delims_error(unmatched, sess).map(|mut e| e.emit()); + } +} -- cgit 1.4.1-3-g733a5 From 8fa8e02c282f6dc4ebcd0f8403a881116a948443 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 12:29:23 +0200 Subject: syntax: simplify imports --- src/libsyntax/parse/lexer/comments.rs | 5 +++-- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/tests.rs | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'src/libsyntax/parse/parser') diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index ac79ce323bf..33415bdcb62 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -1,9 +1,10 @@ pub use CommentStyle::*; +use super::is_block_doc_comment; + use crate::ast; use crate::source_map::SourceMap; -use crate::parse::lexer::is_block_doc_comment; -use crate::parse::lexer::ParseSess; +use crate::sess::ParseSess; use syntax_pos::{BytePos, CharPos, Pos, FileName}; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index cc8738edff7..fcb7c4d1323 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -5,7 +5,7 @@ use crate::ptr::P; use crate::ast::{self, Attribute, Pat, PatKind, FieldPat, RangeEnd, RangeSyntax, Mac}; use crate::ast::{BindingMode, Ident, Mutability, Path, QSelf, Expr, ExprKind}; use crate::mut_visit::{noop_visit_pat, noop_visit_mac, MutVisitor}; -use crate::parse::token::{self}; +use crate::parse::token; use crate::print::pprust; use crate::source_map::{respan, Span, Spanned}; use crate::ThinVec; diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs index 169eb954efa..201eaca947b 100644 --- a/src/libsyntax/parse/tests.rs +++ b/src/libsyntax/parse/tests.rs @@ -2,8 +2,8 @@ use super::*; use crate::ast::{self, Name, PatKind}; use crate::attr::first_attr_value_str_by_name; -use crate::parse::{ParseSess, PResult}; -use crate::parse::new_parser_from_source_str; +use crate::sess::ParseSess; +use crate::parse::{PResult, new_parser_from_source_str}; use crate::parse::token::Token; use crate::print::pprust::item_to_string; use crate::ptr::P; -- cgit 1.4.1-3-g733a5 From 9d6768a478b8a6afa1e16dfebe9d79bd3f508cdf Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 12:46:32 +0200 Subject: syntax::parser::token -> syntax::token --- src/librustc/hir/lowering.rs | 2 +- src/librustc/hir/map/def_collector.rs | 2 +- src/librustc/ich/impls_syntax.rs | 4 +- src/librustc_interface/interface.rs | 2 +- src/librustc_resolve/build_reduced_graph.rs | 2 +- src/librustc_save_analysis/dump_visitor.rs | 2 +- src/librustc_save_analysis/span_utils.rs | 2 +- src/librustdoc/html/highlight.rs | 2 +- src/librustdoc/passes/check_code_block_syntax.rs | 2 +- src/libsyntax/ast.rs | 2 +- src/libsyntax/attr/mod.rs | 2 +- src/libsyntax/feature_gate/check.rs | 2 +- src/libsyntax/lib.rs | 1 + src/libsyntax/mut_visit.rs | 2 +- src/libsyntax/parse/lexer/mod.rs | 2 +- src/libsyntax/parse/lexer/tests.rs | 2 +- src/libsyntax/parse/lexer/tokentrees.rs | 2 +- src/libsyntax/parse/lexer/unicode_chars.rs | 2 +- src/libsyntax/parse/literal.rs | 2 +- src/libsyntax/parse/mod.rs | 3 +- src/libsyntax/parse/parser/attr.rs | 2 +- src/libsyntax/parse/parser/diagnostics.rs | 2 +- src/libsyntax/parse/parser/expr.rs | 2 +- src/libsyntax/parse/parser/generics.rs | 2 +- src/libsyntax/parse/parser/mod.rs | 2 +- src/libsyntax/parse/parser/module.rs | 2 +- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/parser/path.rs | 2 +- src/libsyntax/parse/parser/stmt.rs | 2 +- src/libsyntax/parse/parser/ty.rs | 2 +- src/libsyntax/parse/tests.rs | 2 +- src/libsyntax/parse/token.rs | 728 ----------------------- src/libsyntax/print/pprust.rs | 2 +- src/libsyntax/token.rs | 728 +++++++++++++++++++++++ src/libsyntax/tokenstream.rs | 2 +- src/libsyntax/util/parser.rs | 2 +- src/libsyntax/visit.rs | 2 +- src/libsyntax_expand/base.rs | 2 +- src/libsyntax_expand/expand.rs | 2 +- src/libsyntax_expand/mbe.rs | 2 +- src/libsyntax_expand/mbe/macro_check.rs | 2 +- src/libsyntax_expand/mbe/macro_parser.rs | 2 +- src/libsyntax_expand/mbe/macro_rules.rs | 3 +- src/libsyntax_expand/mbe/quoted.rs | 2 +- src/libsyntax_expand/mbe/transcribe.rs | 2 +- src/libsyntax_expand/proc_macro.rs | 3 +- src/libsyntax_expand/proc_macro_server.rs | 7 +- src/libsyntax_ext/asm.rs | 2 +- src/libsyntax_ext/assert.rs | 2 +- src/libsyntax_ext/cfg.rs | 2 +- src/libsyntax_ext/cmdline_attrs.rs | 3 +- src/libsyntax_ext/concat_idents.rs | 2 +- src/libsyntax_ext/format.rs | 2 +- src/libsyntax_ext/global_asm.rs | 2 +- src/libsyntax_ext/plugin_macro_defs.rs | 4 +- src/libsyntax_ext/source_util.rs | 3 +- src/test/ui-fulldeps/ast_stmt_expr_attr.rs | 2 +- src/test/ui-fulldeps/auxiliary/roman-numerals.rs | 2 +- 58 files changed, 792 insertions(+), 789 deletions(-) delete mode 100644 src/libsyntax/parse/token.rs create mode 100644 src/libsyntax/token.rs (limited to 'src/libsyntax/parse/parser') diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 6344c7a233c..87ad4ace592 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -66,7 +66,7 @@ use syntax::ptr::P as AstP; use syntax::ast::*; use syntax::errors; use syntax::print::pprust; -use syntax::parse::token::{self, Nonterminal, Token}; +use syntax::token::{self, Nonterminal, Token}; use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::sess::ParseSess; use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned}; diff --git a/src/librustc/hir/map/def_collector.rs b/src/librustc/hir/map/def_collector.rs index e9970e30bf9..57c1421bde6 100644 --- a/src/librustc/hir/map/def_collector.rs +++ b/src/librustc/hir/map/def_collector.rs @@ -4,7 +4,7 @@ use crate::hir::def_id::DefIndex; use syntax::ast::*; use syntax::visit; use syntax::symbol::{kw, sym}; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax_pos::hygiene::ExpnId; use syntax_pos::Span; diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 2201c4b0980..c401bd17dd4 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -8,9 +8,9 @@ use std::mem; use syntax::ast; use syntax::feature_gate; -use syntax::parse::token; -use syntax::symbol::SymbolStr; +use syntax::token; use syntax::tokenstream; +use syntax_pos::symbol::SymbolStr; use syntax_pos::SourceFile; use crate::hir::def_id::{DefId, CrateNum, CRATE_DEF_INDEX}; diff --git a/src/librustc_interface/interface.rs b/src/librustc_interface/interface.rs index 4e4d6d982fb..034e861b212 100644 --- a/src/librustc_interface/interface.rs +++ b/src/librustc_interface/interface.rs @@ -16,7 +16,7 @@ use std::result; use std::sync::{Arc, Mutex}; use syntax::{self, parse}; use syntax::ast::{self, MetaItemKind}; -use syntax::parse::token; +use syntax::token; use syntax::source_map::{FileName, FileLoader, SourceMap}; use syntax::sess::ParseSess; use syntax_pos::edition; diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index 55f054d0be3..4239518b879 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -33,7 +33,7 @@ use syntax::attr; use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, NodeId}; use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind}; use syntax::feature_gate::is_builtin_attr; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::print::pprust; use syntax::{span_err, struct_span_err}; use syntax::source_map::{respan, Spanned}; diff --git a/src/librustc_save_analysis/dump_visitor.rs b/src/librustc_save_analysis/dump_visitor.rs index a372106d379..5c5fbcc07de 100644 --- a/src/librustc_save_analysis/dump_visitor.rs +++ b/src/librustc_save_analysis/dump_visitor.rs @@ -24,7 +24,7 @@ use std::path::Path; use std::env; use syntax::ast::{self, Attribute, NodeId, PatKind}; -use syntax::parse::token; +use syntax::token; use syntax::visit::{self, Visitor}; use syntax::print::pprust::{ bounds_to_string, diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs index fb9919d777d..4d0780cf94d 100644 --- a/src/librustc_save_analysis/span_utils.rs +++ b/src/librustc_save_analysis/span_utils.rs @@ -3,7 +3,7 @@ use rustc::session::Session; use crate::generated_code; use syntax::parse::lexer::{self, StringReader}; -use syntax::parse::token::{self, TokenKind}; +use syntax::token::{self, TokenKind}; use syntax_pos::*; #[derive(Clone)] diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 88ba13f2796..4bd72f7e61c 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -13,7 +13,7 @@ use std::io::prelude::*; use syntax::source_map::SourceMap; use syntax::parse::lexer; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; use syntax_pos::{Span, FileName}; diff --git a/src/librustdoc/passes/check_code_block_syntax.rs b/src/librustdoc/passes/check_code_block_syntax.rs index 10e15ab8881..4603e77b0fd 100644 --- a/src/librustdoc/passes/check_code_block_syntax.rs +++ b/src/librustdoc/passes/check_code_block_syntax.rs @@ -1,6 +1,6 @@ use errors::Applicability; use syntax::parse::lexer::{StringReader as Lexer}; -use syntax::parse::token; +use syntax::token; use syntax::sess::ParseSess; use syntax::source_map::FilePathMapping; use syntax_pos::{InnerSpan, FileName}; diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 67d1acbccfb..18151a1586c 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -24,9 +24,9 @@ pub use crate::util::parser::ExprPrecedence; pub use syntax_pos::symbol::{Ident, Symbol as Name}; -use crate::parse::token::{self, DelimToken}; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; +use crate::token::{self, DelimToken}; use crate::tokenstream::TokenStream; use syntax_pos::symbol::{kw, sym, Symbol}; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index c663995eb8f..1534d2723c8 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -17,7 +17,7 @@ use crate::source_map::{BytePos, Spanned}; use crate::parse::lexer::comments::doc_comment_style; use crate::parse; use crate::parse::PResult; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::ptr::P; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs index 5b1493ebc9b..ecff89ad59b 100644 --- a/src/libsyntax/feature_gate/check.rs +++ b/src/libsyntax/feature_gate/check.rs @@ -11,7 +11,7 @@ use crate::attr::{self, check_builtin_attribute}; use crate::source_map::Spanned; use crate::edition::{ALL_EDITIONS, Edition}; use crate::visit::{self, FnKind, Visitor}; -use crate::parse::token; +use crate::token; use crate::sess::ParseSess; use crate::symbol::{Symbol, sym}; use crate::tokenstream::TokenTree; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 7be6e6c7e18..4c134430f30 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -107,6 +107,7 @@ pub mod show_span; pub use syntax_pos::edition; pub use syntax_pos::symbol; pub mod sess; +pub mod token; pub mod tokenstream; pub mod visit; diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 7261601e144..0c90652526d 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -9,7 +9,7 @@ use crate::ast::*; use crate::source_map::{Spanned, respan}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::ptr::P; use crate::ThinVec; use crate::tokenstream::*; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index a6dfed542d6..5499a3cae5f 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, TokenKind}; +use crate::token::{self, Token, TokenKind}; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs index de301b1fc49..b9b82b2bcef 100644 --- a/src/libsyntax/parse/lexer/tests.rs +++ b/src/libsyntax/parse/lexer/tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::symbol::Symbol; use crate::source_map::{SourceMap, FilePathMapping}; -use crate::parse::token; +use crate::token; use crate::with_default_globals; use errors::{Handler, emitter::EmitterWriter}; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index de8ac2c71e8..3c15a5e1dae 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -4,8 +4,8 @@ use syntax_pos::Span; use super::{StringReader, UnmatchedBrace}; use crate::print::pprust::token_to_string; -use crate::parse::token::{self, Token}; use crate::parse::PResult; +use crate::token::{self, Token}; use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint}; impl<'a> StringReader<'a> { diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index 525b4215aff..6eb995b61d3 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -4,7 +4,7 @@ use super::StringReader; use errors::{Applicability, DiagnosticBuilder}; use syntax_pos::{BytePos, Pos, Span, symbol::kw}; -use crate::parse::token; +use crate::token; #[rustfmt::skip] // for line breaks const UNICODE_ARRAY: &[(char, &str, char)] = &[ diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index a8eeac59954..d4c9b7850c5 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -1,8 +1,8 @@ //! Code related to parsing literals. use crate::ast::{self, Lit, LitKind}; -use crate::parse::token::{self, Token}; use crate::symbol::{kw, sym, Symbol}; +use crate::token::{self, Token}; use crate::tokenstream::TokenTree; use log::debug; diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 5b020d3d288..7db0ca0b78d 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -2,7 +2,7 @@ use crate::ast; use crate::parse::parser::{Parser, emit_unclosed_delims, make_unclosed_delims_error}; -use crate::parse::token::Nonterminal; +use crate::token::{self, Nonterminal}; use crate::tokenstream::{self, TokenStream, TokenTree}; use crate::print::pprust; use crate::sess::ParseSess; @@ -25,7 +25,6 @@ mod tests; #[macro_use] pub mod parser; pub mod lexer; -pub mod token; crate mod classify; crate mod literal; diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 1c292661f24..aa0542a09a9 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,7 +1,7 @@ use super::{SeqSep, PResult, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; -use crate::parse::token::{self, Nonterminal, DelimToken}; +use crate::token::{self, Nonterminal, DelimToken}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; diff --git a/src/libsyntax/parse/parser/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index 49a517a5c44..09f0b9b74b2 100644 --- a/src/libsyntax/parse/parser/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -6,7 +6,7 @@ use crate::ast::{ self, Param, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, }; -use crate::parse::token::{self, TokenKind, token_can_begin_expr}; +use crate::token::{self, TokenKind, token_can_begin_expr}; use crate::print::pprust; use crate::ptr::P; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index 509e6482dcc..4b962201e85 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -12,7 +12,7 @@ use crate::ast::{ }; use crate::maybe_recover_from_interpolated_ty_qpath; use crate::parse::classify; -use crate::parse::token::{self, Token, TokenKind}; +use crate::token::{self, Token, TokenKind}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, Span}; diff --git a/src/libsyntax/parse/parser/generics.rs b/src/libsyntax/parse/parser/generics.rs index 3c094750b4d..cfc3768cac5 100644 --- a/src/libsyntax/parse/parser/generics.rs +++ b/src/libsyntax/parse/parser/generics.rs @@ -1,7 +1,7 @@ use super::{Parser, PResult}; use crate::ast::{self, WhereClause, GenericParam, GenericParamKind, GenericBounds, Attribute}; -use crate::parse::token; +use crate::token; use crate::source_map::DUMMY_SP; use syntax_pos::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/mod.rs b/src/libsyntax/parse/parser/mod.rs index 1284e89f195..6498b674a3b 100644 --- a/src/libsyntax/parse/parser/mod.rs +++ b/src/libsyntax/parse/parser/mod.rs @@ -18,7 +18,7 @@ use crate::ast::{ use crate::parse::{PResult, Directory, DirectoryOwnership}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use crate::parse::token::{self, Token, TokenKind, DelimToken}; +use crate::token::{self, Token, TokenKind, DelimToken}; use crate::print::pprust; use crate::ptr::P; use crate::sess::ParseSess; diff --git a/src/libsyntax/parse/parser/module.rs b/src/libsyntax/parse/parser/module.rs index 242a17659a0..70e0dfc3e88 100644 --- a/src/libsyntax/parse/parser/module.rs +++ b/src/libsyntax/parse/parser/module.rs @@ -5,7 +5,7 @@ use super::diagnostics::Error; use crate::attr; use crate::ast::{self, Ident, Attribute, ItemKind, Mod, Crate}; use crate::parse::{new_sub_parser_from_file, DirectoryOwnership}; -use crate::parse::token::{self, TokenKind}; +use crate::token::{self, TokenKind}; use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName}; use crate::symbol::sym; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index fcb7c4d1323..c4a983188fa 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -5,7 +5,7 @@ use crate::ptr::P; use crate::ast::{self, Attribute, Pat, PatKind, FieldPat, RangeEnd, RangeSyntax, Mac}; use crate::ast::{BindingMode, Ident, Mutability, Path, QSelf, Expr, ExprKind}; use crate::mut_visit::{noop_visit_pat, noop_visit_mac, MutVisitor}; -use crate::parse::token; +use crate::token; use crate::print::pprust; use crate::source_map::{respan, Span, Spanned}; use crate::ThinVec; diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index 4438d61d9ee..2659e0c680e 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -3,7 +3,7 @@ use super::{Parser, PResult, TokenType}; use crate::{maybe_whole, ThinVec}; use crate::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs}; use crate::ast::{AnonConst, GenericArg, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::source_map::{Span, BytePos}; use syntax_pos::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index 12c530f3cbb..010fd9c37fd 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -9,7 +9,7 @@ use crate::{maybe_whole, ThinVec}; use crate::ast::{self, DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind}; use crate::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter}; use crate::parse::{classify, DirectoryOwnership}; -use crate::parse::token; +use crate::token; use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index b770b90705c..4183416e628 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -6,7 +6,7 @@ use crate::ptr::P; use crate::ast::{self, Ty, TyKind, MutTy, BareFnTy, FunctionRetTy, GenericParam, Lifetime, Ident}; use crate::ast::{TraitBoundModifier, TraitObjectSyntax, GenericBound, GenericBounds, PolyTraitRef}; use crate::ast::{Mutability, AnonConst, Mac}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::source_map::Span; use crate::symbol::{kw}; diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs index 201eaca947b..27ca2b6472f 100644 --- a/src/libsyntax/parse/tests.rs +++ b/src/libsyntax/parse/tests.rs @@ -4,7 +4,7 @@ use crate::ast::{self, Name, PatKind}; use crate::attr::first_attr_value_str_by_name; use crate::sess::ParseSess; use crate::parse::{PResult, new_parser_from_source_str}; -use crate::parse::token::Token; +use crate::token::Token; use crate::print::pprust::item_to_string; use crate::ptr::P; use crate::source_map::FilePathMapping; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs deleted file mode 100644 index 6f3da344ccf..00000000000 --- a/src/libsyntax/parse/token.rs +++ /dev/null @@ -1,728 +0,0 @@ -pub use BinOpToken::*; -pub use Nonterminal::*; -pub use DelimToken::*; -pub use LitKind::*; -pub use TokenKind::*; - -use crate::ast; -use crate::ptr::P; -use crate::symbol::kw; -use crate::tokenstream::TokenTree; - -use syntax_pos::symbol::Symbol; -use syntax_pos::{self, Span, DUMMY_SP}; - -use std::fmt; -use std::mem; -#[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert_size; -use rustc_data_structures::sync::Lrc; - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum BinOpToken { - Plus, - Minus, - Star, - Slash, - Percent, - Caret, - And, - Or, - Shl, - Shr, -} - -/// A delimiter token. -#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum DelimToken { - /// A round parenthesis (i.e., `(` or `)`). - Paren, - /// A square bracket (i.e., `[` or `]`). - Bracket, - /// A curly brace (i.e., `{` or `}`). - Brace, - /// An empty delimiter. - NoDelim, -} - -impl DelimToken { - pub fn len(self) -> usize { - if self == NoDelim { 0 } else { 1 } - } - - pub fn is_empty(self) -> bool { - self == NoDelim - } -} - -#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub enum LitKind { - Bool, // AST only, must never appear in a `Token` - Byte, - Char, - Integer, - Float, - Str, - StrRaw(u16), // raw string delimited by `n` hash symbols - ByteStr, - ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols - Err, -} - -/// A literal token. -#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Lit { - pub kind: LitKind, - pub symbol: Symbol, - pub suffix: Option, -} - -impl fmt::Display for Lit { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Lit { kind, symbol, suffix } = *self; - match kind { - Byte => write!(f, "b'{}'", symbol)?, - Char => write!(f, "'{}'", symbol)?, - Str => write!(f, "\"{}\"", symbol)?, - StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=symbol)?, - ByteStr => write!(f, "b\"{}\"", symbol)?, - ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=symbol)?, - Integer | - Float | - Bool | - Err => write!(f, "{}", symbol)?, - } - - if let Some(suffix) = suffix { - write!(f, "{}", suffix)?; - } - - Ok(()) - } -} - -impl LitKind { - /// An English article for the literal token kind. - crate fn article(self) -> &'static str { - match self { - Integer | Err => "an", - _ => "a", - } - } - - crate fn descr(self) -> &'static str { - match self { - Bool => panic!("literal token contains `Lit::Bool`"), - Byte => "byte", - Char => "char", - Integer => "integer", - Float => "float", - Str | StrRaw(..) => "string", - ByteStr | ByteStrRaw(..) => "byte string", - Err => "error", - } - } - - crate fn may_have_suffix(self) -> bool { - match self { - Integer | Float | Err => true, - _ => false, - } - } -} - -impl Lit { - pub fn new(kind: LitKind, symbol: Symbol, suffix: Option) -> Lit { - Lit { kind, symbol, suffix } - } -} - -pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { - let ident_token = Token::new(Ident(name, is_raw), span); - token_can_begin_expr(&ident_token) -} - -pub(crate) fn token_can_begin_expr(ident_token: &Token) -> bool { - !ident_token.is_reserved_ident() || - ident_token.is_path_segment_keyword() || - match ident_token.kind { - TokenKind::Ident(ident, _) => [ - kw::Async, - kw::Do, - kw::Box, - kw::Break, - kw::Continue, - kw::False, - kw::For, - kw::If, - kw::Let, - kw::Loop, - kw::Match, - kw::Move, - kw::Return, - kw::True, - kw::Unsafe, - kw::While, - kw::Yield, - kw::Static, - ].contains(&ident), - _=> false, - } -} - -fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { - let ident_token = Token::new(Ident(name, is_raw), span); - - !ident_token.is_reserved_ident() || - ident_token.is_path_segment_keyword() || - [ - kw::Underscore, - kw::For, - kw::Impl, - kw::Fn, - kw::Unsafe, - kw::Extern, - kw::Typeof, - kw::Dyn, - ].contains(&name) -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub enum TokenKind { - /* Expression-operator symbols. */ - Eq, - Lt, - Le, - EqEq, - Ne, - Ge, - Gt, - AndAnd, - OrOr, - Not, - Tilde, - BinOp(BinOpToken), - BinOpEq(BinOpToken), - - /* Structural symbols */ - At, - Dot, - DotDot, - DotDotDot, - DotDotEq, - Comma, - Semi, - Colon, - ModSep, - RArrow, - LArrow, - FatArrow, - Pound, - Dollar, - Question, - /// Used by proc macros for representing lifetimes, not generated by lexer right now. - SingleQuote, - /// An opening delimiter (e.g., `{`). - OpenDelim(DelimToken), - /// A closing delimiter (e.g., `}`). - CloseDelim(DelimToken), - - /* Literals */ - Literal(Lit), - - /* Name components */ - Ident(ast::Name, /* is_raw */ bool), - Lifetime(ast::Name), - - Interpolated(Lrc), - - // Can be expanded into several tokens. - /// A doc comment. - DocComment(ast::Name), - - // Junk. These carry no data because we don't really care about the data - // they *would* carry, and don't really want to allocate a new ident for - // them. Instead, users could extract that from the associated span. - - /// Whitespace. - Whitespace, - /// A comment. - Comment, - Shebang(ast::Name), - /// A completely invalid token which should be skipped. - Unknown(ast::Name), - - Eof, -} - -// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(target_arch = "x86_64")] -static_assert_size!(TokenKind, 16); - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Token { - pub kind: TokenKind, - pub span: Span, -} - -impl TokenKind { - pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { - Literal(Lit::new(kind, symbol, suffix)) - } - - /// Returns tokens that are likely to be typed accidentally instead of the current token. - /// Enables better error recovery when the wrong token is found. - crate fn similar_tokens(&self) -> Option> { - match *self { - Comma => Some(vec![Dot, Lt, Semi]), - Semi => Some(vec![Colon, Comma]), - _ => None - } - } -} - -impl Token { - pub fn new(kind: TokenKind, span: Span) -> Self { - Token { kind, span } - } - - /// Some token that will be thrown away later. - crate fn dummy() -> Self { - Token::new(TokenKind::Whitespace, DUMMY_SP) - } - - /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: ast::Ident) -> Self { - Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) - } - - /// Return this token by value and leave a dummy token in its place. - pub fn take(&mut self) -> Self { - mem::replace(self, Token::dummy()) - } - - crate fn is_op(&self) -> bool { - match self.kind { - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | - Ident(..) | Lifetime(..) | Interpolated(..) | - Whitespace | Comment | Shebang(..) | Eof => false, - _ => true, - } - } - - crate fn is_like_plus(&self) -> bool { - match self.kind { - BinOp(Plus) | BinOpEq(Plus) => true, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of an expression. - pub fn can_begin_expr(&self) -> bool { - match self.kind { - Ident(name, is_raw) => - ident_can_begin_expr(name, self.span, is_raw), // value name or keyword - OpenDelim(..) | // tuple, array or block - Literal(..) | // literal - Not | // operator not - BinOp(Minus) | // unary minus - BinOp(Star) | // dereference - BinOp(Or) | OrOr | // closure - BinOp(And) | // reference - AndAnd | // double reference - // DotDotDot is no longer supported, but we need some way to display the error - DotDot | DotDotDot | DotDotEq | // range notation - Lt | BinOp(Shl) | // associated path - ModSep | // global path - Lifetime(..) | // labeled loop - Pound => true, // expression attributes - Interpolated(ref nt) => match **nt { - NtLiteral(..) | - NtIdent(..) | - NtExpr(..) | - NtBlock(..) | - NtPath(..) | - NtLifetime(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of a type. - pub fn can_begin_type(&self) -> bool { - match self.kind { - Ident(name, is_raw) => - ident_can_begin_type(name, self.span, is_raw), // type name or keyword - OpenDelim(Paren) | // tuple - OpenDelim(Bracket) | // array - Not | // never - BinOp(Star) | // raw pointer - BinOp(And) | // reference - AndAnd | // double reference - Question | // maybe bound in trait object - Lifetime(..) | // lifetime bound in trait object - Lt | BinOp(Shl) | // associated path - ModSep => true, // global path - Interpolated(ref nt) => match **nt { - NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of a const param. - crate fn can_begin_const_arg(&self) -> bool { - match self.kind { - OpenDelim(Brace) => true, - Interpolated(ref nt) => match **nt { - NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, - _ => false, - } - _ => self.can_begin_literal_or_bool(), - } - } - - /// Returns `true` if the token can appear at the start of a generic bound. - crate fn can_begin_bound(&self) -> bool { - self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || - self == &Question || self == &OpenDelim(Paren) - } - - /// Returns `true` if the token is any literal - pub fn is_lit(&self) -> bool { - match self.kind { - Literal(..) => true, - _ => false, - } - } - - /// Returns `true` if the token is any literal, a minus (which can prefix a literal, - /// for example a '-42', or one of the boolean idents). - pub fn can_begin_literal_or_bool(&self) -> bool { - match self.kind { - Literal(..) | BinOp(Minus) => true, - Ident(name, false) if name.is_bool_lit() => true, - Interpolated(ref nt) => match **nt { - NtLiteral(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns an identifier if this token is an identifier. - pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { - match self.kind { - Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), - Interpolated(ref nt) => match **nt { - NtIdent(ident, is_raw) => Some((ident, is_raw)), - _ => None, - }, - _ => None, - } - } - - /// Returns a lifetime identifier if this token is a lifetime. - pub fn lifetime(&self) -> Option { - match self.kind { - Lifetime(name) => Some(ast::Ident::new(name, self.span)), - Interpolated(ref nt) => match **nt { - NtLifetime(ident) => Some(ident), - _ => None, - }, - _ => None, - } - } - - /// Returns `true` if the token is an identifier. - pub fn is_ident(&self) -> bool { - self.ident().is_some() - } - - /// Returns `true` if the token is a lifetime. - crate fn is_lifetime(&self) -> bool { - self.lifetime().is_some() - } - - /// Returns `true` if the token is a identifier whose name is the given - /// string slice. - crate fn is_ident_named(&self, name: Symbol) -> bool { - self.ident().map_or(false, |(ident, _)| ident.name == name) - } - - /// Returns `true` if the token is an interpolated path. - fn is_path(&self) -> bool { - if let Interpolated(ref nt) = self.kind { - if let NtPath(..) = **nt { - return true; - } - } - false - } - - /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? - /// That is, is this a pre-parsed expression dropped into the token stream - /// (which happens while parsing the result of macro expansion)? - crate fn is_whole_expr(&self) -> bool { - if let Interpolated(ref nt) = self.kind { - if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { - return true; - } - } - - false - } - - /// Returns `true` if the token is either the `mut` or `const` keyword. - crate fn is_mutability(&self) -> bool { - self.is_keyword(kw::Mut) || - self.is_keyword(kw::Const) - } - - crate fn is_qpath_start(&self) -> bool { - self == &Lt || self == &BinOp(Shl) - } - - crate fn is_path_start(&self) -> bool { - self == &ModSep || self.is_qpath_start() || self.is_path() || - self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() - } - - /// Returns `true` if the token is a given keyword, `kw`. - pub fn is_keyword(&self, kw: Symbol) -> bool { - self.is_non_raw_ident_where(|id| id.name == kw) - } - - crate fn is_path_segment_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) - } - - // Returns true for reserved identifiers used internally for elided lifetimes, - // unnamed method parameters, crate root module, error recovery etc. - crate fn is_special_ident(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_special) - } - - /// Returns `true` if the token is a keyword used in the language. - crate fn is_used_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_used_keyword) - } - - /// Returns `true` if the token is a keyword reserved for possible future use. - crate fn is_unused_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) - } - - /// Returns `true` if the token is either a special identifier or a keyword. - pub fn is_reserved_ident(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_reserved) - } - - /// Returns `true` if the token is the identifier `true` or `false`. - crate fn is_bool_lit(&self) -> bool { - self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) - } - - /// Returns `true` if the token is a non-raw identifier for which `pred` holds. - fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { - match self.ident() { - Some((id, false)) => pred(id), - _ => false, - } - } - - crate fn glue(&self, joint: &Token) -> Option { - let kind = match self.kind { - Eq => match joint.kind { - Eq => EqEq, - Gt => FatArrow, - _ => return None, - }, - Lt => match joint.kind { - Eq => Le, - Lt => BinOp(Shl), - Le => BinOpEq(Shl), - BinOp(Minus) => LArrow, - _ => return None, - }, - Gt => match joint.kind { - Eq => Ge, - Gt => BinOp(Shr), - Ge => BinOpEq(Shr), - _ => return None, - }, - Not => match joint.kind { - Eq => Ne, - _ => return None, - }, - BinOp(op) => match joint.kind { - Eq => BinOpEq(op), - BinOp(And) if op == And => AndAnd, - BinOp(Or) if op == Or => OrOr, - Gt if op == Minus => RArrow, - _ => return None, - }, - Dot => match joint.kind { - Dot => DotDot, - DotDot => DotDotDot, - _ => return None, - }, - DotDot => match joint.kind { - Dot => DotDotDot, - Eq => DotDotEq, - _ => return None, - }, - Colon => match joint.kind { - Colon => ModSep, - _ => return None, - }, - SingleQuote => match joint.kind { - Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), - _ => return None, - }, - - Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | - DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | - Question | OpenDelim(..) | CloseDelim(..) | - Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, - }; - - Some(Token::new(kind, self.span.to(joint.span))) - } - - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { - if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { - return false - } - match (&self.kind, &other.kind) { - (&Eq, &Eq) | - (&Lt, &Lt) | - (&Le, &Le) | - (&EqEq, &EqEq) | - (&Ne, &Ne) | - (&Ge, &Ge) | - (&Gt, &Gt) | - (&AndAnd, &AndAnd) | - (&OrOr, &OrOr) | - (&Not, &Not) | - (&Tilde, &Tilde) | - (&At, &At) | - (&Dot, &Dot) | - (&DotDot, &DotDot) | - (&DotDotDot, &DotDotDot) | - (&DotDotEq, &DotDotEq) | - (&Comma, &Comma) | - (&Semi, &Semi) | - (&Colon, &Colon) | - (&ModSep, &ModSep) | - (&RArrow, &RArrow) | - (&LArrow, &LArrow) | - (&FatArrow, &FatArrow) | - (&Pound, &Pound) | - (&Dollar, &Dollar) | - (&Question, &Question) | - (&Whitespace, &Whitespace) | - (&Comment, &Comment) | - (&Eof, &Eof) => true, - - (&BinOp(a), &BinOp(b)) | - (&BinOpEq(a), &BinOpEq(b)) => a == b, - - (&OpenDelim(a), &OpenDelim(b)) | - (&CloseDelim(a), &CloseDelim(b)) => a == b, - - (&DocComment(a), &DocComment(b)) | - (&Shebang(a), &Shebang(b)) => a == b, - - (&Literal(a), &Literal(b)) => a == b, - - (&Lifetime(a), &Lifetime(b)) => a == b, - (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || - a == kw::DollarCrate || - c == kw::DollarCrate), - - (&Interpolated(_), &Interpolated(_)) => false, - - _ => panic!("forgot to add a token?"), - } - } -} - -impl PartialEq for Token { - fn eq(&self, rhs: &TokenKind) -> bool { - self.kind == *rhs - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable)] -/// For interpolation during macro expansion. -pub enum Nonterminal { - NtItem(P), - NtBlock(P), - NtStmt(ast::Stmt), - NtPat(P), - NtExpr(P), - NtTy(P), - NtIdent(ast::Ident, /* is_raw */ bool), - NtLifetime(ast::Ident), - NtLiteral(P), - /// Stuff inside brackets for attributes - NtMeta(ast::AttrItem), - NtPath(ast::Path), - NtVis(ast::Visibility), - NtTT(TokenTree), - // Used only for passing items to proc macro attributes (they are not - // strictly necessary for that, `Annotatable` can be converted into - // tokens directly, but doing that naively regresses pretty-printing). - NtTraitItem(ast::TraitItem), - NtImplItem(ast::ImplItem), - NtForeignItem(ast::ForeignItem), -} - -impl PartialEq for Nonterminal { - fn eq(&self, rhs: &Self) -> bool { - match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs, - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them - // correctly based on data from AST. This will prevent them from matching each other - // in macros. The comparison will become possible only when each nonterminal has an - // attached token stream from which it was parsed. - _ => false, - } - } -} - -impl fmt::Debug for Nonterminal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - NtItem(..) => f.pad("NtItem(..)"), - NtBlock(..) => f.pad("NtBlock(..)"), - NtStmt(..) => f.pad("NtStmt(..)"), - NtPat(..) => f.pad("NtPat(..)"), - NtExpr(..) => f.pad("NtExpr(..)"), - NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), - NtLiteral(..) => f.pad("NtLiteral(..)"), - NtMeta(..) => f.pad("NtMeta(..)"), - NtPath(..) => f.pad("NtPath(..)"), - NtTT(..) => f.pad("NtTT(..)"), - NtImplItem(..) => f.pad("NtImplItem(..)"), - NtTraitItem(..) => f.pad("NtTraitItem(..)"), - NtForeignItem(..) => f.pad("NtForeignItem(..)"), - NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), - } - } -} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 1d59c13a9d0..049f34efc26 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -4,7 +4,7 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg}; use crate::util::parser::{self, AssocOp, Fixity}; use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; -use crate::parse::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; +use crate::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; use crate::parse::lexer::comments; use crate::parse; use crate::print::pp::{self, Breaks}; diff --git a/src/libsyntax/token.rs b/src/libsyntax/token.rs new file mode 100644 index 00000000000..6f3da344ccf --- /dev/null +++ b/src/libsyntax/token.rs @@ -0,0 +1,728 @@ +pub use BinOpToken::*; +pub use Nonterminal::*; +pub use DelimToken::*; +pub use LitKind::*; +pub use TokenKind::*; + +use crate::ast; +use crate::ptr::P; +use crate::symbol::kw; +use crate::tokenstream::TokenTree; + +use syntax_pos::symbol::Symbol; +use syntax_pos::{self, Span, DUMMY_SP}; + +use std::fmt; +use std::mem; +#[cfg(target_arch = "x86_64")] +use rustc_data_structures::static_assert_size; +use rustc_data_structures::sync::Lrc; + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +pub enum BinOpToken { + Plus, + Minus, + Star, + Slash, + Percent, + Caret, + And, + Or, + Shl, + Shr, +} + +/// A delimiter token. +#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +pub enum DelimToken { + /// A round parenthesis (i.e., `(` or `)`). + Paren, + /// A square bracket (i.e., `[` or `]`). + Bracket, + /// A curly brace (i.e., `{` or `}`). + Brace, + /// An empty delimiter. + NoDelim, +} + +impl DelimToken { + pub fn len(self) -> usize { + if self == NoDelim { 0 } else { 1 } + } + + pub fn is_empty(self) -> bool { + self == NoDelim + } +} + +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum LitKind { + Bool, // AST only, must never appear in a `Token` + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u16), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols + Err, +} + +/// A literal token. +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Lit { + pub kind: LitKind, + pub symbol: Symbol, + pub suffix: Option, +} + +impl fmt::Display for Lit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Lit { kind, symbol, suffix } = *self; + match kind { + Byte => write!(f, "b'{}'", symbol)?, + Char => write!(f, "'{}'", symbol)?, + Str => write!(f, "\"{}\"", symbol)?, + StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + ByteStr => write!(f, "b\"{}\"", symbol)?, + ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + Integer | + Float | + Bool | + Err => write!(f, "{}", symbol)?, + } + + if let Some(suffix) = suffix { + write!(f, "{}", suffix)?; + } + + Ok(()) + } +} + +impl LitKind { + /// An English article for the literal token kind. + crate fn article(self) -> &'static str { + match self { + Integer | Err => "an", + _ => "a", + } + } + + crate fn descr(self) -> &'static str { + match self { + Bool => panic!("literal token contains `Lit::Bool`"), + Byte => "byte", + Char => "char", + Integer => "integer", + Float => "float", + Str | StrRaw(..) => "string", + ByteStr | ByteStrRaw(..) => "byte string", + Err => "error", + } + } + + crate fn may_have_suffix(self) -> bool { + match self { + Integer | Float | Err => true, + _ => false, + } + } +} + +impl Lit { + pub fn new(kind: LitKind, symbol: Symbol, suffix: Option) -> Lit { + Lit { kind, symbol, suffix } + } +} + +pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); + token_can_begin_expr(&ident_token) +} + +pub(crate) fn token_can_begin_expr(ident_token: &Token) -> bool { + !ident_token.is_reserved_ident() || + ident_token.is_path_segment_keyword() || + match ident_token.kind { + TokenKind::Ident(ident, _) => [ + kw::Async, + kw::Do, + kw::Box, + kw::Break, + kw::Continue, + kw::False, + kw::For, + kw::If, + kw::Let, + kw::Loop, + kw::Match, + kw::Move, + kw::Return, + kw::True, + kw::Unsafe, + kw::While, + kw::Yield, + kw::Static, + ].contains(&ident), + _=> false, + } +} + +fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); + + !ident_token.is_reserved_ident() || + ident_token.is_path_segment_keyword() || + [ + kw::Underscore, + kw::For, + kw::Impl, + kw::Fn, + kw::Unsafe, + kw::Extern, + kw::Typeof, + kw::Dyn, + ].contains(&name) +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum TokenKind { + /* Expression-operator symbols. */ + Eq, + Lt, + Le, + EqEq, + Ne, + Ge, + Gt, + AndAnd, + OrOr, + Not, + Tilde, + BinOp(BinOpToken), + BinOpEq(BinOpToken), + + /* Structural symbols */ + At, + Dot, + DotDot, + DotDotDot, + DotDotEq, + Comma, + Semi, + Colon, + ModSep, + RArrow, + LArrow, + FatArrow, + Pound, + Dollar, + Question, + /// Used by proc macros for representing lifetimes, not generated by lexer right now. + SingleQuote, + /// An opening delimiter (e.g., `{`). + OpenDelim(DelimToken), + /// A closing delimiter (e.g., `}`). + CloseDelim(DelimToken), + + /* Literals */ + Literal(Lit), + + /* Name components */ + Ident(ast::Name, /* is_raw */ bool), + Lifetime(ast::Name), + + Interpolated(Lrc), + + // Can be expanded into several tokens. + /// A doc comment. + DocComment(ast::Name), + + // Junk. These carry no data because we don't really care about the data + // they *would* carry, and don't really want to allocate a new ident for + // them. Instead, users could extract that from the associated span. + + /// Whitespace. + Whitespace, + /// A comment. + Comment, + Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), + + Eof, +} + +// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(target_arch = "x86_64")] +static_assert_size!(TokenKind, 16); + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Token { + pub kind: TokenKind, + pub span: Span, +} + +impl TokenKind { + pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { + Literal(Lit::new(kind, symbol, suffix)) + } + + /// Returns tokens that are likely to be typed accidentally instead of the current token. + /// Enables better error recovery when the wrong token is found. + crate fn similar_tokens(&self) -> Option> { + match *self { + Comma => Some(vec![Dot, Lt, Semi]), + Semi => Some(vec![Colon, Comma]), + _ => None + } + } +} + +impl Token { + pub fn new(kind: TokenKind, span: Span) -> Self { + Token { kind, span } + } + + /// Some token that will be thrown away later. + crate fn dummy() -> Self { + Token::new(TokenKind::Whitespace, DUMMY_SP) + } + + /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. + pub fn from_ast_ident(ident: ast::Ident) -> Self { + Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) + } + + /// Return this token by value and leave a dummy token in its place. + pub fn take(&mut self) -> Self { + mem::replace(self, Token::dummy()) + } + + crate fn is_op(&self) -> bool { + match self.kind { + OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | + Ident(..) | Lifetime(..) | Interpolated(..) | + Whitespace | Comment | Shebang(..) | Eof => false, + _ => true, + } + } + + crate fn is_like_plus(&self) -> bool { + match self.kind { + BinOp(Plus) | BinOpEq(Plus) => true, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of an expression. + pub fn can_begin_expr(&self) -> bool { + match self.kind { + Ident(name, is_raw) => + ident_can_begin_expr(name, self.span, is_raw), // value name or keyword + OpenDelim(..) | // tuple, array or block + Literal(..) | // literal + Not | // operator not + BinOp(Minus) | // unary minus + BinOp(Star) | // dereference + BinOp(Or) | OrOr | // closure + BinOp(And) | // reference + AndAnd | // double reference + // DotDotDot is no longer supported, but we need some way to display the error + DotDot | DotDotDot | DotDotEq | // range notation + Lt | BinOp(Shl) | // associated path + ModSep | // global path + Lifetime(..) | // labeled loop + Pound => true, // expression attributes + Interpolated(ref nt) => match **nt { + NtLiteral(..) | + NtIdent(..) | + NtExpr(..) | + NtBlock(..) | + NtPath(..) | + NtLifetime(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of a type. + pub fn can_begin_type(&self) -> bool { + match self.kind { + Ident(name, is_raw) => + ident_can_begin_type(name, self.span, is_raw), // type name or keyword + OpenDelim(Paren) | // tuple + OpenDelim(Bracket) | // array + Not | // never + BinOp(Star) | // raw pointer + BinOp(And) | // reference + AndAnd | // double reference + Question | // maybe bound in trait object + Lifetime(..) | // lifetime bound in trait object + Lt | BinOp(Shl) | // associated path + ModSep => true, // global path + Interpolated(ref nt) => match **nt { + NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of a const param. + crate fn can_begin_const_arg(&self) -> bool { + match self.kind { + OpenDelim(Brace) => true, + Interpolated(ref nt) => match **nt { + NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, + _ => false, + } + _ => self.can_begin_literal_or_bool(), + } + } + + /// Returns `true` if the token can appear at the start of a generic bound. + crate fn can_begin_bound(&self) -> bool { + self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || + self == &Question || self == &OpenDelim(Paren) + } + + /// Returns `true` if the token is any literal + pub fn is_lit(&self) -> bool { + match self.kind { + Literal(..) => true, + _ => false, + } + } + + /// Returns `true` if the token is any literal, a minus (which can prefix a literal, + /// for example a '-42', or one of the boolean idents). + pub fn can_begin_literal_or_bool(&self) -> bool { + match self.kind { + Literal(..) | BinOp(Minus) => true, + Ident(name, false) if name.is_bool_lit() => true, + Interpolated(ref nt) => match **nt { + NtLiteral(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns an identifier if this token is an identifier. + pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { + match self.kind { + Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), + Interpolated(ref nt) => match **nt { + NtIdent(ident, is_raw) => Some((ident, is_raw)), + _ => None, + }, + _ => None, + } + } + + /// Returns a lifetime identifier if this token is a lifetime. + pub fn lifetime(&self) -> Option { + match self.kind { + Lifetime(name) => Some(ast::Ident::new(name, self.span)), + Interpolated(ref nt) => match **nt { + NtLifetime(ident) => Some(ident), + _ => None, + }, + _ => None, + } + } + + /// Returns `true` if the token is an identifier. + pub fn is_ident(&self) -> bool { + self.ident().is_some() + } + + /// Returns `true` if the token is a lifetime. + crate fn is_lifetime(&self) -> bool { + self.lifetime().is_some() + } + + /// Returns `true` if the token is a identifier whose name is the given + /// string slice. + crate fn is_ident_named(&self, name: Symbol) -> bool { + self.ident().map_or(false, |(ident, _)| ident.name == name) + } + + /// Returns `true` if the token is an interpolated path. + fn is_path(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtPath(..) = **nt { + return true; + } + } + false + } + + /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? + /// That is, is this a pre-parsed expression dropped into the token stream + /// (which happens while parsing the result of macro expansion)? + crate fn is_whole_expr(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { + return true; + } + } + + false + } + + /// Returns `true` if the token is either the `mut` or `const` keyword. + crate fn is_mutability(&self) -> bool { + self.is_keyword(kw::Mut) || + self.is_keyword(kw::Const) + } + + crate fn is_qpath_start(&self) -> bool { + self == &Lt || self == &BinOp(Shl) + } + + crate fn is_path_start(&self) -> bool { + self == &ModSep || self.is_qpath_start() || self.is_path() || + self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() + } + + /// Returns `true` if the token is a given keyword, `kw`. + pub fn is_keyword(&self, kw: Symbol) -> bool { + self.is_non_raw_ident_where(|id| id.name == kw) + } + + crate fn is_path_segment_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) + } + + // Returns true for reserved identifiers used internally for elided lifetimes, + // unnamed method parameters, crate root module, error recovery etc. + crate fn is_special_ident(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_special) + } + + /// Returns `true` if the token is a keyword used in the language. + crate fn is_used_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_used_keyword) + } + + /// Returns `true` if the token is a keyword reserved for possible future use. + crate fn is_unused_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) + } + + /// Returns `true` if the token is either a special identifier or a keyword. + pub fn is_reserved_ident(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_reserved) + } + + /// Returns `true` if the token is the identifier `true` or `false`. + crate fn is_bool_lit(&self) -> bool { + self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) + } + + /// Returns `true` if the token is a non-raw identifier for which `pred` holds. + fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { + match self.ident() { + Some((id, false)) => pred(id), + _ => false, + } + } + + crate fn glue(&self, joint: &Token) -> Option { + let kind = match self.kind { + Eq => match joint.kind { + Eq => EqEq, + Gt => FatArrow, + _ => return None, + }, + Lt => match joint.kind { + Eq => Le, + Lt => BinOp(Shl), + Le => BinOpEq(Shl), + BinOp(Minus) => LArrow, + _ => return None, + }, + Gt => match joint.kind { + Eq => Ge, + Gt => BinOp(Shr), + Ge => BinOpEq(Shr), + _ => return None, + }, + Not => match joint.kind { + Eq => Ne, + _ => return None, + }, + BinOp(op) => match joint.kind { + Eq => BinOpEq(op), + BinOp(And) if op == And => AndAnd, + BinOp(Or) if op == Or => OrOr, + Gt if op == Minus => RArrow, + _ => return None, + }, + Dot => match joint.kind { + Dot => DotDot, + DotDot => DotDotDot, + _ => return None, + }, + DotDot => match joint.kind { + Dot => DotDotDot, + Eq => DotDotEq, + _ => return None, + }, + Colon => match joint.kind { + Colon => ModSep, + _ => return None, + }, + SingleQuote => match joint.kind { + Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), + _ => return None, + }, + + Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | + DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | + Question | OpenDelim(..) | CloseDelim(..) | + Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, + }; + + Some(Token::new(kind, self.span.to(joint.span))) + } + + // See comments in `Nonterminal::to_tokenstream` for why we care about + // *probably* equal here rather than actual equality + crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { + if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { + return false + } + match (&self.kind, &other.kind) { + (&Eq, &Eq) | + (&Lt, &Lt) | + (&Le, &Le) | + (&EqEq, &EqEq) | + (&Ne, &Ne) | + (&Ge, &Ge) | + (&Gt, &Gt) | + (&AndAnd, &AndAnd) | + (&OrOr, &OrOr) | + (&Not, &Not) | + (&Tilde, &Tilde) | + (&At, &At) | + (&Dot, &Dot) | + (&DotDot, &DotDot) | + (&DotDotDot, &DotDotDot) | + (&DotDotEq, &DotDotEq) | + (&Comma, &Comma) | + (&Semi, &Semi) | + (&Colon, &Colon) | + (&ModSep, &ModSep) | + (&RArrow, &RArrow) | + (&LArrow, &LArrow) | + (&FatArrow, &FatArrow) | + (&Pound, &Pound) | + (&Dollar, &Dollar) | + (&Question, &Question) | + (&Whitespace, &Whitespace) | + (&Comment, &Comment) | + (&Eof, &Eof) => true, + + (&BinOp(a), &BinOp(b)) | + (&BinOpEq(a), &BinOpEq(b)) => a == b, + + (&OpenDelim(a), &OpenDelim(b)) | + (&CloseDelim(a), &CloseDelim(b)) => a == b, + + (&DocComment(a), &DocComment(b)) | + (&Shebang(a), &Shebang(b)) => a == b, + + (&Literal(a), &Literal(b)) => a == b, + + (&Lifetime(a), &Lifetime(b)) => a == b, + (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || + a == kw::DollarCrate || + c == kw::DollarCrate), + + (&Interpolated(_), &Interpolated(_)) => false, + + _ => panic!("forgot to add a token?"), + } + } +} + +impl PartialEq for Token { + fn eq(&self, rhs: &TokenKind) -> bool { + self.kind == *rhs + } +} + +#[derive(Clone, RustcEncodable, RustcDecodable)] +/// For interpolation during macro expansion. +pub enum Nonterminal { + NtItem(P), + NtBlock(P), + NtStmt(ast::Stmt), + NtPat(P), + NtExpr(P), + NtTy(P), + NtIdent(ast::Ident, /* is_raw */ bool), + NtLifetime(ast::Ident), + NtLiteral(P), + /// Stuff inside brackets for attributes + NtMeta(ast::AttrItem), + NtPath(ast::Path), + NtVis(ast::Visibility), + NtTT(TokenTree), + // Used only for passing items to proc macro attributes (they are not + // strictly necessary for that, `Annotatable` can be converted into + // tokens directly, but doing that naively regresses pretty-printing). + NtTraitItem(ast::TraitItem), + NtImplItem(ast::ImplItem), + NtForeignItem(ast::ForeignItem), +} + +impl PartialEq for Nonterminal { + fn eq(&self, rhs: &Self) -> bool { + match (self, rhs) { + (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => + ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs, + (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, + (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, + // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them + // correctly based on data from AST. This will prevent them from matching each other + // in macros. The comparison will become possible only when each nonterminal has an + // attached token stream from which it was parsed. + _ => false, + } + } +} + +impl fmt::Debug for Nonterminal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + NtItem(..) => f.pad("NtItem(..)"), + NtBlock(..) => f.pad("NtBlock(..)"), + NtStmt(..) => f.pad("NtStmt(..)"), + NtPat(..) => f.pad("NtPat(..)"), + NtExpr(..) => f.pad("NtExpr(..)"), + NtTy(..) => f.pad("NtTy(..)"), + NtIdent(..) => f.pad("NtIdent(..)"), + NtLiteral(..) => f.pad("NtLiteral(..)"), + NtMeta(..) => f.pad("NtMeta(..)"), + NtPath(..) => f.pad("NtPath(..)"), + NtTT(..) => f.pad("NtTT(..)"), + NtImplItem(..) => f.pad("NtImplItem(..)"), + NtTraitItem(..) => f.pad("NtTraitItem(..)"), + NtForeignItem(..) => f.pad("NtForeignItem(..)"), + NtVis(..) => f.pad("NtVis(..)"), + NtLifetime(..) => f.pad("NtLifetime(..)"), + } + } +} diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index a51d208704a..6e1bb85ce1a 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -13,7 +13,7 @@ //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking //! ownership of the original. -use crate::parse::token::{self, DelimToken, Token, TokenKind}; +use crate::token::{self, DelimToken, Token, TokenKind}; use syntax_pos::{Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index 982755e8680..edb708d7e97 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, BinOpToken}; +use crate::token::{self, Token, BinOpToken}; use crate::symbol::kw; use crate::ast::{self, BinOpKind}; diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 117787d08c7..cfd160fd577 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -14,7 +14,7 @@ //! those that are created by the expansion of a macro. use crate::ast::*; -use crate::parse::token::Token; +use crate::token::Token; use crate::tokenstream::{TokenTree, TokenStream}; use syntax_pos::Span; diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs index d02251eb746..47835c92659 100644 --- a/src/libsyntax_expand/base.rs +++ b/src/libsyntax_expand/base.rs @@ -6,11 +6,11 @@ use syntax::source_map::SourceMap; use syntax::edition::Edition; use syntax::mut_visit::{self, MutVisitor}; use syntax::parse::{self, parser, DirectoryOwnership}; -use syntax::parse::token; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym, Ident, Symbol}; use syntax::{ThinVec, MACRO_ARGUMENTS}; +use syntax::token; use syntax::tokenstream::{self, TokenStream}; use syntax::visit::Visitor; diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs index 7dbc7787010..6d1f0abe49e 100644 --- a/src/libsyntax_expand/expand.rs +++ b/src/libsyntax_expand/expand.rs @@ -13,12 +13,12 @@ use syntax::config::StripUnconfigured; use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; use syntax::mut_visit::*; use syntax::parse::{DirectoryOwnership, PResult}; -use syntax::parse::token; use syntax::parse::parser::Parser; use syntax::print::pprust; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{sym, Symbol}; +use syntax::token; use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::visit::{self, Visitor}; use syntax::util::map_in_place::MapInPlace; diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs index 06e0cde3ad8..6964d01b719 100644 --- a/src/libsyntax_expand/mbe.rs +++ b/src/libsyntax_expand/mbe.rs @@ -10,7 +10,7 @@ crate mod macro_rules; crate mod quoted; use syntax::ast; -use syntax::parse::token::{self, Token, TokenKind}; +use syntax::token::{self, Token, TokenKind}; use syntax::tokenstream::{DelimSpan}; use syntax_pos::Span; diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs index 50abda8d45e..25754ed4217 100644 --- a/src/libsyntax_expand/mbe/macro_check.rs +++ b/src/libsyntax_expand/mbe/macro_check.rs @@ -108,7 +108,7 @@ use crate::mbe::{KleeneToken, TokenTree}; use syntax::ast::NodeId; use syntax::early_buffered_lints::BufferedEarlyLintId; -use syntax::parse::token::{DelimToken, Token, TokenKind}; +use syntax::token::{DelimToken, Token, TokenKind}; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs index 3efe22626a9..950e7c0f286 100644 --- a/src/libsyntax_expand/mbe/macro_parser.rs +++ b/src/libsyntax_expand/mbe/macro_parser.rs @@ -79,10 +79,10 @@ use crate::mbe::{self, TokenTree}; use syntax::ast::{Ident, Name}; use syntax::parse::{Directory, PResult}; use syntax::parse::parser::{Parser, PathStyle}; -use syntax::parse::token::{self, DocComment, Nonterminal, Token}; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, DocComment, Nonterminal, Token}; use syntax::tokenstream::{DelimSpan, TokenStream}; use errors::FatalError; diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs index 55719907403..a5fc301fbf6 100644 --- a/src/libsyntax_expand/mbe/macro_rules.rs +++ b/src/libsyntax_expand/mbe/macro_rules.rs @@ -13,12 +13,11 @@ use syntax::attr::{self, TransparencyError}; use syntax::edition::Edition; use syntax::feature_gate::Features; use syntax::parse::parser::Parser; -use syntax::parse::token::TokenKind::*; -use syntax::parse::token::{self, NtTT, Token}; use syntax::parse::Directory; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, NtTT, Token, TokenKind::*}; use syntax::tokenstream::{DelimSpan, TokenStream}; use errors::{DiagnosticBuilder, FatalError}; diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs index cedd59233ad..dec504c0d97 100644 --- a/src/libsyntax_expand/mbe/quoted.rs +++ b/src/libsyntax_expand/mbe/quoted.rs @@ -2,10 +2,10 @@ use crate::mbe::macro_parser; use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; use syntax::ast; -use syntax::parse::token::{self, Token}; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::symbol::kw; +use syntax::token::{self, Token}; use syntax::tokenstream; use syntax_pos::Span; diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs index 6f060103ef4..4092d4b97de 100644 --- a/src/libsyntax_expand/mbe/transcribe.rs +++ b/src/libsyntax_expand/mbe/transcribe.rs @@ -4,7 +4,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use syntax::ast::{Ident, Mac}; use syntax::mut_visit::{self, MutVisitor}; -use syntax::parse::token::{self, NtTT, Token}; +use syntax::token::{self, NtTT, Token}; use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use smallvec::{smallvec, SmallVec}; diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs index 1f4c481e3ea..51c368bbaa6 100644 --- a/src/libsyntax_expand/proc_macro.rs +++ b/src/libsyntax_expand/proc_macro.rs @@ -4,8 +4,9 @@ use crate::proc_macro_server; use syntax::ast::{self, ItemKind, Attribute, Mac}; use syntax::attr::{mark_used, mark_known}; use syntax::errors::{Applicability, FatalError}; -use syntax::parse::{self, token}; +use syntax::parse; use syntax::symbol::sym; +use syntax::token; use syntax::tokenstream::{self, TokenStream}; use syntax::visit::Visitor; diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs index 4ce99cfe73b..67a15e48205 100644 --- a/src/libsyntax_expand/proc_macro_server.rs +++ b/src/libsyntax_expand/proc_macro_server.rs @@ -1,10 +1,11 @@ use crate::base::ExtCtxt; use syntax::ast; -use syntax::parse::{self, token}; +use syntax::parse::{self}; use syntax::parse::lexer::comments; use syntax::print::pprust; use syntax::sess::ParseSess; +use syntax::token; use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; use errors::Diagnostic; @@ -52,7 +53,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> { fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) -> Self { - use syntax::parse::token::*; + use syntax::token::*; let joint = is_joint == Joint; let Token { kind, span } = match tree { @@ -193,7 +194,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { - use syntax::parse::token::*; + use syntax::token::*; let (ch, joint, span) = match self { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs index 8c9a34713ea..539d777105d 100644 --- a/src/libsyntax_ext/asm.rs +++ b/src/libsyntax_ext/asm.rs @@ -8,7 +8,7 @@ use errors::DiagnosticBuilder; use syntax::ast; use syntax_expand::base::{self, *}; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; use syntax::ast::AsmDialect; diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index f4d1f7fb09c..a15423b7ad8 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -2,7 +2,7 @@ use errors::{Applicability, DiagnosticBuilder}; use syntax::ast::{self, *}; use syntax_expand::base::*; -use syntax::parse::token::{self, TokenKind}; +use syntax::token::{self, TokenKind}; use syntax::parse::parser::Parser; use syntax::print::pprust; use syntax::ptr::P; diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs index 9e693f29c5a..583236d9754 100644 --- a/src/libsyntax_ext/cfg.rs +++ b/src/libsyntax_ext/cfg.rs @@ -8,7 +8,7 @@ use syntax::ast; use syntax_expand::base::{self, *}; use syntax::attr; use syntax::tokenstream::TokenStream; -use syntax::parse::token; +use syntax::token; use syntax_pos::Span; pub fn expand_cfg( diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs index 2d981526a39..171f2405573 100644 --- a/src/libsyntax_ext/cmdline_attrs.rs +++ b/src/libsyntax_ext/cmdline_attrs.rs @@ -2,7 +2,8 @@ use syntax::ast::{self, AttrItem, AttrStyle}; use syntax::attr::mk_attr; -use syntax::parse::{self, token}; +use syntax::parse; +use syntax::token; use syntax::sess::ParseSess; use syntax_expand::panictry; use syntax_pos::FileName; diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs index a132a4136ea..8a1bc56cf1c 100644 --- a/src/libsyntax_ext/concat_idents.rs +++ b/src/libsyntax_ext/concat_idents.rs @@ -2,7 +2,7 @@ use rustc_data_structures::thin_vec::ThinVec; use syntax::ast; use syntax_expand::base::{self, *}; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::ptr::P; use syntax_pos::Span; use syntax_pos::symbol::Symbol; diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs index 314c2eefd4c..e25ba7b1783 100644 --- a/src/libsyntax_ext/format.rs +++ b/src/libsyntax_ext/format.rs @@ -9,7 +9,7 @@ use errors::pluralize; use syntax::ast; use syntax_expand::base::{self, *}; -use syntax::parse::token; +use syntax::token; use syntax::ptr::P; use syntax::symbol::{Symbol, sym}; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs index 879ae1e4215..8a8ce9a7f14 100644 --- a/src/libsyntax_ext/global_asm.rs +++ b/src/libsyntax_ext/global_asm.rs @@ -13,7 +13,7 @@ use errors::DiagnosticBuilder; use syntax::ast; use syntax::source_map::respan; use syntax_expand::base::{self, *}; -use syntax::parse::token; +use syntax::token; use syntax::ptr::P; use syntax_pos::Span; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index 1ca9422eb9d..cee1b97af55 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -4,12 +4,12 @@ use syntax::ast::*; use syntax::attr; use syntax::edition::Edition; -use syntax_expand::base::{Resolver, NamedSyntaxExtension}; -use syntax::parse::token; use syntax::ptr::P; use syntax::source_map::respan; use syntax::symbol::sym; +use syntax::token; use syntax::tokenstream::*; +use syntax_expand::base::{Resolver, NamedSyntaxExtension}; use syntax_pos::{Span, DUMMY_SP}; use syntax_pos::hygiene::{ExpnData, ExpnKind, AstPass}; diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs index f6c58fcdfa1..7e47b40714d 100644 --- a/src/libsyntax_ext/source_util.rs +++ b/src/libsyntax_ext/source_util.rs @@ -1,10 +1,11 @@ use syntax_expand::panictry; use syntax_expand::base::{self, *}; use syntax::ast; -use syntax::parse::{self, token, DirectoryOwnership}; +use syntax::parse::{self, DirectoryOwnership}; use syntax::print::pprust; use syntax::ptr::P; use syntax::symbol::Symbol; +use syntax::token; use syntax::tokenstream::TokenStream; use syntax::early_buffered_lints::BufferedEarlyLintId; diff --git a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs index 927e2c0820e..6277e2e7ea9 100644 --- a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs +++ b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs @@ -16,7 +16,7 @@ use syntax::parse; use syntax::parse::PResult; use syntax::parse::new_parser_from_source_str; use syntax::parse::parser::Parser; -use syntax::parse::token; +use syntax::token; use syntax::ptr::P; use syntax::parse::parser::attr::*; use syntax::print::pprust; diff --git a/src/test/ui-fulldeps/auxiliary/roman-numerals.rs b/src/test/ui-fulldeps/auxiliary/roman-numerals.rs index 3524f449c74..520347faa15 100644 --- a/src/test/ui-fulldeps/auxiliary/roman-numerals.rs +++ b/src/test/ui-fulldeps/auxiliary/roman-numerals.rs @@ -15,7 +15,7 @@ extern crate syntax_pos; extern crate rustc; extern crate rustc_driver; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::tokenstream::{TokenTree, TokenStream}; use syntax_expand::base::{ExtCtxt, MacResult, DummyResult, MacEager}; use syntax_pos::Span; -- cgit 1.4.1-3-g733a5 From 3667e6248ec18740ce57db7997333a30c991929b Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 13:06:36 +0200 Subject: move PResult to librustc_errors --- src/librustc_driver/lib.rs | 3 ++- src/librustc_errors/lib.rs | 10 +++++++++- src/librustc_interface/passes.rs | 3 ++- src/libsyntax/attr/mod.rs | 3 ++- src/libsyntax/parse/lexer/tokentrees.rs | 3 ++- src/libsyntax/parse/mod.rs | 11 +---------- src/libsyntax/parse/parser/attr.rs | 4 +++- src/libsyntax/parse/parser/diagnostics.rs | 8 +++----- src/libsyntax/parse/parser/expr.rs | 4 ++-- src/libsyntax/parse/parser/generics.rs | 4 +++- src/libsyntax/parse/parser/item.rs | 4 ++-- src/libsyntax/parse/parser/mod.rs | 4 ++-- src/libsyntax/parse/parser/module.rs | 4 +++- src/libsyntax/parse/parser/pat.rs | 4 ++-- src/libsyntax/parse/parser/path.rs | 4 ++-- src/libsyntax/parse/parser/stmt.rs | 4 ++-- src/libsyntax/parse/parser/ty.rs | 4 ++-- src/libsyntax/tests.rs | 4 ++-- src/libsyntax_expand/expand.rs | 4 ++-- src/libsyntax_expand/mbe/macro_parser.rs | 4 ++-- src/test/ui-fulldeps/ast_stmt_expr_attr.rs | 3 ++- 21 files changed, 52 insertions(+), 44 deletions(-) (limited to 'src/libsyntax/parse/parser') diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs index 6e8bc11162f..611b891d99a 100644 --- a/src/librustc_driver/lib.rs +++ b/src/librustc_driver/lib.rs @@ -42,6 +42,7 @@ use rustc::ty::TyCtxt; use rustc::util::common::{set_time_depth, time, print_time_passes_entry, ErrorReported}; use rustc_metadata::locator; use rustc_codegen_utils::codegen_backend::CodegenBackend; +use errors::PResult; use rustc_interface::interface; use rustc_interface::util::get_codegen_sysroot; use rustc_data_structures::sync::SeqCst; @@ -64,7 +65,7 @@ use std::time::Instant; use syntax::ast; use syntax::source_map::FileLoader; use syntax::feature_gate::{GatedCfg, UnstableFeatures}; -use syntax::parse::{self, PResult}; +use syntax::parse; use syntax::symbol::sym; use syntax_pos::{DUMMY_SP, FileName}; diff --git a/src/librustc_errors/lib.rs b/src/librustc_errors/lib.rs index 67c180a05e9..fb5cccf61a7 100644 --- a/src/librustc_errors/lib.rs +++ b/src/librustc_errors/lib.rs @@ -15,7 +15,8 @@ use Level::*; use emitter::{Emitter, EmitterWriter, is_case_difference}; use registry::Registry; - +#[cfg(target_arch = "x86_64")] +use rustc_data_structures::static_assert_size; use rustc_data_structures::sync::{self, Lrc, Lock}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_data_structures::stable_hasher::StableHasher; @@ -48,6 +49,13 @@ use syntax_pos::{ SpanSnippetError, }; +pub type PResult<'a, T> = Result>; + +// `PResult` is used a lot. Make sure it doesn't unintentionally get bigger. +// (See also the comment on `DiagnosticBuilderInner`.) +#[cfg(target_arch = "x86_64")] +static_assert_size!(PResult<'_, bool>, 16); + /// Indicates the confidence in the correctness of a suggestion. /// /// All suggestions are marked with an `Applicability`. Tools use the applicability of a suggestion diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index 52332744d1a..ce34caee6fa 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -22,6 +22,7 @@ use rustc_codegen_utils::codegen_backend::CodegenBackend; use rustc_codegen_utils::link::filename_for_metadata; use rustc_data_structures::{box_region_allow_access, declare_box_region_type, parallel}; use rustc_data_structures::sync::{Lrc, ParallelIterator, par_iter}; +use rustc_errors::PResult; use rustc_incremental; use rustc_metadata::cstore; use rustc_mir as mir; @@ -36,7 +37,7 @@ use syntax::{self, ast, visit}; use syntax::early_buffered_lints::BufferedEarlyLint; use syntax_expand::base::{NamedSyntaxExtension, ExtCtxt}; use syntax::mut_visit::MutVisitor; -use syntax::parse::{self, PResult}; +use syntax::parse; use syntax::util::node_count::NodeCounter; use syntax::symbol::Symbol; use syntax_pos::FileName; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 1534d2723c8..d609225a54f 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -16,7 +16,6 @@ use crate::mut_visit::visit_clobber; use crate::source_map::{BytePos, Spanned}; use crate::parse::lexer::comments::doc_comment_style; use crate::parse; -use crate::parse::PResult; use crate::token::{self, Token}; use crate::ptr::P; use crate::sess::ParseSess; @@ -25,6 +24,8 @@ use crate::ThinVec; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use crate::GLOBALS; +use errors::PResult; + use log::debug; use syntax_pos::Span; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 3c15a5e1dae..2b056434d4d 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -4,10 +4,11 @@ use syntax_pos::Span; use super::{StringReader, UnmatchedBrace}; use crate::print::pprust::token_to_string; -use crate::parse::PResult; use crate::token::{self, Token}; use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint}; +use errors::PResult; + impl<'a> StringReader<'a> { crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec) { let mut tt_reader = TokenTreesReader { diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 7db0ca0b78d..badf8bcba57 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -7,9 +7,7 @@ use crate::tokenstream::{self, TokenStream, TokenTree}; use crate::print::pprust; use crate::sess::ParseSess; -use errors::{FatalError, Level, Diagnostic, DiagnosticBuilder}; -#[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert_size; +use errors::{PResult, FatalError, Level, Diagnostic}; use rustc_data_structures::sync::Lrc; use syntax_pos::{Span, SourceFile, FileName}; @@ -29,13 +27,6 @@ pub mod lexer; crate mod classify; crate mod literal; -pub type PResult<'a, T> = Result>; - -// `PResult` is used a lot. Make sure it doesn't unintentionally get bigger. -// (See also the comment on `DiagnosticBuilderInner`.) -#[cfg(target_arch = "x86_64")] -static_assert_size!(PResult<'_, bool>, 16); - #[derive(Clone)] pub struct Directory<'a> { pub path: Cow<'a, Path>, diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index aa0542a09a9..2f1b87da7e9 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,10 +1,12 @@ -use super::{SeqSep, PResult, Parser, TokenType, PathStyle}; +use super::{SeqSep, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; use crate::token::{self, Nonterminal, DelimToken}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; +use errors::PResult; + use log::debug; #[derive(Debug)] diff --git a/src/libsyntax/parse/parser/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index 09f0b9b74b2..26d7f48025e 100644 --- a/src/libsyntax/parse/parser/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -1,7 +1,4 @@ -use super::{ - BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType, - SeqSep, PResult, Parser -}; +use super::{BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType, SeqSep, Parser}; use crate::ast::{ self, Param, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, @@ -12,7 +9,8 @@ use crate::ptr::P; use crate::symbol::{kw, sym}; use crate::ThinVec; use crate::util::parser::AssocOp; -use errors::{Applicability, DiagnosticBuilder, DiagnosticId, pluralize}; + +use errors::{PResult, Applicability, DiagnosticBuilder, DiagnosticId, pluralize}; use rustc_data_structures::fx::FxHashSet; use syntax_pos::{Span, DUMMY_SP, MultiSpan, SpanSnippetError}; use log::{debug, trace}; diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index 4b962201e85..9faa4aefbb6 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult, Restrictions, PrevTokenKind, TokenType, PathStyle, BlockMode}; +use super::{Parser, Restrictions, PrevTokenKind, TokenType, PathStyle, BlockMode}; use super::{SemiColonMode, SeqSep, TokenExpectType}; use super::pat::{GateOr, PARAM_EXPECTED}; use super::diagnostics::Error; @@ -18,7 +18,7 @@ use crate::ptr::P; use crate::source_map::{self, Span}; use crate::util::parser::{AssocOp, Fixity, prec_let_scrutinee_needs_par}; -use errors::Applicability; +use errors::{PResult, Applicability}; use syntax_pos::symbol::{kw, sym}; use syntax_pos::Symbol; use std::mem; diff --git a/src/libsyntax/parse/parser/generics.rs b/src/libsyntax/parse/parser/generics.rs index cfc3768cac5..ae9ecd8fe39 100644 --- a/src/libsyntax/parse/parser/generics.rs +++ b/src/libsyntax/parse/parser/generics.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult}; +use super::Parser; use crate::ast::{self, WhereClause, GenericParam, GenericParamKind, GenericBounds, Attribute}; use crate::token; @@ -6,6 +6,8 @@ use crate::source_map::DUMMY_SP; use syntax_pos::symbol::{kw, sym}; +use errors::PResult; + impl<'a> Parser<'a> { /// Parses bounds of a lifetime parameter `BOUND + BOUND + BOUND`, possibly with trailing `+`. /// diff --git a/src/libsyntax/parse/parser/item.rs b/src/libsyntax/parse/parser/item.rs index 9d543055f23..3c618d75d34 100644 --- a/src/libsyntax/parse/parser/item.rs +++ b/src/libsyntax/parse/parser/item.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult, PathStyle}; +use super::{Parser, PathStyle}; use super::diagnostics::{Error, dummy_arg, ConsumeClosingDelim}; use crate::maybe_whole; @@ -17,7 +17,7 @@ use crate::ThinVec; use log::debug; use std::mem; -use errors::{Applicability, DiagnosticBuilder, DiagnosticId, StashKey}; +use errors::{PResult, Applicability, DiagnosticBuilder, DiagnosticId, StashKey}; use syntax_pos::BytePos; /// Whether the type alias or associated type is a concrete type or an opaque type. diff --git a/src/libsyntax/parse/parser/mod.rs b/src/libsyntax/parse/parser/mod.rs index 6498b674a3b..f7f7b0e83d4 100644 --- a/src/libsyntax/parse/parser/mod.rs +++ b/src/libsyntax/parse/parser/mod.rs @@ -15,7 +15,7 @@ use crate::ast::{ self, Abi, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Ident, IsAsync, MacDelimiter, Mutability, StrStyle, Visibility, VisibilityKind, Unsafety, }; -use crate::parse::{PResult, Directory, DirectoryOwnership}; +use crate::parse::{Directory, DirectoryOwnership}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::token::{self, Token, TokenKind, DelimToken}; @@ -27,7 +27,7 @@ use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; use crate::ThinVec; -use errors::{Applicability, DiagnosticBuilder, DiagnosticId, FatalError}; +use errors::{PResult, Applicability, DiagnosticBuilder, DiagnosticId, FatalError}; use syntax_pos::{Span, BytePos, DUMMY_SP, FileName}; use log::debug; diff --git a/src/libsyntax/parse/parser/module.rs b/src/libsyntax/parse/parser/module.rs index 70e0dfc3e88..72049daaed3 100644 --- a/src/libsyntax/parse/parser/module.rs +++ b/src/libsyntax/parse/parser/module.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult}; +use super::Parser; use super::item::ItemInfo; use super::diagnostics::Error; @@ -9,6 +9,8 @@ use crate::token::{self, TokenKind}; use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName}; use crate::symbol::sym; +use errors::PResult; + use std::path::{self, Path, PathBuf}; /// Information about the path to a module. diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index c4a983188fa..f347300da71 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult, PathStyle}; +use super::{Parser, PathStyle}; use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole}; use crate::ptr::P; @@ -10,7 +10,7 @@ use crate::print::pprust; use crate::source_map::{respan, Span, Spanned}; use crate::ThinVec; use syntax_pos::symbol::{kw, sym}; -use errors::{Applicability, DiagnosticBuilder}; +use errors::{PResult, Applicability, DiagnosticBuilder}; type Expected = Option<&'static str>; diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index 2659e0c680e..9ceb3ba1eb4 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult, TokenType}; +use super::{Parser, TokenType}; use crate::{maybe_whole, ThinVec}; use crate::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs}; @@ -9,7 +9,7 @@ use syntax_pos::symbol::{kw, sym}; use std::mem; use log::debug; -use errors::{Applicability, pluralize}; +use errors::{PResult, Applicability, pluralize}; /// Specifies how to parse a path. #[derive(Copy, Clone, PartialEq)] diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index 010fd9c37fd..2a3d6b05bb9 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult, Restrictions, PrevTokenKind, SemiColonMode, BlockMode}; +use super::{Parser, Restrictions, PrevTokenKind, SemiColonMode, BlockMode}; use super::expr::LhsExpr; use super::path::PathStyle; use super::pat::GateOr; @@ -14,7 +14,7 @@ use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; use std::mem; -use errors::Applicability; +use errors::{PResult, Applicability}; impl<'a> Parser<'a> { /// Parses a statement. This stops just before trailing semicolons on everything but items. diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index 4183416e628..a891634e611 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -1,4 +1,4 @@ -use super::{Parser, PResult, PathStyle, PrevTokenKind, TokenType}; +use super::{Parser, PathStyle, PrevTokenKind, TokenType}; use super::item::ParamCfg; use crate::{maybe_whole, maybe_recover_from_interpolated_ty_qpath}; @@ -10,7 +10,7 @@ use crate::token::{self, Token}; use crate::source_map::Span; use crate::symbol::{kw}; -use errors::{Applicability, pluralize}; +use errors::{PResult, Applicability, pluralize}; /// Returns `true` if `IDENT t` can start a type -- `IDENT::a::b`, `IDENT`, /// `IDENT<::AssocTy>`. diff --git a/src/libsyntax/tests.rs b/src/libsyntax/tests.rs index e73c8b43bcc..ed457c3627f 100644 --- a/src/libsyntax/tests.rs +++ b/src/libsyntax/tests.rs @@ -1,5 +1,5 @@ use crate::ast; -use crate::parse::{PResult, source_file_to_stream}; +use crate::parse::source_file_to_stream; use crate::parse::new_parser_from_source_str; use crate::parse::parser::Parser; use crate::sess::ParseSess; @@ -8,7 +8,7 @@ use crate::tokenstream::TokenStream; use crate::with_default_globals; use errors::emitter::EmitterWriter; -use errors::Handler; +use errors::{PResult, Handler}; use rustc_data_structures::sync::Lrc; use syntax_pos::{BytePos, Span, MultiSpan}; diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs index 6d1f0abe49e..e91dd2aba15 100644 --- a/src/libsyntax_expand/expand.rs +++ b/src/libsyntax_expand/expand.rs @@ -12,7 +12,7 @@ use syntax::configure; use syntax::config::StripUnconfigured; use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; use syntax::mut_visit::*; -use syntax::parse::{DirectoryOwnership, PResult}; +use syntax::parse::DirectoryOwnership; use syntax::parse::parser::Parser; use syntax::print::pprust; use syntax::ptr::P; @@ -23,7 +23,7 @@ use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::visit::{self, Visitor}; use syntax::util::map_in_place::MapInPlace; -use errors::{Applicability, FatalError}; +use errors::{PResult, Applicability, FatalError}; use smallvec::{smallvec, SmallVec}; use syntax_pos::{Span, DUMMY_SP, FileName}; diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs index 950e7c0f286..80bf27e1a94 100644 --- a/src/libsyntax_expand/mbe/macro_parser.rs +++ b/src/libsyntax_expand/mbe/macro_parser.rs @@ -77,7 +77,7 @@ use TokenTreeOrTokenTreeSlice::*; use crate::mbe::{self, TokenTree}; use syntax::ast::{Ident, Name}; -use syntax::parse::{Directory, PResult}; +use syntax::parse::Directory; use syntax::parse::parser::{Parser, PathStyle}; use syntax::print::pprust; use syntax::sess::ParseSess; @@ -85,7 +85,7 @@ use syntax::symbol::{kw, sym, Symbol}; use syntax::token::{self, DocComment, Nonterminal, Token}; use syntax::tokenstream::{DelimSpan, TokenStream}; -use errors::FatalError; +use errors::{PResult, FatalError}; use smallvec::{smallvec, SmallVec}; use syntax_pos::Span; diff --git a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs index 6277e2e7ea9..ac864e76784 100644 --- a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs +++ b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs @@ -6,14 +6,15 @@ #![feature(rustc_private)] extern crate syntax; +extern crate rustc_errors; +use rustc_errors::PResult; use syntax::ast::*; use syntax::attr::*; use syntax::ast; use syntax::sess::ParseSess; use syntax::source_map::{FilePathMapping, FileName}; use syntax::parse; -use syntax::parse::PResult; use syntax::parse::new_parser_from_source_str; use syntax::parse::parser::Parser; use syntax::token; -- cgit 1.4.1-3-g733a5 From 255b12a8d3ef3639b4b389fc56d93bd80f03d087 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 13:17:20 +0200 Subject: move parse::classify -> util::classify --- src/libsyntax/lib.rs | 1 + src/libsyntax/parse/classify.rs | 25 ------------------------- src/libsyntax/parse/mod.rs | 1 - src/libsyntax/parse/parser/expr.rs | 2 +- src/libsyntax/parse/parser/stmt.rs | 3 ++- src/libsyntax/print/pprust.rs | 4 ++-- src/libsyntax/util/classify.rs | 25 +++++++++++++++++++++++++ 7 files changed, 31 insertions(+), 30 deletions(-) delete mode 100644 src/libsyntax/parse/classify.rs create mode 100644 src/libsyntax/util/classify.rs (limited to 'src/libsyntax/parse/parser') diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 4c134430f30..205fbe3c52e 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -85,6 +85,7 @@ pub mod diagnostics { pub mod error_codes; pub mod util { + crate mod classify; pub mod lev_distance; pub mod node_count; pub mod parser; diff --git a/src/libsyntax/parse/classify.rs b/src/libsyntax/parse/classify.rs deleted file mode 100644 index 44560688750..00000000000 --- a/src/libsyntax/parse/classify.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Routines the parser uses to classify AST nodes - -// Predicates on exprs and stmts that the pretty-printer and parser use - -use crate::ast; - -/// Does this expression require a semicolon to be treated -/// as a statement? The negation of this: 'can this expression -/// be used as a statement without a semicolon' -- is used -/// as an early-bail-out in the parser so that, for instance, -/// if true {...} else {...} -/// |x| 5 -/// isn't parsed as (if true {...} else {...} | x) | 5 -pub fn expr_requires_semi_to_be_stmt(e: &ast::Expr) -> bool { - match e.kind { - ast::ExprKind::If(..) | - ast::ExprKind::Match(..) | - ast::ExprKind::Block(..) | - ast::ExprKind::While(..) | - ast::ExprKind::Loop(..) | - ast::ExprKind::ForLoop(..) | - ast::ExprKind::TryBlock(..) => false, - _ => true, - } -} diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index badf8bcba57..44fd39448e5 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -24,7 +24,6 @@ mod tests; pub mod parser; pub mod lexer; -crate mod classify; crate mod literal; #[derive(Clone)] diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index 9faa4aefbb6..cc984f03c7d 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -11,11 +11,11 @@ use crate::ast::{ FunctionRetTy, Param, FnDecl, BinOpKind, BinOp, UnOp, Mac, AnonConst, Field, Lit, }; use crate::maybe_recover_from_interpolated_ty_qpath; -use crate::parse::classify; use crate::token::{self, Token, TokenKind}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, Span}; +use crate::util::classify; use crate::util::parser::{AssocOp, Fixity, prec_let_scrutinee_needs_par}; use errors::{PResult, Applicability}; diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index 2a3d6b05bb9..30e47b7a0b2 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -8,7 +8,8 @@ use crate::ptr::P; use crate::{maybe_whole, ThinVec}; use crate::ast::{self, DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind}; use crate::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter}; -use crate::parse::{classify, DirectoryOwnership}; +use crate::parse::DirectoryOwnership; +use crate::util::classify; use crate::token; use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 049f34efc26..0b72b034c20 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -6,10 +6,10 @@ use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; use crate::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; use crate::parse::lexer::comments; -use crate::parse; use crate::print::pp::{self, Breaks}; use crate::print::pp::Breaks::{Consistent, Inconsistent}; use crate::ptr::P; +use crate::util::classify; use crate::sess::ParseSess; use crate::symbol::{kw, sym}; use crate::tokenstream::{self, TokenStream, TokenTree}; @@ -1659,7 +1659,7 @@ impl<'a> State<'a> { ast::StmtKind::Expr(ref expr) => { self.space_if_not_bol(); self.print_expr_outer_attr_style(expr, false); - if parse::classify::expr_requires_semi_to_be_stmt(expr) { + if classify::expr_requires_semi_to_be_stmt(expr) { self.s.word(";"); } } diff --git a/src/libsyntax/util/classify.rs b/src/libsyntax/util/classify.rs new file mode 100644 index 00000000000..44560688750 --- /dev/null +++ b/src/libsyntax/util/classify.rs @@ -0,0 +1,25 @@ +//! Routines the parser uses to classify AST nodes + +// Predicates on exprs and stmts that the pretty-printer and parser use + +use crate::ast; + +/// Does this expression require a semicolon to be treated +/// as a statement? The negation of this: 'can this expression +/// be used as a statement without a semicolon' -- is used +/// as an early-bail-out in the parser so that, for instance, +/// if true {...} else {...} +/// |x| 5 +/// isn't parsed as (if true {...} else {...} | x) | 5 +pub fn expr_requires_semi_to_be_stmt(e: &ast::Expr) -> bool { + match e.kind { + ast::ExprKind::If(..) | + ast::ExprKind::Match(..) | + ast::ExprKind::Block(..) | + ast::ExprKind::While(..) | + ast::ExprKind::Loop(..) | + ast::ExprKind::ForLoop(..) | + ast::ExprKind::TryBlock(..) => false, + _ => true, + } +} -- cgit 1.4.1-3-g733a5 From a1571b68552b0d56d85080c5f92fdab233775de4 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 14:14:30 +0200 Subject: syntax::attr: remove usage of lexer --- src/libsyntax/attr/mod.rs | 5 ++--- src/libsyntax/parse/parser/attr.rs | 11 +++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'src/libsyntax/parse/parser') diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index d609225a54f..c639431794c 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -14,7 +14,6 @@ use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem}; use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam}; use crate::mut_visit::visit_clobber; use crate::source_map::{BytePos, Spanned}; -use crate::parse::lexer::comments::doc_comment_style; use crate::parse; use crate::token::{self, Token}; use crate::ptr::P; @@ -401,11 +400,11 @@ pub fn mk_attr_outer(item: MetaItem) -> Attribute { mk_attr(AttrStyle::Outer, item.path, item.kind.tokens(item.span), item.span) } -pub fn mk_doc_comment(comment: Symbol, span: Span) -> Attribute { +pub fn mk_doc_comment(style: AttrStyle, comment: Symbol, span: Span) -> Attribute { Attribute { kind: AttrKind::DocComment(comment), id: mk_attr_id(), - style: doc_comment_style(&comment.as_str()), + style, span, } } diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 2f1b87da7e9..920e7a521ef 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,10 +1,12 @@ use super::{SeqSep, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; +use crate::parse::lexer::comments; use crate::token::{self, Nonterminal, DelimToken}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; +use syntax_pos::Symbol; use errors::PResult; use log::debug; @@ -45,7 +47,7 @@ impl<'a> Parser<'a> { just_parsed_doc_comment = false; } token::DocComment(s) => { - let attr = attr::mk_doc_comment(s, self.token.span); + let attr = self.mk_doc_comment(s); if attr.style != ast::AttrStyle::Outer { let mut err = self.fatal("expected outer doc comment"); err.note("inner doc comments like this (starting with \ @@ -62,6 +64,11 @@ impl<'a> Parser<'a> { Ok(attrs) } + fn mk_doc_comment(&self, s: Symbol) -> ast::Attribute { + let style = comments::doc_comment_style(&s.as_str()); + attr::mk_doc_comment(style, s, self.token.span) + } + /// Matches `attribute = # ! [ meta_item ]`. /// /// If `permit_inner` is `true`, then a leading `!` indicates an inner @@ -230,7 +237,7 @@ impl<'a> Parser<'a> { } token::DocComment(s) => { // We need to get the position of this token before we bump. - let attr = attr::mk_doc_comment(s, self.token.span); + let attr = self.mk_doc_comment(s); if attr.style == ast::AttrStyle::Inner { attrs.push(attr); self.bump(); -- cgit 1.4.1-3-g733a5 From 27f97aa468b5079bfd159e6fee9a04d5501a8818 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 14:39:52 +0200 Subject: move syntax::parse::lexer::comments -> syntax::util::comments --- src/librustc_save_analysis/lib.rs | 2 +- src/librustdoc/clean/mod.rs | 2 +- src/libsyntax/lib.rs | 1 + src/libsyntax/parse/lexer/comments.rs | 255 -------------------------- src/libsyntax/parse/lexer/comments/tests.rs | 47 ----- src/libsyntax/parse/lexer/mod.rs | 21 +-- src/libsyntax/parse/lexer/tests.rs | 1 + src/libsyntax/parse/parser/attr.rs | 2 +- src/libsyntax/parse/parser/mod.rs | 2 +- src/libsyntax/print/pprust.rs | 2 +- src/libsyntax/util/comments.rs | 270 ++++++++++++++++++++++++++++ src/libsyntax/util/comments/tests.rs | 47 +++++ src/libsyntax_expand/proc_macro_server.rs | 4 +- 13 files changed, 329 insertions(+), 327 deletions(-) delete mode 100644 src/libsyntax/parse/lexer/comments.rs delete mode 100644 src/libsyntax/parse/lexer/comments/tests.rs create mode 100644 src/libsyntax/util/comments.rs create mode 100644 src/libsyntax/util/comments/tests.rs (limited to 'src/libsyntax/parse/parser') diff --git a/src/librustc_save_analysis/lib.rs b/src/librustc_save_analysis/lib.rs index 9408bbe557a..a2f8837c581 100644 --- a/src/librustc_save_analysis/lib.rs +++ b/src/librustc_save_analysis/lib.rs @@ -29,7 +29,7 @@ use std::path::{Path, PathBuf}; use syntax::ast::{self, Attribute, DUMMY_NODE_ID, NodeId, PatKind}; use syntax::source_map::Spanned; -use syntax::parse::lexer::comments::strip_doc_comment_decoration; +use syntax::util::comments::strip_doc_comment_decoration; use syntax::print::pprust; use syntax::visit::{self, Visitor}; use syntax::print::pprust::{param_to_string, ty_to_string}; diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 0c670e5e717..f6cac8ca48d 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -28,7 +28,7 @@ use rustc::ty::layout::VariantIdx; use rustc::util::nodemap::{FxHashMap, FxHashSet}; use syntax::ast::{self, Attribute, AttrStyle, AttrKind, Ident}; use syntax::attr; -use syntax::parse::lexer::comments; +use syntax::util::comments; use syntax::source_map::DUMMY_SP; use syntax_pos::symbol::{Symbol, kw, sym}; use syntax_pos::hygiene::MacroKind; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 205fbe3c52e..bdad52bfb45 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -86,6 +86,7 @@ pub mod error_codes; pub mod util { crate mod classify; + pub mod comments; pub mod lev_distance; pub mod node_count; pub mod parser; diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs deleted file mode 100644 index 33415bdcb62..00000000000 --- a/src/libsyntax/parse/lexer/comments.rs +++ /dev/null @@ -1,255 +0,0 @@ -pub use CommentStyle::*; - -use super::is_block_doc_comment; - -use crate::ast; -use crate::source_map::SourceMap; -use crate::sess::ParseSess; - -use syntax_pos::{BytePos, CharPos, Pos, FileName}; - -use std::usize; - -#[cfg(test)] -mod tests; - -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum CommentStyle { - /// No code on either side of each line of the comment - Isolated, - /// Code exists to the left of the comment - Trailing, - /// Code before /* foo */ and after the comment - Mixed, - /// Just a manual blank line "\n\n", for layout - BlankLine, -} - -#[derive(Clone)] -pub struct Comment { - pub style: CommentStyle, - pub lines: Vec, - pub pos: BytePos, -} - -fn is_doc_comment(s: &str) -> bool { - (s.starts_with("///") && super::is_doc_comment(s)) || s.starts_with("//!") || - (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!") -} - -pub fn doc_comment_style(comment: &str) -> ast::AttrStyle { - assert!(is_doc_comment(comment)); - if comment.starts_with("//!") || comment.starts_with("/*!") { - ast::AttrStyle::Inner - } else { - ast::AttrStyle::Outer - } -} - -pub fn strip_doc_comment_decoration(comment: &str) -> String { - /// remove whitespace-only lines from the start/end of lines - fn vertical_trim(lines: Vec) -> Vec { - let mut i = 0; - let mut j = lines.len(); - // first line of all-stars should be omitted - if !lines.is_empty() && lines[0].chars().all(|c| c == '*') { - i += 1; - } - - while i < j && lines[i].trim().is_empty() { - i += 1; - } - // like the first, a last line of all stars should be omitted - if j > i && - lines[j - 1] - .chars() - .skip(1) - .all(|c| c == '*') { - j -= 1; - } - - while j > i && lines[j - 1].trim().is_empty() { - j -= 1; - } - - lines[i..j].to_vec() - } - - /// remove a "[ \t]*\*" block from each line, if possible - fn horizontal_trim(lines: Vec) -> Vec { - let mut i = usize::MAX; - let mut can_trim = true; - let mut first = true; - - for line in &lines { - for (j, c) in line.chars().enumerate() { - if j > i || !"* \t".contains(c) { - can_trim = false; - break; - } - if c == '*' { - if first { - i = j; - first = false; - } else if i != j { - can_trim = false; - } - break; - } - } - if i >= line.len() { - can_trim = false; - } - if !can_trim { - break; - } - } - - if can_trim { - lines.iter() - .map(|line| (&line[i + 1..line.len()]).to_string()) - .collect() - } else { - lines - } - } - - // one-line comments lose their prefix - const ONELINERS: &[&str] = &["///!", "///", "//!", "//"]; - - for prefix in ONELINERS { - if comment.starts_with(*prefix) { - return (&comment[prefix.len()..]).to_string(); - } - } - - if comment.starts_with("/*") { - let lines = comment[3..comment.len() - 2] - .lines() - .map(|s| s.to_string()) - .collect::>(); - - let lines = vertical_trim(lines); - let lines = horizontal_trim(lines); - - return lines.join("\n"); - } - - panic!("not a doc-comment: {}", comment); -} - -/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. -/// Otherwise returns `Some(k)` where `k` is first char offset after that leading -/// whitespace. Note that `k` may be outside bounds of `s`. -fn all_whitespace(s: &str, col: CharPos) -> Option { - let mut idx = 0; - for (i, ch) in s.char_indices().take(col.to_usize()) { - if !ch.is_whitespace() { - return None; - } - idx = i + ch.len_utf8(); - } - Some(idx) -} - -fn trim_whitespace_prefix(s: &str, col: CharPos) -> &str { - let len = s.len(); - match all_whitespace(&s, col) { - Some(col) => if col < len { &s[col..] } else { "" }, - None => s, - } -} - -fn split_block_comment_into_lines( - text: &str, - col: CharPos, -) -> Vec { - let mut res: Vec = vec![]; - let mut lines = text.lines(); - // just push the first line - res.extend(lines.next().map(|it| it.to_string())); - // for other lines, strip common whitespace prefix - for line in lines { - res.push(trim_whitespace_prefix(line, col).to_string()) - } - res -} - -// it appears this function is called only from pprust... that's -// probably not a good thing. -crate fn gather_comments(sess: &ParseSess, path: FileName, src: String) -> Vec { - let cm = SourceMap::new(sess.source_map().path_mapping().clone()); - let source_file = cm.new_source_file(path, src); - let text = (*source_file.src.as_ref().unwrap()).clone(); - - let text: &str = text.as_str(); - let start_bpos = source_file.start_pos; - let mut pos = 0; - let mut comments: Vec = Vec::new(); - let mut code_to_the_left = false; - - if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { - comments.push(Comment { - style: Isolated, - lines: vec![text[..shebang_len].to_string()], - pos: start_bpos, - }); - pos += shebang_len; - } - - for token in rustc_lexer::tokenize(&text[pos..]) { - let token_text = &text[pos..pos + token.len]; - match token.kind { - rustc_lexer::TokenKind::Whitespace => { - if let Some(mut idx) = token_text.find('\n') { - code_to_the_left = false; - while let Some(next_newline) = &token_text[idx + 1..].find('\n') { - idx = idx + 1 + next_newline; - comments.push(Comment { - style: BlankLine, - lines: vec![], - pos: start_bpos + BytePos((pos + idx) as u32), - }); - } - } - } - rustc_lexer::TokenKind::BlockComment { terminated: _ } => { - if !is_block_doc_comment(token_text) { - let code_to_the_right = match text[pos + token.len..].chars().next() { - Some('\r') | Some('\n') => false, - _ => true, - }; - let style = match (code_to_the_left, code_to_the_right) { - (true, true) | (false, true) => Mixed, - (false, false) => Isolated, - (true, false) => Trailing, - }; - - // Count the number of chars since the start of the line by rescanning. - let pos_in_file = start_bpos + BytePos(pos as u32); - let line_begin_in_file = source_file.line_begin_pos(pos_in_file); - let line_begin_pos = (line_begin_in_file - start_bpos).to_usize(); - let col = CharPos(text[line_begin_pos..pos].chars().count()); - - let lines = split_block_comment_into_lines(token_text, col); - comments.push(Comment { style, lines, pos: pos_in_file }) - } - } - rustc_lexer::TokenKind::LineComment => { - if !is_doc_comment(token_text) { - comments.push(Comment { - style: if code_to_the_left { Trailing } else { Isolated }, - lines: vec![token_text.to_string()], - pos: start_bpos + BytePos(pos as u32), - }) - } - } - _ => { - code_to_the_left = true; - } - } - pos += token.len; - } - - comments -} diff --git a/src/libsyntax/parse/lexer/comments/tests.rs b/src/libsyntax/parse/lexer/comments/tests.rs deleted file mode 100644 index f9cd69fb50d..00000000000 --- a/src/libsyntax/parse/lexer/comments/tests.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; - -#[test] -fn test_block_doc_comment_1() { - let comment = "/**\n * Test \n ** Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " Test \n* Test\n Test"); -} - -#[test] -fn test_block_doc_comment_2() { - let comment = "/**\n * Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " Test\n Test"); -} - -#[test] -fn test_block_doc_comment_3() { - let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " let a: *i32;\n *a = 5;"); -} - -#[test] -fn test_block_doc_comment_4() { - let comment = "/*******************\n test\n *********************/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " test"); -} - -#[test] -fn test_line_doc_comment() { - let stripped = strip_doc_comment_decoration("/// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("///! test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("///test"); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration("///!test"); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration("//test"); - assert_eq!(stripped, "test"); -} diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 5499a3cae5f..b1b7b08c78a 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,6 +1,7 @@ use crate::token::{self, Token, TokenKind}; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; +use crate::util::comments; use errors::{FatalError, DiagnosticBuilder}; use syntax_pos::{BytePos, Pos, Span}; @@ -15,7 +16,6 @@ use log::debug; #[cfg(test)] mod tests; -pub mod comments; mod tokentrees; mod unicode_chars; mod unescape_error_reporting; @@ -179,7 +179,7 @@ impl<'a> StringReader<'a> { rustc_lexer::TokenKind::LineComment => { let string = self.str_from(start); // comments with only more "/"s are not doc comments - let tok = if is_doc_comment(string) { + let tok = if comments::is_line_doc_comment(string) { self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment"); token::DocComment(Symbol::intern(string)) } else { @@ -192,7 +192,7 @@ impl<'a> StringReader<'a> { let string = self.str_from(start); // block comments starting with "/**" or "/*!" are doc-comments // but comments with only "*"s between two "/"s are not - let is_doc_comment = is_block_doc_comment(string); + let is_doc_comment = comments::is_block_doc_comment(string); if !terminated { let msg = if is_doc_comment { @@ -643,18 +643,3 @@ impl<'a> StringReader<'a> { } } } - -fn is_doc_comment(s: &str) -> bool { - let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') || - s.starts_with("//!"); - debug!("is {:?} a doc comment? {}", s, res); - res -} - -fn is_block_doc_comment(s: &str) -> bool { - // Prevent `/**/` from being parsed as a doc comment - let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') || - s.starts_with("/*!")) && s.len() >= 5; - debug!("is {:?} a doc comment? {}", s, res); - res -} diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs index b9b82b2bcef..baa6fb59537 100644 --- a/src/libsyntax/parse/lexer/tests.rs +++ b/src/libsyntax/parse/lexer/tests.rs @@ -3,6 +3,7 @@ use super::*; use crate::symbol::Symbol; use crate::source_map::{SourceMap, FilePathMapping}; use crate::token; +use crate::util::comments::is_doc_comment; use crate::with_default_globals; use errors::{Handler, emitter::EmitterWriter}; diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 920e7a521ef..31f0a02a483 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,7 +1,7 @@ use super::{SeqSep, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; -use crate::parse::lexer::comments; +use crate::util::comments; use crate::token::{self, Nonterminal, DelimToken}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; diff --git a/src/libsyntax/parse/parser/mod.rs b/src/libsyntax/parse/parser/mod.rs index f7f7b0e83d4..455f4172f5f 100644 --- a/src/libsyntax/parse/parser/mod.rs +++ b/src/libsyntax/parse/parser/mod.rs @@ -17,7 +17,7 @@ use crate::ast::{ }; use crate::parse::{Directory, DirectoryOwnership}; use crate::parse::lexer::UnmatchedBrace; -use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; +use crate::util::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::token::{self, Token, TokenKind, DelimToken}; use crate::print::pprust; use crate::ptr::P; diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 0b72b034c20..4ca4bdeb046 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -2,10 +2,10 @@ use crate::ast::{self, BlockCheckMode, PatKind, RangeEnd, RangeSyntax}; use crate::ast::{SelfKind, GenericBound, TraitBoundModifier}; use crate::ast::{Attribute, MacDelimiter, GenericArg}; use crate::util::parser::{self, AssocOp, Fixity}; +use crate::util::comments; use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; use crate::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; -use crate::parse::lexer::comments; use crate::print::pp::{self, Breaks}; use crate::print::pp::Breaks::{Consistent, Inconsistent}; use crate::ptr::P; diff --git a/src/libsyntax/util/comments.rs b/src/libsyntax/util/comments.rs new file mode 100644 index 00000000000..448b4f3b825 --- /dev/null +++ b/src/libsyntax/util/comments.rs @@ -0,0 +1,270 @@ +pub use CommentStyle::*; + +use crate::ast; +use crate::source_map::SourceMap; +use crate::sess::ParseSess; + +use syntax_pos::{BytePos, CharPos, Pos, FileName}; + +use std::usize; + +use log::debug; + +#[cfg(test)] +mod tests; + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum CommentStyle { + /// No code on either side of each line of the comment + Isolated, + /// Code exists to the left of the comment + Trailing, + /// Code before /* foo */ and after the comment + Mixed, + /// Just a manual blank line "\n\n", for layout + BlankLine, +} + +#[derive(Clone)] +pub struct Comment { + pub style: CommentStyle, + pub lines: Vec, + pub pos: BytePos, +} + +crate fn is_line_doc_comment(s: &str) -> bool { + let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') || + s.starts_with("//!"); + debug!("is {:?} a doc comment? {}", s, res); + res +} + +crate fn is_block_doc_comment(s: &str) -> bool { + // Prevent `/**/` from being parsed as a doc comment + let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') || + s.starts_with("/*!")) && s.len() >= 5; + debug!("is {:?} a doc comment? {}", s, res); + res +} + +crate fn is_doc_comment(s: &str) -> bool { + (s.starts_with("///") && is_line_doc_comment(s)) || s.starts_with("//!") || + (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!") +} + +pub fn doc_comment_style(comment: &str) -> ast::AttrStyle { + assert!(is_doc_comment(comment)); + if comment.starts_with("//!") || comment.starts_with("/*!") { + ast::AttrStyle::Inner + } else { + ast::AttrStyle::Outer + } +} + +pub fn strip_doc_comment_decoration(comment: &str) -> String { + /// remove whitespace-only lines from the start/end of lines + fn vertical_trim(lines: Vec) -> Vec { + let mut i = 0; + let mut j = lines.len(); + // first line of all-stars should be omitted + if !lines.is_empty() && lines[0].chars().all(|c| c == '*') { + i += 1; + } + + while i < j && lines[i].trim().is_empty() { + i += 1; + } + // like the first, a last line of all stars should be omitted + if j > i && + lines[j - 1] + .chars() + .skip(1) + .all(|c| c == '*') { + j -= 1; + } + + while j > i && lines[j - 1].trim().is_empty() { + j -= 1; + } + + lines[i..j].to_vec() + } + + /// remove a "[ \t]*\*" block from each line, if possible + fn horizontal_trim(lines: Vec) -> Vec { + let mut i = usize::MAX; + let mut can_trim = true; + let mut first = true; + + for line in &lines { + for (j, c) in line.chars().enumerate() { + if j > i || !"* \t".contains(c) { + can_trim = false; + break; + } + if c == '*' { + if first { + i = j; + first = false; + } else if i != j { + can_trim = false; + } + break; + } + } + if i >= line.len() { + can_trim = false; + } + if !can_trim { + break; + } + } + + if can_trim { + lines.iter() + .map(|line| (&line[i + 1..line.len()]).to_string()) + .collect() + } else { + lines + } + } + + // one-line comments lose their prefix + const ONELINERS: &[&str] = &["///!", "///", "//!", "//"]; + + for prefix in ONELINERS { + if comment.starts_with(*prefix) { + return (&comment[prefix.len()..]).to_string(); + } + } + + if comment.starts_with("/*") { + let lines = comment[3..comment.len() - 2] + .lines() + .map(|s| s.to_string()) + .collect::>(); + + let lines = vertical_trim(lines); + let lines = horizontal_trim(lines); + + return lines.join("\n"); + } + + panic!("not a doc-comment: {}", comment); +} + +/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. +/// Otherwise returns `Some(k)` where `k` is first char offset after that leading +/// whitespace. Note that `k` may be outside bounds of `s`. +fn all_whitespace(s: &str, col: CharPos) -> Option { + let mut idx = 0; + for (i, ch) in s.char_indices().take(col.to_usize()) { + if !ch.is_whitespace() { + return None; + } + idx = i + ch.len_utf8(); + } + Some(idx) +} + +fn trim_whitespace_prefix(s: &str, col: CharPos) -> &str { + let len = s.len(); + match all_whitespace(&s, col) { + Some(col) => if col < len { &s[col..] } else { "" }, + None => s, + } +} + +fn split_block_comment_into_lines( + text: &str, + col: CharPos, +) -> Vec { + let mut res: Vec = vec![]; + let mut lines = text.lines(); + // just push the first line + res.extend(lines.next().map(|it| it.to_string())); + // for other lines, strip common whitespace prefix + for line in lines { + res.push(trim_whitespace_prefix(line, col).to_string()) + } + res +} + +// it appears this function is called only from pprust... that's +// probably not a good thing. +crate fn gather_comments(sess: &ParseSess, path: FileName, src: String) -> Vec { + let cm = SourceMap::new(sess.source_map().path_mapping().clone()); + let source_file = cm.new_source_file(path, src); + let text = (*source_file.src.as_ref().unwrap()).clone(); + + let text: &str = text.as_str(); + let start_bpos = source_file.start_pos; + let mut pos = 0; + let mut comments: Vec = Vec::new(); + let mut code_to_the_left = false; + + if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { + comments.push(Comment { + style: Isolated, + lines: vec![text[..shebang_len].to_string()], + pos: start_bpos, + }); + pos += shebang_len; + } + + for token in rustc_lexer::tokenize(&text[pos..]) { + let token_text = &text[pos..pos + token.len]; + match token.kind { + rustc_lexer::TokenKind::Whitespace => { + if let Some(mut idx) = token_text.find('\n') { + code_to_the_left = false; + while let Some(next_newline) = &token_text[idx + 1..].find('\n') { + idx = idx + 1 + next_newline; + comments.push(Comment { + style: BlankLine, + lines: vec![], + pos: start_bpos + BytePos((pos + idx) as u32), + }); + } + } + } + rustc_lexer::TokenKind::BlockComment { terminated: _ } => { + if !is_block_doc_comment(token_text) { + let code_to_the_right = match text[pos + token.len..].chars().next() { + Some('\r') | Some('\n') => false, + _ => true, + }; + let style = match (code_to_the_left, code_to_the_right) { + (true, true) | (false, true) => Mixed, + (false, false) => Isolated, + (true, false) => Trailing, + }; + + // Count the number of chars since the start of the line by rescanning. + let pos_in_file = start_bpos + BytePos(pos as u32); + let line_begin_in_file = source_file.line_begin_pos(pos_in_file); + let line_begin_pos = (line_begin_in_file - start_bpos).to_usize(); + let col = CharPos(text[line_begin_pos..pos].chars().count()); + + let lines = split_block_comment_into_lines(token_text, col); + comments.push(Comment { style, lines, pos: pos_in_file }) + } + } + rustc_lexer::TokenKind::LineComment => { + if !is_doc_comment(token_text) { + comments.push(Comment { + style: if code_to_the_left { Trailing } else { Isolated }, + lines: vec![token_text.to_string()], + pos: start_bpos + BytePos(pos as u32), + }) + } + } + _ => { + code_to_the_left = true; + } + } + pos += token.len; + } + + comments +} diff --git a/src/libsyntax/util/comments/tests.rs b/src/libsyntax/util/comments/tests.rs new file mode 100644 index 00000000000..f9cd69fb50d --- /dev/null +++ b/src/libsyntax/util/comments/tests.rs @@ -0,0 +1,47 @@ +use super::*; + +#[test] +fn test_block_doc_comment_1() { + let comment = "/**\n * Test \n ** Test\n * Test\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " Test \n* Test\n Test"); +} + +#[test] +fn test_block_doc_comment_2() { + let comment = "/**\n * Test\n * Test\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " Test\n Test"); +} + +#[test] +fn test_block_doc_comment_3() { + let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " let a: *i32;\n *a = 5;"); +} + +#[test] +fn test_block_doc_comment_4() { + let comment = "/*******************\n test\n *********************/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " test"); +} + +#[test] +fn test_line_doc_comment() { + let stripped = strip_doc_comment_decoration("/// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("///! test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("///test"); + assert_eq!(stripped, "test"); + let stripped = strip_doc_comment_decoration("///!test"); + assert_eq!(stripped, "test"); + let stripped = strip_doc_comment_decoration("//test"); + assert_eq!(stripped, "test"); +} diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs index 67a15e48205..e96b6092787 100644 --- a/src/libsyntax_expand/proc_macro_server.rs +++ b/src/libsyntax_expand/proc_macro_server.rs @@ -1,8 +1,8 @@ use crate::base::ExtCtxt; use syntax::ast; -use syntax::parse::{self}; -use syntax::parse::lexer::comments; +use syntax::parse; +use syntax::util::comments; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::token; -- cgit 1.4.1-3-g733a5 From cc9c139694389c8df158640d4bcc20a2fe31f1ea Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 18:47:14 +0200 Subject: move syntax::{parse::literal -> util::literal} --- src/libsyntax/lib.rs | 1 + src/libsyntax/parse/literal.rs | 305 ------------------------------------- src/libsyntax/parse/mod.rs | 2 - src/libsyntax/parse/parser/expr.rs | 3 +- src/libsyntax/util/literal.rs | 305 +++++++++++++++++++++++++++++++++++++ 5 files changed, 307 insertions(+), 309 deletions(-) delete mode 100644 src/libsyntax/parse/literal.rs create mode 100644 src/libsyntax/util/literal.rs (limited to 'src/libsyntax/parse/parser') diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index bdad52bfb45..1b17de529c4 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -88,6 +88,7 @@ pub mod util { crate mod classify; pub mod comments; pub mod lev_distance; + crate mod literal; pub mod node_count; pub mod parser; pub mod map_in_place; diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs deleted file mode 100644 index d4c9b7850c5..00000000000 --- a/src/libsyntax/parse/literal.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Code related to parsing literals. - -use crate::ast::{self, Lit, LitKind}; -use crate::symbol::{kw, sym, Symbol}; -use crate::token::{self, Token}; -use crate::tokenstream::TokenTree; - -use log::debug; -use rustc_data_structures::sync::Lrc; -use syntax_pos::Span; -use rustc_lexer::unescape::{unescape_char, unescape_byte}; -use rustc_lexer::unescape::{unescape_str, unescape_byte_str}; -use rustc_lexer::unescape::{unescape_raw_str, unescape_raw_byte_str}; - -use std::ascii; - -crate enum LitError { - NotLiteral, - LexerError, - InvalidSuffix, - InvalidIntSuffix, - InvalidFloatSuffix, - NonDecimalFloat(u32), - IntTooLarge, -} - -impl LitKind { - /// Converts literal token into a semantic literal. - fn from_lit_token(lit: token::Lit) -> Result { - let token::Lit { kind, symbol, suffix } = lit; - if suffix.is_some() && !kind.may_have_suffix() { - return Err(LitError::InvalidSuffix); - } - - Ok(match kind { - token::Bool => { - assert!(symbol.is_bool_lit()); - LitKind::Bool(symbol == kw::True) - } - token::Byte => return unescape_byte(&symbol.as_str()) - .map(LitKind::Byte).map_err(|_| LitError::LexerError), - token::Char => return unescape_char(&symbol.as_str()) - .map(LitKind::Char).map_err(|_| LitError::LexerError), - - // There are some valid suffixes for integer and float literals, - // so all the handling is done internally. - token::Integer => return integer_lit(symbol, suffix), - token::Float => return float_lit(symbol, suffix), - - token::Str => { - // If there are no characters requiring special treatment we can - // reuse the symbol from the token. Otherwise, we must generate a - // new symbol because the string in the LitKind is different to the - // string in the token. - let s = symbol.as_str(); - let symbol = if s.contains(&['\\', '\r'][..]) { - let mut buf = String::with_capacity(s.len()); - let mut error = Ok(()); - unescape_str(&s, &mut |_, unescaped_char| { - match unescaped_char { - Ok(c) => buf.push(c), - Err(_) => error = Err(LitError::LexerError), - } - }); - error?; - Symbol::intern(&buf) - } else { - symbol - }; - LitKind::Str(symbol, ast::StrStyle::Cooked) - } - token::StrRaw(n) => { - // Ditto. - let s = symbol.as_str(); - let symbol = if s.contains('\r') { - let mut buf = String::with_capacity(s.len()); - let mut error = Ok(()); - unescape_raw_str(&s, &mut |_, unescaped_char| { - match unescaped_char { - Ok(c) => buf.push(c), - Err(_) => error = Err(LitError::LexerError), - } - }); - error?; - buf.shrink_to_fit(); - Symbol::intern(&buf) - } else { - symbol - }; - LitKind::Str(symbol, ast::StrStyle::Raw(n)) - } - token::ByteStr => { - let s = symbol.as_str(); - let mut buf = Vec::with_capacity(s.len()); - let mut error = Ok(()); - unescape_byte_str(&s, &mut |_, unescaped_byte| { - match unescaped_byte { - Ok(c) => buf.push(c), - Err(_) => error = Err(LitError::LexerError), - } - }); - error?; - buf.shrink_to_fit(); - LitKind::ByteStr(Lrc::new(buf)) - } - token::ByteStrRaw(_) => { - let s = symbol.as_str(); - let bytes = if s.contains('\r') { - let mut buf = Vec::with_capacity(s.len()); - let mut error = Ok(()); - unescape_raw_byte_str(&s, &mut |_, unescaped_byte| { - match unescaped_byte { - Ok(c) => buf.push(c), - Err(_) => error = Err(LitError::LexerError), - } - }); - error?; - buf.shrink_to_fit(); - buf - } else { - symbol.to_string().into_bytes() - }; - - LitKind::ByteStr(Lrc::new(bytes)) - }, - token::Err => LitKind::Err(symbol), - }) - } - - /// Attempts to recover a token from semantic literal. - /// This function is used when the original token doesn't exist (e.g. the literal is created - /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). - pub fn to_lit_token(&self) -> token::Lit { - let (kind, symbol, suffix) = match *self { - LitKind::Str(symbol, ast::StrStyle::Cooked) => { - // Don't re-intern unless the escaped string is different. - let s = symbol.as_str(); - let escaped = s.escape_default().to_string(); - let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) }; - (token::Str, symbol, None) - } - LitKind::Str(symbol, ast::StrStyle::Raw(n)) => { - (token::StrRaw(n), symbol, None) - } - LitKind::ByteStr(ref bytes) => { - let string = bytes.iter().cloned().flat_map(ascii::escape_default) - .map(Into::::into).collect::(); - (token::ByteStr, Symbol::intern(&string), None) - } - LitKind::Byte(byte) => { - let string: String = ascii::escape_default(byte).map(Into::::into).collect(); - (token::Byte, Symbol::intern(&string), None) - } - LitKind::Char(ch) => { - let string: String = ch.escape_default().map(Into::::into).collect(); - (token::Char, Symbol::intern(&string), None) - } - LitKind::Int(n, ty) => { - let suffix = match ty { - ast::LitIntType::Unsigned(ty) => Some(ty.name()), - ast::LitIntType::Signed(ty) => Some(ty.name()), - ast::LitIntType::Unsuffixed => None, - }; - (token::Integer, sym::integer(n), suffix) - } - LitKind::Float(symbol, ty) => { - let suffix = match ty { - ast::LitFloatType::Suffixed(ty) => Some(ty.name()), - ast::LitFloatType::Unsuffixed => None, - }; - (token::Float, symbol, suffix) - } - LitKind::Bool(value) => { - let symbol = if value { kw::True } else { kw::False }; - (token::Bool, symbol, None) - } - LitKind::Err(symbol) => { - (token::Err, symbol, None) - } - }; - - token::Lit::new(kind, symbol, suffix) - } -} - -impl Lit { - /// Converts literal token into an AST literal. - crate fn from_lit_token(token: token::Lit, span: Span) -> Result { - Ok(Lit { token, kind: LitKind::from_lit_token(token)?, span }) - } - - /// Converts arbitrary token into an AST literal. - crate fn from_token(token: &Token) -> Result { - let lit = match token.kind { - token::Ident(name, false) if name.is_bool_lit() => - token::Lit::new(token::Bool, name, None), - token::Literal(lit) => - lit, - token::Interpolated(ref nt) => { - if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt { - if let ast::ExprKind::Lit(lit) = &expr.kind { - return Ok(lit.clone()); - } - } - return Err(LitError::NotLiteral); - } - _ => return Err(LitError::NotLiteral) - }; - - Lit::from_lit_token(lit, token.span) - } - - /// Attempts to recover an AST literal from semantic literal. - /// This function is used when the original token doesn't exist (e.g. the literal is created - /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). - pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit { - Lit { token: kind.to_lit_token(), kind, span } - } - - /// Losslessly convert an AST literal into a token tree. - crate fn token_tree(&self) -> TokenTree { - let token = match self.token.kind { - token::Bool => token::Ident(self.token.symbol, false), - _ => token::Literal(self.token), - }; - TokenTree::token(token, self.span) - } -} - -fn strip_underscores(symbol: Symbol) -> Symbol { - // Do not allocate a new string unless necessary. - let s = symbol.as_str(); - if s.contains('_') { - let mut s = s.to_string(); - s.retain(|c| c != '_'); - return Symbol::intern(&s); - } - symbol -} - -fn filtered_float_lit(symbol: Symbol, suffix: Option, base: u32) - -> Result { - debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base); - if base != 10 { - return Err(LitError::NonDecimalFloat(base)); - } - Ok(match suffix { - Some(suf) => LitKind::Float(symbol, ast::LitFloatType::Suffixed(match suf { - sym::f32 => ast::FloatTy::F32, - sym::f64 => ast::FloatTy::F64, - _ => return Err(LitError::InvalidFloatSuffix), - })), - None => LitKind::Float(symbol, ast::LitFloatType::Unsuffixed) - }) -} - -fn float_lit(symbol: Symbol, suffix: Option) -> Result { - debug!("float_lit: {:?}, {:?}", symbol, suffix); - filtered_float_lit(strip_underscores(symbol), suffix, 10) -} - -fn integer_lit(symbol: Symbol, suffix: Option) -> Result { - debug!("integer_lit: {:?}, {:?}", symbol, suffix); - let symbol = strip_underscores(symbol); - let s = symbol.as_str(); - - let base = match s.as_bytes() { - [b'0', b'x', ..] => 16, - [b'0', b'o', ..] => 8, - [b'0', b'b', ..] => 2, - _ => 10, - }; - - let ty = match suffix { - Some(suf) => match suf { - sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize), - sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8), - sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16), - sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32), - sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64), - sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128), - sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize), - sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8), - sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16), - sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32), - sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64), - sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128), - // `1f64` and `2f32` etc. are valid float literals, and - // `fxxx` looks more like an invalid float literal than invalid integer literal. - _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base), - _ => return Err(LitError::InvalidIntSuffix), - } - _ => ast::LitIntType::Unsuffixed - }; - - let s = &s[if base != 10 { 2 } else { 0 } ..]; - u128::from_str_radix(s, base).map(|i| LitKind::Int(i, ty)).map_err(|_| { - // Small bases are lexed as if they were base 10, e.g, the string - // might be `0b10201`. This will cause the conversion above to fail, - // but these kinds of errors are already reported by the lexer. - let from_lexer = - base < 10 && s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base)); - if from_lexer { LitError::LexerError } else { LitError::IntTooLarge } - }) -} diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 44fd39448e5..18550762017 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -24,8 +24,6 @@ mod tests; pub mod parser; pub mod lexer; -crate mod literal; - #[derive(Clone)] pub struct Directory<'a> { pub path: Cow<'a, Path>, diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index cc984f03c7d..800074035ce 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -3,8 +3,6 @@ use super::{SemiColonMode, SeqSep, TokenExpectType}; use super::pat::{GateOr, PARAM_EXPECTED}; use super::diagnostics::Error; -use crate::parse::literal::LitError; - use crate::ast::{ self, DUMMY_NODE_ID, Attribute, AttrStyle, Ident, CaptureBy, BlockCheckMode, Expr, ExprKind, RangeLimits, Label, Movability, IsAsync, Arm, Ty, TyKind, @@ -16,6 +14,7 @@ use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, Span}; use crate::util::classify; +use crate::util::literal::LitError; use crate::util::parser::{AssocOp, Fixity, prec_let_scrutinee_needs_par}; use errors::{PResult, Applicability}; diff --git a/src/libsyntax/util/literal.rs b/src/libsyntax/util/literal.rs new file mode 100644 index 00000000000..d4c9b7850c5 --- /dev/null +++ b/src/libsyntax/util/literal.rs @@ -0,0 +1,305 @@ +//! Code related to parsing literals. + +use crate::ast::{self, Lit, LitKind}; +use crate::symbol::{kw, sym, Symbol}; +use crate::token::{self, Token}; +use crate::tokenstream::TokenTree; + +use log::debug; +use rustc_data_structures::sync::Lrc; +use syntax_pos::Span; +use rustc_lexer::unescape::{unescape_char, unescape_byte}; +use rustc_lexer::unescape::{unescape_str, unescape_byte_str}; +use rustc_lexer::unescape::{unescape_raw_str, unescape_raw_byte_str}; + +use std::ascii; + +crate enum LitError { + NotLiteral, + LexerError, + InvalidSuffix, + InvalidIntSuffix, + InvalidFloatSuffix, + NonDecimalFloat(u32), + IntTooLarge, +} + +impl LitKind { + /// Converts literal token into a semantic literal. + fn from_lit_token(lit: token::Lit) -> Result { + let token::Lit { kind, symbol, suffix } = lit; + if suffix.is_some() && !kind.may_have_suffix() { + return Err(LitError::InvalidSuffix); + } + + Ok(match kind { + token::Bool => { + assert!(symbol.is_bool_lit()); + LitKind::Bool(symbol == kw::True) + } + token::Byte => return unescape_byte(&symbol.as_str()) + .map(LitKind::Byte).map_err(|_| LitError::LexerError), + token::Char => return unescape_char(&symbol.as_str()) + .map(LitKind::Char).map_err(|_| LitError::LexerError), + + // There are some valid suffixes for integer and float literals, + // so all the handling is done internally. + token::Integer => return integer_lit(symbol, suffix), + token::Float => return float_lit(symbol, suffix), + + token::Str => { + // If there are no characters requiring special treatment we can + // reuse the symbol from the token. Otherwise, we must generate a + // new symbol because the string in the LitKind is different to the + // string in the token. + let s = symbol.as_str(); + let symbol = if s.contains(&['\\', '\r'][..]) { + let mut buf = String::with_capacity(s.len()); + let mut error = Ok(()); + unescape_str(&s, &mut |_, unescaped_char| { + match unescaped_char { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + Symbol::intern(&buf) + } else { + symbol + }; + LitKind::Str(symbol, ast::StrStyle::Cooked) + } + token::StrRaw(n) => { + // Ditto. + let s = symbol.as_str(); + let symbol = if s.contains('\r') { + let mut buf = String::with_capacity(s.len()); + let mut error = Ok(()); + unescape_raw_str(&s, &mut |_, unescaped_char| { + match unescaped_char { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + buf.shrink_to_fit(); + Symbol::intern(&buf) + } else { + symbol + }; + LitKind::Str(symbol, ast::StrStyle::Raw(n)) + } + token::ByteStr => { + let s = symbol.as_str(); + let mut buf = Vec::with_capacity(s.len()); + let mut error = Ok(()); + unescape_byte_str(&s, &mut |_, unescaped_byte| { + match unescaped_byte { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + buf.shrink_to_fit(); + LitKind::ByteStr(Lrc::new(buf)) + } + token::ByteStrRaw(_) => { + let s = symbol.as_str(); + let bytes = if s.contains('\r') { + let mut buf = Vec::with_capacity(s.len()); + let mut error = Ok(()); + unescape_raw_byte_str(&s, &mut |_, unescaped_byte| { + match unescaped_byte { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + buf.shrink_to_fit(); + buf + } else { + symbol.to_string().into_bytes() + }; + + LitKind::ByteStr(Lrc::new(bytes)) + }, + token::Err => LitKind::Err(symbol), + }) + } + + /// Attempts to recover a token from semantic literal. + /// This function is used when the original token doesn't exist (e.g. the literal is created + /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). + pub fn to_lit_token(&self) -> token::Lit { + let (kind, symbol, suffix) = match *self { + LitKind::Str(symbol, ast::StrStyle::Cooked) => { + // Don't re-intern unless the escaped string is different. + let s = symbol.as_str(); + let escaped = s.escape_default().to_string(); + let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) }; + (token::Str, symbol, None) + } + LitKind::Str(symbol, ast::StrStyle::Raw(n)) => { + (token::StrRaw(n), symbol, None) + } + LitKind::ByteStr(ref bytes) => { + let string = bytes.iter().cloned().flat_map(ascii::escape_default) + .map(Into::::into).collect::(); + (token::ByteStr, Symbol::intern(&string), None) + } + LitKind::Byte(byte) => { + let string: String = ascii::escape_default(byte).map(Into::::into).collect(); + (token::Byte, Symbol::intern(&string), None) + } + LitKind::Char(ch) => { + let string: String = ch.escape_default().map(Into::::into).collect(); + (token::Char, Symbol::intern(&string), None) + } + LitKind::Int(n, ty) => { + let suffix = match ty { + ast::LitIntType::Unsigned(ty) => Some(ty.name()), + ast::LitIntType::Signed(ty) => Some(ty.name()), + ast::LitIntType::Unsuffixed => None, + }; + (token::Integer, sym::integer(n), suffix) + } + LitKind::Float(symbol, ty) => { + let suffix = match ty { + ast::LitFloatType::Suffixed(ty) => Some(ty.name()), + ast::LitFloatType::Unsuffixed => None, + }; + (token::Float, symbol, suffix) + } + LitKind::Bool(value) => { + let symbol = if value { kw::True } else { kw::False }; + (token::Bool, symbol, None) + } + LitKind::Err(symbol) => { + (token::Err, symbol, None) + } + }; + + token::Lit::new(kind, symbol, suffix) + } +} + +impl Lit { + /// Converts literal token into an AST literal. + crate fn from_lit_token(token: token::Lit, span: Span) -> Result { + Ok(Lit { token, kind: LitKind::from_lit_token(token)?, span }) + } + + /// Converts arbitrary token into an AST literal. + crate fn from_token(token: &Token) -> Result { + let lit = match token.kind { + token::Ident(name, false) if name.is_bool_lit() => + token::Lit::new(token::Bool, name, None), + token::Literal(lit) => + lit, + token::Interpolated(ref nt) => { + if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt { + if let ast::ExprKind::Lit(lit) = &expr.kind { + return Ok(lit.clone()); + } + } + return Err(LitError::NotLiteral); + } + _ => return Err(LitError::NotLiteral) + }; + + Lit::from_lit_token(lit, token.span) + } + + /// Attempts to recover an AST literal from semantic literal. + /// This function is used when the original token doesn't exist (e.g. the literal is created + /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). + pub fn from_lit_kind(kind: LitKind, span: Span) -> Lit { + Lit { token: kind.to_lit_token(), kind, span } + } + + /// Losslessly convert an AST literal into a token tree. + crate fn token_tree(&self) -> TokenTree { + let token = match self.token.kind { + token::Bool => token::Ident(self.token.symbol, false), + _ => token::Literal(self.token), + }; + TokenTree::token(token, self.span) + } +} + +fn strip_underscores(symbol: Symbol) -> Symbol { + // Do not allocate a new string unless necessary. + let s = symbol.as_str(); + if s.contains('_') { + let mut s = s.to_string(); + s.retain(|c| c != '_'); + return Symbol::intern(&s); + } + symbol +} + +fn filtered_float_lit(symbol: Symbol, suffix: Option, base: u32) + -> Result { + debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base); + if base != 10 { + return Err(LitError::NonDecimalFloat(base)); + } + Ok(match suffix { + Some(suf) => LitKind::Float(symbol, ast::LitFloatType::Suffixed(match suf { + sym::f32 => ast::FloatTy::F32, + sym::f64 => ast::FloatTy::F64, + _ => return Err(LitError::InvalidFloatSuffix), + })), + None => LitKind::Float(symbol, ast::LitFloatType::Unsuffixed) + }) +} + +fn float_lit(symbol: Symbol, suffix: Option) -> Result { + debug!("float_lit: {:?}, {:?}", symbol, suffix); + filtered_float_lit(strip_underscores(symbol), suffix, 10) +} + +fn integer_lit(symbol: Symbol, suffix: Option) -> Result { + debug!("integer_lit: {:?}, {:?}", symbol, suffix); + let symbol = strip_underscores(symbol); + let s = symbol.as_str(); + + let base = match s.as_bytes() { + [b'0', b'x', ..] => 16, + [b'0', b'o', ..] => 8, + [b'0', b'b', ..] => 2, + _ => 10, + }; + + let ty = match suffix { + Some(suf) => match suf { + sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize), + sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8), + sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16), + sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32), + sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64), + sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128), + sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize), + sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8), + sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16), + sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32), + sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64), + sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128), + // `1f64` and `2f32` etc. are valid float literals, and + // `fxxx` looks more like an invalid float literal than invalid integer literal. + _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base), + _ => return Err(LitError::InvalidIntSuffix), + } + _ => ast::LitIntType::Unsuffixed + }; + + let s = &s[if base != 10 { 2 } else { 0 } ..]; + u128::from_str_radix(s, base).map(|i| LitKind::Int(i, ty)).map_err(|_| { + // Small bases are lexed as if they were base 10, e.g, the string + // might be `0b10201`. This will cause the conversion above to fail, + // but these kinds of errors are already reported by the lexer. + let from_lexer = + base < 10 && s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base)); + if from_lexer { LitError::LexerError } else { LitError::IntTooLarge } + }) +} -- cgit 1.4.1-3-g733a5