diff options
| author | Matthias Einwag <matthias.einwag@live.com> | 2019-02-12 22:46:14 -0800 |
|---|---|---|
| committer | Matthias Einwag <matthias.einwag@live.com> | 2019-02-12 22:46:14 -0800 |
| commit | 871338c3aed87cb84f02ebd7fd9b447966d5b05d (patch) | |
| tree | a2e1315d7d17d0b9f3463686ed2fbf36f3238ec4 /src/libsyntax/parse | |
| parent | 1ef34a5a39641846e824b6450a705d6031002beb (diff) | |
| parent | 0f949c2fcc696d0260a99196d5e5400c59a26a54 (diff) | |
| download | rust-871338c3aed87cb84f02ebd7fd9b447966d5b05d.tar.gz rust-871338c3aed87cb84f02ebd7fd9b447966d5b05d.zip | |
Merging master
Diffstat (limited to 'src/libsyntax/parse')
| -rw-r--r-- | src/libsyntax/parse/attr.rs | 18 | ||||
| -rw-r--r-- | src/libsyntax/parse/classify.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/comments.rs | 35 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 58 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/tokentrees.rs | 36 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/unicode_chars.rs | 2 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 159 | ||||
| -rw-r--r-- | src/libsyntax/parse/parser.rs | 991 | ||||
| -rw-r--r-- | src/libsyntax/parse/token.rs | 90 |
9 files changed, 777 insertions, 614 deletions
diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 914a0667ebf..b36ca0574cb 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -1,10 +1,12 @@ -use attr; -use ast; -use source_map::respan; -use parse::{SeqSep, PResult}; -use parse::token::{self, Nonterminal, DelimToken}; -use parse::parser::{Parser, TokenType, PathStyle}; -use tokenstream::{TokenStream, TokenTree}; +use crate::attr; +use crate::ast; +use crate::source_map::respan; +use crate::parse::{SeqSep, PResult}; +use crate::parse::token::{self, Nonterminal, DelimToken}; +use crate::parse::parser::{Parser, TokenType, PathStyle}; +use crate::tokenstream::{TokenStream, TokenTree}; + +use log::debug; #[derive(Debug)] enum InnerAttributeParsePolicy<'a> { @@ -74,7 +76,7 @@ impl<'a> Parser<'a> { /// The same as `parse_attribute`, except it takes in an `InnerAttributeParsePolicy` /// that prescribes how to handle inner attributes. fn parse_attribute_with_inner_parse_policy(&mut self, - inner_parse_policy: InnerAttributeParsePolicy) + inner_parse_policy: InnerAttributeParsePolicy<'_>) -> PResult<'a, ast::Attribute> { debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}", inner_parse_policy, diff --git a/src/libsyntax/parse/classify.rs b/src/libsyntax/parse/classify.rs index a1cdfd9146a..b4103440e35 100644 --- a/src/libsyntax/parse/classify.rs +++ b/src/libsyntax/parse/classify.rs @@ -2,7 +2,7 @@ // Predicates on exprs and stmts that the pretty-printer and parser use -use ast; +use crate::ast; /// Does this expression require a semicolon to be treated /// as a statement? The negation of this: 'can this expression diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index ffc480d829d..74fff3324ea 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -1,11 +1,13 @@ -pub use self::CommentStyle::*; +pub use CommentStyle::*; + +use crate::ast; +use crate::source_map::SourceMap; +use crate::parse::lexer::{is_block_doc_comment, is_pattern_whitespace}; +use crate::parse::lexer::{self, ParseSess, StringReader, TokenAndSpan}; +use crate::print::pprust; -use ast; -use source_map::SourceMap; use syntax_pos::{BytePos, CharPos, Pos, FileName}; -use parse::lexer::{is_block_doc_comment, is_pattern_whitespace}; -use parse::lexer::{self, ParseSess, StringReader, TokenAndSpan}; -use print::pprust; +use log::debug; use std::io::Read; use std::usize; @@ -135,7 +137,7 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String { panic!("not a doc-comment: {}", comment); } -fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) { +fn push_blank_line_comment(rdr: &StringReader<'_>, comments: &mut Vec<Comment>) { debug!(">>> blank-line comment"); comments.push(Comment { style: BlankLine, @@ -144,7 +146,10 @@ fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) { }); } -fn consume_whitespace_counting_blank_lines(rdr: &mut StringReader, comments: &mut Vec<Comment>) { +fn consume_whitespace_counting_blank_lines( + rdr: &mut StringReader<'_>, + comments: &mut Vec<Comment> +) { while is_pattern_whitespace(rdr.ch) && !rdr.is_eof() { if rdr.ch_is('\n') { push_blank_line_comment(rdr, &mut *comments); @@ -153,7 +158,7 @@ fn consume_whitespace_counting_blank_lines(rdr: &mut StringReader, comments: &mu } } -fn read_shebang_comment(rdr: &mut StringReader, +fn read_shebang_comment(rdr: &mut StringReader<'_>, code_to_the_left: bool, comments: &mut Vec<Comment>) { debug!(">>> shebang comment"); @@ -166,7 +171,7 @@ fn read_shebang_comment(rdr: &mut StringReader, }); } -fn read_line_comments(rdr: &mut StringReader, +fn read_line_comments(rdr: &mut StringReader<'_>, code_to_the_left: bool, comments: &mut Vec<Comment>) { debug!(">>> line comments"); @@ -192,9 +197,9 @@ fn read_line_comments(rdr: &mut StringReader, } } -/// Returns None if the first col chars of s contain a non-whitespace char. -/// Otherwise returns Some(k) where k is first char offset after that leading -/// whitespace. Note k may be outside bounds of s. +/// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. +/// Otherwise returns `Some(k)` where `k` is first char offset after that leading +/// whitespace. Note that `k` may be outside bounds of `s`. fn all_whitespace(s: &str, col: CharPos) -> Option<usize> { let mut idx = 0; for (i, ch) in s.char_indices().take(col.to_usize()) { @@ -222,7 +227,7 @@ fn trim_whitespace_prefix_and_push_line(lines: &mut Vec<String>, s: String, col: lines.push(s1); } -fn read_block_comment(rdr: &mut StringReader, +fn read_block_comment(rdr: &mut StringReader<'_>, code_to_the_left: bool, comments: &mut Vec<Comment>) { debug!(">>> block comment"); @@ -312,7 +317,7 @@ fn read_block_comment(rdr: &mut StringReader, } -fn consume_comment(rdr: &mut StringReader, +fn consume_comment(rdr: &mut StringReader<'_>, comments: &mut Vec<Comment>, code_to_the_left: &mut bool, anything_to_the_left: &mut bool) { diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 06f9162a400..9168d4b61c1 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,9 +1,10 @@ -use ast::{self, Ident}; +use crate::ast::{self, Ident}; +use crate::source_map::{SourceMap, FilePathMapping}; +use crate::errors::{Applicability, FatalError, Diagnostic, DiagnosticBuilder}; +use crate::parse::{token, ParseSess}; +use crate::symbol::{Symbol, keywords}; + use syntax_pos::{self, BytePos, CharPos, Pos, Span, NO_EXPANSION}; -use source_map::{SourceMap, FilePathMapping}; -use errors::{Applicability, FatalError, Diagnostic, DiagnosticBuilder}; -use parse::{token, ParseSess}; -use symbol::{Symbol, keywords}; use core::unicode::property::Pattern_White_Space; use std::borrow::Cow; @@ -11,6 +12,7 @@ use std::char; use std::iter; use std::mem::replace; use rustc_data_structures::sync::Lrc; +use log::debug; pub mod comments; mod tokentrees; @@ -31,6 +33,15 @@ impl Default for TokenAndSpan { } } +#[derive(Clone, Debug)] +pub struct UnmatchedBrace { + pub expected_delim: token::DelimToken, + pub found_delim: token::DelimToken, + pub found_span: Span, + pub unclosed_span: Option<Span>, + pub candidate_span: Option<Span>, +} + pub struct StringReader<'a> { pub sess: &'a ParseSess, /// The absolute offset within the source_map of the next character to read @@ -56,6 +67,7 @@ pub struct StringReader<'a> { span_src_raw: Span, /// Stack of open delimiters and their spans. Used for error message. open_braces: Vec<(token::DelimToken, Span)>, + crate unmatched_braces: Vec<UnmatchedBrace>, /// The type and spans for all braces /// /// Used only for error recovery when arriving to EOF with mismatched braces. @@ -100,7 +112,7 @@ impl<'a> StringReader<'a> { self.unwrap_or_abort(res) } - /// Return the next token. EFFECT: advances the string_reader. + /// Returns the next token. EFFECT: advances the string_reader. pub fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { assert!(self.fatal_errs.is_empty()); let ret_val = TokenAndSpan { @@ -220,6 +232,7 @@ impl<'a> StringReader<'a> { span: syntax_pos::DUMMY_SP, span_src_raw: syntax_pos::DUMMY_SP, open_braces: Vec::new(), + unmatched_braces: Vec::new(), matching_delim_spans: Vec::new(), override_span, last_unclosed_found_span: None, @@ -412,7 +425,7 @@ impl<'a> StringReader<'a> { self.with_str_from_to(start, self.pos, f) } - /// Create a Name from a given offset to the current offset, each + /// Creates a Name from a given offset to the current offset, each /// adjusted 1 towards each other (assumes that on either side there is a /// single-byte delimiter). fn name_from(&self, start: BytePos) -> ast::Name { @@ -449,7 +462,7 @@ impl<'a> StringReader<'a> { } return s.into(); - fn translate_crlf_(rdr: &StringReader, + fn translate_crlf_(rdr: &StringReader<'_>, start: BytePos, s: &str, mut j: usize, @@ -657,7 +670,7 @@ impl<'a> StringReader<'a> { } /// If there is whitespace, shebang, or a comment, scan it. Otherwise, - /// return None. + /// return `None`. fn scan_whitespace_or_comment(&mut self) -> Option<TokenAndSpan> { match self.ch.unwrap_or('\0') { // # to handle shebang at start of file -- this is the entry point @@ -907,7 +920,7 @@ impl<'a> StringReader<'a> { /// in a byte, (non-raw) byte string, char, or (non-raw) string literal. /// `start` is the position of `first_source_char`, which is already consumed. /// - /// Returns true if there was a valid char/byte, false otherwise. + /// Returns `true` if there was a valid char/byte. fn scan_char_or_byte(&mut self, start: BytePos, first_source_char: char, @@ -1139,7 +1152,7 @@ impl<'a> StringReader<'a> { } } - /// Check that a base is valid for a floating literal, emitting a nice + /// Checks that a base is valid for a floating literal, emitting a nice /// error if it isn't. fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: usize) { match base { @@ -1172,7 +1185,7 @@ impl<'a> StringReader<'a> { } } - /// Return the next token from the string, advances the input past that + /// Returns the next token from the string, advances the input past that /// token, and updates the interner fn next_token_inner(&mut self) -> Result<token::Token, ()> { let c = self.ch; @@ -1866,19 +1879,20 @@ fn char_at(s: &str, byte: usize) -> char { mod tests { use super::*; - use ast::{Ident, CrateConfig}; - use symbol::Symbol; - use syntax_pos::{BytePos, Span, NO_EXPANSION}; - use source_map::SourceMap; - use errors; - use feature_gate::UnstableFeatures; - use parse::token; + use crate::ast::{Ident, CrateConfig}; + use crate::symbol::Symbol; + use crate::source_map::SourceMap; + use crate::errors; + use crate::feature_gate::UnstableFeatures; + use crate::parse::token; + use crate::diagnostics::plugin::ErrorMap; + use crate::with_globals; use std::io; use std::path::PathBuf; - use diagnostics::plugin::ErrorMap; + use syntax_pos::{BytePos, Span, NO_EXPANSION}; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::sync::Lock; - use with_globals; + fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess { let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), Some(sm.clone()), @@ -1943,7 +1957,7 @@ mod tests { // check that the given reader produces the desired stream // of tokens (stop checking after exhausting the expected vec) - fn check_tokenization(mut string_reader: StringReader, expected: Vec<token::Token>) { + fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<token::Token>) { for expected_tok in &expected { assert_eq!(&string_reader.next_token().tok, expected_tok); } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index d219f29f06c..0db36c84cdf 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -1,7 +1,7 @@ -use print::pprust::token_to_string; -use parse::lexer::StringReader; -use parse::{token, PResult}; -use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint}; +use crate::print::pprust::token_to_string; +use crate::parse::lexer::{StringReader, UnmatchedBrace}; +use crate::parse::{token, PResult}; +use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint}; impl<'a> StringReader<'a> { // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. @@ -101,38 +101,38 @@ impl<'a> StringReader<'a> { } // Incorrect delimiter. token::CloseDelim(other) => { - let token_str = token_to_string(&self.token); + let mut unclosed_delimiter = None; + let mut candidate = None; if self.last_unclosed_found_span != Some(self.span) { // do not complain about the same unclosed delimiter multiple times self.last_unclosed_found_span = Some(self.span); - let msg = format!("incorrect close delimiter: `{}`", token_str); - let mut err = self.sess.span_diagnostic.struct_span_err( - self.span, - &msg, - ); - err.span_label(self.span, "incorrect close delimiter"); // This is a conservative error: only report the last unclosed // delimiter. The previous unclosed delimiters could actually be // closed! The parser just hasn't gotten to them yet. if let Some(&(_, sp)) = self.open_braces.last() { - err.span_label(sp, "un-closed delimiter"); + unclosed_delimiter = Some(sp); }; if let Some(current_padding) = sm.span_to_margin(self.span) { for (brace, brace_span) in &self.open_braces { if let Some(padding) = sm.span_to_margin(*brace_span) { // high likelihood of these two corresponding if current_padding == padding && brace == &other { - err.span_label( - *brace_span, - "close delimiter possibly meant for this", - ); + candidate = Some(*brace_span); } } } } - err.emit(); + let (tok, _) = self.open_braces.pop().unwrap(); + self.unmatched_braces.push(UnmatchedBrace { + expected_delim: tok, + found_delim: other, + found_span: self.span, + unclosed_span: unclosed_delimiter, + candidate_span: candidate, + }); + } else { + self.open_braces.pop(); } - self.open_braces.pop().unwrap(); // If the incorrect delimiter matches an earlier opening // delimiter, then don't consume it (it can be used to diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index 7da4284c0e4..75862178169 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -2,7 +2,7 @@ // http://www.unicode.org/Public/security/10.0.0/confusables.txt use syntax_pos::{Span, NO_EXPANSION}; -use errors::{Applicability, DiagnosticBuilder}; +use crate::errors::{Applicability, DiagnosticBuilder}; use super::StringReader; const UNICODE_ARRAY: &[(char, &str, char)] = &[ diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index c7330004d6d..69940ae621c 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -1,16 +1,19 @@ -//! The main parser interface +//! The main parser interface. + +use crate::ast::{self, CrateConfig, NodeId}; +use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; +use crate::source_map::{SourceMap, FilePathMapping}; +use crate::errors::{FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder}; +use crate::feature_gate::UnstableFeatures; +use crate::parse::parser::Parser; +use crate::symbol::Symbol; +use crate::tokenstream::{TokenStream, TokenTree}; +use crate::diagnostics::plugin::ErrorMap; +use crate::print::pprust::token_to_string; use rustc_data_structures::sync::{Lrc, Lock}; -use ast::{self, CrateConfig, NodeId}; -use early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; -use source_map::{SourceMap, FilePathMapping}; use syntax_pos::{Span, SourceFile, FileName, MultiSpan}; -use errors::{FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder}; -use feature_gate::UnstableFeatures; -use parse::parser::Parser; -use symbol::Symbol; -use tokenstream::{TokenStream, TokenTree}; -use diagnostics::plugin::ErrorMap; +use log::debug; use rustc_data_structures::fx::FxHashSet; use std::borrow::Cow; @@ -35,12 +38,11 @@ pub struct ParseSess { pub unstable_features: UnstableFeatures, pub config: CrateConfig, pub missing_fragment_specifiers: Lock<FxHashSet<Span>>, - /// Places where raw identifiers were used. This is used for feature gating - /// raw identifiers + /// Places where raw identifiers were used. This is used for feature-gating raw identifiers. pub raw_identifier_spans: Lock<Vec<Span>>, - /// The registered diagnostics codes + /// The registered diagnostics codes. crate registered_diagnostics: Lock<ErrorMap>, - /// Used to determine and report recursive mod inclusions + /// Used to determine and report recursive module inclusions. included_mod_stack: Lock<Vec<PathBuf>>, source_map: Lrc<SourceMap>, pub buffered_lints: Lock<Vec<BufferedEarlyLint>>, @@ -125,31 +127,33 @@ pub fn parse_crate_attrs_from_file<'a>(input: &Path, sess: &'a ParseSess) } pub fn parse_crate_from_source_str(name: FileName, source: String, sess: &ParseSess) - -> PResult<ast::Crate> { + -> PResult<'_, ast::Crate> { new_parser_from_source_str(sess, name, source).parse_crate_mod() } pub fn parse_crate_attrs_from_source_str(name: FileName, source: String, sess: &ParseSess) - -> PResult<Vec<ast::Attribute>> { + -> PResult<'_, Vec<ast::Attribute>> { new_parser_from_source_str(sess, name, source).parse_inner_attributes() } -pub fn parse_stream_from_source_str(name: FileName, source: String, sess: &ParseSess, - override_span: Option<Span>) - -> TokenStream { +pub fn parse_stream_from_source_str( + name: FileName, + source: String, + sess: &ParseSess, + override_span: Option<Span>, +) -> (TokenStream, Vec<lexer::UnmatchedBrace>) { source_file_to_stream(sess, sess.source_map().new_source_file(name, source), override_span) } -/// Create a new parser from a source string -pub fn new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String) - -> Parser { +/// Creates a new parser from a source string. +pub fn new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String) -> Parser<'_> { panictry_buffer!(&sess.span_diagnostic, maybe_new_parser_from_source_str(sess, name, source)) } -/// Create a new parser from a source string. Returns any buffered errors from lexing the initial +/// Creates a new parser from a source string. Returns any buffered errors from lexing the initial /// token stream. pub fn maybe_new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String) - -> Result<Parser, Vec<Diagnostic>> + -> Result<Parser<'_>, Vec<Diagnostic>> { let mut parser = maybe_source_file_to_parser(sess, sess.source_map().new_source_file(name, source))?; @@ -157,13 +161,13 @@ pub fn maybe_new_parser_from_source_str(sess: &ParseSess, name: FileName, source Ok(parser) } -/// Create a new parser, handling errors as appropriate +/// Creates a new parser, handling errors as appropriate /// if the file doesn't exist pub fn new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path) -> Parser<'a> { source_file_to_parser(sess, file_to_source_file(sess, path, None)) } -/// Create a new parser, returning buffered diagnostics if the file doesn't +/// Creates a new parser, returning buffered diagnostics if the file doesn't /// exist or from lexing the initial token stream. pub fn maybe_new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path) -> Result<Parser<'a>, Vec<Diagnostic>> { @@ -186,19 +190,21 @@ crate fn new_sub_parser_from_file<'a>(sess: &'a ParseSess, } /// Given a source_file and config, return a parser -fn source_file_to_parser(sess: & ParseSess, source_file: Lrc<SourceFile>) -> Parser { +fn source_file_to_parser(sess: &ParseSess, source_file: Lrc<SourceFile>) -> Parser<'_> { panictry_buffer!(&sess.span_diagnostic, maybe_source_file_to_parser(sess, source_file)) } /// Given a source_file and config, return a parser. Returns any buffered errors from lexing the /// initial token stream. -fn maybe_source_file_to_parser(sess: &ParseSess, source_file: Lrc<SourceFile>) - -> Result<Parser, Vec<Diagnostic>> -{ +fn maybe_source_file_to_parser( + sess: &ParseSess, + source_file: Lrc<SourceFile>, +) -> Result<Parser<'_>, Vec<Diagnostic>> { let end_pos = source_file.end_pos; - let mut parser = stream_to_parser(sess, maybe_file_to_stream(sess, source_file, None)?); - + let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?; + let mut parser = stream_to_parser(sess, stream); + parser.unclosed_delims = unclosed_delims; if parser.token == token::Eof && parser.span.is_dummy() { parser.span = Span::new(end_pos, end_pos, parser.span.ctxt()); } @@ -208,7 +214,7 @@ fn maybe_source_file_to_parser(sess: &ParseSess, source_file: Lrc<SourceFile>) // must preserve old name for now, because quote! from the *existing* // compiler expands into it -pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser { +pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser<'_> { stream_to_parser(sess, tts.into_iter().collect()) } @@ -232,7 +238,7 @@ fn try_file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option<Span>) } /// Given a session and a path and an optional span (for error reporting), -/// add the path to the session's source_map and return the new source_file. +/// add the path to the session's `source_map` and return the new `source_file`. fn file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option<Span>) -> Lrc<SourceFile> { match try_file_to_source_file(sess, path, spanopt) { @@ -244,37 +250,56 @@ fn file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option<Span>) } } -/// Given a source_file, produce a sequence of token-trees -pub fn source_file_to_stream(sess: &ParseSess, - source_file: Lrc<SourceFile>, - override_span: Option<Span>) -> TokenStream { +/// Given a source_file, produces a sequence of token trees. +pub fn source_file_to_stream( + sess: &ParseSess, + source_file: Lrc<SourceFile>, + override_span: Option<Span>, +) -> (TokenStream, Vec<lexer::UnmatchedBrace>) { panictry_buffer!(&sess.span_diagnostic, maybe_file_to_stream(sess, source_file, override_span)) } -/// Given a source file, produce a sequence of token-trees. Returns any buffered errors from +/// Given a source file, produces a sequence of token trees. Returns any buffered errors from /// parsing the token tream. -pub fn maybe_file_to_stream(sess: &ParseSess, - source_file: Lrc<SourceFile>, - override_span: Option<Span>) -> Result<TokenStream, Vec<Diagnostic>> { +pub fn maybe_file_to_stream( + sess: &ParseSess, + source_file: Lrc<SourceFile>, + override_span: Option<Span>, +) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> { let mut srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?; srdr.real_token(); match srdr.parse_all_token_trees() { - Ok(stream) => Ok(stream), + Ok(stream) => Ok((stream, srdr.unmatched_braces)), Err(err) => { let mut buffer = Vec::with_capacity(1); err.buffer(&mut buffer); + // Not using `emit_unclosed_delims` to use `db.buffer` + for unmatched in srdr.unmatched_braces { + let mut db = sess.span_diagnostic.struct_span_err(unmatched.found_span, &format!( + "incorrect close delimiter: `{}`", + token_to_string(&token::Token::CloseDelim(unmatched.found_delim)), + )); + db.span_label(unmatched.found_span, "incorrect close delimiter"); + if let Some(sp) = unmatched.candidate_span { + db.span_label(sp, "close delimiter possibly meant for this"); + } + if let Some(sp) = unmatched.unclosed_span { + db.span_label(sp, "un-closed delimiter"); + } + db.buffer(&mut buffer); + } Err(buffer) } } } -/// Given stream and the `ParseSess`, produce a parser -pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser { +/// Given stream and the `ParseSess`, produces a parser. +pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> { Parser::new(sess, stream, None, true, false) } -/// Parse a string representing a character literal into its final form. +/// Parses a string representing a character literal into its final form. /// Rather than just accepting/rejecting a given literal, unescapes it as /// well. Can take any slice prefixed by a character escape. Returns the /// character and the number of characters consumed. @@ -333,15 +358,14 @@ fn char_lit(lit: &str, diag: Option<(Span, &Handler)>) -> (char, isize) { } } -/// Parse a string representing a string literal into its final form. Does -/// unescaping. +/// Parses a string representing a string literal into its final form. Does unescaping. pub fn str_lit(lit: &str, diag: Option<(Span, &Handler)>) -> String { debug!("str_lit: given {}", lit.escape_default()); let mut res = String::with_capacity(lit.len()); let error = |i| format!("lexer should have rejected {} at {}", lit, i); - /// Eat everything up to a non-whitespace + /// Eat everything up to a non-whitespace. fn eat<'a>(it: &mut iter::Peekable<str::CharIndices<'a>>) { loop { match it.peek().map(|x| x.1) { @@ -402,7 +426,7 @@ pub fn str_lit(lit: &str, diag: Option<(Span, &Handler)>) -> String { res } -/// Parse a string representing a raw string literal into its final form. The +/// Parses a string representing a raw string literal into its final form. The /// only operation this does is convert embedded CRLF into a single LF. fn raw_str_lit(lit: &str) -> String { debug!("raw_str_lit: given {}", lit.escape_default()); @@ -528,7 +552,7 @@ fn float_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>) filtered_float_lit(Symbol::intern(s), suffix, diag) } -/// Parse a string representing a byte literal into its final form. Similar to `char_lit` +/// Parses a string representing a byte literal into its final form. Similar to `char_lit`. fn byte_lit(lit: &str) -> (u8, usize) { let err = |i| format!("lexer accepted invalid byte literal {} step {}", lit, i); @@ -565,7 +589,7 @@ fn byte_str_lit(lit: &str) -> Lrc<Vec<u8>> { let error = |i| panic!("lexer should have rejected {} at {}", lit, i); - /// Eat everything up to a non-whitespace + /// Eat everything up to a non-whitespace. fn eat<I: Iterator<Item=(usize, u8)>>(it: &mut iter::Peekable<I>) { loop { match it.peek().map(|x| x.1) { @@ -732,10 +756,11 @@ fn integer_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>) }) } -/// `SeqSep` : a sequence separator (token) -/// and whether a trailing separator is allowed. +/// A sequence separator. pub struct SeqSep { + /// The seperator token. pub sep: Option<token::Token>, + /// `true` if a trailing separator is allowed. pub trailing_sep_allowed: bool, } @@ -758,22 +783,22 @@ impl SeqSep { #[cfg(test)] mod tests { use super::*; + use crate::ast::{self, Ident, PatKind}; + use crate::attr::first_attr_value_str_by_name; + use crate::ptr::P; + use crate::print::pprust::item_to_string; + use crate::tokenstream::{DelimSpan, TokenTree}; + use crate::util::parser_testing::string_to_stream; + use crate::util::parser_testing::{string_to_expr, string_to_item}; + use crate::with_globals; use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION}; - use ast::{self, Ident, PatKind}; - use attr::first_attr_value_str_by_name; - use ptr::P; - use print::pprust::item_to_string; - use tokenstream::{DelimSpan, TokenTree}; - use util::parser_testing::string_to_stream; - use util::parser_testing::{string_to_expr, string_to_item}; - use with_globals; /// Parses an item. /// /// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` /// when a syntax error occurred. fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) - -> PResult<Option<P<ast::Item>>> { + -> PResult<'_, Option<P<ast::Item>>> { new_parser_from_source_str(sess, name, source).parse_item() } @@ -913,20 +938,20 @@ mod tests { struct PatIdentVisitor { spans: Vec<Span> } - impl<'a> ::visit::Visitor<'a> for PatIdentVisitor { + impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor { fn visit_pat(&mut self, p: &'a ast::Pat) { match p.node { PatKind::Ident(_ , ref spannedident, _) => { self.spans.push(spannedident.span.clone()); } _ => { - ::visit::walk_pat(self, p); + crate::visit::walk_pat(self, p); } } } } let mut v = PatIdentVisitor { spans: Vec::new() }; - ::visit::walk_item(&mut v, &item); + crate::visit::walk_item(&mut v, &item); return v.spans; } @@ -1007,7 +1032,7 @@ mod tests { fn ttdelim_span() { fn parse_expr_from_source_str( name: FileName, source: String, sess: &ParseSess - ) -> PResult<P<ast::Expr>> { + ) -> PResult<'_, P<ast::Expr>> { new_parser_from_source_str(sess, name, source).parse_expr() } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 514b2952c50..67154305735 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1,53 +1,55 @@ +use crate::ast::{AngleBracketedArgs, ParenthesizedArgs, AttrStyle, BareFnTy}; +use crate::ast::{GenericBound, TraitBoundModifier}; +use crate::ast::Unsafety; +use crate::ast::{Mod, AnonConst, Arg, Arm, Guard, Attribute, BindingMode, TraitItemKind}; +use crate::ast::Block; +use crate::ast::{BlockCheckMode, CaptureBy, Movability}; +use crate::ast::{Constness, Crate}; +use crate::ast::Defaultness; +use crate::ast::EnumDef; +use crate::ast::{Expr, ExprKind, RangeLimits}; +use crate::ast::{Field, FnDecl, FnHeader}; +use crate::ast::{ForeignItem, ForeignItemKind, FunctionRetTy}; +use crate::ast::{GenericParam, GenericParamKind}; +use crate::ast::GenericArg; +use crate::ast::{Ident, ImplItem, IsAsync, IsAuto, Item, ItemKind}; +use crate::ast::{Label, Lifetime, Lit, LitKind}; +use crate::ast::Local; +use crate::ast::MacStmtStyle; +use crate::ast::{Mac, Mac_, MacDelimiter}; +use crate::ast::{MutTy, Mutability}; +use crate::ast::{Pat, PatKind, PathSegment}; +use crate::ast::{PolyTraitRef, QSelf}; +use crate::ast::{Stmt, StmtKind}; +use crate::ast::{VariantData, StructField}; +use crate::ast::StrStyle; +use crate::ast::SelfKind; +use crate::ast::{TraitItem, TraitRef, TraitObjectSyntax}; +use crate::ast::{Ty, TyKind, TypeBinding, GenericBounds}; +use crate::ast::{Visibility, VisibilityKind, WhereClause, CrateSugar}; +use crate::ast::{UseTree, UseTreeKind}; +use crate::ast::{BinOpKind, UnOp}; +use crate::ast::{RangeEnd, RangeSyntax}; +use crate::{ast, attr}; +use crate::ext::base::DummyResult; +use crate::source_map::{self, SourceMap, Spanned, respan}; +use crate::errors::{self, Applicability, DiagnosticBuilder, DiagnosticId}; +use crate::parse::{self, SeqSep, classify, token}; +use crate::parse::lexer::{TokenAndSpan, UnmatchedBrace}; +use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; +use crate::parse::token::DelimToken; +use crate::parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership}; +use crate::util::parser::{AssocOp, Fixity}; +use crate::print::pprust; +use crate::ptr::P; +use crate::parse::PResult; +use crate::ThinVec; +use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; +use crate::symbol::{Symbol, keywords}; + use rustc_target::spec::abi::{self, Abi}; -use ast::{AngleBracketedArgs, ParenthesizedArgs, AttrStyle, BareFnTy}; -use ast::{GenericBound, TraitBoundModifier}; -use ast::Unsafety; -use ast::{Mod, AnonConst, Arg, Arm, Guard, Attribute, BindingMode, TraitItemKind}; -use ast::Block; -use ast::{BlockCheckMode, CaptureBy, Movability}; -use ast::{Constness, Crate}; -use ast::Defaultness; -use ast::EnumDef; -use ast::{Expr, ExprKind, RangeLimits}; -use ast::{Field, FnDecl, FnHeader}; -use ast::{ForeignItem, ForeignItemKind, FunctionRetTy}; -use ast::{GenericParam, GenericParamKind}; -use ast::GenericArg; -use ast::{Ident, ImplItem, IsAsync, IsAuto, Item, ItemKind}; -use ast::{Label, Lifetime, Lit, LitKind}; -use ast::Local; -use ast::MacStmtStyle; -use ast::{Mac, Mac_, MacDelimiter}; -use ast::{MutTy, Mutability}; -use ast::{Pat, PatKind, PathSegment}; -use ast::{PolyTraitRef, QSelf}; -use ast::{Stmt, StmtKind}; -use ast::{VariantData, StructField}; -use ast::StrStyle; -use ast::SelfKind; -use ast::{TraitItem, TraitRef, TraitObjectSyntax}; -use ast::{Ty, TyKind, TypeBinding, GenericBounds}; -use ast::{Visibility, VisibilityKind, WhereClause, CrateSugar}; -use ast::{UseTree, UseTreeKind}; -use ast::{BinOpKind, UnOp}; -use ast::{RangeEnd, RangeSyntax}; -use {ast, attr}; -use ext::base::DummyResult; -use source_map::{self, SourceMap, Spanned, respan}; use syntax_pos::{self, Span, MultiSpan, BytePos, FileName}; -use errors::{self, Applicability, DiagnosticBuilder, DiagnosticId}; -use parse::{self, SeqSep, classify, token}; -use parse::lexer::TokenAndSpan; -use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use parse::token::DelimToken; -use parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership}; -use util::parser::{AssocOp, Fixity}; -use print::pprust; -use ptr::P; -use parse::PResult; -use ThinVec; -use tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; -use symbol::{Symbol, keywords}; +use log::{debug, trace}; use std::borrow::Cow; use std::cmp; @@ -64,7 +66,7 @@ pub enum AliasKind { Existential(GenericBounds), } -bitflags! { +bitflags::bitflags! { struct Restrictions: u8 { const STMT_EXPR = 1 << 0; const NO_STRUCT_LITERAL = 1 << 1; @@ -73,7 +75,7 @@ bitflags! { type ItemInfo = (Ident, ItemKind, Option<Vec<Attribute>>); -/// How to parse a path. +/// Specifies how to parse a path. #[derive(Copy, Clone, PartialEq)] pub enum PathStyle { /// In some contexts, notably in expressions, paths with generic arguments are ambiguous @@ -109,7 +111,7 @@ enum BlockMode { Ignore, } -/// Possibly accept an `token::Interpolated` expression (a pre-parsed expression +/// Possibly accepts an `token::Interpolated` expression (a pre-parsed expression /// dropped into the token stream, which happens while parsing the result of /// macro expansion). Placement of these is not as complex as I feared it would /// be. The important thing is to make sure that lookahead doesn't balk at @@ -249,6 +251,11 @@ pub struct Parser<'a> { /// /// See the comments in the `parse_path_segment` function for more details. crate unmatched_angle_bracket_count: u32, + crate max_angle_bracket_count: u32, + /// List of all unclosed delimiters found by the lexer. If an entry is used for error recovery + /// it gets removed from here. Every entry left at the end gets emitted as an independent + /// error. + crate unclosed_delims: Vec<UnmatchedBrace>, } @@ -395,6 +402,7 @@ crate enum TokenType { Ident, Path, Type, + Const, } impl TokenType { @@ -407,15 +415,16 @@ impl TokenType { TokenType::Ident => "identifier".to_string(), TokenType::Path => "path".to_string(), TokenType::Type => "type".to_string(), + TokenType::Const => "const".to_string(), } } } -/// Returns true if `IDENT t` can start a type - `IDENT::a::b`, `IDENT<u8, u8>`, +/// Returns `true` if `IDENT t` can start a type -- `IDENT::a::b`, `IDENT<u8, u8>`, /// `IDENT<<u8 as Trait>::AssocTy>`. /// /// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes -/// that IDENT is not the ident of a fn trait +/// that `IDENT` is not the ident of a fn trait. fn can_continue_type_after_non_fn_ident(t: &token::Token) -> bool { t == &token::ModSep || t == &token::Lt || t == &token::BinOp(token::Shl) @@ -453,7 +462,7 @@ pub enum Error { impl Error { fn span_err<S: Into<MultiSpan>>(self, sp: S, - handler: &errors::Handler) -> DiagnosticBuilder { + handler: &errors::Handler) -> DiagnosticBuilder<'_> { match self { Error::FileNotFoundForModule { ref mod_name, ref default_path, @@ -516,7 +525,7 @@ impl From<P<Expr>> for LhsExpr { } } -/// Create a placeholder argument. +/// Creates a placeholder argument. fn dummy_arg(span: Span) -> Arg { let ident = Ident::new(keywords::Invalid.name(), span); let pat = P(Pat { @@ -571,6 +580,8 @@ impl<'a> Parser<'a> { desugar_doc_comments, cfg_mods: true, unmatched_angle_bracket_count: 0, + max_angle_bracket_count: 0, + unclosed_delims: Vec::new(), }; let tok = parser.next_tok(); @@ -603,7 +614,7 @@ impl<'a> Parser<'a> { next } - /// Convert the current token to a string using self's reader + /// Converts the current token to a string using `self`'s reader. pub fn this_token_to_string(&self) -> String { pprust::token_to_string(&self.token) } @@ -638,13 +649,12 @@ impl<'a> Parser<'a> { } } - /// Expect and consume the token t. Signal an error if - /// the next token is not t. - pub fn expect(&mut self, t: &token::Token) -> PResult<'a, ()> { + /// Expects and consumes the token `t`. Signals an error if the next token is not `t`. + pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> { if self.expected_tokens.is_empty() { if self.token == *t { self.bump(); - Ok(()) + Ok(false) } else { let token_str = pprust::token_to_string(t); let this_token_str = self.this_token_descr(); @@ -659,6 +669,12 @@ impl<'a> Parser<'a> { self.sess.source_map().next_point(self.prev_span) }; let label_exp = format!("expected `{}`", token_str); + match self.recover_closing_delimiter(&[t.clone()], err) { + Err(e) => err = e, + Ok(recovered) => { + return Ok(recovered); + } + } let cm = self.sess.source_map(); match (cm.lookup_line(self.span.lo()), cm.lookup_line(sp.lo())) { (Ok(ref a), Ok(ref b)) if a.line == b.line => { @@ -678,12 +694,64 @@ impl<'a> Parser<'a> { } } + fn recover_closing_delimiter( + &mut self, + tokens: &[token::Token], + mut err: DiagnosticBuilder<'a>, + ) -> PResult<'a, bool> { + let mut pos = None; + // we want to use the last closing delim that would apply + for (i, unmatched) in self.unclosed_delims.iter().enumerate().rev() { + if tokens.contains(&token::CloseDelim(unmatched.expected_delim)) + && Some(self.span) > unmatched.unclosed_span + { + pos = Some(i); + } + } + match pos { + Some(pos) => { + // Recover and assume that the detected unclosed delimiter was meant for + // this location. Emit the diagnostic and act as if the delimiter was + // present for the parser's sake. + + // Don't attempt to recover from this unclosed delimiter more than once. + let unmatched = self.unclosed_delims.remove(pos); + let delim = TokenType::Token(token::CloseDelim(unmatched.expected_delim)); + + // We want to suggest the inclusion of the closing delimiter where it makes + // the most sense, which is immediately after the last token: + // + // {foo(bar {}} + // - ^ + // | | + // | help: `)` may belong here (FIXME: #58270) + // | + // unclosed delimiter + if let Some(sp) = unmatched.unclosed_span { + err.span_label(sp, "unclosed delimiter"); + } + err.span_suggestion_short( + self.sess.source_map().next_point(self.prev_span), + &format!("{} may belong here", delim.to_string()), + delim.to_string(), + Applicability::MaybeIncorrect, + ); + err.emit(); + self.expected_tokens.clear(); // reduce errors + Ok(true) + } + _ => Err(err), + } + } + /// Expect next token to be edible or inedible token. If edible, /// then consume it; if inedible, then return without consuming /// anything. Signal a fatal error if next token is unexpected. - pub fn expect_one_of(&mut self, - edible: &[token::Token], - inedible: &[token::Token]) -> PResult<'a, ()>{ + pub fn expect_one_of( + &mut self, + edible: &[token::Token], + inedible: &[token::Token], + ) -> PResult<'a, bool /* recovered */> { fn tokens_to_string(tokens: &[TokenType]) -> String { let mut i = tokens.iter(); // This might be a sign we need a connect method on Iterator. @@ -703,10 +771,10 @@ impl<'a> Parser<'a> { } if edible.contains(&self.token) { self.bump(); - Ok(()) + Ok(false) } else if inedible.contains(&self.token) { // leave it in the input - Ok(()) + Ok(false) } else { let mut expected = edible.iter() .map(|x| TokenType::Token(x.clone())) @@ -757,6 +825,15 @@ impl<'a> Parser<'a> { } else { label_sp }; + match self.recover_closing_delimiter(&expected.iter().filter_map(|tt| match tt { + TokenType::Token(t) => Some(t.clone()), + _ => None, + }).collect::<Vec<_>>(), err) { + Err(e) => err = e, + Ok(recovered) => { + return Ok(recovered); + } + } let cm = self.sess.source_map(); match (cm.lookup_line(self.span.lo()), cm.lookup_line(sp.lo())) { @@ -789,7 +866,7 @@ impl<'a> Parser<'a> { } } - /// returns the span of expr, if it was not interpolated or the span of the interpolated token + /// Returns the span of expr, if it was not interpolated or the span of the interpolated token. fn interpolated_or_expr_span(&self, expr: PResult<'a, P<Expr>>) -> PResult<'a, (Span, P<Expr>)> { @@ -863,7 +940,7 @@ impl<'a> Parser<'a> { } } - /// Check if the next token is `tok`, and return `true` if so. + /// Checks if the next token is `tok`, and returns `true` if so. /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not /// encountered. @@ -873,8 +950,7 @@ impl<'a> Parser<'a> { is_present } - /// Consume token 'tok' if it exists. Returns true if the given - /// token was present, false otherwise. + /// Consumes a token 'tok' if it exists. Returns whether the given token was present. pub fn eat(&mut self, tok: &token::Token) -> bool { let is_present = self.check(tok); if is_present { self.bump() } @@ -886,8 +962,8 @@ impl<'a> Parser<'a> { self.token.is_keyword(kw) } - /// If the next token is the given keyword, eat it and return - /// true. Otherwise, return false. + /// If the next token is the given keyword, eats it and returns + /// `true`. Otherwise, returns `false`. pub fn eat_keyword(&mut self, kw: keywords::Keyword) -> bool { if self.check_keyword(kw) { self.bump(); @@ -906,9 +982,9 @@ impl<'a> Parser<'a> { } } - /// If the given word is not a keyword, signal an error. - /// If the next token is not the given word, signal an error. - /// Otherwise, eat it. + /// If the given word is not a keyword, signals an error. + /// If the next token is not the given word, signals an error. + /// Otherwise, eats it. fn expect_keyword(&mut self, kw: keywords::Keyword) -> PResult<'a, ()> { if !self.eat_keyword(kw) { self.unexpected() @@ -944,11 +1020,20 @@ impl<'a> Parser<'a> { } } - /// Expect and consume a `+`. if `+=` is seen, replace it with a `=` - /// and continue. If a `+` is not seen, return false. + fn check_const_arg(&mut self) -> bool { + if self.token.can_begin_const_arg() { + true + } else { + self.expected_tokens.push(TokenType::Const); + false + } + } + + /// Expects and consumes a `+`. if `+=` is seen, replaces it with a `=` + /// and continues. If a `+` is not seen, returns `false`. /// - /// This is using when token splitting += into +. - /// See issue 47856 for an example of when this may occur. + /// This is used when token-splitting `+=` into `+`. + /// See issue #47856 for an example of when this may occur. fn eat_plus(&mut self) -> bool { self.expected_tokens.push(TokenType::Token(token::BinOp(token::Plus))); match self.token { @@ -967,7 +1052,7 @@ impl<'a> Parser<'a> { /// Checks to see if the next token is either `+` or `+=`. - /// Otherwise returns false. + /// Otherwise returns `false`. fn check_plus(&mut self) -> bool { if self.token.is_like_plus() { true @@ -978,8 +1063,8 @@ impl<'a> Parser<'a> { } } - /// Expect and consume an `&`. If `&&` is seen, replace it with a single - /// `&` and continue. If an `&` is not seen, signal an error. + /// Expects and consumes an `&`. If `&&` is seen, replaces it with a single + /// `&` and continues. If an `&` is not seen, signals an error. fn expect_and(&mut self) -> PResult<'a, ()> { self.expected_tokens.push(TokenType::Token(token::BinOp(token::And))); match self.token { @@ -995,8 +1080,8 @@ impl<'a> Parser<'a> { } } - /// Expect and consume an `|`. If `||` is seen, replace it with a single - /// `|` and continue. If an `|` is not seen, signal an error. + /// Expects and consumes an `|`. If `||` is seen, replaces it with a single + /// `|` and continues. If an `|` is not seen, signals an error. fn expect_or(&mut self) -> PResult<'a, ()> { self.expected_tokens.push(TokenType::Token(token::BinOp(token::Or))); match self.token { @@ -1028,8 +1113,9 @@ impl<'a> Parser<'a> { } } - /// Attempt to consume a `<`. If `<<` is seen, replace it with a single - /// `<` and continue. If a `<` is not seen, return false. + /// Attempts to consume a `<`. If `<<` is seen, replaces it with a single + /// `<` and continue. If `<-` is seen, replaces it with a single `<` + /// and continue. If a `<` is not seen, returns false. /// /// This is meant to be used when parsing generics on a path to get the /// starting token. @@ -1045,12 +1131,18 @@ impl<'a> Parser<'a> { self.bump_with(token::Lt, span); true } + token::LArrow => { + let span = self.span.with_lo(self.span.lo() + BytePos(1)); + self.bump_with(token::BinOp(token::Minus), span); + true + } _ => false, }; if ate { // See doc comment for `unmatched_angle_bracket_count`. self.unmatched_angle_bracket_count += 1; + self.max_angle_bracket_count += 1; debug!("eat_lt: (increment) count={:?}", self.unmatched_angle_bracket_count); } @@ -1065,9 +1157,8 @@ impl<'a> Parser<'a> { } } - /// Expect and consume a GT. if a >> is seen, replace it - /// with a single > and continue. If a GT is not seen, - /// signal an error. + /// Expects and consumes a single `>` token. if a `>>` is seen, replaces it + /// with a single `>` and continues. If a `>` is not seen, signals an error. fn expect_gt(&mut self) -> PResult<'a, ()> { self.expected_tokens.push(TokenType::Token(token::Gt)); let ate = match self.token { @@ -1091,18 +1182,18 @@ impl<'a> Parser<'a> { }; match ate { - Some(x) => { + Some(_) => { // See doc comment for `unmatched_angle_bracket_count`. self.unmatched_angle_bracket_count -= 1; debug!("expect_gt: (decrement) count={:?}", self.unmatched_angle_bracket_count); - Ok(x) + Ok(()) }, None => self.unexpected(), } } - /// Eat and discard tokens until one of `kets` is encountered. Respects token trees, + /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. fn eat_to_tokens(&mut self, kets: &[&token::Token]) { let handler = self.diagnostic(); @@ -1115,8 +1206,8 @@ impl<'a> Parser<'a> { } } - /// Parse a sequence, including the closing delimiter. The function - /// f must consume tokens until reaching the next separator or + /// Parses a sequence, including the closing delimiter. The function + /// `f` must consume tokens until reaching the next separator or /// closing bracket. pub fn parse_seq_to_end<T, F>(&mut self, ket: &token::Token, @@ -1125,19 +1216,22 @@ impl<'a> Parser<'a> { -> PResult<'a, Vec<T>> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, { - let val = self.parse_seq_to_before_end(ket, sep, f)?; - self.bump(); + let (val, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; + if !recovered { + self.bump(); + } Ok(val) } - /// Parse a sequence, not including the closing delimiter. The function - /// f must consume tokens until reaching the next separator or + /// Parses a sequence, not including the closing delimiter. The function + /// `f` must consume tokens until reaching the next separator or /// closing bracket. - pub fn parse_seq_to_before_end<T, F>(&mut self, - ket: &token::Token, - sep: SeqSep, - f: F) - -> PResult<'a, Vec<T>> + pub fn parse_seq_to_before_end<T, F>( + &mut self, + ket: &token::Token, + sep: SeqSep, + f: F, + ) -> PResult<'a, (Vec<T>, bool)> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T> { self.parse_seq_to_before_tokens(&[ket], sep, TokenExpectType::Expect, f) @@ -1149,10 +1243,11 @@ impl<'a> Parser<'a> { sep: SeqSep, expect: TokenExpectType, mut f: F, - ) -> PResult<'a, Vec<T>> + ) -> PResult<'a, (Vec<T>, bool /* recovered */)> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T> { - let mut first: bool = true; + let mut first = true; + let mut recovered = false; let mut v = vec![]; while !kets.iter().any(|k| { match expect { @@ -1168,23 +1263,30 @@ impl<'a> Parser<'a> { if first { first = false; } else { - if let Err(mut e) = self.expect(t) { - // Attempt to keep parsing if it was a similar separator - if let Some(ref tokens) = t.similar_tokens() { - if tokens.contains(&self.token) { - self.bump(); - } + match self.expect(t) { + Ok(false) => {} + Ok(true) => { + recovered = true; + break; } - e.emit(); - // Attempt to keep parsing if it was an omitted separator - match f(self) { - Ok(t) => { - v.push(t); - continue; - }, - Err(mut e) => { - e.cancel(); - break; + Err(mut e) => { + // Attempt to keep parsing if it was a similar separator + if let Some(ref tokens) = t.similar_tokens() { + if tokens.contains(&self.token) { + self.bump(); + } + } + e.emit(); + // Attempt to keep parsing if it was an omitted separator + match f(self) { + Ok(t) => { + v.push(t); + continue; + }, + Err(mut e) => { + e.cancel(); + break; + } } } } @@ -1203,23 +1305,26 @@ impl<'a> Parser<'a> { v.push(t); } - Ok(v) + Ok((v, recovered)) } - /// Parse a sequence, including the closing delimiter. The function - /// f must consume tokens until reaching the next separator or + /// Parses a sequence, including the closing delimiter. The function + /// `f` must consume tokens until reaching the next separator or /// closing bracket. - fn parse_unspanned_seq<T, F>(&mut self, - bra: &token::Token, - ket: &token::Token, - sep: SeqSep, - f: F) - -> PResult<'a, Vec<T>> where + fn parse_unspanned_seq<T, F>( + &mut self, + bra: &token::Token, + ket: &token::Token, + sep: SeqSep, + f: F, + ) -> PResult<'a, Vec<T>> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, { self.expect(bra)?; - let result = self.parse_seq_to_before_end(ket, sep, f)?; - self.eat(ket); + let (result, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; + if !recovered { + self.eat(ket); + } Ok(result) } @@ -1313,7 +1418,7 @@ impl<'a> Parser<'a> { self.sess.span_diagnostic.span_bug(sp, m) } - fn cancel(&self, err: &mut DiagnosticBuilder) { + fn cancel(&self, err: &mut DiagnosticBuilder<'_>) { self.sess.span_diagnostic.cancel(err) } @@ -1321,15 +1426,14 @@ impl<'a> Parser<'a> { &self.sess.span_diagnostic } - /// Is the current token one of the keywords that signals a bare function - /// type? + /// Is the current token one of the keywords that signals a bare function type? fn token_is_bare_fn_keyword(&mut self) -> bool { self.check_keyword(keywords::Fn) || self.check_keyword(keywords::Unsafe) || self.check_keyword(keywords::Extern) } - /// parse a `TyKind::BareFn` type: + /// Parses a `TyKind::BareFn` type. fn parse_ty_bare_fn(&mut self, generic_params: Vec<GenericParam>) -> PResult<'a, TyKind> { /* @@ -1366,7 +1470,7 @@ impl<'a> Parser<'a> { }))) } - /// Parse asyncness: `async` or nothing + /// Parses asyncness: `async` or nothing. fn parse_asyncness(&mut self) -> IsAsync { if self.eat_keyword(keywords::Async) { IsAsync::Async { @@ -1378,7 +1482,7 @@ impl<'a> Parser<'a> { } } - /// Parse unsafety: `unsafe` or nothing. + /// Parses unsafety: `unsafe` or nothing. fn parse_unsafety(&mut self) -> Unsafety { if self.eat_keyword(keywords::Unsafe) { Unsafety::Unsafe @@ -1387,7 +1491,7 @@ impl<'a> Parser<'a> { } } - /// Parse the items in a trait declaration + /// Parses the items in a trait declaration. pub fn parse_trait_item(&mut self, at_end: &mut bool) -> PResult<'a, TraitItem> { maybe_whole!(self, NtTraitItem, |x| x); let attrs = self.parse_outer_attributes()?; @@ -1504,7 +1608,7 @@ impl<'a> Parser<'a> { }) } - /// Parse optional return type [ -> TY ] in function decl + /// Parses an optional return type `[ -> TY ]` in a function declaration. fn parse_ret_ty(&mut self, allow_plus: bool) -> PResult<'a, FunctionRetTy> { if self.eat(&token::RArrow) { Ok(FunctionRetTy::Ty(self.parse_ty_common(allow_plus, true)?)) @@ -1513,12 +1617,13 @@ impl<'a> Parser<'a> { } } - // Parse a type + /// Parses a type. pub fn parse_ty(&mut self) -> PResult<'a, P<Ty>> { self.parse_ty_common(true, true) } - /// Parse a type in restricted contexts where `+` is not permitted. + /// Parses a type in restricted contexts where `+` is not permitted. + /// /// Example 1: `&'a TYPE` /// `+` is prohibited to maintain operator priority (P(+) < P(&)). /// Example 2: `value1 as TYPE + value2` @@ -1721,7 +1826,7 @@ impl<'a> Parser<'a> { match ty.node { TyKind::Rptr(ref lifetime, ref mut_ty) => { let sum_with_parens = pprust::to_string(|s| { - use print::pprust::PrintState; + use crate::print::pprust::PrintState; s.s.word("&")?; s.print_opt_lifetime(lifetime)?; @@ -1821,7 +1926,8 @@ impl<'a> Parser<'a> { self.look_ahead(offset + 1, |t| t == &token::Colon) } - /// Skip unexpected attributes and doc comments in this position and emit an appropriate error. + /// Skips unexpected attributes and doc comments in this position and emits an appropriate + /// error. fn eat_incorrect_doc_comment(&mut self, applied_to: &str) { if let token::DocComment(_) = self.token { let mut err = self.diagnostic().struct_span_err( @@ -1850,8 +1956,7 @@ impl<'a> Parser<'a> { } } - /// This version of parse arg doesn't necessarily require - /// identifier names. + /// This version of parse arg doesn't necessarily require identifier names. fn parse_arg_general(&mut self, require_name: bool, is_trait_item: bool) -> PResult<'a, Arg> { maybe_whole!(self, NtArg, |x| x); @@ -1959,12 +2064,12 @@ impl<'a> Parser<'a> { Ok(Arg { ty, pat, id: ast::DUMMY_NODE_ID }) } - /// Parse a single function argument + /// Parses a single function argument. crate fn parse_arg(&mut self) -> PResult<'a, Arg> { self.parse_arg_general(true, false) } - /// Parse an argument in a lambda header e.g., |arg, arg| + /// Parses an argument in a lambda header (e.g., `|arg, arg|`). fn parse_fn_block_arg(&mut self) -> PResult<'a, Arg> { let pat = self.parse_pat(Some("argument name"))?; let t = if self.eat(&token::Colon) { @@ -1991,7 +2096,7 @@ impl<'a> Parser<'a> { } } - /// Matches token_lit = LIT_INTEGER | ... + /// Matches `token_lit = LIT_INTEGER | ...`. fn parse_lit_token(&mut self) -> PResult<'a, LitKind> { let out = match self.token { token::Interpolated(ref nt) => match nt.0 { @@ -2057,7 +2162,7 @@ impl<'a> Parser<'a> { Ok(out) } - /// Matches lit = true | false | token_lit + /// Matches `lit = true | false | token_lit`. crate fn parse_lit(&mut self) -> PResult<'a, Lit> { let lo = self.span; let lit = if self.eat_keyword(keywords::True) { @@ -2071,7 +2176,7 @@ impl<'a> Parser<'a> { Ok(source_map::Spanned { node: lit, span: lo.to(self.prev_span) }) } - /// matches '-' lit | lit (cf. ast_validation::AstValidator::check_expr_within_pat) + /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`). crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> { maybe_whole_expr!(self); @@ -2113,7 +2218,7 @@ impl<'a> Parser<'a> { } } - /// Parses qualified path. + /// Parses a qualified path. /// Assumes that the leading `<` has been parsed already. /// /// `qualified_path = <type [as trait_ref]>::path` @@ -2189,8 +2294,9 @@ impl<'a> Parser<'a> { Ok(ast::Path { segments, span: lo.to(self.prev_span) }) } - /// Like `parse_path`, but also supports parsing `Word` meta items into paths for back-compat. - /// This is used when parsing derive macro paths in `#[derive]` attributes. + /// Like `parse_path`, but also supports parsing `Word` meta items into paths for + /// backwards-compatibility. This is used when parsing derive macro paths in `#[derive]` + /// attributes. pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> { let meta_ident = match self.token { token::Interpolated(ref nt) => match nt.0 { @@ -2271,7 +2377,10 @@ impl<'a> Parser<'a> { // We use `style == PathStyle::Expr` to check if this is in a recursion or not. If // it isn't, then we reset the unmatched angle bracket count as we're about to start // parsing a new path. - if style == PathStyle::Expr { self.unmatched_angle_bracket_count = 0; } + if style == PathStyle::Expr { + self.unmatched_angle_bracket_count = 0; + self.max_angle_bracket_count = 0; + } let args = if self.eat_lt() { // `<'a, T, A = U>` @@ -2283,12 +2392,14 @@ impl<'a> Parser<'a> { } else { // `(T, U) -> R` self.bump(); // `(` - let inputs = self.parse_seq_to_before_tokens( + let (inputs, recovered) = self.parse_seq_to_before_tokens( &[&token::CloseDelim(token::Paren)], SeqSep::trailing_allowed(token::Comma), TokenExpectType::Expect, |p| p.parse_ty())?; - self.bump(); // `)` + if !recovered { + self.bump(); // `)` + } let span = lo.to(self.prev_span); let output = if self.eat(&token::RArrow) { Some(self.parse_ty_common(false, false)?) @@ -2310,7 +2421,7 @@ impl<'a> Parser<'a> { self.token.is_lifetime() } - /// Parse single lifetime 'a or panic. + /// Parses a single lifetime `'a` or panics. crate fn expect_lifetime(&mut self) -> Lifetime { if let Some(ident) = self.token.lifetime() { let span = self.span; @@ -2331,7 +2442,7 @@ impl<'a> Parser<'a> { } } - /// Parse mutability (`mut` or nothing). + /// Parses mutability (`mut` or nothing). fn parse_mutability(&mut self) -> Mutability { if self.eat_keyword(keywords::Mut) { Mutability::Mutable @@ -2462,12 +2573,10 @@ impl<'a> Parser<'a> { } /// At the bottom (top?) of the precedence hierarchy, - /// parse things like parenthesized exprs, - /// macros, return, etc. + /// Parses things like parenthesized exprs, macros, `return`, etc. /// - /// N.B., this does not parse outer attributes, - /// and is private because it only works - /// correctly if called from parse_dot_or_call_expr(). + /// N.B., this does not parse outer attributes, and is private because it only works + /// correctly if called from `parse_dot_or_call_expr()`. fn parse_bottom_expr(&mut self) -> PResult<'a, P<Expr>> { maybe_whole_expr!(self); @@ -2494,9 +2603,13 @@ impl<'a> Parser<'a> { // (e,) is a tuple with only one field, e let mut es = vec![]; let mut trailing_comma = false; + let mut recovered = false; while self.token != token::CloseDelim(token::Paren) { es.push(self.parse_expr()?); - self.expect_one_of(&[], &[token::Comma, token::CloseDelim(token::Paren)])?; + recovered = self.expect_one_of( + &[], + &[token::Comma, token::CloseDelim(token::Paren)], + )?; if self.eat(&token::Comma) { trailing_comma = true; } else { @@ -2504,7 +2617,9 @@ impl<'a> Parser<'a> { break; } } - self.bump(); + if !recovered { + self.bump(); + } hi = self.prev_span; ex = if es.len() == 1 && !trailing_comma { @@ -2701,6 +2816,21 @@ impl<'a> Parser<'a> { hi = pth.span; ex = ExprKind::Path(None, pth); } else { + if !self.unclosed_delims.is_empty() && self.check(&token::Semi) { + // Don't complain about bare semicolons after unclosed braces + // recovery in order to keep the error count down. Fixing the + // delimiters will possibly also fix the bare semicolon found in + // expression context. For example, silence the following error: + // ``` + // error: expected expression, found `;` + // --> file.rs:2:13 + // | + // 2 | foo(bar(; + // | ^ expected expression + // ``` + self.bump(); + return Ok(self.mk_expr(self.span, ExprKind::Err, ThinVec::new())); + } match self.parse_literal_maybe_minus() { Ok(expr) => { hi = expr.span; @@ -2800,7 +2930,7 @@ impl<'a> Parser<'a> { match self.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Brace)]) { - Ok(()) => if let Some(f) = parsed_field.or(recovery_field) { + Ok(_) => if let Some(f) = parsed_field.or(recovery_field) { // only include the field if there's no parse error for the field name fields.push(f); } @@ -2831,7 +2961,7 @@ impl<'a> Parser<'a> { } } - /// Parse a block or unsafe block + /// Parses a block or unsafe block. fn parse_block_expr(&mut self, opt_label: Option<Label>, lo: Span, blk_mode: BlockCheckMode, outer_attrs: ThinVec<Attribute>) @@ -2845,7 +2975,7 @@ impl<'a> Parser<'a> { return Ok(self.mk_expr(blk.span, ExprKind::Block(blk, opt_label), attrs)); } - /// parse a.b or a(13) or a[4] or just a + /// Parses `a.b` or `a(13)` or `a[4]` or just `a`. fn parse_dot_or_call_expr(&mut self, already_parsed_attrs: Option<ThinVec<Attribute>>) -> PResult<'a, P<Expr>> { @@ -3063,7 +3193,7 @@ impl<'a> Parser<'a> { None => continue, }; let sugg = pprust::to_string(|s| { - use print::pprust::PrintState; + use crate::print::pprust::PrintState; s.popen()?; s.print_expr(&e)?; s.s.word( ".")?; @@ -3153,7 +3283,7 @@ impl<'a> Parser<'a> { self.span = span; } - /// parse a single token tree from the input. + /// Parses a single token tree from the input. crate fn parse_token_tree(&mut self) -> TokenTree { match self.token { token::OpenDelim(..) => { @@ -3313,7 +3443,7 @@ impl<'a> Parser<'a> { return Ok(self.mk_expr(lo.to(hi), ex, attrs)); } - /// Parse an associative expression + /// Parses an associative expression. /// /// This parses an expression accounting for associativity and precedence of the operators in /// the expression. @@ -3324,7 +3454,7 @@ impl<'a> Parser<'a> { self.parse_assoc_expr_with(0, already_parsed_attrs.into()) } - /// Parse an associative expression with operators of at least `min_prec` precedence + /// Parses an associative expression with operators of at least `min_prec` precedence. fn parse_assoc_expr_with(&mut self, min_prec: usize, lhs: LhsExpr) @@ -3455,6 +3585,14 @@ impl<'a> Parser<'a> { }), }?; + // Make sure that the span of the parent node is larger than the span of lhs and rhs, + // including the attributes. + let lhs_span = lhs + .attrs + .iter() + .filter(|a| a.style == AttrStyle::Outer) + .next() + .map_or(lhs_span, |a| a.span); let span = lhs_span.to(rhs.span); lhs = match op { AssocOp::Add | AssocOp::Subtract | AssocOp::Multiply | AssocOp::Divide | @@ -3651,7 +3789,7 @@ impl<'a> Parser<'a> { } } - /// Parse an 'if' or 'if let' expression ('if' token already eaten) + /// Parses an `if` or `if let` expression (`if` token already eaten). fn parse_if_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { if self.check_keyword(keywords::Let) { return self.parse_if_let_expr(attrs); @@ -3687,7 +3825,7 @@ impl<'a> Parser<'a> { Ok(self.mk_expr(lo.to(hi), ExprKind::If(cond, thn, els), attrs)) } - /// Parse an 'if let' expression ('if' token already eaten) + /// Parses an `if let` expression (`if` token already eaten). fn parse_if_let_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { let lo = self.prev_span; @@ -3705,7 +3843,7 @@ impl<'a> Parser<'a> { Ok(self.mk_expr(lo.to(hi), ExprKind::IfLet(pats, expr, thn, els), attrs)) } - // `move |args| expr` + /// Parses `move |args| expr`. fn parse_lambda_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> @@ -3801,7 +3939,7 @@ impl<'a> Parser<'a> { Ok(self.mk_expr(span_lo.to(hi), ExprKind::ForLoop(pat, expr, loop_block, opt_label), attrs)) } - /// Parse a 'while' or 'while let' expression ('while' token already eaten) + /// Parses a `while` or `while let` expression (`while` token already eaten). fn parse_while_expr(&mut self, opt_label: Option<Label>, span_lo: Span, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { @@ -3815,7 +3953,7 @@ impl<'a> Parser<'a> { return Ok(self.mk_expr(span, ExprKind::While(cond, body, opt_label), attrs)); } - /// Parse a 'while let' expression ('while' token already eaten) + /// Parses a `while let` expression (`while` token already eaten). fn parse_while_let_expr(&mut self, opt_label: Option<Label>, span_lo: Span, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { @@ -3839,7 +3977,7 @@ impl<'a> Parser<'a> { Ok(self.mk_expr(span, ExprKind::Loop(body, opt_label), attrs)) } - /// Parse an `async move {...}` expression + /// Parses an `async move {...}` expression. pub fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { @@ -3857,7 +3995,7 @@ impl<'a> Parser<'a> { ExprKind::Async(capture_clause, ast::DUMMY_NODE_ID, body), attrs)) } - /// Parse a `try {...}` expression (`try` token already eaten) + /// Parses a `try {...}` expression (`try` token already eaten). fn parse_try_block(&mut self, span_lo: Span, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { @@ -3975,15 +4113,15 @@ impl<'a> Parser<'a> { }) } - /// Parse an expression + /// Parses an expression. #[inline] pub fn parse_expr(&mut self) -> PResult<'a, P<Expr>> { self.parse_expr_res(Restrictions::empty(), None) } - /// Evaluate the closure with restrictions in place. + /// Evaluates the closure with restrictions in place. /// - /// After the closure is evaluated, restrictions are reset. + /// Afters the closure is evaluated, restrictions are reset. fn with_res<F, T>(&mut self, r: Restrictions, f: F) -> T where F: FnOnce(&mut Self) -> T { @@ -3995,7 +4133,7 @@ impl<'a> Parser<'a> { } - /// Parse an expression, subject to the given restrictions + /// Parses an expression, subject to the given restrictions. #[inline] fn parse_expr_res(&mut self, r: Restrictions, already_parsed_attrs: Option<ThinVec<Attribute>>) @@ -4003,7 +4141,7 @@ impl<'a> Parser<'a> { self.with_res(r, |this| this.parse_assoc_expr(already_parsed_attrs)) } - /// Parse the RHS of a local variable declaration (e.g., '= 14;') + /// Parses the RHS of a local variable declaration (e.g., '= 14;'). fn parse_initializer(&mut self, skip_eq: bool) -> PResult<'a, Option<P<Expr>>> { if self.eat(&token::Eq) { Ok(Some(self.parse_expr()?)) @@ -4014,7 +4152,7 @@ impl<'a> Parser<'a> { } } - /// Parse patterns, separated by '|' s + /// Parses patterns, separated by '|' s. fn parse_pats(&mut self) -> PResult<'a, Vec<P<Pat>>> { // Allow a '|' before the pats (RFC 1925 + RFC 2530) self.eat(&token::BinOp(token::Or)); @@ -4204,7 +4342,7 @@ impl<'a> Parser<'a> { }) } - /// Parse the fields of a struct-like pattern + /// Parses the fields of a struct-like pattern. fn parse_pat_fields(&mut self) -> PResult<'a, (Vec<source_map::Spanned<ast::FieldPat>>, bool)> { let mut fields = Vec::new(); let mut etc = false; @@ -4396,13 +4534,13 @@ impl<'a> Parser<'a> { Ok(pat) } - /// Parse a pattern. + /// Parses a pattern. pub fn parse_pat(&mut self, expected: Option<&'static str>) -> PResult<'a, P<Pat>> { self.parse_pat_with_range_pat(true, expected) } - /// Parse a pattern, with a setting whether modern range patterns e.g., `a..=b`, `a..b` are - /// allowed. + /// Parses a pattern, with a setting whether modern range patterns (e.g., `a..=b`, `a..b` are + /// allowed). fn parse_pat_with_range_pat( &mut self, allow_range_pat: bool, @@ -4612,9 +4750,9 @@ impl<'a> Parser<'a> { Ok(P(pat)) } - /// Parse ident or ident @ pat + /// Parses `ident` or `ident @ pat`. /// used by the copy foo and ref foo patterns to give a good - /// error message when parsing mistakes like ref foo(a,b) + /// error message when parsing mistakes like `ref foo(a, b)`. fn parse_pat_ident(&mut self, binding_mode: ast::BindingMode) -> PResult<'a, PatKind> { @@ -4640,7 +4778,7 @@ impl<'a> Parser<'a> { Ok(PatKind::Ident(binding_mode, ident, sub)) } - /// Parse a local variable declaration + /// Parses a local variable declaration. fn parse_local(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Local>> { let lo = self.prev_span; let pat = self.parse_top_level_pat()?; @@ -4713,7 +4851,7 @@ impl<'a> Parser<'a> { })) } - /// Parse a structure field + /// Parses a structure field. fn parse_name_and_ty(&mut self, lo: Span, vis: Visibility, @@ -4732,7 +4870,7 @@ impl<'a> Parser<'a> { }) } - /// Emit an expected item after attributes error. + /// Emits an expected-item-after-attributes error. fn expected_item_err(&mut self, attrs: &[Attribute]) -> PResult<'a, ()> { let message = match attrs.last() { Some(&Attribute { is_sugared_doc: true, .. }) => "expected item after doc comment", @@ -5155,13 +5293,13 @@ impl<'a> Parser<'a> { })) } - /// Is this expression a successfully-parsed statement? + /// Checks if this expression is a successfully parsed statement. fn expr_is_complete(&mut self, e: &Expr) -> bool { self.restrictions.contains(Restrictions::STMT_EXPR) && !classify::expr_requires_semi_to_be_stmt(e) } - /// Parse a block. No inner attrs are allowed. + /// Parses a block. No inner attributes are allowed. pub fn parse_block(&mut self) -> PResult<'a, P<Block>> { maybe_whole!(self, NtBlock, |x| x); @@ -5212,7 +5350,7 @@ impl<'a> Parser<'a> { stmt_span = stmt_span.with_hi(self.prev_span.hi()); } let sugg = pprust::to_string(|s| { - use print::pprust::{PrintState, INDENT_UNIT}; + use crate::print::pprust::{PrintState, INDENT_UNIT}; s.ibox(INDENT_UNIT)?; s.bopen()?; s.print_stmt(&stmt)?; @@ -5239,7 +5377,7 @@ impl<'a> Parser<'a> { self.parse_block_tail(lo, BlockCheckMode::Default) } - /// Parse a block. Inner attrs are allowed. + /// Parses a block. Inner attributes are allowed. fn parse_inner_attrs_and_block(&mut self) -> PResult<'a, (Vec<Attribute>, P<Block>)> { maybe_whole!(self, NtBlock, |x| (Vec::new(), x)); @@ -5249,7 +5387,7 @@ impl<'a> Parser<'a> { self.parse_block_tail(lo, BlockCheckMode::Default)?)) } - /// Parse the rest of a block expression or function body + /// Parses the rest of a block expression or function body. /// Precondition: already parsed the '{'. fn parse_block_tail(&mut self, lo: Span, s: BlockCheckMode) -> PResult<'a, P<Block>> { let mut stmts = vec![]; @@ -5283,7 +5421,7 @@ impl<'a> Parser<'a> { })) } - /// Parse a statement, including the trailing semicolon. + /// Parses a statement, including the trailing semicolon. crate fn parse_full_stmt(&mut self, macro_legacy_warnings: bool) -> PResult<'a, Option<Stmt>> { // skip looking for a trailing semicolon when we have an interpolated statement maybe_whole!(self, NtStmt, |x| Some(x)); @@ -5345,11 +5483,14 @@ impl<'a> Parser<'a> { ).emit(); } - // Parse bounds of a type parameter `BOUND + BOUND + BOUND`, possibly with trailing `+`. - // BOUND = TY_BOUND | LT_BOUND - // LT_BOUND = LIFETIME (e.g., `'a`) - // TY_BOUND = TY_BOUND_NOPAREN | (TY_BOUND_NOPAREN) - // TY_BOUND_NOPAREN = [?] [for<LT_PARAM_DEFS>] SIMPLE_PATH (e.g., `?for<'a: 'b> m::Trait<'a>`) + /// Parses bounds of a type parameter `BOUND + BOUND + ...`, possibly with trailing `+`. + /// + /// ``` + /// BOUND = TY_BOUND | LT_BOUND + /// LT_BOUND = LIFETIME (e.g., `'a`) + /// TY_BOUND = TY_BOUND_NOPAREN | (TY_BOUND_NOPAREN) + /// TY_BOUND_NOPAREN = [?] [for<LT_PARAM_DEFS>] SIMPLE_PATH (e.g., `?for<'a: 'b> m::Trait<'a>`) + /// ``` fn parse_generic_bounds_common(&mut self, allow_plus: bool) -> PResult<'a, GenericBounds> { let mut bounds = Vec::new(); loop { @@ -5403,8 +5544,11 @@ impl<'a> Parser<'a> { self.parse_generic_bounds_common(true) } - // Parse bounds of a lifetime parameter `BOUND + BOUND + BOUND`, possibly with trailing `+`. - // BOUND = LT_BOUND (e.g., `'a`) + /// Parses bounds of a lifetime parameter `BOUND + BOUND + BOUND`, possibly with trailing `+`. + /// + /// ``` + /// BOUND = LT_BOUND (e.g., `'a`) + /// ``` fn parse_lt_param_bounds(&mut self) -> GenericBounds { let mut lifetimes = Vec::new(); while self.check_lifetime() { @@ -5417,7 +5561,7 @@ impl<'a> Parser<'a> { lifetimes } - /// Matches typaram = IDENT (`?` unbound)? optbounds ( EQ ty )? + /// Matches `typaram = IDENT (`?` unbound)? optbounds ( EQ ty )?`. fn parse_ty_param(&mut self, preceding_attrs: Vec<Attribute>) -> PResult<'a, GenericParam> { @@ -5448,6 +5592,7 @@ impl<'a> Parser<'a> { } /// Parses the following grammar: + /// /// TraitItemAssocTy = Ident ["<"...">"] [":" [GenericBounds]] ["where" ...] ["=" Ty] fn parse_trait_item_assoc_ty(&mut self) -> PResult<'a, (Ident, TraitItemKind, ast::Generics)> { @@ -5472,15 +5617,27 @@ impl<'a> Parser<'a> { Ok((ident, TraitItemKind::Type(bounds, default), generics)) } - /// Parses (possibly empty) list of lifetime and type parameters, possibly including - /// trailing comma and erroneous trailing attributes. + fn parse_const_param(&mut self, preceding_attrs: Vec<Attribute>) -> PResult<'a, GenericParam> { + self.expect_keyword(keywords::Const)?; + let ident = self.parse_ident()?; + self.expect(&token::Colon)?; + let ty = self.parse_ty()?; + + Ok(GenericParam { + ident, + id: ast::DUMMY_NODE_ID, + attrs: preceding_attrs.into(), + bounds: Vec::new(), + kind: GenericParamKind::Const { + ty, + } + }) + } + + /// Parses a (possibly empty) list of lifetime and type parameters, possibly including + /// a trailing comma and erroneous trailing attributes. crate fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> { - let mut lifetimes = Vec::new(); let mut params = Vec::new(); - let mut seen_ty_param: Option<Span> = None; - let mut last_comma_span = None; - let mut bad_lifetime_pos = vec![]; - let mut suggestions = vec![]; loop { let attrs = self.parse_outer_attributes()?; if self.check_lifetime() { @@ -5491,39 +5648,40 @@ impl<'a> Parser<'a> { } else { Vec::new() }; - lifetimes.push(ast::GenericParam { + params.push(ast::GenericParam { ident: lifetime.ident, id: lifetime.id, attrs: attrs.into(), bounds, kind: ast::GenericParamKind::Lifetime, }); - if let Some(sp) = seen_ty_param { - let remove_sp = last_comma_span.unwrap_or(self.prev_span).to(self.prev_span); - bad_lifetime_pos.push(self.prev_span); - if let Ok(snippet) = self.sess.source_map().span_to_snippet(self.prev_span) { - suggestions.push((remove_sp, String::new())); - suggestions.push(( - sp.shrink_to_lo(), - format!("{}, ", snippet))); - } - } + } else if self.check_keyword(keywords::Const) { + // Parse const parameter. + params.push(self.parse_const_param(attrs)?); } else if self.check_ident() { // Parse type parameter. params.push(self.parse_ty_param(attrs)?); - if seen_ty_param.is_none() { - seen_ty_param = Some(self.prev_span); - } } else { // Check for trailing attributes and stop parsing. if !attrs.is_empty() { - let param_kind = if seen_ty_param.is_some() { "type" } else { "lifetime" }; - self.struct_span_err( - attrs[0].span, - &format!("trailing attribute after {} parameters", param_kind), - ) - .span_label(attrs[0].span, "attributes must go before parameters") - .emit(); + if !params.is_empty() { + self.struct_span_err( + attrs[0].span, + &format!("trailing attribute after generic parameter"), + ) + .span_label(attrs[0].span, "attributes must go before parameters") + .emit(); + } else { + self.struct_span_err( + attrs[0].span, + &format!("attribute without generic parameters"), + ) + .span_label( + attrs[0].span, + "attributes are only permitted when preceding parameters", + ) + .emit(); + } } break } @@ -5531,27 +5689,11 @@ impl<'a> Parser<'a> { if !self.eat(&token::Comma) { break } - last_comma_span = Some(self.prev_span); - } - if !bad_lifetime_pos.is_empty() { - let mut err = self.struct_span_err( - bad_lifetime_pos, - "lifetime parameters must be declared prior to type parameters", - ); - if !suggestions.is_empty() { - err.multipart_suggestion( - "move the lifetime parameter prior to the first type parameter", - suggestions, - Applicability::MachineApplicable, - ); - } - err.emit(); } - lifetimes.extend(params); // ensure the correct order of lifetimes and type params - Ok(lifetimes) + Ok(params) } - /// Parse a set of optional generic type parameter declarations. Where + /// Parses a set of optional generic type parameter declarations. Where /// clauses are not parsed here, and must be added later via /// `parse_where_clause()`. /// @@ -5579,7 +5721,7 @@ impl<'a> Parser<'a> { } } - /// Parse generic args (within a path segment) with recovery for extra leading angle brackets. + /// Parses generic args (within a path segment) with recovery for extra leading angle brackets. /// For the purposes of understanding the parsing logic of generic arguments, this function /// can be thought of being the same as just calling `self.parse_generic_args()` if the source /// had the correct amount of leading angle brackets. @@ -5730,35 +5872,16 @@ impl<'a> Parser<'a> { fn parse_generic_args(&mut self) -> PResult<'a, (Vec<GenericArg>, Vec<TypeBinding>)> { let mut args = Vec::new(); let mut bindings = Vec::new(); + let mut misplaced_assoc_ty_bindings: Vec<Span> = Vec::new(); + let mut assoc_ty_bindings: Vec<Span> = Vec::new(); - let mut seen_type = false; - let mut seen_binding = false; + let args_lo = self.span; - let mut last_comma_span = None; - let mut first_type_or_binding_span: Option<Span> = None; - let mut first_binding_span: Option<Span> = None; - - let mut bad_lifetime_pos = vec![]; - let mut bad_type_pos = vec![]; - - let mut lifetime_suggestions = vec![]; - let mut type_suggestions = vec![]; loop { if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) { // Parse lifetime argument. args.push(GenericArg::Lifetime(self.expect_lifetime())); - - if seen_type || seen_binding { - let remove_sp = last_comma_span.unwrap_or(self.prev_span).to(self.prev_span); - bad_lifetime_pos.push(self.prev_span); - - if let Ok(snippet) = self.sess.source_map().span_to_snippet(self.prev_span) { - lifetime_suggestions.push((remove_sp, String::new())); - lifetime_suggestions.push(( - first_type_or_binding_span.unwrap().shrink_to_lo(), - format!("{}, ", snippet))); - } - } + misplaced_assoc_ty_bindings.append(&mut assoc_ty_bindings); } else if self.check_ident() && self.look_ahead(1, |t| t == &token::Eq) { // Parse associated type binding. let lo = self.span; @@ -5772,134 +5895,67 @@ impl<'a> Parser<'a> { ty, span, }); - - seen_binding = true; - if first_type_or_binding_span.is_none() { - first_type_or_binding_span = Some(span); - } - if first_binding_span.is_none() { - first_binding_span = Some(span); - } + assoc_ty_bindings.push(span); + } else if self.check_const_arg() { + // FIXME(const_generics): to distinguish between idents for types and consts, + // we should introduce a GenericArg::Ident in the AST and distinguish when + // lowering to the HIR. For now, idents for const args are not permitted. + + // Parse const argument. + let expr = if let token::OpenDelim(token::Brace) = self.token { + self.parse_block_expr(None, self.span, BlockCheckMode::Default, ThinVec::new())? + } else if self.token.is_ident() { + // FIXME(const_generics): to distinguish between idents for types and consts, + // we should introduce a GenericArg::Ident in the AST and distinguish when + // lowering to the HIR. For now, idents for const args are not permitted. + return Err( + self.fatal("identifiers may currently not be used for const generics") + ); + } else { + // FIXME(const_generics): this currently conflicts with emplacement syntax + // with negative integer literals. + self.parse_literal_maybe_minus()? + }; + let value = AnonConst { + id: ast::DUMMY_NODE_ID, + value: expr, + }; + args.push(GenericArg::Const(value)); + misplaced_assoc_ty_bindings.append(&mut assoc_ty_bindings); } else if self.check_type() { // Parse type argument. - let ty_param = self.parse_ty()?; - if seen_binding { - let remove_sp = last_comma_span.unwrap_or(self.prev_span).to(self.prev_span); - bad_type_pos.push(self.prev_span); - - if let Ok(snippet) = self.sess.source_map().span_to_snippet(self.prev_span) { - type_suggestions.push((remove_sp, String::new())); - type_suggestions.push(( - first_binding_span.unwrap().shrink_to_lo(), - format!("{}, ", snippet))); - } - } - - if first_type_or_binding_span.is_none() { - first_type_or_binding_span = Some(ty_param.span); - } - args.push(GenericArg::Type(ty_param)); - seen_type = true; + args.push(GenericArg::Type(self.parse_ty()?)); + misplaced_assoc_ty_bindings.append(&mut assoc_ty_bindings); } else { break } if !self.eat(&token::Comma) { break - } else { - last_comma_span = Some(self.prev_span); - } - } - - self.maybe_report_incorrect_generic_argument_order( - bad_lifetime_pos, bad_type_pos, lifetime_suggestions, type_suggestions - ); - - Ok((args, bindings)) - } - - /// Maybe report an error about incorrect generic argument order - "lifetime parameters - /// must be declared before type parameters", "type parameters must be declared before - /// associated type bindings" or both. - fn maybe_report_incorrect_generic_argument_order( - &self, - bad_lifetime_pos: Vec<Span>, - bad_type_pos: Vec<Span>, - lifetime_suggestions: Vec<(Span, String)>, - type_suggestions: Vec<(Span, String)>, - ) { - let mut err = if !bad_lifetime_pos.is_empty() && !bad_type_pos.is_empty() { - let mut positions = bad_lifetime_pos.clone(); - positions.extend_from_slice(&bad_type_pos); - - self.struct_span_err( - positions, - "generic arguments must declare lifetimes, types and associated type bindings in \ - that order", - ) - } else if !bad_lifetime_pos.is_empty() { - self.struct_span_err( - bad_lifetime_pos.clone(), - "lifetime parameters must be declared prior to type parameters" - ) - } else if !bad_type_pos.is_empty() { - self.struct_span_err( - bad_type_pos.clone(), - "type parameters must be declared prior to associated type bindings" - ) - } else { - return; - }; - - if !bad_lifetime_pos.is_empty() { - for sp in &bad_lifetime_pos { - err.span_label(*sp, "must be declared prior to type parameters"); - } - } - - if !bad_type_pos.is_empty() { - for sp in &bad_type_pos { - err.span_label(*sp, "must be declared prior to associated type bindings"); } } - if !lifetime_suggestions.is_empty() && !type_suggestions.is_empty() { - let mut suggestions = lifetime_suggestions; - suggestions.extend_from_slice(&type_suggestions); - - let plural = bad_lifetime_pos.len() + bad_type_pos.len() > 1; - err.multipart_suggestion( - &format!( - "move the parameter{}", - if plural { "s" } else { "" }, - ), - suggestions, - Applicability::MachineApplicable, - ); - } else if !lifetime_suggestions.is_empty() { - err.multipart_suggestion( - &format!( - "move the lifetime parameter{} prior to the first type parameter", - if bad_lifetime_pos.len() > 1 { "s" } else { "" }, - ), - lifetime_suggestions, - Applicability::MachineApplicable, - ); - } else if !type_suggestions.is_empty() { - err.multipart_suggestion( - &format!( - "move the type parameter{} prior to the first associated type binding", - if bad_type_pos.len() > 1 { "s" } else { "" }, - ), - type_suggestions, - Applicability::MachineApplicable, + // FIXME: we would like to report this in ast_validation instead, but we currently do not + // preserve ordering of generic parameters with respect to associated type binding, so we + // lose that information after parsing. + if misplaced_assoc_ty_bindings.len() > 0 { + let mut err = self.struct_span_err( + args_lo.to(self.prev_span), + "associated type bindings must be declared after generic parameters", ); + for span in misplaced_assoc_ty_bindings { + err.span_label( + span, + "this associated type binding should be moved after the generic parameters", + ); + } + err.emit(); } - err.emit(); + Ok((args, bindings)) } - /// Parses an optional `where` clause and places it in `generics`. + /// Parses an optional where-clause and places it in `generics`. /// /// ```ignore (only-for-syntax-highlight) /// where T : Trait<U, V> + 'b, 'a : 'b @@ -6001,7 +6057,7 @@ impl<'a> Parser<'a> { let sp = self.span; let mut variadic = false; - let args: Vec<Option<Arg>> = + let (args, recovered): (Vec<Option<Arg>>, bool) = self.parse_seq_to_before_end( &token::CloseDelim(token::Paren), SeqSep::trailing_allowed(token::Comma), @@ -6049,7 +6105,9 @@ impl<'a> Parser<'a> { } )?; - self.eat(&token::CloseDelim(token::Paren)); + if !recovered { + self.eat(&token::CloseDelim(token::Paren)); + } let args: Vec<_> = args.into_iter().filter_map(|x| x).collect(); @@ -6061,7 +6119,7 @@ impl<'a> Parser<'a> { Ok((args, variadic)) } - /// Parse the argument list and result type of a function declaration + /// Parses the argument list and result type of a function declaration. fn parse_fn_decl(&mut self, allow_variadic: bool) -> PResult<'a, P<FnDecl>> { let (args, variadic) = self.parse_fn_args(true, allow_variadic)?; @@ -6183,7 +6241,7 @@ impl<'a> Parser<'a> { Ok(Some(Arg::from_self(eself, eself_ident))) } - /// Parse the parameter list and result type of a function that may have a `self` parameter. + /// Parses the parameter list and result type of a function that may have a `self` parameter. fn parse_fn_decl_with_self<F>(&mut self, parse_arg_fn: F) -> PResult<'a, P<FnDecl>> where F: FnMut(&mut Parser<'a>) -> PResult<'a, Arg>, { @@ -6194,15 +6252,15 @@ impl<'a> Parser<'a> { // Parse the rest of the function parameter list. let sep = SeqSep::trailing_allowed(token::Comma); - let fn_inputs = if let Some(self_arg) = self_arg { + let (fn_inputs, recovered) = if let Some(self_arg) = self_arg { if self.check(&token::CloseDelim(token::Paren)) { - vec![self_arg] + (vec![self_arg], false) } else if self.eat(&token::Comma) { let mut fn_inputs = vec![self_arg]; - fn_inputs.append(&mut self.parse_seq_to_before_end( - &token::CloseDelim(token::Paren), sep, parse_arg_fn)? - ); - fn_inputs + let (mut input, recovered) = self.parse_seq_to_before_end( + &token::CloseDelim(token::Paren), sep, parse_arg_fn)?; + fn_inputs.append(&mut input); + (fn_inputs, recovered) } else { return self.unexpected(); } @@ -6210,8 +6268,10 @@ impl<'a> Parser<'a> { self.parse_seq_to_before_end(&token::CloseDelim(token::Paren), sep, parse_arg_fn)? }; - // Parse closing paren and return type. - self.expect(&token::CloseDelim(token::Paren))?; + if !recovered { + // Parse closing paren and return type. + self.expect(&token::CloseDelim(token::Paren))?; + } Ok(P(FnDecl { inputs: fn_inputs, output: self.parse_ret_ty(true)?, @@ -6219,7 +6279,7 @@ impl<'a> Parser<'a> { })) } - // parse the |arg, arg| header on a lambda + /// Parses the `|arg, arg|` header of a closure. fn parse_fn_block_decl(&mut self) -> PResult<'a, P<FnDecl>> { let inputs_captures = { if self.eat(&token::OrOr) { @@ -6231,7 +6291,7 @@ impl<'a> Parser<'a> { SeqSep::trailing_allowed(token::Comma), TokenExpectType::NoExpect, |p| p.parse_fn_block_arg() - )?; + )?.0; self.expect_or()?; args } @@ -6245,7 +6305,7 @@ impl<'a> Parser<'a> { })) } - /// Parse the name and optional generic types of a function header. + /// Parses the name and optional generic types of a function header. fn parse_fn_header(&mut self) -> PResult<'a, (Ident, ast::Generics)> { let id = self.parse_ident()?; let generics = self.parse_generics()?; @@ -6265,7 +6325,7 @@ impl<'a> Parser<'a> { }) } - /// Parse an item-position function declaration. + /// Parses an item-position function declaration. fn parse_item_fn(&mut self, unsafety: Unsafety, asyncness: IsAsync, @@ -6280,21 +6340,22 @@ impl<'a> Parser<'a> { Ok((ident, ItemKind::Fn(decl, header, generics, body), Some(inner_attrs))) } - /// true if we are looking at `const ID`, false for things like `const fn` etc + /// Returns `true` if we are looking at `const ID` + /// (returns `false` for things like `const fn`, etc.). fn is_const_item(&mut self) -> bool { self.token.is_keyword(keywords::Const) && !self.look_ahead(1, |t| t.is_keyword(keywords::Fn)) && !self.look_ahead(1, |t| t.is_keyword(keywords::Unsafe)) } - /// parses all the "front matter" for a `fn` declaration, up to + /// Parses all the "front matter" for a `fn` declaration, up to /// and including the `fn` keyword: /// /// - `const fn` /// - `unsafe fn` /// - `const unsafe fn` /// - `extern fn` - /// - etc + /// - etc. fn parse_fn_front_matter(&mut self) -> PResult<'a, ( Spanned<Constness>, @@ -6321,7 +6382,7 @@ impl<'a> Parser<'a> { Ok((constness, unsafety, asyncness, abi)) } - /// Parse an impl item. + /// Parses an impl item. pub fn parse_impl_item(&mut self, at_end: &mut bool) -> PResult<'a, ImplItem> { maybe_whole!(self, NtImplItem, |x| x); let attrs = self.parse_outer_attributes()?; @@ -6460,7 +6521,7 @@ impl<'a> Parser<'a> { } } - /// Parse `trait Foo { ... }` or `trait Foo = Bar;` + /// Parses `trait Foo { ... }` or `trait Foo = Bar;`. fn parse_item_trait(&mut self, is_auto: IsAuto, unsafety: Unsafety) -> PResult<'a, ItemInfo> { let ident = self.parse_ident()?; let mut tps = self.parse_generics()?; @@ -6516,6 +6577,7 @@ impl<'a> Parser<'a> { // `<` (LIFETIME|IDENT) `,` - first generic parameter in a list // `<` (LIFETIME|IDENT) `:` - generic parameter with bounds // `<` (LIFETIME|IDENT) `=` - generic parameter with a default + // `<` const - generic const parameter // The only truly ambiguous case is // `<` IDENT `>` `::` IDENT ... // we disambiguate it in favor of generics (`impl<T> ::absolute::Path<T> { ... }`) @@ -6525,7 +6587,8 @@ impl<'a> Parser<'a> { (self.look_ahead(1, |t| t == &token::Pound || t == &token::Gt) || self.look_ahead(1, |t| t.is_lifetime() || t.is_ident()) && self.look_ahead(2, |t| t == &token::Gt || t == &token::Comma || - t == &token::Colon || t == &token::Eq)) + t == &token::Colon || t == &token::Eq) || + self.look_ahead(1, |t| t.is_keyword(keywords::Const))) } fn parse_impl_body(&mut self) -> PResult<'a, (Vec<ImplItem>, Vec<Attribute>)> { @@ -6549,9 +6612,11 @@ impl<'a> Parser<'a> { } /// Parses an implementation item, `impl` keyword is already parsed. + /// /// impl<'a, T> TYPE { /* impl items */ } /// impl<'a, T> TRAIT for TYPE { /* impl items */ } /// impl<'a, T> !TRAIT for TYPE { /* impl items */ } + /// /// We actually parse slightly more relaxed grammar for better error reporting and recovery. /// `impl` GENERICS `!`? TYPE `for`? (TYPE | `..`) (`where` PREDICATES)? `{` BODY `}` /// `impl` GENERICS `!`? TYPE (`where` PREDICATES)? `{` BODY `}` @@ -6643,7 +6708,7 @@ impl<'a> Parser<'a> { } } - /// Parse struct Foo { ... } + /// Parses `struct Foo { ... }`. fn parse_item_struct(&mut self) -> PResult<'a, ItemInfo> { let class_name = self.parse_ident()?; @@ -6697,7 +6762,7 @@ impl<'a> Parser<'a> { Ok((class_name, ItemKind::Struct(vdata, generics), None)) } - /// Parse union Foo { ... } + /// Parses `union Foo { ... }`. fn parse_item_union(&mut self) -> PResult<'a, ItemInfo> { let class_name = self.parse_ident()?; @@ -6791,7 +6856,7 @@ impl<'a> Parser<'a> { Ok(fields) } - /// Parse a structure field declaration + /// Parses a structure field declaration. fn parse_single_struct_field(&mut self, lo: Span, vis: Visibility, @@ -6853,7 +6918,7 @@ impl<'a> Parser<'a> { Ok(a_var) } - /// Parse an element of a struct definition + /// Parses an element of a struct declaration. fn parse_struct_decl_field(&mut self) -> PResult<'a, StructField> { let attrs = self.parse_outer_attributes()?; let lo = self.span; @@ -6861,11 +6926,11 @@ impl<'a> Parser<'a> { self.parse_single_struct_field(lo, vis, attrs) } - /// Parse `pub`, `pub(crate)` and `pub(in path)` plus shortcuts `crate` for `pub(crate)`, + /// Parses `pub`, `pub(crate)` and `pub(in path)` plus shortcuts `crate` for `pub(crate)`, /// `pub(self)` for `pub(in self)` and `pub(super)` for `pub(in super)`. - /// If the following element can't be a tuple (i.e., it's a function definition, - /// it's not a tuple struct field) and the contents within the parens - /// isn't valid, emit a proper diagnostic. + /// If the following element can't be a tuple (i.e., it's a function definition), then + /// it's not a tuple struct field), and the contents within the parentheses isn't valid, + /// so emit a proper diagnostic. pub fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { maybe_whole!(self, NtVis, |x| x); @@ -6946,7 +7011,7 @@ impl<'a> Parser<'a> { Ok(respan(lo, VisibilityKind::Public)) } - /// Parse defaultness: `default` or nothing. + /// Parses defaultness (i.e., `default` or nothing). fn parse_defaultness(&mut self) -> Defaultness { // `pub` is included for better error messages if self.check_keyword(keywords::Default) && @@ -6995,7 +7060,7 @@ impl<'a> Parser<'a> { } } - /// Given a termination token, parse all of the items in a module + /// Given a termination token, parses all of the items in a module. fn parse_mod_items(&mut self, term: &token::Token, inner_lo: Span) -> PResult<'a, Mod> { let mut items = vec![]; while let Some(item) = self.parse_item()? { @@ -7042,11 +7107,12 @@ impl<'a> Parser<'a> { /// Parse a `mod <foo> { ... }` or `mod <foo>;` item fn parse_item_mod(&mut self, outer_attrs: &[Attribute]) -> PResult<'a, ItemInfo> { let (in_cfg, outer_attrs) = { - let mut strip_unconfigured = ::config::StripUnconfigured { + let mut strip_unconfigured = crate::config::StripUnconfigured { sess: self.sess, features: None, // don't perform gated feature checking }; - let outer_attrs = strip_unconfigured.process_cfg_attrs(outer_attrs.to_owned()); + let mut outer_attrs = outer_attrs.to_owned(); + strip_unconfigured.process_cfg_attrs(&mut outer_attrs); (!self.cfg_mods || strip_unconfigured.in_cfg(&outer_attrs), outer_attrs) }; @@ -7131,7 +7197,7 @@ impl<'a> Parser<'a> { } } - /// Returns either a path to a module, or . + /// Returns a path to a module. pub fn default_submod_path( id: ast::Ident, relative: Option<ast::Ident>, @@ -7274,7 +7340,7 @@ impl<'a> Parser<'a> { } } - /// Read a module from a source file. + /// Reads a module from a source file. fn eval_src_mod(&mut self, path: PathBuf, directory_ownership: DirectoryOwnership, @@ -7306,7 +7372,7 @@ impl<'a> Parser<'a> { Ok((m0, mod_attrs)) } - /// Parse a function declaration from a foreign module + /// Parses a function declaration from a foreign module. fn parse_item_foreign_fn(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, ForeignItem> { self.expect_keyword(keywords::Fn)?; @@ -7326,7 +7392,7 @@ impl<'a> Parser<'a> { }) } - /// Parse a static item from a foreign module. + /// Parses a static item from a foreign module. /// Assumes that the `static` keyword is already parsed. fn parse_item_foreign_static(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, ForeignItem> { @@ -7346,7 +7412,7 @@ impl<'a> Parser<'a> { }) } - /// Parse a type from a foreign module + /// Parses a type from a foreign module. fn parse_item_foreign_type(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, ForeignItem> { self.expect_keyword(keywords::Type)?; @@ -7405,12 +7471,14 @@ impl<'a> Parser<'a> { Ok(ident) } - /// Parse extern crate links + /// Parses `extern crate` links. /// /// # Examples /// + /// ``` /// extern crate foo; /// extern crate bar as foo; + /// ``` fn parse_item_extern_crate(&mut self, lo: Span, visibility: Visibility, @@ -7429,16 +7497,17 @@ impl<'a> Parser<'a> { Ok(self.mk_item(span, item_name, ItemKind::ExternCrate(orig_name), visibility, attrs)) } - /// Parse `extern` for foreign ABIs - /// modules. + /// Parses `extern` for foreign ABIs modules. /// /// `extern` is expected to have been - /// consumed before calling this method + /// consumed before calling this method. /// - /// # Examples: + /// # Examples /// + /// ```ignore (only-for-syntax-highlight) /// extern "C" {} /// extern {} + /// ``` fn parse_item_foreign_mod(&mut self, lo: Span, opt_abi: Option<Abi>, @@ -7465,11 +7534,12 @@ impl<'a> Parser<'a> { Ok(self.mk_item(lo.to(prev_span), invalid, ItemKind::ForeignMod(m), visibility, attrs)) } - /// Parse `type Foo = Bar;` + /// Parses `type Foo = Bar;` /// or /// `existential type Foo: Bar;` /// or - /// `return None` without modifying the parser state + /// `return `None`` + /// without modifying the parser state. fn eat_type(&mut self) -> Option<PResult<'a, (Ident, AliasKind, ast::Generics)>> { // This parses the grammar: // Ident ["<"...">"] ["where" ...] ("=" | ":") Ty ";" @@ -7484,7 +7554,7 @@ impl<'a> Parser<'a> { } } - /// Parse type alias or existential type + /// Parses a type alias or existential type. fn parse_existential_or_alias( &mut self, existential: bool, @@ -7505,7 +7575,7 @@ impl<'a> Parser<'a> { Ok((ident, alias, tps)) } - /// Parse the part of an "enum" decl following the '{' + /// Parses the part of an enum declaration following the `{`. fn parse_enum_def(&mut self, _generics: &ast::Generics) -> PResult<'a, EnumDef> { let mut variants = Vec::new(); let mut all_nullary = true; @@ -7564,7 +7634,7 @@ impl<'a> Parser<'a> { Ok(ast::EnumDef { variants }) } - /// Parse an "enum" declaration + /// Parses an enum declaration. fn parse_item_enum(&mut self) -> PResult<'a, ItemInfo> { let id = self.parse_ident()?; let mut generics = self.parse_generics()?; @@ -7660,7 +7730,7 @@ impl<'a> Parser<'a> { })) } - /// Parse one of the items allowed by the flags. + /// Parses one of the items allowed by the flags. fn parse_item_implementation( &mut self, attrs: Vec<Attribute>, @@ -8085,7 +8155,7 @@ impl<'a> Parser<'a> { self.parse_macro_use_or_failure(attrs, macros_allowed, attributes_allowed, lo, visibility) } - /// Parse a foreign item. + /// Parses a foreign item. crate fn parse_foreign_item(&mut self) -> PResult<'a, ForeignItem> { maybe_whole!(self, NtForeignItem, |ni| ni); @@ -8201,7 +8271,7 @@ impl<'a> Parser<'a> { Ok(None) } - /// Parse a macro invocation inside a `trait`, `impl` or `extern` block + /// Parses a macro invocation inside a `trait`, `impl` or `extern` block. fn parse_assoc_macro_invoc(&mut self, item_kind: &str, vis: Option<&Visibility>, at_end: &mut bool) -> PResult<'a, Option<Mac>> { @@ -8227,7 +8297,7 @@ impl<'a> Parser<'a> { // eat a matched-delimiter token tree: let (delim, tts) = self.expect_delimited_token_tree()?; if delim != MacDelimiter::Brace { - self.expect(&token::Semi)? + self.expect(&token::Semi)?; } Ok(Some(respan(lo.to(self.prev_span), Mac_ { path: pth, tts, delim }))) @@ -8304,13 +8374,15 @@ impl<'a> Parser<'a> { *t == token::BinOp(token::Star)) } - /// Parse UseTree + /// Parses a `UseTree`. /// + /// ``` /// USE_TREE = [`::`] `*` | /// [`::`] `{` USE_TREE_LIST `}` | /// PATH `::` `*` | /// PATH `::` `{` USE_TREE_LIST `}` | /// PATH [`as` IDENT] + /// ``` fn parse_use_tree(&mut self) -> PResult<'a, UseTree> { let lo = self.span; @@ -8349,9 +8421,11 @@ impl<'a> Parser<'a> { Ok(UseTree { prefix, kind, span: lo.to(self.prev_span) }) } - /// Parse UseTreeKind::Nested(list) + /// Parses a `UseTreeKind::Nested(list)`. /// + /// ``` /// USE_TREE_LIST = Ø | (USE_TREE `,`)* USE_TREE [`,`] + /// ``` fn parse_use_tree_list(&mut self) -> PResult<'a, Vec<(UseTree, ast::NodeId)>> { self.parse_unspanned_seq(&token::OpenDelim(token::Brace), &token::CloseDelim(token::Brace), @@ -8368,15 +8442,17 @@ impl<'a> Parser<'a> { } } - /// Parses a source module as a crate. This is the main - /// entry point for the parser. + /// Parses a source module as a crate. This is the main entry point for the parser. pub fn parse_crate_mod(&mut self) -> PResult<'a, Crate> { let lo = self.span; - Ok(ast::Crate { + let krate = Ok(ast::Crate { attrs: self.parse_inner_attributes()?, module: self.parse_mod_items(&token::Eof, lo)?, span: lo.to(self.span), - }) + }); + emit_unclosed_delims(&self.unclosed_delims, self.diagnostic()); + self.unclosed_delims.clear(); + krate } pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> { @@ -8405,3 +8481,20 @@ impl<'a> Parser<'a> { } } } + +pub fn emit_unclosed_delims(unclosed_delims: &[UnmatchedBrace], handler: &errors::Handler) { + for unmatched in unclosed_delims { + let mut err = handler.struct_span_err(unmatched.found_span, &format!( + "incorrect close delimiter: `{}`", + pprust::token_to_string(&token::Token::CloseDelim(unmatched.found_delim)), + )); + err.span_label(unmatched.found_span, "incorrect close delimiter"); + if let Some(sp) = unmatched.candidate_span { + err.span_label(sp, "close delimiter possibly meant for this"); + } + if let Some(sp) = unmatched.unclosed_span { + err.span_label(sp, "un-closed delimiter"); + } + err.emit(); + } +} diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index f06e975a6d9..ff7f3e0bfae 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -1,22 +1,27 @@ -pub use self::BinOpToken::*; -pub use self::Nonterminal::*; -pub use self::DelimToken::*; -pub use self::Lit::*; -pub use self::Token::*; - -use ast::{self}; -use parse::ParseSess; -use print::pprust; -use ptr::P; +pub use BinOpToken::*; +pub use Nonterminal::*; +pub use DelimToken::*; +pub use Lit::*; +pub use Token::*; + +use crate::ast::{self}; +use crate::parse::ParseSess; +use crate::print::pprust; +use crate::ptr::P; +use crate::symbol::keywords; +use crate::syntax::parse::parse_stream_from_source_str; +use crate::syntax::parse::parser::emit_unclosed_delims; +use crate::tokenstream::{self, DelimSpan, TokenStream, TokenTree}; + use serialize::{Decodable, Decoder, Encodable, Encoder}; -use symbol::keywords; -use syntax::parse::parse_stream_from_source_str; -use syntax_pos::{self, Span, FileName}; use syntax_pos::symbol::{self, Symbol}; -use tokenstream::{self, DelimSpan, TokenStream, TokenTree}; +use syntax_pos::{self, Span, FileName}; +use log::info; use std::{cmp, fmt}; use std::mem; +#[cfg(target_arch = "x86_64")] +use rustc_data_structures::static_assert; use rustc_data_structures::sync::{Lrc, Lock}; #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] @@ -33,16 +38,16 @@ pub enum BinOpToken { Shr, } -/// A delimiter token +/// A delimiter token. #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum DelimToken { - /// A round parenthesis: `(` or `)` + /// A round parenthesis (i.e., `(` or `)`). Paren, - /// A square bracket: `[` or `]` + /// A square bracket (i.e., `[` or `]`). Bracket, - /// A curly brace: `{` or `}` + /// A curly brace (i.e., `{` or `}`). Brace, - /// An empty delimiter + /// An empty delimiter. NoDelim, } @@ -167,9 +172,9 @@ pub enum Token { Question, /// Used by proc macros for representing lifetimes, not generated by lexer right now. SingleQuote, - /// An opening delimiter, eg. `{` + /// An opening delimiter (e.g., `{`). OpenDelim(DelimToken), - /// A closing delimiter, eg. `}` + /// A closing delimiter (e.g., `}`). CloseDelim(DelimToken), /* Literals */ @@ -183,16 +188,16 @@ pub enum Token { // and so the `LazyTokenStream` can be ignored by Eq, Hash, etc. Interpolated(Lrc<(Nonterminal, LazyTokenStream)>), // Can be expanded into several tokens. - /// Doc comment + /// A doc comment. DocComment(ast::Name), // Junk. These carry no data because we don't really care about the data // they *would* carry, and don't really want to allocate a new ident for // them. Instead, users could extract that from the associated span. - /// Whitespace + /// Whitespace. Whitespace, - /// Comment + /// A comment. Comment, Shebang(ast::Name), @@ -275,6 +280,20 @@ impl Token { } } + /// Returns `true` if the token can appear at the start of a const param. + pub fn can_begin_const_arg(&self) -> bool { + match self { + OpenDelim(Brace) => true, + Interpolated(ref nt) => match nt.0 { + NtExpr(..) => true, + NtBlock(..) => true, + NtLiteral(..) => true, + _ => false, + } + _ => self.can_begin_literal_or_bool(), + } + } + /// Returns `true` if the token can appear at the start of a generic bound. crate fn can_begin_bound(&self) -> bool { self.is_path_start() || self.is_lifetime() || self.is_keyword(keywords::For) || @@ -289,7 +308,7 @@ impl Token { } } - /// Returns `true` if the token is any literal, a minus (which can follow a literal, + /// Returns `true` if the token is any literal, a minus (which can prefix a literal, /// for example a '-42', or one of the boolean idents). crate fn can_begin_literal_or_bool(&self) -> bool { match *self { @@ -483,8 +502,8 @@ impl Token { /// Enables better error recovery when the wrong token is found. crate fn similar_tokens(&self) -> Option<Vec<Token>> { match *self { - Comma => Some(vec![Dot, Lt]), - Semi => Some(vec![Colon]), + Comma => Some(vec![Dot, Lt, Semi]), + Semi => Some(vec![Colon, Comma]), _ => None } } @@ -541,7 +560,10 @@ impl Token { // FIXME(#43081): Avoid this pretty-print + reparse hack let source = pprust::token_to_string(self); let filename = FileName::macro_expansion_source_code(&source); - parse_stream_from_source_str(filename, source, sess, Some(span)) + let (tokens, errors) = parse_stream_from_source_str( + filename, source, sess, Some(span)); + emit_unclosed_delims(&errors, &sess.span_diagnostic); + tokens }); // During early phases of the compiler the AST could get modified @@ -683,7 +705,7 @@ impl PartialEq for Nonterminal { } impl fmt::Debug for Nonterminal { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { NtItem(..) => f.pad("NtItem(..)"), NtBlock(..) => f.pad("NtBlock(..)"), @@ -729,13 +751,13 @@ impl PartialEq for LazyTokenStream { } impl fmt::Debug for LazyTokenStream { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.clone().0.into_inner(), f) } } impl LazyTokenStream { - fn new() -> Self { + pub fn new() -> Self { LazyTokenStream(Lock::new(None)) } @@ -782,12 +804,13 @@ fn prepend_attrs(sess: &ParseSess, let source = pprust::attr_to_string(attr); let macro_filename = FileName::macro_expansion_source_code(&source); if attr.is_sugared_doc { - let stream = parse_stream_from_source_str( + let (stream, errors) = parse_stream_from_source_str( macro_filename, source, sess, Some(span), ); + emit_unclosed_delims(&errors, &sess.span_diagnostic); builder.push(stream); continue } @@ -804,12 +827,13 @@ fn prepend_attrs(sess: &ParseSess, // ... and for more complicated paths, fall back to a reparse hack that // should eventually be removed. } else { - let stream = parse_stream_from_source_str( + let (stream, errors) = parse_stream_from_source_str( macro_filename, source, sess, Some(span), ); + emit_unclosed_delims(&errors, &sess.span_diagnostic); brackets.push(stream); } |
