diff options
Diffstat (limited to 'compiler/rustc_parse/src')
| -rw-r--r-- | compiler/rustc_parse/src/lexer/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/lib.rs | 62 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/attr.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/diagnostics.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/expr.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/item.rs | 41 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/mod.rs | 118 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/nonterminal.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/pat.rs | 33 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/path.rs | 117 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/stmt.rs | 124 |
11 files changed, 397 insertions, 137 deletions
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 0dfacd78908..b5b34c7338d 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -510,7 +510,7 @@ impl<'a> StringReader<'a> { FatalError.raise() } - /// Note: It was decided to not add a test case, because it would be to big. + /// Note: It was decided to not add a test case, because it would be too big. /// <https://github.com/rust-lang/rust/pull/50296#issuecomment-392135180> fn report_too_many_hashes(&self, start: BytePos, found: usize) -> ! { self.fatal_span_( diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index f125a12147a..44999c9b63a 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -1,21 +1,23 @@ //! The main parser interface. -#![feature(bool_to_option)] #![feature(crate_visibility_modifier)] #![feature(bindings_after_at)] #![feature(iter_order_by)] #![feature(or_patterns)] use rustc_ast as ast; +use rustc_ast::attr::HasAttrs; use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{self, LazyTokenStream, TokenStream, TokenTree}; use rustc_ast_pretty::pprust; +use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::sync::Lrc; use rustc_errors::{Diagnostic, FatalError, Level, PResult}; use rustc_session::parse::ParseSess; use rustc_span::{symbol::kw, FileName, SourceFile, Span, DUMMY_SP}; use smallvec::SmallVec; +use std::cell::RefCell; use std::mem; use std::path::Path; use std::str; @@ -250,29 +252,23 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // before we fall back to the stringification. let convert_tokens = - |tokens: &Option<LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream()); + |tokens: Option<&LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream()); let tokens = match *nt { Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()), - Nonterminal::NtBlock(ref block) => convert_tokens(&block.tokens), - Nonterminal::NtStmt(ref stmt) => { - // FIXME: We currently only collect tokens for `:stmt` - // matchers in `macro_rules!` macros. When we start collecting - // tokens for attributes on statements, we will need to prepend - // attributes here - convert_tokens(&stmt.tokens) - } - Nonterminal::NtPat(ref pat) => convert_tokens(&pat.tokens), - Nonterminal::NtTy(ref ty) => convert_tokens(&ty.tokens), + Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.as_ref()), + Nonterminal::NtStmt(ref stmt) => prepend_attrs(stmt.attrs(), stmt.tokens()), + Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()), + Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()), Nonterminal::NtIdent(ident, is_raw) => { Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) } Nonterminal::NtLifetime(ident) => { Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) } - Nonterminal::NtMeta(ref attr) => convert_tokens(&attr.tokens), - Nonterminal::NtPath(ref path) => convert_tokens(&path.tokens), - Nonterminal::NtVis(ref vis) => convert_tokens(&vis.tokens), + Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()), + Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()), + Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()), Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => { if expr.tokens.is_none() { @@ -282,6 +278,25 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke } }; + // Caches the stringification of 'good' `TokenStreams` which passed + // `tokenstream_probably_equal_for_proc_macro`. This allows us to avoid + // repeatedly stringifying and comparing the same `TokenStream` for deeply + // nested nonterminals. + // + // We cache by the strinification instead of the `TokenStream` to avoid + // needing to implement `Hash` for `TokenStream`. Note that it's possible to + // have two distinct `TokenStream`s that stringify to the same result + // (e.g. if they differ only in hygiene information). However, any + // information lost during the stringification process is also intentionally + // ignored by `tokenstream_probably_equal_for_proc_macro`, so it's fine + // that a single cache entry may 'map' to multiple distinct `TokenStream`s. + // + // This is a temporary hack to prevent compilation blowup on certain inputs. + // The entire pretty-print/retokenize process will be removed soon. + thread_local! { + static GOOD_TOKEN_CACHE: RefCell<FxHashSet<String>> = Default::default(); + } + // FIXME(#43081): Avoid this pretty-print + reparse hack // Pretty-print the AST struct without inserting any parenthesis // beyond those explicitly written by the user (e.g. `ExpnKind::Paren`). @@ -289,7 +304,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // ever used for a comparison against the capture tokenstream. let source = pprust::nonterminal_to_string_no_extra_parens(nt); let filename = FileName::macro_expansion_source_code(&source); - let reparsed_tokens = parse_stream_from_source_str(filename, source, sess, Some(span)); + let reparsed_tokens = parse_stream_from_source_str(filename, source.clone(), sess, Some(span)); // During early phases of the compiler the AST could get modified // directly (e.g., attributes added or removed) and the internal cache @@ -315,8 +330,13 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // modifications, including adding/removing typically non-semantic // tokens such as extra braces and commas, don't happen. if let Some(tokens) = tokens { + if GOOD_TOKEN_CACHE.with(|cache| cache.borrow().contains(&source)) { + return tokens; + } + // Compare with a non-relaxed delim match to start. if tokenstream_probably_equal_for_proc_macro(&tokens, &reparsed_tokens, sess, false) { + GOOD_TOKEN_CACHE.with(|cache| cache.borrow_mut().insert(source.clone())); return tokens; } @@ -325,6 +345,11 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // token stream to match up with inserted parenthesis in the reparsed stream. let source_with_parens = pprust::nonterminal_to_string(nt); let filename_with_parens = FileName::macro_expansion_source_code(&source_with_parens); + + if GOOD_TOKEN_CACHE.with(|cache| cache.borrow().contains(&source_with_parens)) { + return tokens; + } + let reparsed_tokens_with_parens = parse_stream_from_source_str( filename_with_parens, source_with_parens, @@ -340,6 +365,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke sess, true, ) { + GOOD_TOKEN_CACHE.with(|cache| cache.borrow_mut().insert(source.clone())); return tokens; } @@ -419,9 +445,9 @@ pub fn tokenstream_probably_equal_for_proc_macro( // to iterate breaking tokens mutliple times. For example: // '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]' let mut token_trees: SmallVec<[_; 2]>; - if let TokenTree::Token(token) = &tree { + if let TokenTree::Token(token) = tree { let mut out = SmallVec::<[_; 2]>::new(); - out.push(token.clone()); + out.push(token); // Iterate to fixpoint: // * We start off with 'out' containing our initial token, and `temp` empty // * If we are able to break any tokens in `out`, then `out` will have diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 3738fbaeac8..fae09fa6fec 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -8,8 +8,9 @@ use rustc_span::{sym, Span}; use tracing::debug; +// Public for rustfmt usage #[derive(Debug)] -pub(super) enum InnerAttrPolicy<'a> { +pub enum InnerAttrPolicy<'a> { Permitted, Forbidden { reason: &'a str, saw_doc_comment: bool, prev_attr_sp: Option<Span> }, } @@ -78,7 +79,8 @@ impl<'a> Parser<'a> { /// Matches `attribute = # ! [ meta_item ]`. /// `inner_parse_policy` prescribes how to handle inner attributes. - fn parse_attribute( + // Public for rustfmt usage. + pub fn parse_attribute( &mut self, inner_parse_policy: InnerAttrPolicy<'_>, ) -> PResult<'a, ast::Attribute> { @@ -312,14 +314,13 @@ impl<'a> Parser<'a> { } pub fn maybe_needs_tokens(attrs: &[ast::Attribute]) -> bool { + // One of the attributes may either itself be a macro, or apply derive macros (`derive`), + // or expand to macro attributes (`cfg_attr`). attrs.iter().any(|attr| { - if let Some(ident) = attr.ident() { + attr.ident().map_or(true, |ident| { ident.name == sym::derive - // This might apply a custom attribute/derive - || ident.name == sym::cfg_attr - || !rustc_feature::is_builtin_attr_name(ident.name) - } else { - true - } + || ident.name == sym::cfg_attr + || !rustc_feature::is_builtin_attr_name(ident.name) + }) }) } diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index cd3b8db2303..350a372a684 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -1808,9 +1808,13 @@ impl<'a> Parser<'a> { return Ok(false); // Don't continue. } - /// Handle a generic const argument that had not been enclosed in braces, and suggest enclosing - /// it braces. In this situation, unlike in `handle_ambiguous_unbraced_const_arg`, this is - /// almost certainly a const argument, so we always offer a suggestion. + /// Attempt to parse a generic const argument that has not been enclosed in braces. + /// There are a limited number of expressions that are permitted without being encoded + /// in braces: + /// - Literals. + /// - Single-segment paths (i.e. standalone generic const parameters). + /// All other expressions that can be parsed will emit an error suggesting the expression be + /// wrapped in braces. pub fn handle_unambiguous_unbraced_const_arg(&mut self) -> PResult<'a, P<Expr>> { let start = self.token.span; let expr = self.parse_expr_res(Restrictions::CONST_EXPR, None).map_err(|mut err| { diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index ffbf786491d..93be478fc8c 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1450,10 +1450,10 @@ impl<'a> Parser<'a> { .help("valid widths are 8, 16, 32, 64 and 128") .emit(); } else { - let msg = format!("invalid suffix `{}` for integer literal", suf); + let msg = format!("invalid suffix `{}` for number literal", suf); self.struct_span_err(span, &msg) .span_label(span, format!("invalid suffix `{}`", suf)) - .help("the suffix must be one of the integral types (`u32`, `isize`, etc)") + .help("the suffix must be one of the numeric types (`u32`, `isize`, `f32`, etc.)") .emit(); } } diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 5954b370e6d..4c92c198679 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -247,9 +247,14 @@ impl<'a> Parser<'a> { (ident, ItemKind::Static(ty, m, expr)) } else if let Const::Yes(const_span) = self.parse_constness() { // CONST ITEM - self.recover_const_mut(const_span); - let (ident, ty, expr) = self.parse_item_global(None)?; - (ident, ItemKind::Const(def(), ty, expr)) + if self.token.is_keyword(kw::Impl) { + // recover from `const impl`, suggest `impl const` + self.recover_const_impl(const_span, attrs, def())? + } else { + self.recover_const_mut(const_span); + let (ident, ty, expr) = self.parse_item_global(None)?; + (ident, ItemKind::Const(def(), ty, expr)) + } } else if self.check_keyword(kw::Trait) || self.check_auto_or_unsafe_trait_item() { // TRAIT ITEM self.parse_item_trait(attrs, lo)? @@ -988,6 +993,36 @@ impl<'a> Parser<'a> { } } + /// Recover on `const impl` with `const` already eaten. + fn recover_const_impl( + &mut self, + const_span: Span, + attrs: &mut Vec<Attribute>, + defaultness: Defaultness, + ) -> PResult<'a, ItemInfo> { + let impl_span = self.token.span; + let mut err = self.expected_ident_found(); + let mut impl_info = self.parse_item_impl(attrs, defaultness)?; + match impl_info.1 { + // only try to recover if this is implementing a trait for a type + ItemKind::Impl { of_trait: Some(ref trai), ref mut constness, .. } => { + *constness = Const::Yes(const_span); + + let before_trait = trai.path.span.shrink_to_lo(); + let const_up_to_impl = const_span.with_hi(impl_span.lo()); + err.multipart_suggestion( + "you might have meant to write a const trait impl", + vec![(const_up_to_impl, "".to_owned()), (before_trait, "const ".to_owned())], + Applicability::MaybeIncorrect, + ) + .emit(); + } + ItemKind::Impl { .. } => return Err(err), + _ => unreachable!(), + } + Ok(impl_info) + } + /// Parse `["const" | ("static" "mut"?)] $ident ":" $ty (= $expr)?` with /// `["const" | ("static" "mut"?)]` already parsed and stored in `m`. /// diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 40aa2db58c7..d51a0fcbf09 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -17,12 +17,13 @@ pub use path::PathStyle; use rustc_ast::ptr::P; use rustc_ast::token::{self, DelimToken, Token, TokenKind}; use rustc_ast::tokenstream::{self, DelimSpan, LazyTokenStream, Spacing}; -use rustc_ast::tokenstream::{CreateTokenStream, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{CreateTokenStream, TokenStream, TokenTree, TreeAndSpacing}; use rustc_ast::DUMMY_NODE_ID; use rustc_ast::{self as ast, AnonConst, AttrStyle, AttrVec, Const, CrateSugar, Extern, Unsafe}; use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacDelimiter, Mutability, StrLit}; use rustc_ast::{Visibility, VisibilityKind}; use rustc_ast_pretty::pprust; +use rustc_data_structures::sync::Lrc; use rustc_errors::PResult; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError}; use rustc_session::parse::ParseSess; @@ -131,6 +132,28 @@ struct TokenCursor { // Counts the number of calls to `next` or `next_desugared`, // depending on whether `desugar_doc_comments` is set. num_next_calls: usize, + // During parsing, we may sometimes need to 'unglue' a + // glued token into two component tokens + // (e.g. '>>' into '>' and '>), so that the parser + // can consume them one at a time. This process + // bypasses the normal capturing mechanism + // (e.g. `num_next_calls` will not be incremented), + // since the 'unglued' tokens due not exist in + // the original `TokenStream`. + // + // If we end up consuming both unglued tokens, + // then this is not an issue - we'll end up + // capturing the single 'glued' token. + // + // However, in certain circumstances, we may + // want to capture just the first 'unglued' token. + // For example, capturing the `Vec<u8>` + // in `Option<Vec<u8>>` requires us to unglue + // the trailing `>>` token. The `append_unglued_token` + // field is used to track this token - it gets + // appended to the captured stream when + // we evaluate a `LazyTokenStream` + append_unglued_token: Option<TreeAndSpacing>, } #[derive(Clone)] @@ -264,7 +287,7 @@ impl TokenType { TokenType::Ident => "identifier".to_string(), TokenType::Path => "path".to_string(), TokenType::Type => "type".to_string(), - TokenType::Const => "const".to_string(), + TokenType::Const => "a const expression".to_string(), } } } @@ -335,6 +358,7 @@ impl<'a> Parser<'a> { stack: Vec::new(), num_next_calls: 0, desugar_doc_comments, + append_unglued_token: None, }, desugar_doc_comments, unmatched_angle_bracket_count: 0, @@ -358,6 +382,10 @@ impl<'a> Parser<'a> { self.token_cursor.next() }; self.token_cursor.num_next_calls += 1; + // We've retrieved an token from the underlying + // cursor, so we no longer need to worry about + // an unglued token. See `break_and_eat` for more details + self.token_cursor.append_unglued_token = None; if next.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. next.span = fallback_span.with_ctxt(next.span.ctxt()); @@ -554,6 +582,14 @@ impl<'a> Parser<'a> { let first_span = self.sess.source_map().start_point(self.token.span); let second_span = self.token.span.with_lo(first_span.hi()); self.token = Token::new(first, first_span); + // Keep track of this token - if we end token capturing now, + // we'll want to append this token to the captured stream. + // + // If we consume any additional tokens, then this token + // is not needed (we'll capture the entire 'glued' token), + // and `next_tok` will set this field to `None` + self.token_cursor.append_unglued_token = + Some((TokenTree::Token(self.token.clone()), Spacing::Alone)); // Use the spacing of the glued token as the spacing // of the unglued second token. self.bump_with((Token::new(second, second_span), self.token_spacing)); @@ -873,7 +909,8 @@ impl<'a> Parser<'a> { id: DUMMY_NODE_ID, value: self.mk_expr(blk.span, ExprKind::Block(blk, None), AttrVec::new()), }; - Ok(self.mk_expr(span, ExprKind::ConstBlock(anon_const), AttrVec::new())) + let blk_span = anon_const.value.span; + Ok(self.mk_expr(span.to(blk_span), ExprKind::ConstBlock(anon_const), AttrVec::new())) } /// Parses mutability (`mut` or nothing). @@ -934,16 +971,24 @@ impl<'a> Parser<'a> { is_interpolated_expr = true; } } - let token_tree = if is_interpolated_expr { - // We need to accept arbitrary interpolated expressions to continue - // supporting things like `doc = $expr` that work on stable. - // Non-literal interpolated expressions are rejected after expansion. - self.parse_token_tree() - } else { - self.parse_unsuffixed_lit()?.token_tree() - }; - MacArgs::Eq(eq_span, token_tree.into()) + // The value here is never passed to macros as tokens by itself (not as a part + // of the whole attribute), so we don't collect tokens here. If this changes, + // then token will need to be collected. One catch here is that we are using + // a nonterminal for keeping the expression, but this nonterminal should not + // be wrapped into a group when converting to token stream. + let expr = self.parse_expr()?; + let span = expr.span; + + match &expr.kind { + // Not gated to supporte things like `doc = $expr` that work on stable. + _ if is_interpolated_expr => {} + ExprKind::Lit(lit) if lit.kind.is_unsuffixed() => {} + _ => self.sess.gated_spans.gate(sym::extended_key_value_attributes, span), + } + + let token = token::Interpolated(Lrc::new(token::NtExpr(expr))); + MacArgs::Eq(eq_span, TokenTree::token(token, span).into()) } else { MacArgs::Empty } @@ -1213,14 +1258,21 @@ impl<'a> Parser<'a> { // // This also makes `Parser` very cheap to clone, since // there is no intermediate collection buffer to clone. + #[derive(Clone)] struct LazyTokenStreamImpl { start_token: (Token, Spacing), cursor_snapshot: TokenCursor, num_calls: usize, desugar_doc_comments: bool, + trailing_semi: bool, + append_unglued_token: Option<TreeAndSpacing>, } impl CreateTokenStream for LazyTokenStreamImpl { fn create_token_stream(&self) -> TokenStream { + let mut num_calls = self.num_calls; + if self.trailing_semi { + num_calls += 1; + } // The token produced by the final call to `next` or `next_desugared` // was not actually consumed by the callback. The combination // of chaining the initial token and using `take` produces the desired @@ -1228,16 +1280,30 @@ impl<'a> Parser<'a> { // and omit the final token otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); let tokens = std::iter::once(self.start_token.clone()) - .chain((0..self.num_calls).map(|_| { + .chain((0..num_calls).map(|_| { if self.desugar_doc_comments { cursor_snapshot.next_desugared() } else { cursor_snapshot.next() } })) - .take(self.num_calls); + .take(num_calls); - make_token_stream(tokens) + make_token_stream(tokens, self.append_unglued_token.clone()) + } + fn add_trailing_semi(&self) -> Box<dyn CreateTokenStream> { + if self.trailing_semi { + panic!("Called `add_trailing_semi` twice!"); + } + if self.append_unglued_token.is_some() { + panic!( + "Cannot call `add_trailing_semi` when we have an unglued token {:?}", + self.append_unglued_token + ); + } + let mut new = self.clone(); + new.trailing_semi = true; + Box::new(new) } } @@ -1246,6 +1312,8 @@ impl<'a> Parser<'a> { num_calls: self.token_cursor.num_next_calls - cursor_snapshot.num_next_calls, cursor_snapshot, desugar_doc_comments: self.desugar_doc_comments, + trailing_semi: false, + append_unglued_token: self.token_cursor.append_unglued_token.clone(), }; Ok((ret, Some(LazyTokenStream::new(lazy_impl)))) } @@ -1300,7 +1368,10 @@ pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, sess: &Pa /// Converts a flattened iterator of tokens (including open and close delimiter tokens) /// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair /// of open and close delims. -fn make_token_stream(tokens: impl Iterator<Item = (Token, Spacing)>) -> TokenStream { +fn make_token_stream( + tokens: impl Iterator<Item = (Token, Spacing)>, + append_unglued_token: Option<TreeAndSpacing>, +) -> TokenStream { #[derive(Debug)] struct FrameData { open: Span, @@ -1323,14 +1394,17 @@ fn make_token_stream(tokens: impl Iterator<Item = (Token, Spacing)>) -> TokenStr .inner .push((delimited, Spacing::Alone)); } - token => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push((TokenTree::Token(token), spacing)), + token => { + stack + .last_mut() + .expect("Bottom token frame is missing!") + .inner + .push((TokenTree::Token(token), spacing)); + } } } - let final_buf = stack.pop().expect("Missing final buf!"); + let mut final_buf = stack.pop().expect("Missing final buf!"); + final_buf.inner.extend(append_unglued_token); assert!(stack.is_empty(), "Stack should be empty: final_buf={:?} stack={:?}", final_buf, stack); TokenStream::new(final_buf.inner) } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index ab88362dad9..c007f96a798 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -117,8 +117,8 @@ impl<'a> Parser<'a> { let (stmt, tokens) = self.collect_tokens(|this| this.parse_stmt())?; match stmt { Some(mut s) => { - if s.tokens.is_none() { - s.tokens = tokens; + if s.tokens().is_none() { + s.set_tokens(tokens); } token::NtStmt(s) } diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index ee9a6dca5ad..b62c7373800 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -18,7 +18,7 @@ pub(super) const PARAM_EXPECTED: Expected = Some("parameter name"); const WHILE_PARSING_OR_MSG: &str = "while parsing this or-pattern starting here"; /// Whether or not an or-pattern should be gated when occurring in the current context. -#[derive(PartialEq)] +#[derive(PartialEq, Clone, Copy)] pub(super) enum GateOr { Yes, No, @@ -94,7 +94,7 @@ impl<'a> Parser<'a> { ) -> PResult<'a, P<Pat>> { // Parse the first pattern (`p_0`). let first_pat = self.parse_pat(expected)?; - self.maybe_recover_unexpected_comma(first_pat.span, rc)?; + self.maybe_recover_unexpected_comma(first_pat.span, rc, gate_or)?; // If the next token is not a `|`, // this is not an or-pattern and we should exit here. @@ -110,7 +110,7 @@ impl<'a> Parser<'a> { err.span_label(lo, WHILE_PARSING_OR_MSG); err })?; - self.maybe_recover_unexpected_comma(pat.span, rc)?; + self.maybe_recover_unexpected_comma(pat.span, rc, gate_or)?; pats.push(pat); } let or_pattern_span = lo.to(self.prev_token.span); @@ -190,7 +190,12 @@ impl<'a> Parser<'a> { /// Some special error handling for the "top-level" patterns in a match arm, /// `for` loop, `let`, &c. (in contrast to subpatterns within such). - fn maybe_recover_unexpected_comma(&mut self, lo: Span, rc: RecoverComma) -> PResult<'a, ()> { + fn maybe_recover_unexpected_comma( + &mut self, + lo: Span, + rc: RecoverComma, + gate_or: GateOr, + ) -> PResult<'a, ()> { if rc == RecoverComma::No || self.token != token::Comma { return Ok(()); } @@ -209,18 +214,24 @@ impl<'a> Parser<'a> { let seq_span = lo.to(self.prev_token.span); let mut err = self.struct_span_err(comma_span, "unexpected `,` in pattern"); if let Ok(seq_snippet) = self.span_to_snippet(seq_span) { + const MSG: &str = "try adding parentheses to match on a tuple..."; + + let or_suggestion = + gate_or == GateOr::No || !self.sess.gated_spans.is_ungated(sym::or_patterns); err.span_suggestion( seq_span, - "try adding parentheses to match on a tuple...", + if or_suggestion { MSG } else { MSG.trim_end_matches('.') }, format!("({})", seq_snippet), Applicability::MachineApplicable, - ) - .span_suggestion( - seq_span, - "...or a vertical bar to match on multiple alternatives", - seq_snippet.replace(",", " |"), - Applicability::MachineApplicable, ); + if or_suggestion { + err.span_suggestion( + seq_span, + "...or a vertical bar to match on multiple alternatives", + seq_snippet.replace(",", " |"), + Applicability::MachineApplicable, + ); + } } Err(err) } diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 79e73749038..17e5bcf7605 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -3,10 +3,9 @@ use super::{Parser, TokenType}; use crate::maybe_whole; use rustc_ast::ptr::P; use rustc_ast::token::{self, Token}; -use rustc_ast::{ - self as ast, AngleBracketedArg, AngleBracketedArgs, GenericArg, ParenthesizedArgs, -}; +use rustc_ast::{self as ast, AngleBracketedArg, AngleBracketedArgs, ParenthesizedArgs}; use rustc_ast::{AnonConst, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode}; +use rustc_ast::{GenericArg, GenericArgs}; use rustc_ast::{Path, PathSegment, QSelf}; use rustc_errors::{pluralize, Applicability, PResult}; use rustc_span::source_map::{BytePos, Span}; @@ -414,32 +413,40 @@ impl<'a> Parser<'a> { /// Parses a single argument in the angle arguments `<...>` of a path segment. fn parse_angle_arg(&mut self) -> PResult<'a, Option<AngleBracketedArg>> { - if self.check_ident() && self.look_ahead(1, |t| matches!(t.kind, token::Eq | token::Colon)) - { - // Parse associated type constraint. - let lo = self.token.span; - let ident = self.parse_ident()?; - let kind = if self.eat(&token::Eq) { - let ty = self.parse_assoc_equality_term(ident, self.prev_token.span)?; - AssocTyConstraintKind::Equality { ty } - } else if self.eat(&token::Colon) { - let bounds = self.parse_generic_bounds(Some(self.prev_token.span))?; - AssocTyConstraintKind::Bound { bounds } - } else { - unreachable!(); - }; + let lo = self.token.span; + let arg = self.parse_generic_arg()?; + match arg { + Some(arg) => { + if self.check(&token::Colon) | self.check(&token::Eq) { + let (ident, gen_args) = self.get_ident_from_generic_arg(arg, lo)?; + let kind = if self.eat(&token::Colon) { + // Parse associated type constraint bound. + + let bounds = self.parse_generic_bounds(Some(self.prev_token.span))?; + AssocTyConstraintKind::Bound { bounds } + } else if self.eat(&token::Eq) { + // Parse associated type equality constraint + + let ty = self.parse_assoc_equality_term(ident, self.prev_token.span)?; + AssocTyConstraintKind::Equality { ty } + } else { + unreachable!(); + }; - let span = lo.to(self.prev_token.span); + let span = lo.to(self.prev_token.span); - // Gate associated type bounds, e.g., `Iterator<Item: Ord>`. - if let AssocTyConstraintKind::Bound { .. } = kind { - self.sess.gated_spans.gate(sym::associated_type_bounds, span); + // Gate associated type bounds, e.g., `Iterator<Item: Ord>`. + if let AssocTyConstraintKind::Bound { .. } = kind { + self.sess.gated_spans.gate(sym::associated_type_bounds, span); + } + let constraint = + AssocTyConstraint { id: ast::DUMMY_NODE_ID, ident, gen_args, kind, span }; + Ok(Some(AngleBracketedArg::Constraint(constraint))) + } else { + Ok(Some(AngleBracketedArg::Arg(arg))) + } } - - let constraint = AssocTyConstraint { id: ast::DUMMY_NODE_ID, ident, kind, span }; - Ok(Some(AngleBracketedArg::Constraint(constraint))) - } else { - Ok(self.parse_generic_arg()?.map(AngleBracketedArg::Arg)) + _ => Ok(None), } } @@ -489,6 +496,7 @@ impl<'a> Parser<'a> { /// - An expression surrounded in `{}`. /// - A literal. /// - A numeric literal prefixed by `-`. + /// - A single-segment path. pub(super) fn expr_is_valid_const_arg(&self, expr: &P<rustc_ast::Expr>) -> bool { match &expr.kind { ast::ExprKind::Block(_, _) | ast::ExprKind::Lit(_) => true, @@ -496,6 +504,13 @@ impl<'a> Parser<'a> { ast::ExprKind::Lit(_) => true, _ => false, }, + // We can only resolve single-segment paths at the moment, because multi-segment paths + // require type-checking: see `visit_generic_arg` in `src/librustc_resolve/late.rs`. + ast::ExprKind::Path(None, path) + if path.segments.len() == 1 && path.segments[0].args.is_none() => + { + true + } _ => false, } } @@ -534,4 +549,54 @@ impl<'a> Parser<'a> { }; Ok(Some(arg)) } + + fn get_ident_from_generic_arg( + &self, + gen_arg: GenericArg, + lo: Span, + ) -> PResult<'a, (Ident, Option<GenericArgs>)> { + let gen_arg_span = gen_arg.span(); + match gen_arg { + GenericArg::Type(t) => match t.into_inner().kind { + ast::TyKind::Path(qself, mut path) => { + if let Some(qself) = qself { + let mut err = self.struct_span_err( + gen_arg_span, + "qualified paths cannot be used in associated type constraints", + ); + err.span_label( + qself.path_span, + "not allowed in associated type constraints", + ); + return Err(err); + } + if path.segments.len() == 1 { + let path_seg = path.segments.remove(0); + let ident = path_seg.ident; + let gen_args = path_seg.args.map(|args| args.into_inner()); + return Ok((ident, gen_args)); + } + let err = self.struct_span_err( + path.span, + "paths with multiple segments cannot be used in associated type constraints", + ); + return Err(err); + } + _ => { + let span = lo.to(self.prev_token.span); + let err = self.struct_span_err( + span, + "only path types can be used in associated type constraints", + ); + return Err(err); + } + }, + _ => { + let span = lo.to(self.prev_token.span); + let err = self + .struct_span_err(span, "only types can be used in associated type constraints"); + return Err(err); + } + } + } } diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 131ff1ae6b3..e974556f43a 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -7,8 +7,10 @@ use super::{BlockMode, Parser, Restrictions, SemiColonMode}; use crate::maybe_whole; use rustc_ast as ast; +use rustc_ast::attr::HasAttrs; use rustc_ast::ptr::P; use rustc_ast::token::{self, TokenKind}; +use rustc_ast::tokenstream::LazyTokenStream; use rustc_ast::util::classify; use rustc_ast::{AttrStyle, AttrVec, Attribute, MacCall, MacCallStmt, MacStmtStyle}; use rustc_ast::{Block, BlockCheckMode, Expr, ExprKind, Local, Stmt, StmtKind, DUMMY_NODE_ID}; @@ -31,45 +33,75 @@ impl<'a> Parser<'a> { } fn parse_stmt_without_recovery(&mut self) -> PResult<'a, Option<Stmt>> { - maybe_whole!(self, NtStmt, |x| Some(x)); - - let attrs = self.parse_outer_attributes()?; + let mut attrs = self.parse_outer_attributes()?; + let has_attrs = !attrs.is_empty(); let lo = self.token.span; - let stmt = if self.eat_keyword(kw::Let) { - self.parse_local_mk(lo, attrs.into())? - } else if self.is_kw_followed_by_ident(kw::Mut) { - self.recover_stmt_local(lo, attrs.into(), "missing keyword", "let mut")? - } else if self.is_kw_followed_by_ident(kw::Auto) { - self.bump(); // `auto` - let msg = "write `let` instead of `auto` to introduce a new variable"; - self.recover_stmt_local(lo, attrs.into(), msg, "let")? - } else if self.is_kw_followed_by_ident(sym::var) { - self.bump(); // `var` - let msg = "write `let` instead of `var` to introduce a new variable"; - self.recover_stmt_local(lo, attrs.into(), msg, "let")? - } else if self.check_path() && !self.token.is_qpath_start() && !self.is_path_start_item() { - // We have avoided contextual keywords like `union`, items with `crate` visibility, - // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something - // that starts like a path (1 token), but it fact not a path. - // Also, we avoid stealing syntax from `parse_item_`. - self.parse_stmt_path_start(lo, attrs)? - } else if let Some(item) = self.parse_item_common(attrs.clone(), false, true, |_| true)? { - // FIXME: Bad copy of attrs - self.mk_stmt(lo.to(item.span), StmtKind::Item(P(item))) - } else if self.eat(&token::Semi) { - // Do not attempt to parse an expression if we're done here. - self.error_outer_attrs(&attrs); - self.mk_stmt(lo, StmtKind::Empty) - } else if self.token != token::CloseDelim(token::Brace) { - // Remainder are line-expr stmts. - let e = self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs.into()))?; - self.mk_stmt(lo.to(e.span), StmtKind::Expr(e)) + maybe_whole!(self, NtStmt, |stmt| { + let mut stmt = stmt; + stmt.visit_attrs(|stmt_attrs| { + mem::swap(stmt_attrs, &mut attrs); + stmt_attrs.extend(attrs); + }); + Some(stmt) + }); + + let parse_stmt_inner = |this: &mut Self| { + let stmt = if this.eat_keyword(kw::Let) { + this.parse_local_mk(lo, attrs.into())? + } else if this.is_kw_followed_by_ident(kw::Mut) { + this.recover_stmt_local(lo, attrs.into(), "missing keyword", "let mut")? + } else if this.is_kw_followed_by_ident(kw::Auto) { + this.bump(); // `auto` + let msg = "write `let` instead of `auto` to introduce a new variable"; + this.recover_stmt_local(lo, attrs.into(), msg, "let")? + } else if this.is_kw_followed_by_ident(sym::var) { + this.bump(); // `var` + let msg = "write `let` instead of `var` to introduce a new variable"; + this.recover_stmt_local(lo, attrs.into(), msg, "let")? + } else if this.check_path() + && !this.token.is_qpath_start() + && !this.is_path_start_item() + { + // We have avoided contextual keywords like `union`, items with `crate` visibility, + // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something + // that starts like a path (1 token), but it fact not a path. + // Also, we avoid stealing syntax from `parse_item_`. + this.parse_stmt_path_start(lo, attrs)? + } else if let Some(item) = + this.parse_item_common(attrs.clone(), false, true, |_| true)? + { + // FIXME: Bad copy of attrs + this.mk_stmt(lo.to(item.span), StmtKind::Item(P(item))) + } else if this.eat(&token::Semi) { + // Do not attempt to parse an expression if we're done here. + this.error_outer_attrs(&attrs); + this.mk_stmt(lo, StmtKind::Empty) + } else if this.token != token::CloseDelim(token::Brace) { + // Remainder are line-expr stmts. + let e = this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs.into()))?; + this.mk_stmt(lo.to(e.span), StmtKind::Expr(e)) + } else { + this.error_outer_attrs(&attrs); + return Ok(None); + }; + Ok(Some(stmt)) + }; + + let stmt = if has_attrs { + let (mut stmt, tokens) = self.collect_tokens(parse_stmt_inner)?; + if let Some(stmt) = &mut stmt { + // If we already have tokens (e.g. due to encounting an `NtStmt`), + // use those instead. + if stmt.tokens().is_none() { + stmt.set_tokens(tokens); + } + } + stmt } else { - self.error_outer_attrs(&attrs); - return Ok(None); + parse_stmt_inner(self)? }; - Ok(Some(stmt)) + Ok(stmt) } fn parse_stmt_path_start(&mut self, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, Stmt> { @@ -107,7 +139,7 @@ impl<'a> Parser<'a> { let kind = if delim == token::Brace || self.token == token::Semi || self.token == token::Eof { - StmtKind::MacCall(P(MacCallStmt { mac, style, attrs })) + StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None })) } else { // Since none of the above applied, this is an expression statement macro. let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new()); @@ -219,7 +251,7 @@ impl<'a> Parser<'a> { } }; let hi = if self.token == token::Semi { self.token.span } else { self.prev_token.span }; - Ok(P(ast::Local { ty, pat, init, id: DUMMY_NODE_ID, span: lo.to(hi), attrs })) + Ok(P(ast::Local { ty, pat, init, id: DUMMY_NODE_ID, span: lo.to(hi), attrs, tokens: None })) } /// Parses the RHS of a local variable declaration (e.g., '= 14;'). @@ -376,6 +408,12 @@ impl<'a> Parser<'a> { None => return Ok(None), }; + let add_semi_token = |tokens: Option<&mut LazyTokenStream>| { + if let Some(tokens) = tokens { + *tokens = tokens.add_trailing_semi(); + } + }; + let mut eat_semi = true; match stmt.kind { // Expression without semicolon. @@ -417,6 +455,7 @@ impl<'a> Parser<'a> { *expr = self.mk_expr_err(sp); } } + StmtKind::Expr(_) | StmtKind::MacCall(_) => {} StmtKind::Local(ref mut local) => { if let Err(e) = self.expect_semi() { // We might be at the `,` in `let x = foo<bar, baz>;`. Try to recover. @@ -430,13 +469,18 @@ impl<'a> Parser<'a> { } } eat_semi = false; + // We just checked that there's a semicolon in the tokenstream, + // so capture it + add_semi_token(local.tokens.as_mut()); } - StmtKind::Empty => eat_semi = false, - _ => {} + StmtKind::Empty | StmtKind::Item(_) | StmtKind::Semi(_) => eat_semi = false, } if eat_semi && self.eat(&token::Semi) { stmt = stmt.add_trailing_semicolon(); + // We just checked that we have a semicolon in the tokenstream, + // so capture it + add_semi_token(stmt.tokens_mut()); } stmt.span = stmt.span.to(self.prev_token.span); Ok(Some(stmt)) @@ -447,7 +491,7 @@ impl<'a> Parser<'a> { } pub(super) fn mk_stmt(&self, span: Span, kind: StmtKind) -> Stmt { - Stmt { id: DUMMY_NODE_ID, kind, span, tokens: None } + Stmt { id: DUMMY_NODE_ID, kind, span } } pub(super) fn mk_stmt_err(&self, span: Span) -> Stmt { |