diff options
Diffstat (limited to 'compiler/rustc_parse/src/parser')
| -rw-r--r-- | compiler/rustc_parse/src/parser/attr.rs | 28 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/attr_wrapper.rs | 112 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/diagnostics.rs | 74 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/expr.rs | 176 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/generics.rs | 160 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/item.rs | 189 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/mod.rs | 31 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/pat.rs | 53 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/path.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/stmt.rs | 65 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/ty.rs | 17 |
11 files changed, 505 insertions, 410 deletions
diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 8fdfbcee385..3d06017fcf3 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -8,7 +8,10 @@ use rustc_span::{sym, BytePos, Span}; use thin_vec::ThinVec; use tracing::debug; -use super::{AttrWrapper, Capturing, FnParseMode, ForceCollect, Parser, ParserRange, PathStyle}; +use super::{ + AttrWrapper, Capturing, FnParseMode, ForceCollect, Parser, ParserRange, PathStyle, Trailing, + UsePreAttrPos, +}; use crate::{errors, fluent_generated as fluent, maybe_whole}; // Public for rustfmt usage @@ -162,7 +165,7 @@ impl<'a> Parser<'a> { } loop { // skip any other attributes, we want the item - if snapshot.token.kind == token::Pound { + if snapshot.token == token::Pound { if let Err(err) = snapshot.parse_attribute(InnerAttrPolicy::Permitted) { err.cancel(); return Some(replacement_span); @@ -257,7 +260,8 @@ impl<'a> Parser<'a> { pub fn parse_attr_item(&mut self, force_collect: ForceCollect) -> PResult<'a, ast::AttrItem> { maybe_whole!(self, NtMeta, |attr| attr.into_inner()); - let do_parse = |this: &mut Self, _empty_attrs| { + // Attr items don't have attributes. + self.collect_tokens(None, AttrWrapper::empty(), force_collect, |this, _empty_attrs| { let is_unsafe = this.eat_keyword(kw::Unsafe); let unsafety = if is_unsafe { let unsafe_span = this.prev_token.span; @@ -273,10 +277,12 @@ impl<'a> Parser<'a> { if is_unsafe { this.expect(&token::CloseDelim(Delimiter::Parenthesis))?; } - Ok((ast::AttrItem { unsafety, path, args, tokens: None }, false)) - }; - // Attr items don't have attributes. - self.collect_tokens_trailing_token(AttrWrapper::empty(), force_collect, do_parse) + Ok(( + ast::AttrItem { unsafety, path, args, tokens: None }, + Trailing::No, + UsePreAttrPos::No, + )) + }) } /// Parses attributes that appear after the opening of an item. These should @@ -309,8 +315,8 @@ impl<'a> Parser<'a> { }; if let Some(attr) = attr { // If we are currently capturing tokens (i.e. we are within a call to - // `Parser::collect_tokens_trailing_tokens`) record the token positions of this - // inner attribute, for possible later processing in a `LazyAttrTokenStream`. + // `Parser::collect_tokens`) record the token positions of this inner attribute, + // for possible later processing in a `LazyAttrTokenStream`. if let Capturing::Yes = self.capture_state.capturing { let end_pos = self.num_bump_calls; let parser_range = ParserRange(start_pos..end_pos); @@ -343,7 +349,7 @@ impl<'a> Parser<'a> { // Presumably, the majority of the time there will only be one attr. let mut expanded_attrs = Vec::with_capacity(1); - while self.token.kind != token::Eof { + while self.token != token::Eof { let lo = self.token.span; let item = self.parse_attr_item(ForceCollect::Yes)?; expanded_attrs.push((item, lo.to(self.prev_token.span))); @@ -359,7 +365,7 @@ impl<'a> Parser<'a> { pub(crate) fn parse_meta_seq_top(&mut self) -> PResult<'a, ThinVec<ast::NestedMetaItem>> { // Presumably, the majority of the time there will only be one attr. let mut nmis = ThinVec::with_capacity(1); - while self.token.kind != token::Eof { + while self.token != token::Eof { nmis.push(self.parse_meta_item_inner()?); if !self.eat(&token::Comma) { break; diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index abf61036c2d..49df2811d52 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -12,9 +12,23 @@ use rustc_span::{sym, Span, DUMMY_SP}; use super::{ Capturing, FlatToken, ForceCollect, NodeRange, NodeReplacement, Parser, ParserRange, - TokenCursor, + TokenCursor, Trailing, }; +// When collecting tokens, this fully captures the start point. Usually its +// just after outer attributes, but occasionally it's before. +#[derive(Clone, Debug)] +pub(super) struct CollectPos { + start_token: (Token, Spacing), + cursor_snapshot: TokenCursor, + start_pos: u32, +} + +pub(super) enum UsePreAttrPos { + No, + Yes, +} + /// A wrapper type to ensure that the parser handles outer attributes correctly. /// When we parse outer attributes, we need to ensure that we capture tokens /// for the attribute target. This allows us to perform cfg-expansion on @@ -22,30 +36,32 @@ use super::{ /// /// This wrapper prevents direct access to the underlying `ast::AttrVec`. /// Parsing code can only get access to the underlying attributes -/// by passing an `AttrWrapper` to `collect_tokens_trailing_token`. +/// by passing an `AttrWrapper` to `collect_tokens`. /// This makes it difficult to accidentally construct an AST node /// (which stores an `ast::AttrVec`) without first collecting tokens. /// /// This struct has its own module, to ensure that the parser code /// cannot directly access the `attrs` field. #[derive(Debug, Clone)] -pub struct AttrWrapper { +pub(super) struct AttrWrapper { attrs: AttrVec, // The start of the outer attributes in the parser's token stream. // This lets us create a `NodeReplacement` for the entire attribute - // target, including outer attributes. - start_pos: u32, + // target, including outer attributes. `None` if there are no outer + // attributes. + start_pos: Option<u32>, } impl AttrWrapper { pub(super) fn new(attrs: AttrVec, start_pos: u32) -> AttrWrapper { - AttrWrapper { attrs, start_pos } + AttrWrapper { attrs, start_pos: Some(start_pos) } } - pub fn empty() -> AttrWrapper { - AttrWrapper { attrs: AttrVec::new(), start_pos: u32::MAX } + + pub(super) fn empty() -> AttrWrapper { + AttrWrapper { attrs: AttrVec::new(), start_pos: None } } - pub(crate) fn take_for_recovery(self, psess: &ParseSess) -> AttrVec { + pub(super) fn take_for_recovery(self, psess: &ParseSess) -> AttrVec { psess.dcx().span_delayed_bug( self.attrs.get(0).map(|attr| attr.span).unwrap_or(DUMMY_SP), "AttrVec is taken for recovery but no error is produced", @@ -56,12 +72,12 @@ impl AttrWrapper { /// Prepend `self.attrs` to `attrs`. // FIXME: require passing an NT to prevent misuse of this method - pub(crate) fn prepend_to_nt_inner(mut self, attrs: &mut AttrVec) { + pub(super) fn prepend_to_nt_inner(mut self, attrs: &mut AttrVec) { mem::swap(attrs, &mut self.attrs); attrs.extend(self.attrs); } - pub fn is_empty(&self) -> bool { + pub(super) fn is_empty(&self) -> bool { self.attrs.is_empty() } } @@ -77,7 +93,7 @@ fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool { } // From a value of this type we can reconstruct the `TokenStream` seen by the -// `f` callback passed to a call to `Parser::collect_tokens_trailing_token`, by +// `f` callback passed to a call to `Parser::collect_tokens`, by // replaying the getting of the tokens. This saves us producing a `TokenStream` // if it is never needed, e.g. a captured `macro_rules!` argument that is never // passed to a proc macro. In practice, token stream creation happens rarely @@ -166,16 +182,30 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { } impl<'a> Parser<'a> { + pub(super) fn collect_pos(&self) -> CollectPos { + CollectPos { + start_token: (self.token.clone(), self.token_spacing), + cursor_snapshot: self.token_cursor.clone(), + start_pos: self.num_bump_calls, + } + } + /// Parses code with `f`. If appropriate, it records the tokens (in /// `LazyAttrTokenStream` form) that were parsed in the result, accessible - /// via the `HasTokens` trait. The second (bool) part of the callback's + /// via the `HasTokens` trait. The `Trailing` part of the callback's /// result indicates if an extra token should be captured, e.g. a comma or - /// semicolon. + /// semicolon. The `UsePreAttrPos` part of the callback's result indicates + /// if we should use `pre_attr_pos` as the collection start position (only + /// required in a few cases). /// /// The `attrs` passed in are in `AttrWrapper` form, which is opaque. The /// `AttrVec` within is passed to `f`. See the comment on `AttrWrapper` for /// details. /// + /// `pre_attr_pos` is the position before the outer attributes (or the node + /// itself, if no outer attributes are present). It is only needed if `f` + /// can return `UsePreAttrPos::Yes`. + /// /// Note: If your callback consumes an opening delimiter (including the /// case where `self.token` is an opening delimiter on entry to this /// function), you must also consume the corresponding closing delimiter. @@ -197,11 +227,12 @@ impl<'a> Parser<'a> { /// } // 32..33 /// } // 33..34 /// ``` - pub fn collect_tokens_trailing_token<R: HasAttrs + HasTokens>( + pub(super) fn collect_tokens<R: HasAttrs + HasTokens>( &mut self, + pre_attr_pos: Option<CollectPos>, attrs: AttrWrapper, force_collect: ForceCollect, - f: impl FnOnce(&mut Self, ast::AttrVec) -> PResult<'a, (R, bool)>, + f: impl FnOnce(&mut Self, AttrVec) -> PResult<'a, (R, Trailing, UsePreAttrPos)>, ) -> PResult<'a, R> { // We must collect if anything could observe the collected tokens, i.e. // if any of the following conditions hold. @@ -220,23 +251,20 @@ impl<'a> Parser<'a> { return Ok(f(self, attrs.attrs)?.0); } - let start_token = (self.token.clone(), self.token_spacing); - let cursor_snapshot = self.token_cursor.clone(); - let start_pos = self.num_bump_calls; + let mut collect_pos = self.collect_pos(); let has_outer_attrs = !attrs.attrs.is_empty(); let parser_replacements_start = self.capture_state.parser_replacements.len(); // We set and restore `Capturing::Yes` on either side of the call to - // `f`, so we can distinguish the outermost call to - // `collect_tokens_trailing_token` (e.g. parsing `m` in the example - // above) from any inner (indirectly recursive) calls (e.g. parsing `g` - // in the example above). This distinction is used below and in - // `Parser::parse_inner_attributes`. - let (mut ret, capture_trailing) = { + // `f`, so we can distinguish the outermost call to `collect_tokens` + // (e.g. parsing `m` in the example above) from any inner (indirectly + // recursive) calls (e.g. parsing `g` in the example above). This + // distinction is used below and in `Parser::parse_inner_attributes`. + let (mut ret, capture_trailing, use_pre_attr_pos) = { let prev_capturing = mem::replace(&mut self.capture_state.capturing, Capturing::Yes); - let ret_and_trailing = f(self, attrs.attrs); + let res = f(self, attrs.attrs); self.capture_state.capturing = prev_capturing; - ret_and_trailing? + res? }; // When we're not in `capture_cfg` mode, then skip collecting and @@ -279,10 +307,18 @@ impl<'a> Parser<'a> { return Ok(ret); } + // Replace the post-attribute collection start position with the + // pre-attribute position supplied, if `f` indicated it is necessary. + // (The caller is responsible for providing a non-`None` `pre_attr_pos` + // if this is a possibility.) + if matches!(use_pre_attr_pos, UsePreAttrPos::Yes) { + collect_pos = pre_attr_pos.unwrap(); + } + let parser_replacements_end = self.capture_state.parser_replacements.len(); assert!( - !(self.break_last_token && capture_trailing), + !(self.break_last_token && matches!(capture_trailing, Trailing::Yes)), "Cannot set break_last_token and have trailing token" ); @@ -294,7 +330,7 @@ impl<'a> Parser<'a> { // `AttrTokenStream`, we will create the proper token. + self.break_last_token as u32; - let num_calls = end_pos - start_pos; + let num_calls = end_pos - collect_pos.start_pos; // Take the captured `ParserRange`s for any inner attributes that we parsed in // `Parser::parse_inner_attributes`, and pair them in a `ParserReplacement` with `None`, @@ -328,7 +364,9 @@ impl<'a> Parser<'a> { .iter() .cloned() .chain(inner_attr_parser_replacements.iter().cloned()) - .map(|(parser_range, data)| (NodeRange::new(parser_range, start_pos), data)) + .map(|(parser_range, data)| { + (NodeRange::new(parser_range, collect_pos.start_pos), data) + }) .collect() }; @@ -355,9 +393,9 @@ impl<'a> Parser<'a> { // - `tokens`: lazy tokens for `g` (with its inner attr deleted). let tokens = LazyAttrTokenStream::new(LazyAttrTokenStreamImpl { - start_token, + start_token: collect_pos.start_token, + cursor_snapshot: collect_pos.cursor_snapshot, num_calls, - cursor_snapshot, break_last_token: self.break_last_token, node_replacements, }); @@ -368,9 +406,9 @@ impl<'a> Parser<'a> { } // If `capture_cfg` is set and we're inside a recursive call to - // `collect_tokens_trailing_token`, then we need to register a replace range - // if we have `#[cfg]` or `#[cfg_attr]`. This allows us to run eager cfg-expansion - // on the captured token stream. + // `collect_tokens`, then we need to register a replace range if we + // have `#[cfg]` or `#[cfg_attr]`. This allows us to run eager + // cfg-expansion on the captured token stream. if self.capture_cfg && matches!(self.capture_state.capturing, Capturing::Yes) && has_cfg_or_cfg_attr(ret.attrs()) @@ -389,7 +427,8 @@ impl<'a> Parser<'a> { // Set things up so that the entire AST node that we just parsed, including attributes, // will be replaced with `target` in the lazy token stream. This will allow us to // cfg-expand this AST node. - let start_pos = if has_outer_attrs { attrs.start_pos } else { start_pos }; + let start_pos = + if has_outer_attrs { attrs.start_pos.unwrap() } else { collect_pos.start_pos }; let target = AttrsTarget { attrs: ret.attrs().iter().cloned().collect(), tokens }; self.capture_state .parser_replacements @@ -490,7 +529,6 @@ mod size_asserts { use super::*; // tidy-alphabetical-start - static_assert_size!(AttrWrapper, 16); static_assert_size!(LazyAttrTokenStreamImpl, 96); // tidy-alphabetical-end } diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 47ca85ba060..ef1387c50fa 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -474,8 +474,8 @@ impl<'a> Parser<'a> { // If this isn't the case however, and the suggestion is a token the // content of which is the same as the found token's, we remove it as well. if !eq { - if let TokenType::Token(kind) = &token { - if kind == &self.token.kind { + if let TokenType::Token(kind) = token { + if self.token == *kind { return false; } } @@ -506,7 +506,7 @@ impl<'a> Parser<'a> { } else if !sm.is_multiline(self.prev_token.span.until(self.token.span)) { // The current token is in the same line as the prior token, not recoverable. } else if [token::Comma, token::Colon].contains(&self.token.kind) - && self.prev_token.kind == token::CloseDelim(Delimiter::Parenthesis) + && self.prev_token == token::CloseDelim(Delimiter::Parenthesis) { // Likely typo: The current token is on a new line and is expected to be // `.`, `;`, `?`, or an operator after a close delimiter token. @@ -518,7 +518,7 @@ impl<'a> Parser<'a> { // https://github.com/rust-lang/rust/issues/72253 } else if self.look_ahead(1, |t| { t == &token::CloseDelim(Delimiter::Brace) - || t.can_begin_expr() && t.kind != token::Colon + || t.can_begin_expr() && *t != token::Colon }) && [token::Comma, token::Colon].contains(&self.token.kind) { // Likely typo: `,` → `;` or `:` → `;`. This is triggered if the current token is @@ -562,7 +562,7 @@ impl<'a> Parser<'a> { } } - if self.token.kind == TokenKind::EqEq + if self.token == TokenKind::EqEq && self.prev_token.is_ident() && expected.iter().any(|tok| matches!(tok, TokenType::Token(TokenKind::Eq))) { @@ -655,9 +655,9 @@ impl<'a> Parser<'a> { // positive for a `cr#` that wasn't intended to start a c-string literal, but identifying // that in the parser requires unbounded lookahead, so we only add a hint to the existing // error rather than replacing it entirely. - if ((self.prev_token.kind == TokenKind::Ident(sym::c, IdentIsRaw::No) + if ((self.prev_token == TokenKind::Ident(sym::c, IdentIsRaw::No) && matches!(&self.token.kind, TokenKind::Literal(token::Lit { kind: token::Str, .. }))) - || (self.prev_token.kind == TokenKind::Ident(sym::cr, IdentIsRaw::No) + || (self.prev_token == TokenKind::Ident(sym::cr, IdentIsRaw::No) && matches!( &self.token.kind, TokenKind::Literal(token::Lit { kind: token::Str, .. }) | token::Pound @@ -673,7 +673,7 @@ impl<'a> Parser<'a> { // `pub` may be used for an item or `pub(crate)` if self.prev_token.is_ident_named(sym::public) && (self.token.can_begin_item() - || self.token.kind == TokenKind::OpenDelim(Delimiter::Parenthesis)) + || self.token == TokenKind::OpenDelim(Delimiter::Parenthesis)) { err.span_suggestion_short( self.prev_token.span, @@ -772,7 +772,7 @@ impl<'a> Parser<'a> { ), ); if self.token == token::Pound - && self.look_ahead(1, |t| t.kind == token::OpenDelim(Delimiter::Bracket)) + && self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Bracket)) { // We have // #[attr] @@ -867,7 +867,7 @@ impl<'a> Parser<'a> { let str_span = self.prev_token.span; let mut span = self.token.span; let mut count = 0; - while self.token.kind == TokenKind::Pound + while self.token == TokenKind::Pound && !sm.is_multiline(span.shrink_to_hi().until(self.token.span.shrink_to_lo())) { span = span.with_hi(self.token.span.hi()); @@ -1167,7 +1167,7 @@ impl<'a> Parser<'a> { return; } - if token::PathSep == self.token.kind && segment.args.is_none() { + if self.token == token::PathSep && segment.args.is_none() { let snapshot = self.create_snapshot_for_diagnostic(); self.bump(); let lo = self.token.span; @@ -1176,13 +1176,11 @@ impl<'a> Parser<'a> { let span = lo.to(self.prev_token.span); // Detect trailing `>` like in `x.collect::Vec<_>>()`. let mut trailing_span = self.prev_token.span.shrink_to_hi(); - while self.token.kind == token::BinOp(token::Shr) - || self.token.kind == token::Gt - { + while self.token == token::BinOp(token::Shr) || self.token == token::Gt { trailing_span = trailing_span.to(self.token.span); self.bump(); } - if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) { + if self.token == token::OpenDelim(Delimiter::Parenthesis) { // Recover from bad turbofish: `foo.collect::Vec<_>()`. segment.args = Some(AngleBracketedArgs { args, span }.into()); @@ -1430,7 +1428,7 @@ impl<'a> Parser<'a> { self.restore_snapshot(snapshot); } } - return if token::PathSep == self.token.kind { + return if self.token == token::PathSep { // We have some certainty that this was a bad turbofish at this point. // `foo< bar >::` if let ExprKind::Binary(o, ..) = inner_op.kind @@ -1462,7 +1460,7 @@ impl<'a> Parser<'a> { Err(self.dcx().create_err(err)) } } - } else if token::OpenDelim(Delimiter::Parenthesis) == self.token.kind { + } else if self.token == token::OpenDelim(Delimiter::Parenthesis) { // We have high certainty that this was a bad turbofish at this point. // `foo< bar >(` if let ExprKind::Binary(o, ..) = inner_op.kind @@ -1528,7 +1526,7 @@ impl<'a> Parser<'a> { ]; self.consume_tts(1, &modifiers); - if self.token.kind == token::Eof { + if self.token == token::Eof { // Not entirely sure that what we consumed were fn arguments, rollback. self.restore_snapshot(snapshot); Err(()) @@ -1811,7 +1809,7 @@ impl<'a> Parser<'a> { /// This function gets called in places where a semicolon is NOT expected and if there's a /// semicolon it emits the appropriate error and returns true. pub fn maybe_consume_incorrect_semicolon(&mut self, previous_item: Option<&Item>) -> bool { - if self.token.kind != TokenKind::Semi { + if self.token != TokenKind::Semi { return false; } @@ -2405,10 +2403,10 @@ impl<'a> Parser<'a> { modifier: &[(token::TokenKind, i64)], ) { while acc > 0 { - if let Some((_, val)) = modifier.iter().find(|(t, _)| *t == self.token.kind) { + if let Some((_, val)) = modifier.iter().find(|(t, _)| self.token == *t) { acc += *val; } - if self.token.kind == token::Eof { + if self.token == token::Eof { break; } self.bump(); @@ -2489,13 +2487,14 @@ impl<'a> Parser<'a> { pub(super) fn handle_unambiguous_unbraced_const_arg(&mut self) -> PResult<'a, P<Expr>> { let start = self.token.span; let attrs = self.parse_outer_attributes()?; - let expr = self.parse_expr_res(Restrictions::CONST_EXPR, attrs).map_err(|mut err| { - err.span_label( - start.shrink_to_lo(), - "while parsing a const generic argument starting here", - ); - err - })?; + let (expr, _) = + self.parse_expr_res(Restrictions::CONST_EXPR, attrs).map_err(|mut err| { + err.span_label( + start.shrink_to_lo(), + "while parsing a const generic argument starting here", + ); + err + })?; if !self.expr_is_valid_const_arg(&expr) { self.dcx().emit_err(ConstGenericWithoutBraces { span: expr.span, @@ -2598,7 +2597,7 @@ impl<'a> Parser<'a> { } }) .is_some() - || self.token.kind == TokenKind::Dot; + || self.token == TokenKind::Dot; // This will be true when a trait object type `Foo +` or a path which was a `const fn` with // type params has been parsed. let was_op = @@ -2615,9 +2614,9 @@ impl<'a> Parser<'a> { let attrs = self.parse_outer_attributes()?; self.parse_expr_res(Restrictions::CONST_EXPR, attrs) })() { - Ok(expr) => { + Ok((expr, _)) => { // Find a mistake like `MyTrait<Assoc == S::Assoc>`. - if token::EqEq == snapshot.token.kind { + if snapshot.token == token::EqEq { err.span_suggestion( snapshot.token.span, "if you meant to use an associated type binding, replace `==` with `=`", @@ -2627,7 +2626,7 @@ impl<'a> Parser<'a> { let guar = err.emit(); let value = self.mk_expr_err(start.to(expr.span), guar); return Ok(GenericArg::Const(AnonConst { id: ast::DUMMY_NODE_ID, value })); - } else if token::Colon == snapshot.token.kind + } else if snapshot.token == token::Colon && expr.span.lo() == snapshot.token.span.hi() && matches!(expr.kind, ExprKind::Path(..)) { @@ -2642,8 +2641,7 @@ impl<'a> Parser<'a> { return Ok(GenericArg::Type( self.mk_ty(start.to(expr.span), TyKind::Err(guar)), )); - } else if token::Comma == self.token.kind || self.token.kind.should_end_const_arg() - { + } else if self.token == token::Comma || self.token.kind.should_end_const_arg() { // Avoid the following output by checking that we consumed a full const arg: // help: expressions must be enclosed in braces to be used as const generic // arguments @@ -2674,7 +2672,7 @@ impl<'a> Parser<'a> { })() { // Since we don't know the exact reason why we failed to parse the type or the // expression, employ a simple heuristic to weed out some pathological cases. - Ok(expr) if let token::Comma | token::Gt = snapshot.token.kind => { + Ok((expr, _)) if let token::Comma | token::Gt = snapshot.token.kind => { self.restore_snapshot(snapshot); Some(expr) } @@ -2846,8 +2844,8 @@ impl<'a> Parser<'a> { pub(crate) fn maybe_recover_unexpected_block_label(&mut self) -> bool { // Check for `'a : {` if !(self.check_lifetime() - && self.look_ahead(1, |tok| tok.kind == token::Colon) - && self.look_ahead(2, |tok| tok.kind == token::OpenDelim(Delimiter::Brace))) + && self.look_ahead(1, |t| *t == token::Colon) + && self.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace))) { return false; } @@ -3001,7 +2999,7 @@ impl<'a> Parser<'a> { // >>>>>>> let mut end = None; loop { - if self.token.kind == TokenKind::Eof { + if self.token == TokenKind::Eof { break; } if let Some(span) = self.conflict_marker(&TokenKind::OrOr, &TokenKind::BinOp(token::Or)) diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index cf5d65708ab..e0917ba43e4 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -36,7 +36,7 @@ use super::pat::{CommaRecoveryMode, Expected, RecoverColon, RecoverComma}; use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign}; use super::{ AttrWrapper, BlockMode, ClosureSpans, ForceCollect, Parser, PathStyle, Restrictions, - SemiColonMode, SeqSep, TokenType, Trailing, + SemiColonMode, SeqSep, TokenType, Trailing, UsePreAttrPos, }; use crate::{errors, maybe_recover_from_interpolated_ty_qpath}; @@ -59,15 +59,30 @@ impl<'a> Parser<'a> { self.current_closure.take(); let attrs = self.parse_outer_attributes()?; - self.parse_expr_res(Restrictions::empty(), attrs) + self.parse_expr_res(Restrictions::empty(), attrs).map(|res| res.0) } /// Parses an expression, forcing tokens to be collected. pub fn parse_expr_force_collect(&mut self) -> PResult<'a, P<Expr>> { self.current_closure.take(); + // If the expression is associative (e.g. `1 + 2`), then any preceding + // outer attribute actually belongs to the first inner sub-expression. + // In which case we must use the pre-attr pos to include the attribute + // in the collected tokens for the outer expression. + let pre_attr_pos = self.collect_pos(); let attrs = self.parse_outer_attributes()?; - self.collect_tokens_no_attrs(|this| this.parse_expr_res(Restrictions::empty(), attrs)) + self.collect_tokens( + Some(pre_attr_pos), + AttrWrapper::empty(), + ForceCollect::Yes, + |this, _empty_attrs| { + let (expr, is_assoc) = this.parse_expr_res(Restrictions::empty(), attrs)?; + let use_pre_attr_pos = + if is_assoc { UsePreAttrPos::Yes } else { UsePreAttrPos::No }; + Ok((expr, Trailing::No, use_pre_attr_pos)) + }, + ) } pub fn parse_expr_anon_const(&mut self) -> PResult<'a, AnonConst> { @@ -77,7 +92,7 @@ impl<'a> Parser<'a> { fn parse_expr_catch_underscore(&mut self, restrictions: Restrictions) -> PResult<'a, P<Expr>> { let attrs = self.parse_outer_attributes()?; match self.parse_expr_res(restrictions, attrs) { - Ok(expr) => Ok(expr), + Ok((expr, _)) => Ok(expr), Err(err) => match self.token.ident() { Some((Ident { name: kw::Underscore, .. }, IdentIsRaw::No)) if self.may_recover() && self.look_ahead(1, |t| t == &token::Comma) => @@ -104,18 +119,20 @@ impl<'a> Parser<'a> { &mut self, r: Restrictions, attrs: AttrWrapper, - ) -> PResult<'a, P<Expr>> { + ) -> PResult<'a, (P<Expr>, bool)> { self.with_res(r, |this| this.parse_expr_assoc_with(0, attrs)) } /// Parses an associative expression with operators of at least `min_prec` precedence. + /// The `bool` in the return value indicates if it was an assoc expr, i.e. with an operator + /// followed by a subexpression (e.g. `1 + 2`). pub(super) fn parse_expr_assoc_with( &mut self, min_prec: usize, attrs: AttrWrapper, - ) -> PResult<'a, P<Expr>> { + ) -> PResult<'a, (P<Expr>, bool)> { let lhs = if self.token.is_range_separator() { - return self.parse_expr_prefix_range(attrs); + return self.parse_expr_prefix_range(attrs).map(|res| (res, false)); } else { self.parse_expr_prefix(attrs)? }; @@ -123,15 +140,17 @@ impl<'a> Parser<'a> { } /// Parses the rest of an associative expression (i.e. the part after the lhs) with operators - /// of at least `min_prec` precedence. + /// of at least `min_prec` precedence. The `bool` in the return value indicates if something + /// was actually parsed. pub(super) fn parse_expr_assoc_rest_with( &mut self, min_prec: usize, starts_stmt: bool, mut lhs: P<Expr>, - ) -> PResult<'a, P<Expr>> { + ) -> PResult<'a, (P<Expr>, bool)> { + let mut parsed_something = false; if !self.should_continue_as_assoc_expr(&lhs) { - return Ok(lhs); + return Ok((lhs, parsed_something)); } self.expected_tokens.push(TokenType::Operator); @@ -156,16 +175,17 @@ impl<'a> Parser<'a> { self.err_larrow_operator(self.token.span); } + parsed_something = true; self.bump(); if op.node.is_comparison() { if let Some(expr) = self.check_no_chained_comparison(&lhs, &op)? { - return Ok(expr); + return Ok((expr, parsed_something)); } } // Look for JS' `===` and `!==` and recover if (op.node == AssocOp::Equal || op.node == AssocOp::NotEqual) - && self.token.kind == token::Eq + && self.token == token::Eq && self.prev_token.span.hi() == self.token.span.lo() { let sp = op.span.to(self.token.span); @@ -190,7 +210,7 @@ impl<'a> Parser<'a> { // Look for PHP's `<>` and recover if op.node == AssocOp::Less - && self.token.kind == token::Gt + && self.token == token::Gt && self.prev_token.span.hi() == self.token.span.lo() { let sp = op.span.to(self.token.span); @@ -208,7 +228,7 @@ impl<'a> Parser<'a> { // Look for C++'s `<=>` and recover if op.node == AssocOp::LessEqual - && self.token.kind == token::Gt + && self.token == token::Gt && self.prev_token.span.hi() == self.token.span.lo() { let sp = op.span.to(self.token.span); @@ -263,7 +283,7 @@ impl<'a> Parser<'a> { // the special cases. The code is here only for future convenience. Fixity::None => 1, }; - let rhs = self.with_res(restrictions - Restrictions::STMT_EXPR, |this| { + let (rhs, _) = self.with_res(restrictions - Restrictions::STMT_EXPR, |this| { let attrs = this.parse_outer_attributes()?; this.parse_expr_assoc_with(prec + prec_adjustment, attrs) })?; @@ -319,7 +339,7 @@ impl<'a> Parser<'a> { } } - Ok(lhs) + Ok((lhs, parsed_something)) } fn should_continue_as_assoc_expr(&mut self, lhs: &Expr) -> bool { @@ -441,7 +461,8 @@ impl<'a> Parser<'a> { let attrs = self.parse_outer_attributes()?; Some( self.parse_expr_assoc_with(prec + 1, attrs) - .map_err(|err| self.maybe_err_dotdotlt_syntax(maybe_lt, err))?, + .map_err(|err| self.maybe_err_dotdotlt_syntax(maybe_lt, err))? + .0, ) } else { None @@ -498,7 +519,7 @@ impl<'a> Parser<'a> { // RHS must be parsed with more associativity than the dots. let attrs = this.parse_outer_attributes()?; this.parse_expr_assoc_with(op.unwrap().precedence() + 1, attrs) - .map(|x| (lo.to(x.span), Some(x))) + .map(|(x, _)| (lo.to(x.span), Some(x))) .map_err(|err| this.maybe_err_dotdotlt_syntax(maybe_lt, err))? } else { (lo, None) @@ -882,7 +903,7 @@ impl<'a> Parser<'a> { let mut res = ensure_sufficient_stack(|| { loop { let has_question = - if self.prev_token.kind == TokenKind::Ident(kw::Return, IdentIsRaw::No) { + if self.prev_token == TokenKind::Ident(kw::Return, IdentIsRaw::No) { // We are using noexpect here because we don't expect a `?` directly after // a `return` which could be suggested otherwise. self.eat_noexpect(&token::Question) @@ -894,20 +915,19 @@ impl<'a> Parser<'a> { e = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Try(e)); continue; } - let has_dot = - if self.prev_token.kind == TokenKind::Ident(kw::Return, IdentIsRaw::No) { - // We are using noexpect here because we don't expect a `.` directly after - // a `return` which could be suggested otherwise. - self.eat_noexpect(&token::Dot) - } else if self.token.kind == TokenKind::RArrow && self.may_recover() { - // Recovery for `expr->suffix`. - self.bump(); - let span = self.prev_token.span; - self.dcx().emit_err(errors::ExprRArrowCall { span }); - true - } else { - self.eat(&token::Dot) - }; + let has_dot = if self.prev_token == TokenKind::Ident(kw::Return, IdentIsRaw::No) { + // We are using noexpect here because we don't expect a `.` directly after + // a `return` which could be suggested otherwise. + self.eat_noexpect(&token::Dot) + } else if self.token == TokenKind::RArrow && self.may_recover() { + // Recovery for `expr->suffix`. + self.bump(); + let span = self.prev_token.span; + self.dcx().emit_err(errors::ExprRArrowCall { span }); + true + } else { + self.eat(&token::Dot) + }; if has_dot { // expr.f e = self.parse_dot_suffix_expr(lo, e)?; @@ -1206,7 +1226,7 @@ impl<'a> Parser<'a> { } fn mk_expr_tuple_field_access( - &mut self, + &self, lo: Span, ident_span: Span, base: P<Expr>, @@ -1221,7 +1241,7 @@ impl<'a> Parser<'a> { /// Parse a function call expression, `expr(...)`. fn parse_expr_fn_call(&mut self, lo: Span, fun: P<Expr>) -> P<Expr> { - let snapshot = if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) { + let snapshot = if self.token == token::OpenDelim(Delimiter::Parenthesis) { Some((self.create_snapshot_for_diagnostic(), fun.kind.clone())) } else { None @@ -1585,7 +1605,7 @@ impl<'a> Parser<'a> { // Suggests using '<=' if there is an error parsing qpath when the previous token // is an '=' token. Only emits suggestion if the '<' token and '=' token are // directly adjacent (i.e. '=<') - if maybe_eq_tok.kind == TokenKind::Eq && maybe_eq_tok.span.hi() == lt_span.lo() { + if maybe_eq_tok == TokenKind::Eq && maybe_eq_tok.span.hi() == lt_span.lo() { let eq_lt = maybe_eq_tok.span.to(lt_span); err.span_suggestion(eq_lt, "did you mean", "<=", Applicability::Unspecified); } @@ -2230,7 +2250,7 @@ impl<'a> Parser<'a> { return Ok(()); } - if self.token.kind == token::Comma { + if self.token == token::Comma { if !self.psess.source_map().is_multiline(prev_span.until(self.token.span)) { return Ok(()); } @@ -2336,7 +2356,7 @@ impl<'a> Parser<'a> { let token = self.token.clone(); let attrs = self.parse_outer_attributes()?; match self.parse_expr_res(restrictions, attrs) { - Ok(expr) => expr, + Ok((expr, _)) => expr, Err(err) => self.recover_closure_body(err, before, prev, token, lo, decl_hi)?, } } @@ -2360,7 +2380,7 @@ impl<'a> Parser<'a> { None => {} } - if self.token.kind == TokenKind::Semi + if self.token == TokenKind::Semi && matches!(self.token_cursor.stack.last(), Some((.., Delimiter::Parenthesis))) && self.may_recover() { @@ -2446,7 +2466,7 @@ impl<'a> Parser<'a> { fn parse_fn_block_param(&mut self) -> PResult<'a, Param> { let lo = self.token.span; let attrs = self.parse_outer_attributes()?; - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let pat = this.parse_pat_no_top_alt(Some(Expected::ParameterName), None)?; let ty = if this.eat(&token::Colon) { this.parse_ty()? @@ -2463,7 +2483,8 @@ impl<'a> Parser<'a> { id: DUMMY_NODE_ID, is_placeholder: false, }, - this.token == token::Comma, + Trailing::from(this.token == token::Comma), + UsePreAttrPos::No, )) }) } @@ -2557,7 +2578,7 @@ impl<'a> Parser<'a> { ); } else { // Look for usages of '=>' where '>=' might be intended - if maybe_fatarrow.kind == token::FatArrow { + if maybe_fatarrow == token::FatArrow { err.span_suggestion( maybe_fatarrow.span, "you might have meant to write a \"greater than or equal to\" comparison", @@ -2584,7 +2605,7 @@ impl<'a> Parser<'a> { /// Parses the condition of a `if` or `while` expression. fn parse_expr_cond(&mut self) -> PResult<'a, P<Expr>> { let attrs = self.parse_outer_attributes()?; - let mut cond = + let (mut cond, _) = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL | Restrictions::ALLOW_LET, attrs)?; CondChecker::new(self).visit_expr(&mut cond); @@ -2606,7 +2627,7 @@ impl<'a> Parser<'a> { missing_let: None, comparison: None, }; - if self.prev_token.kind == token::BinOp(token::Or) { + if self.prev_token == token::BinOp(token::Or) { // This was part of a closure, the that part of the parser recover. return Err(self.dcx().create_err(err)); } else { @@ -2633,7 +2654,7 @@ impl<'a> Parser<'a> { self.expect(&token::Eq)?; } let attrs = self.parse_outer_attributes()?; - let expr = self.parse_expr_assoc_with(1 + prec_let_scrutinee_needs_par(), attrs)?; + let (expr, _) = self.parse_expr_assoc_with(1 + prec_let_scrutinee_needs_par(), attrs)?; let span = lo.to(expr.span); Ok(self.mk_expr(span, ExprKind::Let(pat, expr, span, recovered))) } @@ -2742,7 +2763,7 @@ impl<'a> Parser<'a> { } fn parse_for_head(&mut self) -> PResult<'a, (P<Pat>, P<Expr>)> { - let begin_paren = if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) { + let begin_paren = if self.token == token::OpenDelim(Delimiter::Parenthesis) { // Record whether we are about to parse `for (`. // This is used below for recovery in case of `for ( $stuff ) $block` // in which case we will suggest `for $stuff $block`. @@ -2767,7 +2788,7 @@ impl<'a> Parser<'a> { // We know for sure we have seen `for ($SOMETHING in`. In the happy path this would // happen right before the return of this method. let attrs = self.parse_outer_attributes()?; - let expr = match self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs) { + let (expr, _) = match self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs) { Ok(expr) => expr, Err(expr_err) => { // We don't know what followed the `in`, so cancel and bubble up the @@ -2776,7 +2797,7 @@ impl<'a> Parser<'a> { return Err(err); } }; - return if self.token.kind == token::CloseDelim(Delimiter::Parenthesis) { + return if self.token == token::CloseDelim(Delimiter::Parenthesis) { // We know for sure we have seen `for ($SOMETHING in $EXPR)`, so we recover the // parser state and emit a targeted suggestion. let span = vec![start_span, self.token.span]; @@ -2802,7 +2823,7 @@ impl<'a> Parser<'a> { } self.check_for_for_in_in_typo(self.prev_token.span); let attrs = self.parse_outer_attributes()?; - let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs)?; + let (expr, _) = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs)?; Ok((pat, expr)) } @@ -2922,7 +2943,7 @@ impl<'a> Parser<'a> { fn parse_expr_match(&mut self) -> PResult<'a, P<Expr>> { let match_span = self.prev_token.span; let attrs = self.parse_outer_attributes()?; - let scrutinee = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs)?; + let (scrutinee, _) = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs)?; self.parse_match_block(match_span, match_span, scrutinee, MatchKind::Prefix) } @@ -2995,7 +3016,7 @@ impl<'a> Parser<'a> { first_expr: &P<Expr>, arrow_span: Span, ) -> Option<(Span, ErrorGuaranteed)> { - if self.token.kind != token::Semi { + if self.token != token::Semi { return None; } let start_snapshot = self.create_snapshot_for_diagnostic(); @@ -3024,18 +3045,18 @@ impl<'a> Parser<'a> { // We might have either a `,` -> `;` typo, or a block without braces. We need // a more subtle parsing strategy. loop { - if self.token.kind == token::CloseDelim(Delimiter::Brace) { + if self.token == token::CloseDelim(Delimiter::Brace) { // We have reached the closing brace of the `match` expression. return Some(err(self, stmts)); } - if self.token.kind == token::Comma { + if self.token == token::Comma { self.restore_snapshot(start_snapshot); return None; } let pre_pat_snapshot = self.create_snapshot_for_diagnostic(); match self.parse_pat_no_top_alt(None, None) { Ok(_pat) => { - if self.token.kind == token::FatArrow { + if self.token == token::FatArrow { // Reached arm end. self.restore_snapshot(pre_pat_snapshot); return Some(err(self, stmts)); @@ -3070,7 +3091,7 @@ impl<'a> Parser<'a> { pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> { let attrs = self.parse_outer_attributes()?; - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let lo = this.token.span; let (pat, guard) = this.parse_match_arm_pat_and_guard()?; @@ -3127,7 +3148,7 @@ impl<'a> Parser<'a> { let arm_start_span = this.token.span; let attrs = this.parse_outer_attributes()?; - let expr = + let (expr, _) = this.parse_expr_res(Restrictions::STMT_EXPR, attrs).map_err(|mut err| { err.span_label(arrow_span, "while parsing the `match` arm starting here"); err @@ -3244,7 +3265,8 @@ impl<'a> Parser<'a> { id: DUMMY_NODE_ID, is_placeholder: false, }, - false, + Trailing::No, + UsePreAttrPos::No, )) }) } @@ -3286,7 +3308,7 @@ impl<'a> Parser<'a> { } fn parse_match_arm_pat_and_guard(&mut self) -> PResult<'a, (P<Pat>, Option<P<Expr>>)> { - if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) { + if self.token == token::OpenDelim(Delimiter::Parenthesis) { // Detect and recover from `($pat if $cond) => $arm`. let left = self.token.span; match self.parse_pat_allow_top_alt( @@ -3335,8 +3357,9 @@ impl<'a> Parser<'a> { fn parse_match_guard_condition(&mut self) -> PResult<'a, P<Expr>> { let attrs = self.parse_outer_attributes()?; - self.parse_expr_res(Restrictions::ALLOW_LET | Restrictions::IN_IF_GUARD, attrs).map_err( - |mut err| { + match self.parse_expr_res(Restrictions::ALLOW_LET | Restrictions::IN_IF_GUARD, attrs) { + Ok((expr, _)) => Ok(expr), + Err(mut err) => { if self.prev_token == token::OpenDelim(Delimiter::Brace) { let sugg_sp = self.prev_token.span.shrink_to_lo(); // Consume everything within the braces, let's avoid further parse @@ -3344,7 +3367,7 @@ impl<'a> Parser<'a> { self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore); let msg = "you might have meant to start a match arm after the match guard"; if self.eat(&token::CloseDelim(Delimiter::Brace)) { - let applicability = if self.token.kind != token::FatArrow { + let applicability = if self.token != token::FatArrow { // We have high confidence that we indeed didn't have a struct // literal in the match guard, but rather we had some operation // that ended in a path, immediately followed by a block that was @@ -3356,9 +3379,9 @@ impl<'a> Parser<'a> { err.span_suggestion_verbose(sugg_sp, msg, "=> ", applicability); } } - err - }, - ) + Err(err) + } + } } pub(crate) fn is_builtin(&self) -> bool { @@ -3565,7 +3588,7 @@ impl<'a> Parser<'a> { && self.look_ahead(1, |t| { AssocOp::from_token(t).is_some() || matches!(t.kind, token::OpenDelim(_)) - || t.kind == token::Dot + || *t == token::Dot }) { // Looks like they tried to write a shorthand, complex expression. @@ -3709,7 +3732,7 @@ impl<'a> Parser<'a> { fn parse_expr_field(&mut self) -> PResult<'a, ExprField> { let attrs = self.parse_outer_attributes()?; self.recover_vcs_conflict_marker(); - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let lo = this.token.span; // Check if a colon exists one ahead. This means we're parsing a fieldname. @@ -3753,7 +3776,8 @@ impl<'a> Parser<'a> { id: DUMMY_NODE_ID, is_placeholder: false, }, - this.token == token::Comma, + Trailing::from(this.token == token::Comma), + UsePreAttrPos::No, )) }) } @@ -3847,15 +3871,17 @@ impl<'a> Parser<'a> { attrs: AttrWrapper, f: impl FnOnce(&mut Self, ast::AttrVec) -> PResult<'a, P<Expr>>, ) -> PResult<'a, P<Expr>> { - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let res = f(this, attrs)?; - let trailing = (this.restrictions.contains(Restrictions::STMT_EXPR) - && this.token.kind == token::Semi) - // FIXME: pass an additional condition through from the place - // where we know we need a comma, rather than assuming that - // `#[attr] expr,` always captures a trailing comma. - || this.token.kind == token::Comma; - Ok((res, trailing)) + let trailing = Trailing::from( + this.restrictions.contains(Restrictions::STMT_EXPR) + && this.token == token::Semi + // FIXME: pass an additional condition through from the place + // where we know we need a comma, rather than assuming that + // `#[attr] expr,` always captures a trailing comma. + || this.token == token::Comma, + ); + Ok((res, trailing, UsePreAttrPos::No)) }) } } diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs index 9124c15707d..af3b6f740e3 100644 --- a/compiler/rustc_parse/src/parser/generics.rs +++ b/compiler/rustc_parse/src/parser/generics.rs @@ -7,7 +7,7 @@ use rustc_span::symbol::{kw, Ident}; use rustc_span::Span; use thin_vec::ThinVec; -use super::{ForceCollect, Parser}; +use super::{ForceCollect, Parser, Trailing, UsePreAttrPos}; use crate::errors::{ self, MultipleWhereClauses, UnexpectedDefaultValueForLifetimeInGenericParameters, UnexpectedSelfInGenericParameters, WhereClauseBeforeTupleStructBody, @@ -169,94 +169,88 @@ impl<'a> Parser<'a> { let mut done = false; while !done { let attrs = self.parse_outer_attributes()?; - let param = - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { - if this.eat_keyword_noexpect(kw::SelfUpper) { - // `Self` as a generic param is invalid. Here we emit the diagnostic and continue parsing - // as if `Self` never existed. - this.dcx().emit_err(UnexpectedSelfInGenericParameters { - span: this.prev_token.span, - }); + let param = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { + if this.eat_keyword_noexpect(kw::SelfUpper) { + // `Self` as a generic param is invalid. Here we emit the diagnostic and continue parsing + // as if `Self` never existed. + this.dcx() + .emit_err(UnexpectedSelfInGenericParameters { span: this.prev_token.span }); + + // Eat a trailing comma, if it exists. + let _ = this.eat(&token::Comma); + } + + let param = if this.check_lifetime() { + let lifetime = this.expect_lifetime(); + // Parse lifetime parameter. + let (colon_span, bounds) = if this.eat(&token::Colon) { + (Some(this.prev_token.span), this.parse_lt_param_bounds()) + } else { + (None, Vec::new()) + }; - // Eat a trailing comma, if it exists. - let _ = this.eat(&token::Comma); + if this.check_noexpect(&token::Eq) && this.look_ahead(1, |t| t.is_lifetime()) { + let lo = this.token.span; + // Parse `= 'lifetime`. + this.bump(); // `=` + this.bump(); // `'lifetime` + let span = lo.to(this.prev_token.span); + this.dcx().emit_err(UnexpectedDefaultValueForLifetimeInGenericParameters { + span, + }); } - let param = if this.check_lifetime() { - let lifetime = this.expect_lifetime(); - // Parse lifetime parameter. - let (colon_span, bounds) = if this.eat(&token::Colon) { - (Some(this.prev_token.span), this.parse_lt_param_bounds()) - } else { - (None, Vec::new()) - }; - - if this.check_noexpect(&token::Eq) - && this.look_ahead(1, |t| t.is_lifetime()) - { - let lo = this.token.span; - // Parse `= 'lifetime`. - this.bump(); // `=` - this.bump(); // `'lifetime` - let span = lo.to(this.prev_token.span); - this.dcx().emit_err( - UnexpectedDefaultValueForLifetimeInGenericParameters { span }, - ); + Some(ast::GenericParam { + ident: lifetime.ident, + id: lifetime.id, + attrs, + bounds, + kind: ast::GenericParamKind::Lifetime, + is_placeholder: false, + colon_span, + }) + } else if this.check_keyword(kw::Const) { + // Parse const parameter. + Some(this.parse_const_param(attrs)?) + } else if this.check_ident() { + // Parse type parameter. + Some(this.parse_ty_param(attrs)?) + } else if this.token.can_begin_type() { + // Trying to write an associated type bound? (#26271) + let snapshot = this.create_snapshot_for_diagnostic(); + match this.parse_ty_where_predicate() { + Ok(where_predicate) => { + this.dcx().emit_err(errors::BadAssocTypeBounds { + span: where_predicate.span(), + }); + // FIXME - try to continue parsing other generics? } - - Some(ast::GenericParam { - ident: lifetime.ident, - id: lifetime.id, - attrs, - bounds, - kind: ast::GenericParamKind::Lifetime, - is_placeholder: false, - colon_span, - }) - } else if this.check_keyword(kw::Const) { - // Parse const parameter. - Some(this.parse_const_param(attrs)?) - } else if this.check_ident() { - // Parse type parameter. - Some(this.parse_ty_param(attrs)?) - } else if this.token.can_begin_type() { - // Trying to write an associated type bound? (#26271) - let snapshot = this.create_snapshot_for_diagnostic(); - match this.parse_ty_where_predicate() { - Ok(where_predicate) => { - this.dcx().emit_err(errors::BadAssocTypeBounds { - span: where_predicate.span(), - }); - // FIXME - try to continue parsing other generics? - return Ok((None, false)); - } - Err(err) => { - err.cancel(); - // FIXME - maybe we should overwrite 'self' outside of `collect_tokens`? - this.restore_snapshot(snapshot); - return Ok((None, false)); - } + Err(err) => { + err.cancel(); + // FIXME - maybe we should overwrite 'self' outside of `collect_tokens`? + this.restore_snapshot(snapshot); } - } else { - // Check for trailing attributes and stop parsing. - if !attrs.is_empty() { - if !params.is_empty() { - this.dcx() - .emit_err(errors::AttrAfterGeneric { span: attrs[0].span }); - } else { - this.dcx() - .emit_err(errors::AttrWithoutGenerics { span: attrs[0].span }); - } + } + return Ok((None, Trailing::No, UsePreAttrPos::No)); + } else { + // Check for trailing attributes and stop parsing. + if !attrs.is_empty() { + if !params.is_empty() { + this.dcx().emit_err(errors::AttrAfterGeneric { span: attrs[0].span }); + } else { + this.dcx() + .emit_err(errors::AttrWithoutGenerics { span: attrs[0].span }); } - return Ok((None, false)); - }; - - if !this.eat(&token::Comma) { - done = true; } - // We just ate the comma, so no need to capture the trailing token. - Ok((param, false)) - })?; + return Ok((None, Trailing::No, UsePreAttrPos::No)); + }; + + if !this.eat(&token::Comma) { + done = true; + } + // We just ate the comma, so no need to capture the trailing token. + Ok((param, Trailing::No, UsePreAttrPos::No)) + })?; if let Some(param) = param { params.push(param); @@ -393,7 +387,7 @@ impl<'a> Parser<'a> { if let Some(struct_) = struct_ && self.may_recover() - && self.token.kind == token::OpenDelim(Delimiter::Parenthesis) + && self.token == token::OpenDelim(Delimiter::Parenthesis) { snapshot = Some((struct_, self.create_snapshot_for_diagnostic())); }; diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 8775d792c3d..47820e93c23 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -20,7 +20,9 @@ use tracing::debug; use super::diagnostics::{dummy_arg, ConsumeClosingDelim}; use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign}; -use super::{AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Trailing}; +use super::{ + AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, Trailing, UsePreAttrPos, +}; use crate::errors::{self, MacroExpandsToAdtField}; use crate::{fluent_generated as fluent, maybe_whole}; @@ -127,7 +129,7 @@ impl<'a> Parser<'a> { Some(item.into_inner()) }); - self.collect_tokens_trailing_token(attrs, force_collect, |this, mut attrs| { + self.collect_tokens(None, attrs, force_collect, |this, mut attrs| { let lo = this.token.span; let vis = this.parse_visibility(FollowedByType::No)?; let mut def = this.parse_defaultness(); @@ -145,7 +147,7 @@ impl<'a> Parser<'a> { let span = lo.to(this.prev_token.span); let id = DUMMY_NODE_ID; let item = Item { ident, attrs, id, kind, vis, span, tokens: None }; - return Ok((Some(item), false)); + return Ok((Some(item), Trailing::No, UsePreAttrPos::No)); } // At this point, we have failed to parse an item. @@ -160,7 +162,7 @@ impl<'a> Parser<'a> { if !attrs_allowed { this.recover_attrs_no_item(&attrs)?; } - Ok((None, false)) + Ok((None, Trailing::No, UsePreAttrPos::No)) }) } @@ -354,7 +356,7 @@ impl<'a> Parser<'a> { fn is_reuse_path_item(&mut self) -> bool { // no: `reuse ::path` for compatibility reasons with macro invocations self.token.is_keyword(kw::Reuse) - && self.look_ahead(1, |t| t.is_path_start() && t.kind != token::PathSep) + && self.look_ahead(1, |t| t.is_path_start() && *t != token::PathSep) } /// Are we sure this could not possibly be a macro invocation? @@ -499,7 +501,7 @@ impl<'a> Parser<'a> { let mut err = self.dcx().struct_span_err(end.span, msg); if end.is_doc_comment() { err.span_label(end.span, "this doc comment doesn't document anything"); - } else if self.token.kind == TokenKind::Semi { + } else if self.token == TokenKind::Semi { err.span_suggestion_verbose( self.token.span, "consider removing this semicolon", @@ -777,12 +779,12 @@ impl<'a> Parser<'a> { && self .span_to_snippet(self.prev_token.span) .is_ok_and(|snippet| snippet == "}") - && self.token.kind == token::Semi; + && self.token == token::Semi; let mut semicolon_span = self.token.span; if !is_unnecessary_semicolon { // #105369, Detect spurious `;` before assoc fn body is_unnecessary_semicolon = self.token == token::OpenDelim(Delimiter::Brace) - && self.prev_token.kind == token::Semi; + && self.prev_token == token::Semi; semicolon_span = self.prev_token.span; } // We have to bail or we'll potentially never make progress. @@ -1194,7 +1196,7 @@ impl<'a> Parser<'a> { // FIXME: This recovery should be tested better. if safety == Safety::Default && self.token.is_keyword(kw::Unsafe) - && self.look_ahead(1, |t| t.kind == token::OpenDelim(Delimiter::Brace)) + && self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace)) { self.expect(&token::OpenDelim(Delimiter::Brace)).unwrap_err().emit(); safety = Safety::Unsafe(self.token.span); @@ -1258,7 +1260,7 @@ impl<'a> Parser<'a> { && self.is_keyword_ahead(1, &[kw::Extern]) && self.look_ahead( 2 + self.look_ahead(2, |t| t.can_begin_string_literal() as usize), - |t| t.kind == token::OpenDelim(Delimiter::Brace), + |t| *t == token::OpenDelim(Delimiter::Brace), ) } @@ -1343,7 +1345,7 @@ impl<'a> Parser<'a> { ) -> PResult<'a, (Ident, StaticItem)> { let ident = self.parse_ident()?; - if self.token.kind == TokenKind::Lt && self.may_recover() { + if self.token == TokenKind::Lt && self.may_recover() { let generics = self.parse_generics()?; self.dcx().emit_err(errors::StaticWithGenerics { span: generics.span }); } @@ -1546,86 +1548,82 @@ impl<'a> Parser<'a> { self.recover_vcs_conflict_marker(); let help = "enum variants can be `Variant`, `Variant = <integer>`, \ `Variant(Type, ..., TypeN)` or `Variant { fields: Types }`"; - self.collect_tokens_trailing_token( - variant_attrs, - ForceCollect::No, - |this, variant_attrs| { - let vlo = this.token.span; - - let vis = this.parse_visibility(FollowedByType::No)?; - if !this.recover_nested_adt_item(kw::Enum)? { - return Ok((None, false)); - } - let ident = this.parse_field_ident("enum", vlo)?; - - if this.token == token::Not { - if let Err(err) = this.unexpected() { - err.with_note(fluent::parse_macro_expands_to_enum_variant).emit(); - } + self.collect_tokens(None, variant_attrs, ForceCollect::No, |this, variant_attrs| { + let vlo = this.token.span; - this.bump(); - this.parse_delim_args()?; + let vis = this.parse_visibility(FollowedByType::No)?; + if !this.recover_nested_adt_item(kw::Enum)? { + return Ok((None, Trailing::No, UsePreAttrPos::No)); + } + let ident = this.parse_field_ident("enum", vlo)?; - return Ok((None, this.token == token::Comma)); + if this.token == token::Not { + if let Err(err) = this.unexpected() { + err.with_note(fluent::parse_macro_expands_to_enum_variant).emit(); } - let struct_def = if this.check(&token::OpenDelim(Delimiter::Brace)) { - // Parse a struct variant. - let (fields, recovered) = - match this.parse_record_struct_body("struct", ident.span, false) { - Ok((fields, recovered)) => (fields, recovered), - Err(mut err) => { - if this.token == token::Colon { - // We handle `enum` to `struct` suggestion in the caller. - return Err(err); - } - this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]); - this.bump(); // } - err.span_label(span, "while parsing this enum"); - err.help(help); - let guar = err.emit(); - (thin_vec![], Recovered::Yes(guar)) - } - }; - VariantData::Struct { fields, recovered: recovered.into() } - } else if this.check(&token::OpenDelim(Delimiter::Parenthesis)) { - let body = match this.parse_tuple_struct_body() { - Ok(body) => body, + this.bump(); + this.parse_delim_args()?; + + return Ok((None, Trailing::from(this.token == token::Comma), UsePreAttrPos::No)); + } + + let struct_def = if this.check(&token::OpenDelim(Delimiter::Brace)) { + // Parse a struct variant. + let (fields, recovered) = + match this.parse_record_struct_body("struct", ident.span, false) { + Ok((fields, recovered)) => (fields, recovered), Err(mut err) => { if this.token == token::Colon { // We handle `enum` to `struct` suggestion in the caller. return Err(err); } - this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis)]); - this.bump(); // ) + this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]); + this.bump(); // } err.span_label(span, "while parsing this enum"); err.help(help); - err.emit(); - thin_vec![] + let guar = err.emit(); + (thin_vec![], Recovered::Yes(guar)) } }; - VariantData::Tuple(body, DUMMY_NODE_ID) - } else { - VariantData::Unit(DUMMY_NODE_ID) + VariantData::Struct { fields, recovered: recovered.into() } + } else if this.check(&token::OpenDelim(Delimiter::Parenthesis)) { + let body = match this.parse_tuple_struct_body() { + Ok(body) => body, + Err(mut err) => { + if this.token == token::Colon { + // We handle `enum` to `struct` suggestion in the caller. + return Err(err); + } + this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis)]); + this.bump(); // ) + err.span_label(span, "while parsing this enum"); + err.help(help); + err.emit(); + thin_vec![] + } }; + VariantData::Tuple(body, DUMMY_NODE_ID) + } else { + VariantData::Unit(DUMMY_NODE_ID) + }; - let disr_expr = - if this.eat(&token::Eq) { Some(this.parse_expr_anon_const()?) } else { None }; + let disr_expr = + if this.eat(&token::Eq) { Some(this.parse_expr_anon_const()?) } else { None }; - let vr = ast::Variant { - ident, - vis, - id: DUMMY_NODE_ID, - attrs: variant_attrs, - data: struct_def, - disr_expr, - span: vlo.to(this.prev_token.span), - is_placeholder: false, - }; + let vr = ast::Variant { + ident, + vis, + id: DUMMY_NODE_ID, + attrs: variant_attrs, + data: struct_def, + disr_expr, + span: vlo.to(this.prev_token.span), + is_placeholder: false, + }; - Ok((Some(vr), this.token == token::Comma)) - }, - ) + Ok((Some(vr), Trailing::from(this.token == token::Comma), UsePreAttrPos::No)) + }) .map_err(|mut err| { err.help(help); err @@ -1777,7 +1775,7 @@ impl<'a> Parser<'a> { // Unit like structs are handled in parse_item_struct function self.parse_paren_comma_seq(|p| { let attrs = p.parse_outer_attributes()?; - p.collect_tokens_trailing_token(attrs, ForceCollect::No, |p, attrs| { + p.collect_tokens(None, attrs, ForceCollect::No, |p, attrs| { let mut snapshot = None; if p.is_vcs_conflict_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt) { // Account for `<<<<<<<` diff markers. We can't proactively error here because @@ -1815,7 +1813,8 @@ impl<'a> Parser<'a> { attrs, is_placeholder: false, }, - p.token == token::Comma, + Trailing::from(p.token == token::Comma), + UsePreAttrPos::No, )) }) }) @@ -1827,10 +1826,11 @@ impl<'a> Parser<'a> { self.recover_vcs_conflict_marker(); let attrs = self.parse_outer_attributes()?; self.recover_vcs_conflict_marker(); - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let lo = this.token.span; let vis = this.parse_visibility(FollowedByType::No)?; - this.parse_single_struct_field(adt_ty, lo, vis, attrs).map(|field| (field, false)) + this.parse_single_struct_field(adt_ty, lo, vis, attrs) + .map(|field| (field, Trailing::No, UsePreAttrPos::No)) }) } @@ -1914,7 +1914,7 @@ impl<'a> Parser<'a> { let mut err = self.dcx().struct_span_err(sp, msg); if self.token.is_ident() - || (self.token.kind == TokenKind::Pound + || (self.token == TokenKind::Pound && (self.look_ahead(1, |t| t == &token::OpenDelim(Delimiter::Bracket)))) { // This is likely another field, TokenKind::Pound is used for `#[..]` @@ -1937,8 +1937,8 @@ impl<'a> Parser<'a> { fn expect_field_ty_separator(&mut self) -> PResult<'a, ()> { if let Err(err) = self.expect(&token::Colon) { let sm = self.psess.source_map(); - let eq_typo = self.token.kind == token::Eq && self.look_ahead(1, |t| t.is_path_start()); - let semi_typo = self.token.kind == token::Semi + let eq_typo = self.token == token::Eq && self.look_ahead(1, |t| t.is_path_start()); + let semi_typo = self.token == token::Semi && self.look_ahead(1, |t| { t.is_path_start() // We check that we are in a situation like `foo; bar` to avoid bad suggestions @@ -1974,7 +1974,7 @@ impl<'a> Parser<'a> { attrs: AttrVec, ) -> PResult<'a, FieldDef> { let name = self.parse_field_ident(adt_ty, lo)?; - if self.token.kind == token::Not { + if self.token == token::Not { if let Err(mut err) = self.unexpected() { // Encounter the macro invocation err.subdiagnostic(MacroExpandsToAdtField { adt_ty }); @@ -1983,10 +1983,10 @@ impl<'a> Parser<'a> { } self.expect_field_ty_separator()?; let ty = self.parse_ty_for_field_def()?; - if self.token.kind == token::Colon && self.look_ahead(1, |tok| tok.kind != token::Colon) { + if self.token == token::Colon && self.look_ahead(1, |t| *t != token::Colon) { self.dcx().emit_err(errors::SingleColonStructType { span: self.token.span }); } - if self.token.kind == token::Eq { + if self.token == token::Eq { self.bump(); let const_expr = self.parse_expr_anon_const()?; let sp = ty.span.shrink_to_hi().to(const_expr.value.span); @@ -2064,7 +2064,7 @@ impl<'a> Parser<'a> { .parse_ident_common(false) // Cancel this error, we don't need it. .map_err(|err| err.cancel()) - && self.token.kind == TokenKind::Colon + && self.token == TokenKind::Colon { err.span_suggestion( removal_span, @@ -2367,12 +2367,12 @@ impl<'a> Parser<'a> { match self.expected_one_of_not_found(&[], expected) { Ok(error_guaranteed) => Ok(error_guaranteed), Err(mut err) => { - if self.token.kind == token::CloseDelim(Delimiter::Brace) { + if self.token == token::CloseDelim(Delimiter::Brace) { // The enclosing `mod`, `trait` or `impl` is being closed, so keep the `fn` in // the AST for typechecking. err.span_label(ident_span, "while parsing this `fn`"); Ok(err.emit()) - } else if self.token.kind == token::RArrow + } else if self.token == token::RArrow && let Some(fn_params_end) = fn_params_end { // Instead of a function body, the parser has encountered a right arrow @@ -2445,7 +2445,7 @@ impl<'a> Parser<'a> { fn_params_end: Option<Span>, ) -> PResult<'a, Option<P<Block>>> { let has_semi = if req_body { - self.token.kind == TokenKind::Semi + self.token == TokenKind::Semi } else { // Only include `;` in list of expected tokens if body is not required self.check(&TokenKind::Semi) @@ -2458,7 +2458,7 @@ impl<'a> Parser<'a> { } else if self.check(&token::OpenDelim(Delimiter::Brace)) || self.token.is_whole_block() { self.parse_block_common(self.token.span, BlockCheckMode::Default, false) .map(|(attrs, body)| (attrs, Some(body)))? - } else if self.token.kind == token::Eq { + } else if self.token == token::Eq { // Recover `fn foo() = $expr;`. self.bump(); // `=` let eq_sp = self.prev_token.span; @@ -2761,7 +2761,7 @@ impl<'a> Parser<'a> { pub(super) fn parse_fn_params(&mut self, req_name: ReqName) -> PResult<'a, ThinVec<Param>> { let mut first_param = true; // Parse the arguments, starting out with `self` being allowed... - if self.token.kind != TokenKind::OpenDelim(Delimiter::Parenthesis) + if self.token != TokenKind::OpenDelim(Delimiter::Parenthesis) // might be typo'd trait impl, handled elsewhere && !self.token.is_keyword(kw::For) { @@ -2805,12 +2805,12 @@ impl<'a> Parser<'a> { fn parse_param_general(&mut self, req_name: ReqName, first_param: bool) -> PResult<'a, Param> { let lo = self.token.span; let attrs = self.parse_outer_attributes()?; - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { // Possibly parse `self`. Recover if we parsed it and it wasn't allowed here. if let Some(mut param) = this.parse_self_param()? { param.attrs = attrs; let res = if first_param { Ok(param) } else { this.recover_bad_self_param(param) }; - return Ok((res?, false)); + return Ok((res?, Trailing::No, UsePreAttrPos::No)); } let is_name_required = match this.token.kind { @@ -2826,7 +2826,7 @@ impl<'a> Parser<'a> { this.parameter_without_type(&mut err, pat, is_name_required, first_param) { let guar = err.emit(); - Ok((dummy_arg(ident, guar), false)) + Ok((dummy_arg(ident, guar), Trailing::No, UsePreAttrPos::No)) } else { Err(err) }; @@ -2869,7 +2869,8 @@ impl<'a> Parser<'a> { Ok(( Param { attrs, id: ast::DUMMY_NODE_ID, is_placeholder: false, pat, span, ty }, - false, + Trailing::No, + UsePreAttrPos::No, )) }) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 9b3b6d5f9ad..61e3fa2e6c5 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -14,7 +14,7 @@ use std::assert_matches::debug_assert_matches; use std::ops::Range; use std::{fmt, mem, slice}; -use attr_wrapper::AttrWrapper; +use attr_wrapper::{AttrWrapper, UsePreAttrPos}; pub use diagnostics::AttemptLocalParseRecovery; pub(crate) use expr::ForbiddenLetReason; pub(crate) use item::FnParseMode; @@ -238,6 +238,7 @@ impl NodeRange { // is the position of the function's start token. This gives // `NodeRange(10..15)`. fn new(ParserRange(parser_range): ParserRange, start_pos: u32) -> NodeRange { + assert!(parser_range.start >= start_pos && parser_range.end >= start_pos); NodeRange((parser_range.start - start_pos)..(parser_range.end - start_pos)) } } @@ -253,7 +254,7 @@ enum Capturing { Yes, } -// This state is used by `Parser::collect_tokens_trailing_token`. +// This state is used by `Parser::collect_tokens`. #[derive(Clone, Debug)] struct CaptureState { capturing: Capturing, @@ -388,6 +389,12 @@ enum Trailing { Yes, } +impl From<bool> for Trailing { + fn from(b: bool) -> Trailing { + if b { Trailing::Yes } else { Trailing::No } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum TokenDescription { ReservedIdentifier, @@ -459,8 +466,8 @@ impl<'a> Parser<'a> { parser.bump(); // Change this from 1 back to 0 after the bump. This eases debugging of - // `Parser::collect_tokens_trailing_token` nicer because it makes the - // token positions 0-indexed which is nicer than 1-indexed. + // `Parser::collect_tokens` because 0-indexed token positions are nicer + // than 1-indexed token positions. parser.num_bump_calls = 0; parser @@ -527,7 +534,7 @@ impl<'a> Parser<'a> { } else if inedible.contains(&self.token.kind) { // leave it in the input Ok(Recovered::No) - } else if self.token.kind != token::Eof + } else if self.token != token::Eof && self.last_unexpected_token_span == Some(self.token.span) { FatalError.raise(); @@ -756,7 +763,7 @@ impl<'a> Parser<'a> { /// compound tokens like multi-character operators in process. /// Returns `true` if the token was eaten. fn break_and_eat(&mut self, expected: TokenKind) -> bool { - if self.token.kind == expected { + if self.token == expected { self.bump(); return true; } @@ -882,7 +889,7 @@ impl<'a> Parser<'a> { let token_str = pprust::token_kind_to_string(t); match self.current_closure.take() { - Some(closure_spans) if self.token.kind == TokenKind::Semi => { + Some(closure_spans) if self.token == TokenKind::Semi => { // Finding a semicolon instead of a comma // after a closure body indicates that the // closure body may be a block but the user @@ -910,7 +917,7 @@ impl<'a> Parser<'a> { // If this was a missing `@` in a binding pattern // bail with a suggestion // https://github.com/rust-lang/rust/issues/72373 - if self.prev_token.is_ident() && self.token.kind == token::DotDot { + if self.prev_token.is_ident() && self.token == token::DotDot { let msg = format!( "if you meant to bind the contents of the rest of the array \ pattern into `{}`, use `@`", @@ -1546,11 +1553,9 @@ impl<'a> Parser<'a> { ) -> PResult<'a, R> { // The only reason to call `collect_tokens_no_attrs` is if you want tokens, so use // `ForceCollect::Yes` - self.collect_tokens_trailing_token( - AttrWrapper::empty(), - ForceCollect::Yes, - |this, _attrs| Ok((f(this)?, false)), - ) + self.collect_tokens(None, AttrWrapper::empty(), ForceCollect::Yes, |this, _attrs| { + Ok((f(this)?, Trailing::No, UsePreAttrPos::No)) + }) } /// `::{` or `::*` diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index 5bfb8bdf776..eb9a957032f 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -13,7 +13,7 @@ use rustc_span::symbol::{kw, sym, Ident}; use rustc_span::{BytePos, ErrorGuaranteed, Span}; use thin_vec::{thin_vec, ThinVec}; -use super::{ForceCollect, Parser, PathStyle, Restrictions, Trailing}; +use super::{ForceCollect, Parser, PathStyle, Restrictions, Trailing, UsePreAttrPos}; use crate::errors::{ self, AmbiguousRangePattern, DotDotDotForRemainingFields, DotDotDotRangeToPatternNotAllowed, DotDotDotRestPattern, EnumPatternInsteadOfIdentifier, ExpectedBindingLeftOfAt, @@ -369,7 +369,7 @@ impl<'a> Parser<'a> { .and_then(|(ident, _)| ident.name.as_str().chars().next()) .is_some_and(char::is_lowercase) }) - && self.look_ahead(2, |tok| tok.kind == token::OpenDelim(Delimiter::Parenthesis)); + && self.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Parenthesis)); // Check for operators. // `|` is excluded as it is used in pattern alternatives and lambdas, @@ -377,9 +377,9 @@ impl<'a> Parser<'a> { // `[` is included for indexing operations, // `[]` is excluded as `a[]` isn't an expression and should be recovered as `a, []` (cf. `tests/ui/parser/pat-lt-bracket-7.rs`) let has_trailing_operator = matches!(self.token.kind, token::BinOp(op) if op != BinOpToken::Or) - || self.token.kind == token::Question - || (self.token.kind == token::OpenDelim(Delimiter::Bracket) - && self.look_ahead(1, |tok| tok.kind != token::CloseDelim(Delimiter::Bracket))); + || self.token == token::Question + || (self.token == token::OpenDelim(Delimiter::Bracket) + && self.look_ahead(1, |t| *t != token::CloseDelim(Delimiter::Bracket))); if !has_trailing_method && !has_trailing_operator { // Nothing to recover here. @@ -403,7 +403,7 @@ impl<'a> Parser<'a> { // Parse an associative expression such as `+ expr`, `% expr`, ... // Assignements, ranges and `|` are disabled by [`Restrictions::IS_PAT`]. - if let Ok(expr) = + if let Ok((expr, _)) = snapshot.parse_expr_assoc_rest_with(0, false, expr).map_err(|err| err.cancel()) { // We got a valid expression. @@ -413,7 +413,7 @@ impl<'a> Parser<'a> { let is_bound = is_end_bound // is_start_bound: either `..` or `)..` || self.token.is_range_separator() - || self.token.kind == token::CloseDelim(Delimiter::Parenthesis) + || self.token == token::CloseDelim(Delimiter::Parenthesis) && self.look_ahead(1, Token::is_range_separator); // Check that `parse_expr_assoc_with` didn't eat a rhs. @@ -450,7 +450,7 @@ impl<'a> Parser<'a> { lo = self.token.span; } - let pat = if self.check(&token::BinOp(token::And)) || self.token.kind == token::AndAnd { + let pat = if self.check(&token::BinOp(token::And)) || self.token == token::AndAnd { self.parse_pat_deref(expected)? } else if self.check(&token::OpenDelim(Delimiter::Parenthesis)) { self.parse_pat_tuple_or_parens()? @@ -625,7 +625,7 @@ impl<'a> Parser<'a> { /// /// [and]: https://docs.microsoft.com/en-us/dotnet/fsharp/language-reference/pattern-matching fn recover_intersection_pat(&mut self, lhs: P<Pat>) -> PResult<'a, P<Pat>> { - if self.token.kind != token::At { + if self.token != token::At { // Next token is not `@` so it's not going to be an intersection pattern. return Ok(lhs); } @@ -958,14 +958,14 @@ impl<'a> Parser<'a> { self.check_inline_const(dist) || self.look_ahead(dist, |t| { t.is_path_start() // e.g. `MY_CONST`; - || t.kind == token::Dot // e.g. `.5` for recovery; + || *t == token::Dot // e.g. `.5` for recovery; || matches!(t.kind, token::Literal(..) | token::BinOp(token::Minus)) || t.is_bool_lit() || t.is_whole_expr() || t.is_lifetime() // recover `'a` instead of `'a'` || (self.may_recover() // recover leading `(` - && t.kind == token::OpenDelim(Delimiter::Parenthesis) - && self.look_ahead(dist + 1, |t| t.kind != token::OpenDelim(Delimiter::Parenthesis)) + && *t == token::OpenDelim(Delimiter::Parenthesis) + && self.look_ahead(dist + 1, |t| *t != token::OpenDelim(Delimiter::Parenthesis)) && self.is_pat_range_end_start(dist + 1)) }) } @@ -1302,24 +1302,23 @@ impl<'a> Parser<'a> { } } - let field = - self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { - let field = match this.parse_pat_field(lo, attrs) { - Ok(field) => Ok(field), - Err(err) => { - if let Some(delayed_err) = delayed_err.take() { - delayed_err.emit(); - } - return Err(err); + let field = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { + let field = match this.parse_pat_field(lo, attrs) { + Ok(field) => Ok(field), + Err(err) => { + if let Some(delayed_err) = delayed_err.take() { + delayed_err.emit(); } - }?; - ate_comma = this.eat(&token::Comma); + return Err(err); + } + }?; + ate_comma = this.eat(&token::Comma); - last_non_comma_dotdot_span = Some(this.prev_token.span); + last_non_comma_dotdot_span = Some(this.prev_token.span); - // We just ate a comma, so there's no need to capture a trailing token. - Ok((field, false)) - })?; + // We just ate a comma, so there's no need to capture a trailing token. + Ok((field, Trailing::No, UsePreAttrPos::No)) + })?; fields.push(field) } diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 6f82d6b9826..b58f398efed 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -358,9 +358,9 @@ impl<'a> Parser<'a> { })?; let span = lo.to(self.prev_token.span); AngleBracketedArgs { args, span }.into() - } else if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) + } else if self.token == token::OpenDelim(Delimiter::Parenthesis) // FIXME(return_type_notation): Could also recover `...` here. - && self.look_ahead(1, |tok| tok.kind == token::DotDot) + && self.look_ahead(1, |t| *t == token::DotDot) { self.bump(); // ( self.bump(); // .. @@ -384,7 +384,7 @@ impl<'a> Parser<'a> { let token_before_parsing = self.token.clone(); let mut snapshot = None; if self.may_recover() - && prev_token_before_parsing.kind == token::PathSep + && prev_token_before_parsing == token::PathSep && (style == PathStyle::Expr && self.token.can_begin_expr() || style == PathStyle::Pat && self.token.can_begin_pattern()) { @@ -393,7 +393,7 @@ impl<'a> Parser<'a> { let (inputs, _) = match self.parse_paren_comma_seq(|p| p.parse_ty()) { Ok(output) => output, - Err(mut error) if prev_token_before_parsing.kind == token::PathSep => { + Err(mut error) if prev_token_before_parsing == token::PathSep => { error.span_label( prev_token_before_parsing.span.to(token_before_parsing.span), "while parsing this parenthesized list of type arguments starting here", @@ -913,7 +913,7 @@ impl<'a> Parser<'a> { let snapshot = self.create_snapshot_for_diagnostic(); let attrs = self.parse_outer_attributes()?; match self.parse_expr_res(Restrictions::CONST_EXPR, attrs) { - Ok(expr) => { + Ok((expr, _)) => { return Ok(Some(self.dummy_const_arg_needs_braces( self.dcx().struct_span_err(expr.span, "invalid const generic expression"), expr.span, diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index b206f134f0e..69044192780 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -21,6 +21,7 @@ use super::pat::{PatternLocation, RecoverComma}; use super::path::PathStyle; use super::{ AttrWrapper, BlockMode, FnParseMode, ForceCollect, Parser, Restrictions, SemiColonMode, + Trailing, UsePreAttrPos, }; use crate::errors::MalformedLoopLabel; use crate::{errors, maybe_whole}; @@ -45,6 +46,7 @@ impl<'a> Parser<'a> { capture_semi: bool, force_collect: ForceCollect, ) -> PResult<'a, Option<Stmt>> { + let pre_attr_pos = self.collect_pos(); let attrs = self.parse_outer_attributes()?; let lo = self.token.span; @@ -65,11 +67,15 @@ impl<'a> Parser<'a> { } Ok(Some(if self.token.is_keyword(kw::Let) { - self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| { + self.collect_tokens(None, attrs, force_collect, |this, attrs| { this.expect_keyword(kw::Let)?; let local = this.parse_local(attrs)?; - let trailing = capture_semi && this.token.kind == token::Semi; - Ok((this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), trailing)) + let trailing = Trailing::from(capture_semi && this.token == token::Semi); + Ok(( + this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), + trailing, + UsePreAttrPos::No, + )) })? } else if self.is_kw_followed_by_ident(kw::Mut) && self.may_recover() { self.recover_stmt_local_after_let( @@ -103,10 +109,18 @@ impl<'a> Parser<'a> { // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something // that starts like a path (1 token), but it fact not a path. // Also, we avoid stealing syntax from `parse_item_`. - let stmt = self.collect_tokens_trailing_token( + // + // `UsePreAttrPos::Yes` here means the attribute belongs unconditionally to the + // expression, not the statement. (But the statement attributes/tokens are obtained + // from the expression anyway, because `Stmt` delegates `HasAttrs`/`HasTokens` to + // the things within `StmtKind`.) + let stmt = self.collect_tokens( + Some(pre_attr_pos), AttrWrapper::empty(), force_collect, - |this, _empty_attrs| Ok((this.parse_stmt_path_start(lo, attrs)?, false)), + |this, _empty_attrs| { + Ok((this.parse_stmt_path_start(lo, attrs)?, Trailing::No, UsePreAttrPos::Yes)) + }, ); match stmt { Ok(stmt) => stmt, @@ -128,12 +142,15 @@ impl<'a> Parser<'a> { self.error_outer_attrs(attrs); self.mk_stmt(lo, StmtKind::Empty) } else if self.token != token::CloseDelim(Delimiter::Brace) { - // Remainder are line-expr stmts. - let e = self.collect_tokens_trailing_token( + // Remainder are line-expr stmts. This is similar to the `parse_stmt_path_start` case + // above. + let e = self.collect_tokens( + Some(pre_attr_pos), AttrWrapper::empty(), force_collect, |this, _empty_attrs| { - Ok((this.parse_expr_res(Restrictions::STMT_EXPR, attrs)?, false)) + let (expr, _) = this.parse_expr_res(Restrictions::STMT_EXPR, attrs)?; + Ok((expr, Trailing::No, UsePreAttrPos::Yes)) }, )?; if matches!(e.kind, ExprKind::Assign(..)) && self.eat_keyword(kw::Else) { @@ -150,12 +167,16 @@ impl<'a> Parser<'a> { } fn parse_stmt_path_start(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a, Stmt> { - let stmt = self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| { + let stmt = self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let path = this.parse_path(PathStyle::Expr)?; if this.eat(&token::Not) { let stmt_mac = this.parse_stmt_mac(lo, attrs, path)?; - return Ok((stmt_mac, this.token == token::Semi)); + return Ok(( + stmt_mac, + Trailing::from(this.token == token::Semi), + UsePreAttrPos::No, + )); } let expr = if this.eat(&token::OpenDelim(Delimiter::Brace)) { @@ -169,13 +190,17 @@ impl<'a> Parser<'a> { this.parse_expr_dot_or_call_with(attrs, expr, lo) })?; // `DUMMY_SP` will get overwritten later in this function - Ok((this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)), false)) + Ok(( + this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)), + Trailing::No, + UsePreAttrPos::No, + )) })?; if let StmtKind::Expr(expr) = stmt.kind { - // Perform this outside of the `collect_tokens_trailing_token` closure, - // since our outer attributes do not apply to this part of the expression - let expr = self.with_res(Restrictions::STMT_EXPR, |this| { + // Perform this outside of the `collect_tokens` closure, since our + // outer attributes do not apply to this part of the expression. + let (expr, _) = self.with_res(Restrictions::STMT_EXPR, |this| { this.parse_expr_assoc_rest_with(0, true, expr) })?; Ok(self.mk_stmt(lo.to(self.prev_token.span), StmtKind::Expr(expr))) @@ -209,7 +234,7 @@ impl<'a> Parser<'a> { let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac)); let e = self.maybe_recover_from_bad_qpath(e)?; let e = self.parse_expr_dot_or_call_with(attrs, e, lo)?; - let e = self.parse_expr_assoc_rest_with(0, false, e)?; + let (e, _) = self.parse_expr_assoc_rest_with(0, false, e)?; StmtKind::Expr(e) }; Ok(self.mk_stmt(lo.to(hi), kind)) @@ -239,10 +264,14 @@ impl<'a> Parser<'a> { subdiagnostic: fn(Span) -> errors::InvalidVariableDeclarationSub, force_collect: ForceCollect, ) -> PResult<'a, Stmt> { - let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| { + let stmt = self.collect_tokens(None, attrs, force_collect, |this, attrs| { let local = this.parse_local(attrs)?; // FIXME - maybe capture semicolon in recovery? - Ok((this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), false)) + Ok(( + this.mk_stmt(lo.to(this.prev_token.span), StmtKind::Let(local)), + Trailing::No, + UsePreAttrPos::No, + )) })?; self.dcx() .emit_err(errors::InvalidVariableDeclaration { span: lo, sub: subdiagnostic(lo) }); @@ -760,7 +789,7 @@ impl<'a> Parser<'a> { ) ), ); - let suggest_eq = if self.token.kind == token::Dot + let suggest_eq = if self.token == token::Dot && let _ = self.bump() && let mut snapshot = self.create_snapshot_for_diagnostic() && let Ok(_) = snapshot diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs index 352ddd9eac4..a8e8270673a 100644 --- a/compiler/rustc_parse/src/parser/ty.rs +++ b/compiler/rustc_parse/src/parser/ty.rs @@ -420,7 +420,7 @@ impl<'a> Parser<'a> { let mut trailing_plus = false; let (ts, trailing) = self.parse_paren_comma_seq(|p| { let ty = p.parse_ty()?; - trailing_plus = p.prev_token.kind == TokenKind::BinOp(token::Plus); + trailing_plus = p.prev_token == TokenKind::BinOp(token::Plus); Ok(ty) })?; @@ -499,8 +499,8 @@ impl<'a> Parser<'a> { let elt_ty = match self.parse_ty() { Ok(ty) => ty, Err(err) - if self.look_ahead(1, |t| t.kind == token::CloseDelim(Delimiter::Bracket)) - | self.look_ahead(1, |t| t.kind == token::Semi) => + if self.look_ahead(1, |t| *t == token::CloseDelim(Delimiter::Bracket)) + | self.look_ahead(1, |t| *t == token::Semi) => { // Recover from `[LIT; EXPR]` and `[LIT]` self.bump(); @@ -601,7 +601,7 @@ impl<'a> Parser<'a> { let span_start = self.token.span; let ast::FnHeader { ext, safety, constness, coroutine_kind } = self.parse_fn_front_matter(&inherited_vis, Case::Sensitive)?; - if self.may_recover() && self.token.kind == TokenKind::Lt { + if self.may_recover() && self.token == TokenKind::Lt { self.recover_fn_ptr_with_generics(lo, &mut params, param_insertion_point)?; } let decl = self.parse_fn_decl(|_| false, AllowPlus::No, recover_return_sign)?; @@ -681,7 +681,7 @@ impl<'a> Parser<'a> { // Always parse bounds greedily for better error recovery. let bounds = self.parse_generic_bounds()?; - *impl_dyn_multi = bounds.len() > 1 || self.prev_token.kind == TokenKind::BinOp(token::Plus); + *impl_dyn_multi = bounds.len() > 1 || self.prev_token == TokenKind::BinOp(token::Plus); Ok(TyKind::ImplTrait(ast::DUMMY_NODE_ID, bounds)) } @@ -727,8 +727,7 @@ impl<'a> Parser<'a> { self.check_keyword(kw::Dyn) && (self.token.uninterpolated_span().at_least_rust_2018() || self.look_ahead(1, |t| { - (can_begin_dyn_bound_in_edition_2015(t) - || t.kind == TokenKind::BinOp(token::Star)) + (can_begin_dyn_bound_in_edition_2015(t) || *t == TokenKind::BinOp(token::Star)) && !can_continue_type_after_non_fn_ident(t) })) } @@ -750,7 +749,7 @@ impl<'a> Parser<'a> { // Always parse bounds greedily for better error recovery. let bounds = self.parse_generic_bounds()?; - *impl_dyn_multi = bounds.len() > 1 || self.prev_token.kind == TokenKind::BinOp(token::Plus); + *impl_dyn_multi = bounds.len() > 1 || self.prev_token == TokenKind::BinOp(token::Plus); Ok(TyKind::TraitObject(bounds, syntax)) } @@ -1060,7 +1059,7 @@ impl<'a> Parser<'a> { } let mut path = if self.token.is_keyword(kw::Fn) - && self.look_ahead(1, |tok| tok.kind == TokenKind::OpenDelim(Delimiter::Parenthesis)) + && self.look_ahead(1, |t| *t == TokenKind::OpenDelim(Delimiter::Parenthesis)) && let Some(path) = self.recover_path_from_fn() { path |