diff options
Diffstat (limited to 'compiler/rustc_parse/src/parser/mod.rs')
| -rw-r--r-- | compiler/rustc_parse/src/parser/mod.rs | 84 |
1 files changed, 29 insertions, 55 deletions
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 57778d67098..c5b46b809b1 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -25,7 +25,7 @@ use rustc_ast::util::case::Case; use rustc_ast::AttrId; use rustc_ast::DUMMY_NODE_ID; use rustc_ast::{self as ast, AnonConst, Const, DelimArgs, Extern}; -use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, MacDelimiter, Mutability, StrLit}; +use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, Mutability, StrLit}; use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind}; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; @@ -135,9 +135,24 @@ pub struct Parser<'a> { pub capture_cfg: bool, restrictions: Restrictions, expected_tokens: Vec<TokenType>, - // Important: This must only be advanced from `bump` to ensure that - // `token_cursor.num_next_calls` is updated properly. token_cursor: TokenCursor, + // The number of calls to `bump`, i.e. the position in the token stream. + num_bump_calls: usize, + // During parsing we may sometimes need to 'unglue' a glued token into two + // component tokens (e.g. '>>' into '>' and '>), so the parser can consume + // them one at a time. This process bypasses the normal capturing mechanism + // (e.g. `num_bump_calls` will not be incremented), since the 'unglued' + // tokens due not exist in the original `TokenStream`. + // + // If we end up consuming both unglued tokens, this is not an issue. We'll + // end up capturing the single 'glued' token. + // + // However, sometimes we may want to capture just the first 'unglued' + // token. For example, capturing the `Vec<u8>` in `Option<Vec<u8>>` + // requires us to unglue the trailing `>>` token. The `break_last_token` + // field is used to track this token. It gets appended to the captured + // stream when we evaluate a `LazyAttrTokenStream`. + break_last_token: bool, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error /// appropriately. @@ -161,7 +176,7 @@ pub struct Parser<'a> { // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // it doesn't unintentionally get bigger. #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(Parser<'_>, 272); +rustc_data_structures::static_assert_size!(Parser<'_>, 264); /// Stores span information about a closure. #[derive(Clone)] @@ -223,32 +238,6 @@ struct TokenCursor { // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters // because it's the outermost token stream which never has delimiters. stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>, - - // Counts the number of calls to `{,inlined_}next`. - num_next_calls: usize, - - // During parsing, we may sometimes need to 'unglue' a - // glued token into two component tokens - // (e.g. '>>' into '>' and '>), so that the parser - // can consume them one at a time. This process - // bypasses the normal capturing mechanism - // (e.g. `num_next_calls` will not be incremented), - // since the 'unglued' tokens due not exist in - // the original `TokenStream`. - // - // If we end up consuming both unglued tokens, - // then this is not an issue - we'll end up - // capturing the single 'glued' token. - // - // However, in certain circumstances, we may - // want to capture just the first 'unglued' token. - // For example, capturing the `Vec<u8>` - // in `Option<Vec<u8>>` requires us to unglue - // the trailing `>>` token. The `break_last_token` - // field is used to track this token - it gets - // appended to the captured stream when - // we evaluate a `LazyAttrTokenStream`. - break_last_token: bool, } impl TokenCursor { @@ -399,12 +388,9 @@ impl<'a> Parser<'a> { capture_cfg: false, restrictions: Restrictions::empty(), expected_tokens: Vec::new(), - token_cursor: TokenCursor { - tree_cursor: stream.into_trees(), - stack: Vec::new(), - num_next_calls: 0, - break_last_token: false, - }, + token_cursor: TokenCursor { tree_cursor: stream.into_trees(), stack: Vec::new() }, + num_bump_calls: 0, + break_last_token: false, unmatched_angle_bracket_count: 0, max_angle_bracket_count: 0, last_unexpected_token_span: None, @@ -707,7 +693,7 @@ impl<'a> Parser<'a> { // If we consume any additional tokens, then this token // is not needed (we'll capture the entire 'glued' token), // and `bump` will set this field to `None` - self.token_cursor.break_last_token = true; + self.break_last_token = true; // Use the spacing of the glued token as the spacing // of the unglued second token. self.bump_with((Token::new(second, second_span), self.token_spacing)); @@ -1049,11 +1035,11 @@ impl<'a> Parser<'a> { // Note: destructuring here would give nicer code, but it was found in #96210 to be slower // than `.0`/`.1` access. let mut next = self.token_cursor.inlined_next(); - self.token_cursor.num_next_calls += 1; + self.num_bump_calls += 1; // We've retrieved an token from the underlying // cursor, so we no longer need to worry about // an unglued token. See `break_and_eat` for more details - self.token_cursor.break_last_token = false; + self.break_last_token = false; if next.0.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. let fallback_span = self.token.span; @@ -1230,12 +1216,10 @@ impl<'a> Parser<'a> { || self.check(&token::OpenDelim(Delimiter::Brace)); delimited.then(|| { - // We've confirmed above that there is a delimiter so unwrapping is OK. let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else { unreachable!() }; - - DelimArgs { dspan, delim: MacDelimiter::from_token(delim).unwrap(), tokens } + DelimArgs { dspan, delim, tokens } }) } @@ -1251,12 +1235,11 @@ impl<'a> Parser<'a> { } /// Parses a single token tree from the input. - pub(crate) fn parse_token_tree(&mut self) -> TokenTree { + pub fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { token::OpenDelim(..) => { // Grab the tokens within the delimiters. - let tree_cursor = &self.token_cursor.tree_cursor; - let stream = tree_cursor.stream.clone(); + let stream = self.token_cursor.tree_cursor.stream.clone(); let (_, delim, span) = *self.token_cursor.stack.last().unwrap(); // Advance the token cursor through the entire delimited @@ -1287,15 +1270,6 @@ impl<'a> Parser<'a> { } } - /// Parses a stream of tokens into a list of `TokenTree`s, up to EOF. - pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec<TokenTree>> { - let mut tts = Vec::new(); - while self.token != token::Eof { - tts.push(self.parse_token_tree()); - } - Ok(tts) - } - pub fn parse_tokens(&mut self) -> TokenStream { let mut result = Vec::new(); loop { @@ -1455,7 +1429,7 @@ impl<'a> Parser<'a> { } pub fn approx_token_stream_pos(&self) -> usize { - self.token_cursor.num_next_calls + self.num_bump_calls } } |