From a47ae57a187eb9537fe22a50692088cc655f357f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 14:47:12 +1000 Subject: Use an `@` pattern to shorten some code. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'compiler/rustc_parse/src/parser') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 38f18022e3c..a627bc89aa0 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -325,12 +325,9 @@ impl<'a> Parser<'a> { replace_ranges, }); - // If we support tokens at all - if let Some(target_tokens) = ret.tokens_mut() { - if target_tokens.is_none() { - // Store our newly captured tokens into the AST node. - *target_tokens = Some(tokens.clone()); - } + // If we support tokens and don't already have them, store the newly captured tokens. + if let Some(target_tokens @ None) = ret.tokens_mut() { + *target_tokens = Some(tokens.clone()); } let final_attrs = ret.attrs(); -- cgit 1.4.1-3-g733a5 From b16201317eefa9054d545dc5237b1bc830448258 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 17:41:05 +1000 Subject: Use iterator normally in `make_attr_token_stream`. In a `for` loop, instead of a `while` loop. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'compiler/rustc_parse/src/parser') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a627bc89aa0..16261feca91 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -367,7 +367,7 @@ impl<'a> Parser<'a> { /// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and /// close delims. fn make_attr_token_stream( - mut iter: impl Iterator, + iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { #[derive(Debug)] @@ -377,8 +377,7 @@ fn make_attr_token_stream( inner: Vec, } let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; - let mut token_and_spacing = iter.next(); - while let Some((token, spacing)) = token_and_spacing { + for (token, spacing) in iter { match token { FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { stack @@ -416,7 +415,6 @@ fn make_attr_token_stream( .push(AttrTokenTree::AttrsTarget(target)), FlatToken::Empty => {} } - token_and_spacing = iter.next(); } let mut final_buf = stack.pop().expect("Missing final buf!"); if break_last_token { -- cgit 1.4.1-3-g733a5 From a88c4d67d9ea8a9f450638f207be867fa984bfce Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 18:05:18 +1000 Subject: Split the stack in `make_attr_token_stream`. It makes for shorter code, and fewer allocations. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 43 ++++++++++--------------- 1 file changed, 17 insertions(+), 26 deletions(-) (limited to 'compiler/rustc_parse/src/parser') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 16261feca91..5a4259eb1e7 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -376,18 +376,19 @@ fn make_attr_token_stream( open_delim_sp: Option<(Delimiter, Span, Spacing)>, inner: Vec, } - let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; + // The stack always has at least one element. Storing it separately makes for shorter code. + let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] }; + let mut stack_rest = vec![]; for (token, spacing) in iter { match token { FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { - stack - .push(FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }); + stack_rest.push(mem::replace( + &mut stack_top, + FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, + )); } FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { - let frame_data = stack - .pop() - .unwrap_or_else(|| panic!("Token stack was empty for token: {token:?}")); - + let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); assert_eq!( open_delim, delim, @@ -397,28 +398,18 @@ fn make_attr_token_stream( let dspacing = DelimSpacing::new(open_spacing, spacing); let stream = AttrTokenStream::new(frame_data.inner); let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); - stack - .last_mut() - .unwrap_or_else(|| panic!("Bottom token frame is missing for token: {token:?}")) - .inner - .push(delimited); + stack_top.inner.push(delimited); + } + FlatToken::Token(token) => stack_top.inner.push(AttrTokenTree::Token(token, spacing)), + FlatToken::AttrsTarget(target) => { + stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) } - FlatToken::Token(token) => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push(AttrTokenTree::Token(token, spacing)), - FlatToken::AttrsTarget(target) => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push(AttrTokenTree::AttrsTarget(target)), FlatToken::Empty => {} } } - let mut final_buf = stack.pop().expect("Missing final buf!"); + if break_last_token { - let last_token = final_buf.inner.pop().unwrap(); + let last_token = stack_top.inner.pop().unwrap(); if let AttrTokenTree::Token(last_token, spacing) = last_token { let unglued_first = last_token.kind.break_two_token_op().unwrap().0; @@ -426,14 +417,14 @@ fn make_attr_token_stream( let mut first_span = last_token.span.shrink_to_lo(); first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1)); - final_buf + stack_top .inner .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing)); } else { panic!("Unexpected last token {last_token:?}") } } - AttrTokenStream::new(final_buf.inner) + AttrTokenStream::new(stack_top.inner) } // Some types are used a lot. Make sure they don't unintentionally get bigger. -- cgit 1.4.1-3-g733a5 From f5527949f2e819c056551a25d8fab7008facf6da Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 19:32:21 +1000 Subject: Move `Spacing` into `FlatToken`. It's only needed for the `FlatToken::Token` variant. This makes things a little more concise. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 28 +++++++++++-------------- compiler/rustc_parse/src/parser/mod.rs | 2 +- 2 files changed, 13 insertions(+), 17 deletions(-) (limited to 'compiler/rustc_parse/src/parser') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 5a4259eb1e7..67d2424bb48 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -103,11 +103,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { // produce an empty `TokenStream` if no calls were made, and omit the // final token otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); - let tokens = iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) - .chain(iter::repeat_with(|| { - let token = cursor_snapshot.next(); - (FlatToken::Token(token.0), token.1) - })) + let tokens = iter::once(FlatToken::Token(self.start_token.clone())) + .chain(iter::repeat_with(|| FlatToken::Token(cursor_snapshot.next()))) .take(self.num_calls as usize); if self.replace_ranges.is_empty() { @@ -156,11 +153,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { (range.start as usize)..(range.end as usize), target .into_iter() - .map(|target| (FlatToken::AttrsTarget(target), Spacing::Alone)) - .chain( - iter::repeat((FlatToken::Empty, Spacing::Alone)) - .take(range.len() - target_len), - ), + .map(|target| FlatToken::AttrsTarget(target)) + .chain(iter::repeat(FlatToken::Empty).take(range.len() - target_len)), ); } make_attr_token_stream(tokens.into_iter(), self.break_last_token) @@ -367,7 +361,7 @@ impl<'a> Parser<'a> { /// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and /// close delims. fn make_attr_token_stream( - iter: impl Iterator, + iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { #[derive(Debug)] @@ -379,15 +373,15 @@ fn make_attr_token_stream( // The stack always has at least one element. Storing it separately makes for shorter code. let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] }; let mut stack_rest = vec![]; - for (token, spacing) in iter { - match token { - FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { + for flat_token in iter { + match flat_token { + FlatToken::Token((Token { kind: TokenKind::OpenDelim(delim), span }, spacing)) => { stack_rest.push(mem::replace( &mut stack_top, FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, )); } - FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { + FlatToken::Token((Token { kind: TokenKind::CloseDelim(delim), span }, spacing)) => { let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); assert_eq!( @@ -400,7 +394,9 @@ fn make_attr_token_stream( let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); stack_top.inner.push(delimited); } - FlatToken::Token(token) => stack_top.inner.push(AttrTokenTree::Token(token, spacing)), + FlatToken::Token((token, spacing)) => { + stack_top.inner.push(AttrTokenTree::Token(token, spacing)) + } FlatToken::AttrsTarget(target) => { stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 45ca267fe5d..1482c665a16 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1607,7 +1607,7 @@ pub(crate) fn make_unclosed_delims_error( enum FlatToken { /// A token - this holds both delimiter (e.g. '{' and '}') /// and non-delimiter tokens - Token(Token), + Token((Token, Spacing)), /// Holds the `AttrsTarget` for an AST node. The `AttrsTarget` is inserted /// directly into the constructed `AttrTokenStream` as an /// `AttrTokenTree::AttrsTarget`. -- cgit 1.4.1-3-g733a5 From 8a390bae06dbb5686b581797b8f46cb0353dc255 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 10 Jul 2024 14:41:39 +1000 Subject: Change empty replace range condition. The new condition is equivalent in practice, but it's much more obvious that it would result in an empty range, because the condition lines up with the contents of the iterator. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 31 +++++++++++++------------ 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'compiler/rustc_parse/src/parser') diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 67d2424bb48..e90568f177c 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -295,21 +295,22 @@ impl<'a> Parser<'a> { let num_calls = end_pos - start_pos; - // If we have no attributes, then we will never need to - // use any replace ranges. - let replace_ranges: Box<[ReplaceRange]> = if ret.attrs().is_empty() && !self.capture_cfg { - Box::new([]) - } else { - // Grab any replace ranges that occur *inside* the current AST node. - // We will perform the actual replacement when we convert the `LazyAttrTokenStream` - // to an `AttrTokenStream`. - self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] - .iter() - .cloned() - .chain(inner_attr_replace_ranges.iter().cloned()) - .map(|(range, data)| ((range.start - start_pos)..(range.end - start_pos), data)) - .collect() - }; + // This is hot enough for `deep-vector` that checking the conditions for an empty iterator + // is measurably faster than actually executing the iterator. + let replace_ranges: Box<[ReplaceRange]> = + if replace_ranges_start == replace_ranges_end && inner_attr_replace_ranges.is_empty() { + Box::new([]) + } else { + // Grab any replace ranges that occur *inside* the current AST node. + // We will perform the actual replacement when we convert the `LazyAttrTokenStream` + // to an `AttrTokenStream`. + self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] + .iter() + .cloned() + .chain(inner_attr_replace_ranges.iter().cloned()) + .map(|(range, data)| ((range.start - start_pos)..(range.end - start_pos), data)) + .collect() + }; let tokens = LazyAttrTokenStream::new(LazyAttrTokenStreamImpl { start_token, -- cgit 1.4.1-3-g733a5