Auto merge of #83580 - Dylan-DPC:rollup-1zod4p7, r=Dylan-DPC

Rollup of 8 pull requests

Successful merges:

 - #81351 (combine: stop eagerly evaluating consts)
 - #82525 (make unaligned_references future-incompat lint warn-by-default)
 - #82626 (update array missing `IntoIterator` msg)
 - #82917 (Add function core::iter::zip)
 - #82993 (rustdoc: Use diagnostics for error when including sources)
 - #83522 (Improve fs error open_from unix)
 - #83548 (Always preserve `None`-delimited groups in a captured `TokenStream`)
 - #83555 (Add #[inline] to io::Error methods)

Failed merges:

 - #83130 (escape_ascii take 2)

r? `@ghost`
`@rustbot` modify labels: rollup

2 files changed, 55 insertions, 15 deletions

diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
index 7512f46988c..36a0fda6458 100644
--- a/compiler/rustc_parse/src/parser/attr_wrapper.rs
+++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -98,21 +98,46 @@ impl<'a> Parser<'a> {
         }
         impl CreateTokenStream for LazyTokenStreamImpl {
             fn create_token_stream(&self) -> TokenStream {
-                // The token produced by the final call to `next` or `next_desugared`
-                // was not actually consumed by the callback. The combination
-                // of chaining the initial token and using `take` produces the desired
-                // result - we produce an empty `TokenStream` if no calls were made,
-                // and omit the final token otherwise.
+                if self.num_calls == 0 {
+                    return TokenStream::new(vec![]);
+                }
+
                 let mut cursor_snapshot = self.cursor_snapshot.clone();
-                let tokens = std::iter::once(self.start_token.clone())
-                    .chain((0..self.num_calls).map(|_| {
-                        if self.desugar_doc_comments {
+                // Don't skip `None` delimiters, since we want to pass them to
+                // proc macros. Normally, we'll end up capturing `TokenKind::Interpolated`,
+                // which gets converted to a `None`-delimited group when we invoke
+                // a proc-macro. However, it's possible to already have a `None`-delimited
+                // group in the stream (such as when parsing the output of a proc-macro,
+                // or in certain unusual cases with cross-crate `macro_rules!` macros).
+                cursor_snapshot.skip_none_delims = false;
+
+                // The token produced by the final call to `next` or `next_desugared`
+                // was not actually consumed by the callback.
+                let num_calls = self.num_calls - 1;
+                let mut i = 0;
+                let tokens =
+                    std::iter::once(self.start_token.clone()).chain(std::iter::from_fn(|| {
+                        if i >= num_calls {
+                            return None;
+                        }
+
+                        let token = if self.desugar_doc_comments {
                             cursor_snapshot.next_desugared()
                         } else {
                             cursor_snapshot.next()
+                        };
+
+                        // When the `LazyTokenStreamImpl` was original produced, we did *not*
+                        // include `NoDelim` tokens in `num_calls`, since they are normally ignored
+                        // by the parser. Therefore, we only increment our counter for other types of tokens.
+                        if !matches!(
+                            token.0.kind,
+                            token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim)
+                        ) {
+                            i += 1;
                         }
-                    }))
-                    .take(self.num_calls);
+                        Some(token)
+                    }));
 
                 make_token_stream(tokens, self.append_unglued_token.clone())
             }
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index f0ee76d328c..748a8e2bb49 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -172,6 +172,13 @@ struct TokenCursor {
     // appended to the captured stream when
     // we evaluate a `LazyTokenStream`
     append_unglued_token: Option<TreeAndSpacing>,
+    // If `true`, skip the delimiters for `None`-delimited groups,
+    // and just yield the inner tokens. This is `true` during
+    // normal parsing, since the parser code is not currently prepared
+    // to handle `None` delimiters. When capturing a `TokenStream`,
+    // however, we want to handle `None`-delimiters, since
+    // proc-macros always see `None`-delimited groups.
+    skip_none_delims: bool,
 }
 
 #[derive(Clone)]
@@ -184,13 +191,13 @@ struct TokenCursorFrame {
 }
 
 impl TokenCursorFrame {
-    fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self {
+    fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream, skip_none_delims: bool) -> Self {
         TokenCursorFrame {
             delim,
             span,
-            open_delim: delim == token::NoDelim,
+            open_delim: delim == token::NoDelim && skip_none_delims,
             tree_cursor: tts.into_trees(),
-            close_delim: delim == token::NoDelim,
+            close_delim: delim == token::NoDelim && skip_none_delims,
         }
     }
 }
@@ -218,7 +225,7 @@ impl TokenCursor {
                     return (token, spacing);
                 }
                 TokenTree::Delimited(sp, delim, tts) => {
-                    let frame = TokenCursorFrame::new(sp, delim, tts);
+                    let frame = TokenCursorFrame::new(sp, delim, tts, self.skip_none_delims);
                     self.stack.push(mem::replace(&mut self.frame, frame));
                 }
             }
@@ -276,6 +283,7 @@ impl TokenCursor {
                         .cloned()
                         .collect::<TokenStream>()
                 },
+                self.skip_none_delims,
             ),
         ));
 
@@ -371,12 +379,19 @@ impl<'a> Parser<'a> {
             prev_token: Token::dummy(),
             restrictions: Restrictions::empty(),
             expected_tokens: Vec::new(),
+            // Skip over the delimiters for `None`-delimited groups
             token_cursor: TokenCursor {
-                frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens),
+                frame: TokenCursorFrame::new(
+                    DelimSpan::dummy(),
+                    token::NoDelim,
+                    tokens,
+                    /* skip_none_delims */ true,
+                ),
                 stack: Vec::new(),
                 num_next_calls: 0,
                 desugar_doc_comments,
                 append_unglued_token: None,
+                skip_none_delims: true,
             },
             desugar_doc_comments,
             unmatched_angle_bracket_count: 0,