Remove `NtIdent` and `NtLifetime`.

The extra span is now recorded in the new `TokenKind::NtIdent` and
`TokenKind::NtLifetime`. These both consist of a single token, and so
there's no operator precedence problems with inserting them directly
into the token stream.

The other way to do this would be to wrap the ident/lifetime in invisible
delimiters, but there's a lot of code that assumes an interpolated
ident/lifetime fits in a single token, and changing all that code to work with
invisible delimiters would have been a pain. (Maybe it could be done in a
follow-up.)

This change might not seem like much of a win, but it's a first step toward the
much bigger and long-desired removal of `Nonterminal` and
`TokenKind::Interpolated`. That change is big and complex enough that it's
worth doing this piece separately. (Indeed, this commit is based on part of a
late commit in #114647, a prior attempt at that big and complex change.)

1 files changed, 14 insertions, 11 deletions

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index d532a884184..fb666550e93 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -466,12 +466,6 @@ impl TokenStream {
 
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
         match nt {
-            Nonterminal::NtIdent(ident, is_raw) => {
-                TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
-            }
-            Nonterminal::NtLifetime(ident) => {
-                TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
-            }
             Nonterminal::NtItem(item) => TokenStream::from_ast(item),
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
             Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
@@ -489,11 +483,17 @@ impl TokenStream {
     }
 
     fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
-        match &token.kind {
-            token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => {
-                TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing)
+        match token.kind {
+            token::NtIdent(ident, is_raw) => {
+                TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
             }
-            token::Interpolated(nt) => TokenTree::Delimited(
+            token::NtLifetime(ident) => TokenTree::Delimited(
+                DelimSpan::from_single(token.span),
+                DelimSpacing::new(Spacing::JointHidden, spacing),
+                Delimiter::Invisible,
+                TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
+            ),
+            token::Interpolated(ref nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
                 Delimiter::Invisible,
@@ -516,7 +516,10 @@ impl TokenStream {
     pub fn flattened(&self) -> TokenStream {
         fn can_skip(stream: &TokenStream) -> bool {
             stream.trees().all(|tree| match tree {
-                TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
+                TokenTree::Token(token, _) => !matches!(
+                    token.kind,
+                    token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
+                ),
                 TokenTree::Delimited(.., inner) => can_skip(inner),
             })
         }