Rollup merge of #69384 - petrochenkov:nounnorm, r=Centril

parser: `token` -> `normalized_token`, `nonnormalized_token` -> `token`

So, after https://github.com/rust-lang/rust/pull/69006, its follow-ups and an attempt to remove `Parser::prev_span` I came to the conclusion that the unnormalized token and its span is what you want in most cases, so it should be default.

Normalization only makes difference in few cases where we are checking against `token::Ident` or `token::Lifetime` specifically.
This PR uses `normalized_token` for those cases.

Using normalization explicitly means that people writing code should remember about `NtIdent` and `NtLifetime` in general. (That is alleviated by the fact that `token.ident()` and `fn parse_ident_*` are already written.)
Remembering about `NtIdent`, was, however, already the case, kind of, because the implicit normalization was performed only for the current/previous token, but not for things like `look_ahead`.
As a result, most of token classification methods in `token.rs` already take `NtIdent` into account (this PR fixes a few pre-existing minor mistakes though).

The next step is removing `normalized(_prev)_token` entirely and replacing it with `token.ident()` (mostly) and `token.normalize()` (occasionally).
I want to make it a separate PR for that and run it though perf.
`normalized_token` filled on every bump has both a potential to avoid repeated normalization, and to do unnecessary work in advance (it probably doesn't matter anyway, the normalization is very cheap).

r? @Centril

2 files changed, 35 insertions, 32 deletions

diff --git a/src/libsyntax/token.rs b/src/libsyntax/token.rs
index 52bf50604fb..a8a2c9b2fb3 100644
--- a/src/libsyntax/token.rs
+++ b/src/libsyntax/token.rs
@@ -147,36 +147,30 @@ impl Lit {
 
 pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool {
     let ident_token = Token::new(Ident(name, is_raw), span);
-    token_can_begin_expr(&ident_token)
-}
 
-pub fn token_can_begin_expr(ident_token: &Token) -> bool {
     !ident_token.is_reserved_ident()
         || ident_token.is_path_segment_keyword()
-        || match ident_token.kind {
-            TokenKind::Ident(ident, _) => [
-                kw::Async,
-                kw::Do,
-                kw::Box,
-                kw::Break,
-                kw::Continue,
-                kw::False,
-                kw::For,
-                kw::If,
-                kw::Let,
-                kw::Loop,
-                kw::Match,
-                kw::Move,
-                kw::Return,
-                kw::True,
-                kw::Unsafe,
-                kw::While,
-                kw::Yield,
-                kw::Static,
-            ]
-            .contains(&ident),
-            _ => false,
-        }
+        || [
+            kw::Async,
+            kw::Do,
+            kw::Box,
+            kw::Break,
+            kw::Continue,
+            kw::False,
+            kw::For,
+            kw::If,
+            kw::Let,
+            kw::Loop,
+            kw::Match,
+            kw::Move,
+            kw::Return,
+            kw::True,
+            kw::Unsafe,
+            kw::While,
+            kw::Yield,
+            kw::Static,
+        ]
+        .contains(&name)
 }
 
 fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool {
@@ -369,8 +363,8 @@ impl Token {
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
             Interpolated(ref nt) => match **nt {
+                NtIdent(ident, is_raw) => ident_can_begin_expr(ident.name, ident.span, is_raw),
                 NtLiteral(..) |
-                NtIdent(..)   |
                 NtExpr(..)    |
                 NtBlock(..)   |
                 NtPath(..)    |
@@ -397,7 +391,8 @@ impl Token {
             Lt | BinOp(Shl)             | // associated path
             ModSep                      => true, // global path
             Interpolated(ref nt) => match **nt {
-                NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true,
+                NtIdent(ident, is_raw) => ident_can_begin_type(ident.name, ident.span, is_raw),
+                NtTy(..) | NtPath(..) | NtLifetime(..) => true,
                 _ => false,
             },
             _ => false,
@@ -442,6 +437,7 @@ impl Token {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
             Interpolated(ref nt) => match &**nt {
+                NtIdent(ident, false) if ident.name.is_bool_lit() => true,
                 NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
                 _ => false,
             },
diff --git a/src/libsyntax/util/literal.rs b/src/libsyntax/util/literal.rs
index 0c611adc06b..ecf17efc4e0 100644
--- a/src/libsyntax/util/literal.rs
+++ b/src/libsyntax/util/literal.rs
@@ -197,10 +197,17 @@ impl Lit {
             }
             token::Literal(lit) => lit,
             token::Interpolated(ref nt) => {
-                if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
-                    if let ast::ExprKind::Lit(lit) = &expr.kind {
-                        return Ok(lit.clone());
+                match &**nt {
+                    token::NtIdent(ident, false) if ident.name.is_bool_lit() => {
+                        let lit = token::Lit::new(token::Bool, ident.name, None);
+                        return Lit::from_lit_token(lit, ident.span);
                     }
+                    token::NtExpr(expr) | token::NtLiteral(expr) => {
+                        if let ast::ExprKind::Lit(lit) = &expr.kind {
+                            return Ok(lit.clone());
+                        }
+                    }
+                    _ => {}
                 }
                 return Err(LitError::NotLiteral);
             }