Use `token::Lit` in `ast::ExprKind::Lit`.

Instead of `ast::Lit`.

Literal lowering now happens at two different times. Expression literals
are lowered when HIR is crated. Attribute literals are lowered during
parsing.

This commit changes the language very slightly. Some programs that used
to not compile now will compile. This is because some invalid literals
that are removed by `cfg` or attribute macros will no longer trigger
errors. See this comment for more details:
https://github.com/rust-lang/rust/pull/102944#issuecomment-1277476773

5 files changed, 108 insertions, 89 deletions

diff --git a/src/test/ui/codemap_tests/unicode_2.stderr b/src/test/ui/codemap_tests/unicode_2.stderr
index a776a4a1e7e..19aae1d3c95 100644
--- a/src/test/ui/codemap_tests/unicode_2.stderr
+++ b/src/test/ui/codemap_tests/unicode_2.stderr
@@ -1,3 +1,9 @@
+error[E0425]: cannot find value `a̐é` in this scope
+  --> $DIR/unicode_2.rs:4:13
+   |
+LL |     let _ = a̐é;
+   |             ^^ not found in this scope
+
 error: invalid width `7` for integer literal
   --> $DIR/unicode_2.rs:2:25
    |
@@ -14,12 +20,6 @@ LL |     let _ = ("아あ", 1i42);
    |
    = help: valid widths are 8, 16, 32, 64 and 128
 
-error[E0425]: cannot find value `a̐é` in this scope
-  --> $DIR/unicode_2.rs:4:13
-   |
-LL |     let _ = a̐é;
-   |             ^^ not found in this scope
-
 error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0425`.
diff --git a/src/test/ui/lexer/error-stage.rs b/src/test/ui/lexer/error-stage.rs
index 05ae0e82bfb..c8d88f745a1 100644
--- a/src/test/ui/lexer/error-stage.rs
+++ b/src/test/ui/lexer/error-stage.rs
@@ -1,31 +1,80 @@
+// This test is about the treatment of invalid literals. In particular, some
+// literals are only considered invalid if they survive to HIR lowering.
+//
+// Literals with bad suffixes
+// --------------------------
+// Literals consist of a primary part and an optional suffix.
+// https://doc.rust-lang.org/reference/tokens.html#suffixes says:
+//
+//   Any kind of literal (string, integer, etc) with any suffix is valid as a
+//   token, and can be passed to a macro without producing an error. The macro
+//   itself will decide how to interpret such a token and whether to produce an
+//   error or not.
+//
+//   ```
+//   macro_rules! blackhole { ($tt:tt) => () }
+//   blackhole!("string"suffix); // OK
+//   ```
+//
+//   However, suffixes on literal tokens parsed as Rust code are restricted.
+//   Any suffixes are rejected on non-numeric literal tokens, and numeric
+//   literal tokens are accepted only with suffixes from the list below.
+//
+//   Integer: u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, usize, isize
+//   Floating-point: f32, f64
+//
+// This means that something like `"string"any_suffix` is a token accepted by
+// the lexer, but rejected later for being an invalid combination of primary
+// part and suffix.
+//
+// `0b10f32` is a similar case. `0b10` is a valid primary part that is a valid
+// *integer* literal when no suffix is present. It only causes an error later
+// when combined with the `f32` float suffix.
+//
+// However, `0b10.0f32` is different. It is rejected by the lexer because
+// `0b10.0` is not a valid token even on its own.
+//
+// This difference is unfortunate, but it's baked into the language now.
+//
+// Too-large integer literals
+// --------------------------
+// https://doc.rust-lang.org/reference/tokens.html#integer-literals says that
+// literals like `128_i8` and `256_u8` "are too big for their type, but are
+// still valid tokens".
+
 macro_rules! sink {
     ($($x:tt;)*) => {()}
 }
 
-// The invalid literals are ignored because the macro consumes them.
+// The invalid literals are ignored because the macro consumes them. Except for
+// `0b10.0f32` because it's a lexer error.
 const _: () = sink! {
     "string"any_suffix; // OK
     10u123; // OK
     10.0f123; // OK
     0b10f32; // OK
+    0b10.0f32; //~ ERROR binary float literal is not supported
     999340282366920938463463374607431768211455999; // OK
 };
 
-// The invalid literals cause errors.
+// The invalid literals used to cause errors, but this was changed by #102944.
+// Except for `0b010.0f32`, because it's a lexer error.
 #[cfg(FALSE)]
 fn configured_out() {
-    "string"any_suffix; //~ ERROR suffixes on string literals are invalid
-    10u123; //~ ERROR invalid width `123` for integer literal
-    10.0f123; //~ ERROR invalid width `123` for float literal
-    0b10f32; //~ ERROR binary float literal is not supported
-    999340282366920938463463374607431768211455999; //~ ERROR integer literal is too large
+    "string"any_suffix; // OK
+    10u123; // OK
+    10.0f123; // OK
+    0b10f32; // OK
+    0b10.0f32; //~ ERROR binary float literal is not supported
+    999340282366920938463463374607431768211455999; // OK
 }
 
-// The invalid literals cause errors.
+// All the invalid literals cause errors.
 fn main() {
     "string"any_suffix; //~ ERROR suffixes on string literals are invalid
     10u123; //~ ERROR invalid width `123` for integer literal
     10.0f123; //~ ERROR invalid width `123` for float literal
     0b10f32; //~ ERROR binary float literal is not supported
+    0b10.0f32; //~ ERROR binary float literal is not supported
     999340282366920938463463374607431768211455999; //~ ERROR integer literal is too large
 }
diff --git a/src/test/ui/lexer/error-stage.stderr b/src/test/ui/lexer/error-stage.stderr
index 024b7d94037..697a7c28da1 100644
--- a/src/test/ui/lexer/error-stage.stderr
+++ b/src/test/ui/lexer/error-stage.stderr
@@ -1,45 +1,29 @@
-error: suffixes on string literals are invalid
-  --> $DIR/error-stage.rs:17:5
-   |
-LL |     "string"any_suffix;
-   |     ^^^^^^^^^^^^^^^^^^ invalid suffix `any_suffix`
-
-error: invalid width `123` for integer literal
-  --> $DIR/error-stage.rs:18:5
+error: binary float literal is not supported
+  --> $DIR/error-stage.rs:56:5
    |
-LL |     10u123;
+LL |     0b10.0f32;
    |     ^^^^^^
-   |
-   = help: valid widths are 8, 16, 32, 64 and 128
-
-error: invalid width `123` for float literal
-  --> $DIR/error-stage.rs:19:5
-   |
-LL |     10.0f123;
-   |     ^^^^^^^^
-   |
-   = help: valid widths are 32 and 64
 
 error: binary float literal is not supported
-  --> $DIR/error-stage.rs:20:5
+  --> $DIR/error-stage.rs:68:5
    |
-LL |     0b10f32;
-   |     ^^^^^^^ not supported
+LL |     0b10.0f32;
+   |     ^^^^^^
 
-error: integer literal is too large
-  --> $DIR/error-stage.rs:21:5
+error: binary float literal is not supported
+  --> $DIR/error-stage.rs:78:5
    |
-LL |     999340282366920938463463374607431768211455999;
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |     0b10.0f32;
+   |     ^^^^^^
 
 error: suffixes on string literals are invalid
-  --> $DIR/error-stage.rs:26:5
+  --> $DIR/error-stage.rs:74:5
    |
 LL |     "string"any_suffix;
    |     ^^^^^^^^^^^^^^^^^^ invalid suffix `any_suffix`
 
 error: invalid width `123` for integer literal
-  --> $DIR/error-stage.rs:27:5
+  --> $DIR/error-stage.rs:75:5
    |
 LL |     10u123;
    |     ^^^^^^
@@ -47,7 +31,7 @@ LL |     10u123;
    = help: valid widths are 8, 16, 32, 64 and 128
 
 error: invalid width `123` for float literal
-  --> $DIR/error-stage.rs:28:5
+  --> $DIR/error-stage.rs:76:5
    |
 LL |     10.0f123;
    |     ^^^^^^^^
@@ -55,16 +39,16 @@ LL |     10.0f123;
    = help: valid widths are 32 and 64
 
 error: binary float literal is not supported
-  --> $DIR/error-stage.rs:29:5
+  --> $DIR/error-stage.rs:77:5
    |
 LL |     0b10f32;
    |     ^^^^^^^ not supported
 
 error: integer literal is too large
-  --> $DIR/error-stage.rs:30:5
+  --> $DIR/error-stage.rs:79:5
    |
 LL |     999340282366920938463463374607431768211455999;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error: aborting due to 10 previous errors
+error: aborting due to 8 previous errors
 
diff --git a/src/test/ui/parser/bad-lit-suffixes.rs b/src/test/ui/parser/bad-lit-suffixes.rs
index 58b30872677..8cb9ef7e0c9 100644
--- a/src/test/ui/parser/bad-lit-suffixes.rs
+++ b/src/test/ui/parser/bad-lit-suffixes.rs
@@ -28,12 +28,11 @@ fn main() {
 }
 
 #[rustc_dummy = "string"suffix]
-//~^ ERROR suffixes on string literals are invalid
+//~^ ERROR unexpected expression: `"string"suffix`
 fn f() {}
 
 #[must_use = "string"suffix]
-//~^ ERROR suffixes on string literals are invalid
-//~^^ ERROR malformed `must_use` attribute input
+//~^ ERROR unexpected expression: `"string"suffix`
 fn g() {}
 
 #[link(name = "string"suffix)]
diff --git a/src/test/ui/parser/bad-lit-suffixes.stderr b/src/test/ui/parser/bad-lit-suffixes.stderr
index 14c0eda81be..756f99ab12c 100644
--- a/src/test/ui/parser/bad-lit-suffixes.stderr
+++ b/src/test/ui/parser/bad-lit-suffixes.stderr
@@ -10,6 +10,32 @@ error: suffixes on string literals are invalid
 LL |     "C"suffix
    |     ^^^^^^^^^ invalid suffix `suffix`
 
+error: unexpected expression: `"string"suffix`
+  --> $DIR/bad-lit-suffixes.rs:30:17
+   |
+LL | #[rustc_dummy = "string"suffix]
+   |                 ^^^^^^^^^^^^^^
+
+error: unexpected expression: `"string"suffix`
+  --> $DIR/bad-lit-suffixes.rs:34:14
+   |
+LL | #[must_use = "string"suffix]
+   |              ^^^^^^^^^^^^^^
+
+error: suffixes on string literals are invalid
+  --> $DIR/bad-lit-suffixes.rs:38:15
+   |
+LL | #[link(name = "string"suffix)]
+   |               ^^^^^^^^^^^^^^ invalid suffix `suffix`
+
+error: invalid suffix `suffix` for number literal
+  --> $DIR/bad-lit-suffixes.rs:42:41
+   |
+LL | #[rustc_layout_scalar_valid_range_start(0suffix)]
+   |                                         ^^^^^^^ invalid suffix `suffix`
+   |
+   = help: the suffix must be one of the numeric types (`u32`, `isize`, `f32`, etc.)
+
 error: suffixes on string literals are invalid
   --> $DIR/bad-lit-suffixes.rs:12:5
    |
@@ -110,44 +136,5 @@ LL |     1.0e10suffix;
    |
    = help: valid suffixes are `f32` and `f64`
 
-error: suffixes on string literals are invalid
-  --> $DIR/bad-lit-suffixes.rs:30:17
-   |
-LL | #[rustc_dummy = "string"suffix]
-   |                 ^^^^^^^^^^^^^^ invalid suffix `suffix`
-
-error: suffixes on string literals are invalid
-  --> $DIR/bad-lit-suffixes.rs:34:14
-   |
-LL | #[must_use = "string"suffix]
-   |              ^^^^^^^^^^^^^^ invalid suffix `suffix`
-
-error: malformed `must_use` attribute input
-  --> $DIR/bad-lit-suffixes.rs:34:1
-   |
-LL | #[must_use = "string"suffix]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-help: the following are the possible correct uses
-   |
-LL | #[must_use = "reason"]
-   |
-LL | #[must_use]
-   |
-
-error: suffixes on string literals are invalid
-  --> $DIR/bad-lit-suffixes.rs:39:15
-   |
-LL | #[link(name = "string"suffix)]
-   |               ^^^^^^^^^^^^^^ invalid suffix `suffix`
-
-error: invalid suffix `suffix` for number literal
-  --> $DIR/bad-lit-suffixes.rs:43:41
-   |
-LL | #[rustc_layout_scalar_valid_range_start(0suffix)]
-   |                                         ^^^^^^^ invalid suffix `suffix`
-   |
-   = help: the suffix must be one of the numeric types (`u32`, `isize`, `f32`, etc.)
-
-error: aborting due to 21 previous errors
+error: aborting due to 20 previous errors