Auto merge of #75773 - matklad:snapshot-tests, r=Mark-Simulacrum

Introduce expect snapshot testing library into rustc

Snapshot testing is a technique for writing maintainable unit tests.
Unlike usual `assert_eq!` tests, snapshot tests allow
to *automatically* upgrade expected values on test failure.
In a sense, snapshot tests are inline-version of our beloved
UI-tests.

Example:

![expect](https://user-images.githubusercontent.com/1711539/90888810-3bcc8180-e3b7-11ea-9626-d06e89e1a0bb.gif)

A particular library we use, `expect_test` provides an `expect!`
macro, which creates a sort of self-updating string literal (by using
`file!` macro). Self-update is triggered by setting `UPDATE_EXPECT`
environmental variable (this info is printed during the test failure).
This library was extracted from rust-analyzer, where we use it for
most of our tests.

There are some other, more popular snapshot testing libraries:

* https://github.com/mitsuhiko/insta
* https://github.com/aaronabramov/k9

The main differences of `expect` are:

* first-class snapshot objects (so, tests can be written as functions,
  rather than as macros)
* focus on inline-snapshots (but file snapshots are also supported)
* restricted feature set (only `assert_eq` and `assert_debug_eq`)
* no extra runtime (ie, no `cargo insta`)

See rust-analyzer/rust-analyzer#5101 for a
an extended comparison.

It is unclear if this testing style will stick with rustc in the long
run. At the moment, rustc is mainly tested via integrated UI tests.
But in the library-ified world, unit-tests will become somewhat more
important (that's why use use `rustc_lexer` library-ified library as
an example in this PR). Given that the cost of removal shouldn't be
too high, it probably makes sense to just see if this flies!

6 files changed, 67 insertions, 0 deletions

diff --git a/Cargo.lock b/Cargo.lock
index da19714fe69..1bbae2cbd80 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1009,6 +1009,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "expect-test"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3e383741ea1982866572109d1a8c807bd36aad91fca701489fdca56ef92b3b8"
+dependencies = [
+ "difference",
+ "once_cell",
+]
+
+[[package]]
 name = "failure"
 version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3653,6 +3663,7 @@ dependencies = [
 name = "rustc_lexer"
 version = "0.1.0"
 dependencies = [
+ "expect-test",
  "unicode-xid",
 ]
 
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index ac833a55d4c..afa72b5d58c 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -1754,6 +1754,11 @@ impl Step for Crate {
             cargo.arg("--quiet");
         }
 
+        if builder.config.cmd.bless() {
+            // Bless `expect!` tests.
+            cargo.env("UPDATE_EXPECT", "1");
+        }
+
         if target.contains("emscripten") {
             cargo.env(
                 format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)),
diff --git a/src/librustc_lexer/Cargo.toml b/src/librustc_lexer/Cargo.toml
index 950771f0a69..28b56f6fef4 100644
--- a/src/librustc_lexer/Cargo.toml
+++ b/src/librustc_lexer/Cargo.toml
@@ -19,3 +19,6 @@ name = "rustc_lexer"
 # Note that this crate purposefully does not depend on other rustc crates
 [dependencies]
 unicode-xid = "0.2.0"
+
+[dev-dependencies]
+expect-test = "0.1"
diff --git a/src/librustc_lexer/src/lib.rs b/src/librustc_lexer/src/lib.rs
index 2d80ca5a4de..b7d6194cd77 100644
--- a/src/librustc_lexer/src/lib.rs
+++ b/src/librustc_lexer/src/lib.rs
@@ -35,6 +35,7 @@ use std::convert::TryFrom;
 /// Parsed token.
 /// It doesn't contain information about data that has been parsed,
 /// only the type of the token and its size.
+#[derive(Debug)]
 pub struct Token {
     pub kind: TokenKind,
     pub len: usize,
diff --git a/src/librustc_lexer/src/tests.rs b/src/librustc_lexer/src/tests.rs
index b0209ac2899..a1ea5ceb1f6 100644
--- a/src/librustc_lexer/src/tests.rs
+++ b/src/librustc_lexer/src/tests.rs
@@ -1,5 +1,7 @@
 use super::*;
 
+use expect_test::{expect, Expect};
+
 fn check_raw_str(s: &str, expected_hashes: u16, expected_err: Option<RawStrError>) {
     let s = &format!("r{}", s);
     let mut cursor = Cursor::new(s);
@@ -120,3 +122,46 @@ fn test_shebang_followed_by_attrib() {
     let input = "#!/bin/rust-scripts\n#![allow_unused(true)]";
     assert_eq!(strip_shebang(input), Some(19));
 }
+
+fn check_lexing(src: &str, expect: Expect) {
+    let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect();
+    expect.assert_eq(&actual)
+}
+
+#[test]
+fn comment_flavors() {
+    check_lexing(
+        r"
+// line
+//// line as well
+/// outer doc line
+//! inner doc line
+/* block */
+/**/
+/*** also block */
+/** outer doc block */
+/*! inner doc block */
+",
+        expect![[r#"
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: None }, len: 7 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: None }, len: 17 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 }
+                Token { kind: Whitespace, len: 1 }
+                Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 }
+                Token { kind: Whitespace, len: 1 }
+            "#]],
+    )
+}
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 4f98944e4c8..af3fb403703 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -85,11 +85,13 @@ const PERMITTED_DEPENDENCIES: &[&str] = &[
     "crossbeam-queue",
     "crossbeam-utils",
     "datafrog",
+    "difference",
     "digest",
     "dlmalloc",
     "either",
     "ena",
     "env_logger",
+    "expect-test",
     "fake-simd",
     "filetime",
     "flate2",