Auto merge of #13848 - ian-h-chamberlain:feature/color-compiler-diagnostics, r=ian-h-chamberlain

Colorize `cargo check` diagnostics in VSCode via text decorations

Fixes #13648

![colored-rustc-diagnostics](https://user-images.githubusercontent.com/11131775/209479884-10eef8ca-37b4-4aae-88f7-3591ac01b25e.gif)

Use ANSI control characters to display text decorations matching the VScode terminal theme, and strip them out when providing text content for rustc diagnostics.

This adds the small [`anser`](https://www.npmjs.com/package/anser) library (MIT license, no dependencies) to parse the control codes, and it also supports HTML output so it should be fairly easy to switch to a rendered HTML/webview implementation in the future

I also updated the default `cargo check` command to use the rendered ANSI diagnostics, although I'm not sure if it makes sense to put this kind of thing behind a feature flag, or whether it might have any issues on Windows (as I believe ANSI codes are not used for colorization there)?

9 files changed, 328 insertions, 31 deletions

diff --git a/crates/flycheck/src/lib.rs b/crates/flycheck/src/lib.rs
index 590a93fbaa1..11f7b068ecb 100644
--- a/crates/flycheck/src/lib.rs
+++ b/crates/flycheck/src/lib.rs
@@ -47,6 +47,7 @@ pub enum FlycheckConfig {
         features: Vec<String>,
         extra_args: Vec<String>,
         extra_env: FxHashMap<String, String>,
+        ansi_color_output: bool,
     },
     CustomCommand {
         command: String,
@@ -293,12 +294,21 @@ impl FlycheckActor {
                 extra_args,
                 features,
                 extra_env,
+                ansi_color_output,
             } => {
                 let mut cmd = Command::new(toolchain::cargo());
                 cmd.arg(command);
                 cmd.current_dir(&self.root);
-                cmd.args(["--workspace", "--message-format=json", "--manifest-path"])
-                    .arg(self.root.join("Cargo.toml").as_os_str());
+                cmd.arg("--workspace");
+
+                cmd.arg(if *ansi_color_output {
+                    "--message-format=json-diagnostic-rendered-ansi"
+                } else {
+                    "--message-format=json"
+                });
+
+                cmd.arg("--manifest-path");
+                cmd.arg(self.root.join("Cargo.toml").as_os_str());
 
                 for target in target_triples {
                     cmd.args(["--target", target.as_str()]);
diff --git a/crates/rust-analyzer/src/config.rs b/crates/rust-analyzer/src/config.rs
index cb55a327583..b0afbdc9a42 100644
--- a/crates/rust-analyzer/src/config.rs
+++ b/crates/rust-analyzer/src/config.rs
@@ -160,7 +160,9 @@ config_data! {
         check_noDefaultFeatures | checkOnSave_noDefaultFeatures: Option<bool>         = "null",
         /// Override the command rust-analyzer uses instead of `cargo check` for
         /// diagnostics on save. The command is required to output json and
-        /// should therefore include `--message-format=json` or a similar option.
+        /// should therefore include `--message-format=json` or a similar option
+        /// (if your client supports the `colorDiagnosticOutput` experimental
+        /// capability, you can use `--message-format=json-diagnostic-rendered-ansi`).
         ///
         /// If you're changing this because you're using some tool wrapping
         /// Cargo, you might also want to change
@@ -1006,6 +1008,11 @@ impl Config {
         self.experimental("serverStatusNotification")
     }
 
+    /// Whether the client supports colored output for full diagnostics from `checkOnSave`.
+    pub fn color_diagnostic_output(&self) -> bool {
+        self.experimental("colorDiagnosticOutput")
+    }
+
     pub fn publish_diagnostics(&self) -> bool {
         self.data.diagnostics_enable
     }
@@ -1204,6 +1211,7 @@ impl Config {
                 },
                 extra_args: self.data.check_extraArgs.clone(),
                 extra_env: self.check_on_save_extra_env(),
+                ansi_color_output: self.color_diagnostic_output(),
             },
         }
     }
diff --git a/docs/dev/lsp-extensions.md b/docs/dev/lsp-extensions.md
index 1bbb4c2323c..a4780af1a26 100644
--- a/docs/dev/lsp-extensions.md
+++ b/docs/dev/lsp-extensions.md
@@ -792,3 +792,29 @@ export interface ClientCommandOptions {
     commands: string[];
 }
 ```
+
+## Colored Diagnostic Output
+
+**Experimental Client Capability:** `{ "colorDiagnosticOutput": boolean }`
+
+If this capability is set, the "full compiler diagnostics" provided by `checkOnSave`
+will include ANSI color and style codes to render the diagnostic in a similar manner
+as `cargo`. This is translated into `--message-format=json-diagnostic-rendered-ansi`
+when flycheck is run, instead of the default `--message-format=json`.
+
+The full compiler rendered diagnostics are included in the server response
+regardless of this capability:
+
+```typescript
+// https://microsoft.github.io/language-server-protocol/specifications/specification-current#diagnostic
+export interface Diagnostic {
+    ...
+    data?: {
+        /**
+         * The human-readable compiler output as it would be printed to a terminal.
+         * Includes ANSI color and style codes if the client has set the experimental
+         * `colorDiagnosticOutput` capability.
+         */
+        rendered?: string;
+    };
+}
diff --git a/docs/user/generated_config.adoc b/docs/user/generated_config.adoc
index 755c69e12cf..b33a2e79525 100644
--- a/docs/user/generated_config.adoc
+++ b/docs/user/generated_config.adoc
@@ -173,7 +173,9 @@ Whether to pass `--no-default-features` to Cargo. Defaults to
 --
 Override the command rust-analyzer uses instead of `cargo check` for
 diagnostics on save. The command is required to output json and
-should therefore include `--message-format=json` or a similar option.
+should therefore include `--message-format=json` or a similar option
+(if your client supports the `colorDiagnosticOutput` experimental
+capability, you can use `--message-format=json-diagnostic-rendered-ansi`).
 
 If you're changing this because you're using some tool wrapping
 Cargo, you might also want to change
diff --git a/editors/code/package-lock.json b/editors/code/package-lock.json
index ee69d224760..4844837a06f 100644
--- a/editors/code/package-lock.json
+++ b/editors/code/package-lock.json
@@ -9,6 +9,7 @@
             "version": "0.5.0-dev",
             "license": "MIT OR Apache-2.0",
             "dependencies": {
+                "anser": "^2.1.1",
                 "d3": "^7.6.1",
                 "d3-graphviz": "^5.0.2",
                 "vscode-languageclient": "^8.0.2"
@@ -394,6 +395,11 @@
                 "url": "https://github.com/sponsors/epoberezkin"
             }
         },
+        "node_modules/anser": {
+            "version": "2.1.1",
+            "resolved": "https://registry.npmjs.org/anser/-/anser-2.1.1.tgz",
+            "integrity": "sha512-nqLm4HxOTpeLOxcmB3QWmV5TcDFhW9y/fyQ+hivtDFcK4OQ+pQ5fzPnXHM1Mfcm0VkLtvVi1TCPr++Qy0Q/3EQ=="
+        },
         "node_modules/ansi-regex": {
             "version": "5.0.1",
             "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
@@ -4096,6 +4102,11 @@
                 "uri-js": "^4.2.2"
             }
         },
+        "anser": {
+            "version": "2.1.1",
+            "resolved": "https://registry.npmjs.org/anser/-/anser-2.1.1.tgz",
+            "integrity": "sha512-nqLm4HxOTpeLOxcmB3QWmV5TcDFhW9y/fyQ+hivtDFcK4OQ+pQ5fzPnXHM1Mfcm0VkLtvVi1TCPr++Qy0Q/3EQ=="
+        },
         "ansi-regex": {
             "version": "5.0.1",
             "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
diff --git a/editors/code/package.json b/editors/code/package.json
index 468368668fc..930564bd7ca 100644
--- a/editors/code/package.json
+++ b/editors/code/package.json
@@ -35,6 +35,7 @@
         "test": "cross-env TEST_VARIABLE=test node ./out/tests/runTests.js"
     },
     "dependencies": {
+        "anser": "^2.1.1",
         "d3": "^7.6.1",
         "d3-graphviz": "^5.0.2",
         "vscode-languageclient": "^8.0.2"
@@ -643,7 +644,7 @@
                     ]
                 },
                 "rust-analyzer.check.overrideCommand": {
-                    "markdownDescription": "Override the command rust-analyzer uses instead of `cargo check` for\ndiagnostics on save. The command is required to output json and\nshould therefore include `--message-format=json` or a similar option.\n\nIf you're changing this because you're using some tool wrapping\nCargo, you might also want to change\n`#rust-analyzer.cargo.buildScripts.overrideCommand#`.\n\nIf there are multiple linked projects, this command is invoked for\neach of them, with the working directory being the project root\n(i.e., the folder containing the `Cargo.toml`).\n\nAn example command would be:\n\n```bash\ncargo check --workspace --message-format=json --all-targets\n```\n.",
+                    "markdownDescription": "Override the command rust-analyzer uses instead of `cargo check` for\ndiagnostics on save. The command is required to output json and\nshould therefore include `--message-format=json` or a similar option\n(if your client supports the `colorDiagnosticOutput` experimental\ncapability, you can use `--message-format=json-diagnostic-rendered-ansi`).\n\nIf you're changing this because you're using some tool wrapping\nCargo, you might also want to change\n`#rust-analyzer.cargo.buildScripts.overrideCommand#`.\n\nIf there are multiple linked projects, this command is invoked for\neach of them, with the working directory being the project root\n(i.e., the folder containing the `Cargo.toml`).\n\nAn example command would be:\n\n```bash\ncargo check --workspace --message-format=json --all-targets\n```\n.",
                     "default": null,
                     "type": [
                         "null",
diff --git a/editors/code/src/client.ts b/editors/code/src/client.ts
index e6595340aae..82cdf0390ac 100644
--- a/editors/code/src/client.ts
+++ b/editors/code/src/client.ts
@@ -1,8 +1,10 @@
+import * as anser from "anser";
 import * as lc from "vscode-languageclient/node";
 import * as vscode from "vscode";
 import * as ra from "../src/lsp_ext";
 import * as Is from "vscode-languageclient/lib/common/utils/is";
 import { assert } from "./util";
+import * as diagnostics from "./diagnostics";
 import { WorkspaceEdit } from "vscode";
 import { Config, substituteVSCodeVariables } from "./config";
 import { randomUUID } from "crypto";
@@ -120,12 +122,12 @@ export async function createClient(
             },
             async handleDiagnostics(
                 uri: vscode.Uri,
-                diagnostics: vscode.Diagnostic[],
+                diagnosticList: vscode.Diagnostic[],
                 next: lc.HandleDiagnosticsSignature
             ) {
                 const preview = config.previewRustcOutput;
                 const errorCode = config.useRustcErrorCode;
-                diagnostics.forEach((diag, idx) => {
+                diagnosticList.forEach((diag, idx) => {
                     // Abuse the fact that VSCode leaks the LSP diagnostics data field through the
                     // Diagnostic class, if they ever break this we are out of luck and have to go
                     // back to the worst diagnostics experience ever:)
@@ -138,9 +140,10 @@ export async function createClient(
                         ?.rendered;
                     if (rendered) {
                         if (preview) {
+                            const decolorized = anser.ansiToText(rendered);
                             const index =
-                                rendered.match(/^(note|help):/m)?.index || rendered.length;
-                            diag.message = rendered
+                                decolorized.match(/^(note|help):/m)?.index || rendered.length;
+                            diag.message = decolorized
                                 .substring(0, index)
                                 .replace(/^ -->[^\n]+\n/m, "");
                         }
@@ -154,8 +157,8 @@ export async function createClient(
                         }
                         diag.code = {
                             target: vscode.Uri.from({
-                                scheme: "rust-analyzer-diagnostics-view",
-                                path: "/diagnostic message",
+                                scheme: diagnostics.URI_SCHEME,
+                                path: `/diagnostic message [${idx.toString()}]`,
                                 fragment: uri.toString(),
                                 query: idx.toString(),
                             }),
@@ -163,7 +166,7 @@ export async function createClient(
                         };
                     }
                 });
-                return next(uri, diagnostics);
+                return next(uri, diagnosticList);
             },
             async provideHover(
                 document: vscode.TextDocument,
@@ -330,6 +333,7 @@ class ExperimentalFeatures implements lc.StaticFeature {
         caps.codeActionGroup = true;
         caps.hoverActions = true;
         caps.serverStatusNotification = true;
+        caps.colorDiagnosticOutput = true;
         caps.commands = {
             commands: [
                 "rust-analyzer.runSingle",
diff --git a/editors/code/src/diagnostics.ts b/editors/code/src/diagnostics.ts
new file mode 100644
index 00000000000..9695d8bf26d
--- /dev/null
+++ b/editors/code/src/diagnostics.ts
@@ -0,0 +1,212 @@
+import * as anser from "anser";
+import * as vscode from "vscode";
+import { ProviderResult, Range, TextEditorDecorationType, ThemeColor, window } from "vscode";
+import { Ctx } from "./ctx";
+
+export const URI_SCHEME = "rust-analyzer-diagnostics-view";
+
+export class TextDocumentProvider implements vscode.TextDocumentContentProvider {
+    private _onDidChange = new vscode.EventEmitter<vscode.Uri>();
+
+    public constructor(private readonly ctx: Ctx) {}
+
+    get onDidChange(): vscode.Event<vscode.Uri> {
+        return this._onDidChange.event;
+    }
+
+    triggerUpdate(uri: vscode.Uri) {
+        if (uri.scheme === URI_SCHEME) {
+            this._onDidChange.fire(uri);
+        }
+    }
+
+    dispose() {
+        this._onDidChange.dispose();
+    }
+
+    async provideTextDocumentContent(uri: vscode.Uri): Promise<string> {
+        const contents = getRenderedDiagnostic(this.ctx, uri);
+        return anser.ansiToText(contents);
+    }
+}
+
+function getRenderedDiagnostic(ctx: Ctx, uri: vscode.Uri): string {
+    const diags = ctx.client?.diagnostics?.get(vscode.Uri.parse(uri.fragment, true));
+    if (!diags) {
+        return "Unable to find original rustc diagnostic";
+    }
+
+    const diag = diags[parseInt(uri.query)];
+    if (!diag) {
+        return "Unable to find original rustc diagnostic";
+    }
+    const rendered = (diag as unknown as { data?: { rendered?: string } }).data?.rendered;
+
+    if (!rendered) {
+        return "Unable to find original rustc diagnostic";
+    }
+
+    return rendered;
+}
+
+interface AnserStyle {
+    fg: string;
+    bg: string;
+    fg_truecolor: string;
+    bg_truecolor: string;
+    decorations: Array<anser.DecorationName>;
+}
+
+export class AnsiDecorationProvider implements vscode.Disposable {
+    private _decorationTypes = new Map<AnserStyle, TextEditorDecorationType>();
+
+    public constructor(private readonly ctx: Ctx) {}
+
+    dispose(): void {
+        for (const decorationType of this._decorationTypes.values()) {
+            decorationType.dispose();
+        }
+
+        this._decorationTypes.clear();
+    }
+
+    async provideDecorations(editor: vscode.TextEditor) {
+        if (editor.document.uri.scheme !== URI_SCHEME) {
+            return;
+        }
+
+        const decorations = (await this._getDecorations(editor.document.uri)) || [];
+        for (const [decorationType, ranges] of decorations) {
+            editor.setDecorations(decorationType, ranges);
+        }
+    }
+
+    private _getDecorations(
+        uri: vscode.Uri
+    ): ProviderResult<[TextEditorDecorationType, Range[]][]> {
+        const stringContents = getRenderedDiagnostic(this.ctx, uri);
+        const lines = stringContents.split("\n");
+
+        const result = new Map<TextEditorDecorationType, Range[]>();
+        // Populate all known decoration types in the result. This forces any
+        // lingering decorations to be cleared if the text content changes to
+        // something without ANSI codes for a given decoration type.
+        for (const decorationType of this._decorationTypes.values()) {
+            result.set(decorationType, []);
+        }
+
+        for (const [lineNumber, line] of lines.entries()) {
+            const totalEscapeLength = 0;
+
+            // eslint-disable-next-line camelcase
+            const parsed = anser.ansiToJson(line, { use_classes: true });
+
+            let offset = 0;
+
+            for (const span of parsed) {
+                const { content, ...style } = span;
+
+                const range = new Range(
+                    lineNumber,
+                    offset - totalEscapeLength,
+                    lineNumber,
+                    offset + content.length - totalEscapeLength
+                );
+
+                offset += content.length;
+
+                const decorationType = this._getDecorationType(style);
+
+                if (!result.has(decorationType)) {
+                    result.set(decorationType, []);
+                }
+
+                result.get(decorationType)!.push(range);
+            }
+        }
+
+        return [...result];
+    }
+
+    private _getDecorationType(style: AnserStyle): TextEditorDecorationType {
+        let decorationType = this._decorationTypes.get(style);
+
+        if (decorationType) {
+            return decorationType;
+        }
+
+        const fontWeight = style.decorations.find((s) => s === "bold");
+        const fontStyle = style.decorations.find((s) => s === "italic");
+        const textDecoration = style.decorations.find((s) => s === "underline");
+
+        decorationType = window.createTextEditorDecorationType({
+            backgroundColor: AnsiDecorationProvider._convertColor(style.bg, style.bg_truecolor),
+            color: AnsiDecorationProvider._convertColor(style.fg, style.fg_truecolor),
+            fontWeight,
+            fontStyle,
+            textDecoration,
+        });
+
+        this._decorationTypes.set(style, decorationType);
+
+        return decorationType;
+    }
+
+    // NOTE: This could just be a kebab-case to camelCase conversion, but I think it's
+    // a short enough list to just write these by hand
+    static readonly _anserToThemeColor: Record<string, ThemeColor> = {
+        "ansi-black": "ansiBlack",
+        "ansi-white": "ansiWhite",
+        "ansi-red": "ansiRed",
+        "ansi-green": "ansiGreen",
+        "ansi-yellow": "ansiYellow",
+        "ansi-blue": "ansiBlue",
+        "ansi-magenta": "ansiMagenta",
+        "ansi-cyan": "ansiCyan",
+
+        "ansi-bright-black": "ansiBrightBlack",
+        "ansi-bright-white": "ansiBrightWhite",
+        "ansi-bright-red": "ansiBrightRed",
+        "ansi-bright-green": "ansiBrightGreen",
+        "ansi-bright-yellow": "ansiBrightYellow",
+        "ansi-bright-blue": "ansiBrightBlue",
+        "ansi-bright-magenta": "ansiBrightMagenta",
+        "ansi-bright-cyan": "ansiBrightCyan",
+    };
+
+    private static _convertColor(
+        color?: string,
+        truecolor?: string
+    ): ThemeColor | string | undefined {
+        if (!color) {
+            return undefined;
+        }
+
+        if (color === "ansi-truecolor") {
+            if (!truecolor) {
+                return undefined;
+            }
+            return `rgb(${truecolor})`;
+        }
+
+        const paletteMatch = color.match(/ansi-palette-(.+)/);
+        if (paletteMatch) {
+            const paletteColor = paletteMatch[1];
+            // anser won't return both the RGB and the color name at the same time,
+            // so just fake a single foreground control char with the palette number:
+            const spans = anser.ansiToJson(`\x1b[38;5;${paletteColor}m`);
+            const rgb = spans[1].fg;
+
+            if (rgb) {
+                return `rgb(${rgb})`;
+            }
+        }
+
+        const themeColor = AnsiDecorationProvider._anserToThemeColor[color];
+        if (themeColor) {
+            return new ThemeColor("terminal." + themeColor);
+        }
+
+        return undefined;
+    }
+}
diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts
index 9a9667b2cd2..dd439317c70 100644
--- a/editors/code/src/main.ts
+++ b/editors/code/src/main.ts
@@ -3,6 +3,7 @@ import * as lc from "vscode-languageclient/node";
 
 import * as commands from "./commands";
 import { CommandFactory, Ctx, fetchWorkspace } from "./ctx";
+import * as diagnostics from "./diagnostics";
 import { activateTaskProvider } from "./tasks";
 import { setContextValue } from "./util";
 
@@ -48,30 +49,52 @@ async function activateServer(ctx: Ctx): Promise<RustAnalyzerExtensionApi> {
         ctx.pushExtCleanup(activateTaskProvider(ctx.config));
     }
 
+    const diagnosticProvider = new diagnostics.TextDocumentProvider(ctx);
     ctx.pushExtCleanup(
         vscode.workspace.registerTextDocumentContentProvider(
-            "rust-analyzer-diagnostics-view",
-            new (class implements vscode.TextDocumentContentProvider {
-                async provideTextDocumentContent(uri: vscode.Uri): Promise<string> {
-                    const diags = ctx.client?.diagnostics?.get(
-                        vscode.Uri.parse(uri.fragment, true)
-                    );
-                    if (!diags) {
-                        return "Unable to find original rustc diagnostic";
-                    }
-
-                    const diag = diags[parseInt(uri.query)];
-                    if (!diag) {
-                        return "Unable to find original rustc diagnostic";
-                    }
-                    const rendered = (diag as unknown as { data?: { rendered?: string } }).data
-                        ?.rendered;
-                    return rendered ?? "Unable to find original rustc diagnostic";
-                }
-            })()
+            diagnostics.URI_SCHEME,
+            diagnosticProvider
         )
     );
 
+    const decorationProvider = new diagnostics.AnsiDecorationProvider(ctx);
+    ctx.pushExtCleanup(decorationProvider);
+
+    async function decorateVisibleEditors(document: vscode.TextDocument) {
+        for (const editor of vscode.window.visibleTextEditors) {
+            if (document === editor.document) {
+                await decorationProvider.provideDecorations(editor);
+            }
+        }
+    }
+
+    vscode.workspace.onDidChangeTextDocument(
+        async (event) => await decorateVisibleEditors(event.document),
+        null,
+        ctx.subscriptions
+    );
+    vscode.workspace.onDidOpenTextDocument(decorateVisibleEditors, null, ctx.subscriptions);
+    vscode.window.onDidChangeActiveTextEditor(
+        async (editor) => {
+            if (editor) {
+                diagnosticProvider.triggerUpdate(editor.document.uri);
+                await decorateVisibleEditors(editor.document);
+            }
+        },
+        null,
+        ctx.subscriptions
+    );
+    vscode.window.onDidChangeVisibleTextEditors(
+        async (visibleEditors) => {
+            for (const editor of visibleEditors) {
+                diagnosticProvider.triggerUpdate(editor.document.uri);
+                await decorationProvider.provideDecorations(editor);
+            }
+        },
+        null,
+        ctx.subscriptions
+    );
+
     vscode.workspace.onDidChangeWorkspaceFolders(
         async (_) => ctx.onWorkspaceFolderChanges(),
         null,