Rollup merge of #133453 - ferrocene:check-license-metadata, r=Kobzol

Commit license-metadata.json to git and check it's correct in CI

This PR adds `license-metadata.json` to the root of the git repo, and changes `mingw-check` to check that the file is still up-to-date.

By committing this file, we remove the need for developers to a) have reuse installed or b) run an expensive ~90 second analysis of the files on disk when they want generate the COPYRIGHT.html files which depend on this license metadata.

The file will need updating whenever `REUSE.toml` changes, or when git submodules are added, or when git submodules change their license information (as detected by REUSE).

You can now run:

* `./x run collect-license-metadata` to update the `./license-metadata.json` file
* `./x test collect-license-metadata` to test the `./license-metadata.json` file for correctness

The comparison is done with two `serde_json::Value` objects, so the map objects they contain should ignore differences in ordering.

3 files changed, 39 insertions, 12 deletions

diff --git a/src/tools/collect-license-metadata/src/main.rs b/src/tools/collect-license-metadata/src/main.rs
index dce36bb17b6..08a30d0b899 100644
--- a/src/tools/collect-license-metadata/src/main.rs
+++ b/src/tools/collect-license-metadata/src/main.rs
@@ -4,7 +4,7 @@ mod reuse;
 
 use std::path::PathBuf;
 
-use anyhow::Error;
+use anyhow::{Context, Error};
 
 use crate::licenses::LicensesInterner;
 
@@ -12,10 +12,12 @@ use crate::licenses::LicensesInterner;
 ///
 /// You should probably let `bootstrap` execute this program instead of running it directly.
 ///
-/// Run `x.py run collect-license-metadata`
+/// * Run `x.py run collect-license-metadata` to re-regenerate the file.
+/// * Run `x.py test collect-license-metadata` to check if the file you have is correct.
 fn main() -> Result<(), Error> {
     let reuse_exe: PathBuf = std::env::var_os("REUSE_EXE").expect("Missing REUSE_EXE").into();
     let dest: PathBuf = std::env::var_os("DEST").expect("Missing DEST").into();
+    let only_check = std::env::var_os("ONLY_CHECK").is_some();
 
     let mut interner = LicensesInterner::new();
     let paths = crate::reuse::collect(&reuse_exe, &mut interner)?;
@@ -23,15 +25,32 @@ fn main() -> Result<(), Error> {
     let mut tree = crate::path_tree::build(paths);
     tree.simplify();
 
-    if let Some(parent) = dest.parent() {
-        std::fs::create_dir_all(parent)?;
+    let output = serde_json::json!({
+        "files": crate::path_tree::expand_interned_licenses(tree, &interner)
+    });
+
+    if only_check {
+        println!("loading existing license information");
+        let existing = std::fs::read_to_string(&dest).with_context(|| {
+            format!("Failed to read existing license JSON at {}", dest.display())
+        })?;
+        let existing_json: serde_json::Value =
+            serde_json::from_str(&existing).with_context(|| {
+                format!("Failed to read existing license JSON at {}", dest.display())
+            })?;
+        if existing_json != output {
+            eprintln!("The existing {} file is out of date.", dest.display());
+            eprintln!("Run ./x run collect-license-metadata to update it.");
+            anyhow::bail!("The existing {} file doesn't match what REUSE reports.", dest.display());
+        }
+        println!("license information matches");
+    } else {
+        if let Some(parent) = dest.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        std::fs::write(&dest, &serde_json::to_vec_pretty(&output)?)?;
+        println!("license information written to {}", dest.display());
     }
-    std::fs::write(
-        &dest,
-        &serde_json::to_vec_pretty(&serde_json::json!({
-            "files": crate::path_tree::expand_interned_licenses(tree, &interner),
-        }))?,
-    )?;
 
     Ok(())
 }
diff --git a/src/tools/collect-license-metadata/src/reuse.rs b/src/tools/collect-license-metadata/src/reuse.rs
index e5ee8f0da5e..dbe46781b7c 100644
--- a/src/tools/collect-license-metadata/src/reuse.rs
+++ b/src/tools/collect-license-metadata/src/reuse.rs
@@ -10,10 +10,10 @@ pub(crate) fn collect(
     reuse_exe: &Path,
     interner: &mut LicensesInterner,
 ) -> Result<Vec<(PathBuf, LicenseId)>, Error> {
-    eprintln!("gathering license information from REUSE");
+    println!("gathering license information from REUSE (this might take a minute...)");
     let start = Instant::now();
     let raw = &obtain_spdx_document(reuse_exe)?;
-    eprintln!("finished gathering the license information from REUSE in {:.2?}", start.elapsed());
+    println!("finished gathering the license information from REUSE in {:.2?}", start.elapsed());
 
     let document = spdx_rs::parsers::spdx_from_tag_value(&raw)?;
 
diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs
index f9d96b59462..f83d16d0cab 100644
--- a/src/tools/generate-copyright/src/main.rs
+++ b/src/tools/generate-copyright/src/main.rs
@@ -57,6 +57,10 @@ fn main() -> Result<(), Error> {
         dependencies: collected_cargo_metadata,
     };
     let output = template.render()?;
+    // Git stores text files with \n, but this file may contain \r\n in files
+    // copied from dependencies. Normalise them before we write them out, for
+    // consistency.
+    let output = output.replace("\r\n", "\n");
     std::fs::write(&dest_file, output)?;
 
     // Output libstd subset file
@@ -65,6 +69,10 @@ fn main() -> Result<(), Error> {
         dependencies: library_collected_cargo_metadata,
     };
     let output = template.render()?;
+    // Git stores text files with \n, but this file may contain \r\n in files
+    // copied from dependencies. Normalise them before we write them out, for
+    // consistency.
+    let output = output.replace("\r\n", "\n");
     std::fs::write(&libstd_dest_file, output)?;
 
     Ok(())