use std::collections::{HashMap, HashSet}; use std::fs::{self}; use std::io::{self, ErrorKind}; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; use serde::Deserialize; use walkdir::{DirEntry, WalkDir}; use crate::{Crate, LINTCHECK_DOWNLOADS, LINTCHECK_SOURCES}; const DEFAULT_DOCS_LINK: &str = "https://docs.rs/{krate}/{version}/src/{krate_}/{file}.html#{line}"; const DEFAULT_GITHUB_LINK: &str = "{url}/blob/{hash}/src/{file}#L{line}"; const DEFAULT_PATH_LINK: &str = "{path}/src/{file}:{line}"; /// List of sources to check, loaded from a .toml file #[derive(Debug, Deserialize)] pub struct SourceList { crates: HashMap, #[serde(default)] recursive: RecursiveOptions, } #[derive(Debug, Deserialize, Default)] pub struct RecursiveOptions { pub ignore: HashSet, } /// A crate source stored inside the .toml /// will be translated into on one of the `CrateSource` variants #[derive(Debug, Deserialize)] struct TomlCrate { name: String, version: Option, git_url: Option, git_hash: Option, path: Option, options: Option>, /// Magic values: /// * `{krate}` will be replaced by `self.name` /// * `{krate_}` will be replaced by `self.name` with all `-` replaced by `_` /// * `{version}` will be replaced by `self.version` /// * `{url}` will be replaced with `self.git_url` /// * `{hash}` will be replaced with `self.git_hash` /// * `{path}` will be replaced with `self.path` /// * `{file}` will be replaced by the path after `src/` /// * `{line}` will be replaced by the line /// /// If unset, this will be filled by [`read_crates`] since it depends on /// the source. online_link: Option, } impl TomlCrate { fn file_link(&self, default: &str) -> String { let mut link = self.online_link.clone().unwrap_or_else(|| default.to_string()); link = link.replace("{krate}", &self.name); link = link.replace("{krate_}", &self.name.replace('-', "_")); if let Some(version) = &self.version { link = link.replace("{version}", version); } if let Some(url) = &self.git_url { link = link.replace("{url}", url); } if let Some(hash) = &self.git_hash { link = link.replace("{hash}", hash); } if let Some(path) = &self.path { link = link.replace("{path}", path); } link } } /// Represents an archive we download from crates.io, or a git repo, or a local repo/folder /// Once processed (downloaded/extracted/cloned/copied...), this will be translated into a `Crate` #[derive(Debug, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)] pub struct CrateWithSource { pub name: String, pub source: CrateSource, pub file_link: String, pub options: Option>, } #[derive(Debug, Deserialize, Eq, Hash, PartialEq, Ord, PartialOrd)] pub enum CrateSource { CratesIo { version: String }, Git { url: String, commit: String }, Path { path: PathBuf }, } /// Read a `lintcheck_crates.toml` file pub fn read_crates(toml_path: &Path) -> (Vec, RecursiveOptions) { let toml_content: String = fs::read_to_string(toml_path).unwrap_or_else(|_| panic!("Failed to read {}", toml_path.display())); let crate_list: SourceList = toml::from_str(&toml_content).unwrap_or_else(|e| panic!("Failed to parse {}: \n{e}", toml_path.display())); // parse the hashmap of the toml file into a list of crates let toml_crates: Vec = crate_list.crates.into_values().collect(); // flatten TomlCrates into CrateSources (one TomlCrates may represent several versions of a crate => // multiple CrateSources) let mut crate_sources = Vec::new(); for tk in toml_crates { if let Some(ref path) = tk.path { crate_sources.push(CrateWithSource { name: tk.name.clone(), source: CrateSource::Path { path: PathBuf::from(path), }, file_link: tk.file_link(DEFAULT_PATH_LINK), options: tk.options.clone(), }); } else if let Some(ref version) = tk.version { crate_sources.push(CrateWithSource { name: tk.name.clone(), source: CrateSource::CratesIo { version: version.to_string(), }, file_link: tk.file_link(DEFAULT_DOCS_LINK), options: tk.options.clone(), }); } else if tk.git_url.is_some() && tk.git_hash.is_some() { // otherwise, we should have a git source crate_sources.push(CrateWithSource { name: tk.name.clone(), source: CrateSource::Git { url: tk.git_url.clone().unwrap(), commit: tk.git_hash.clone().unwrap(), }, file_link: tk.file_link(DEFAULT_GITHUB_LINK), options: tk.options.clone(), }); } else { panic!("Invalid crate source: {tk:?}"); } // if we have a version as well as a git data OR only one git data, something is funky if tk.version.is_some() && (tk.git_url.is_some() || tk.git_hash.is_some()) || tk.git_hash.is_some() != tk.git_url.is_some() { eprintln!("tomlkrate: {tk:?}"); assert_eq!( tk.git_hash.is_some(), tk.git_url.is_some(), "Error: Encountered TomlCrate with only one of git_hash and git_url!" ); assert!( tk.path.is_none() || (tk.git_hash.is_none() && tk.version.is_none()), "Error: TomlCrate can only have one of 'git_.*', 'version' or 'path' fields" ); unreachable!("Failed to translate TomlCrate into CrateSource!"); } } // sort the crates crate_sources.sort(); (crate_sources, crate_list.recursive) } impl CrateWithSource { pub fn download_and_prepare(&self) -> Crate { let krate = self.download_and_extract(); // Downloaded crates might contain a `rust-toolchain` file. This file // seems to be accessed when `build.rs` files are present. This access // results in build errors since lintcheck and clippy will most certainly // use a different toolchain. // Lintcheck simply removes these files and assumes that our toolchain // is more up to date. let _ = fs::remove_file(krate.path.join("rust-toolchain")); let _ = fs::remove_file(krate.path.join("rust-toolchain.toml")); krate } /// Makes the sources available on the disk for clippy to check. /// Clones a git repo and checks out the specified commit or downloads a crate from crates.io or /// copies a local folder #[expect(clippy::too_many_lines)] fn download_and_extract(&self) -> Crate { #[allow(clippy::result_large_err)] fn get(path: &str) -> Result { const MAX_RETRIES: u8 = 4; let mut retries = 0; loop { match ureq::get(path).call() { Ok(res) => return Ok(res), Err(e) if retries >= MAX_RETRIES => return Err(e), Err(ureq::Error::Transport(e)) => eprintln!("Error: {e}"), Err(e) => return Err(e), } eprintln!("retrying in {retries} seconds..."); std::thread::sleep(Duration::from_secs(u64::from(retries))); retries += 1; } } let name = &self.name; let options = &self.options; let file_link = &self.file_link; match &self.source { CrateSource::CratesIo { version } => { let extract_dir = PathBuf::from(LINTCHECK_SOURCES); let krate_download_dir = PathBuf::from(LINTCHECK_DOWNLOADS); // url to download the crate from crates.io let url = format!("https://crates.io/api/v1/crates/{name}/{version}/download"); println!("Downloading and extracting {name} {version} from {url}"); create_dirs(&krate_download_dir, &extract_dir); let krate_file_path = krate_download_dir.join(format!("{name}-{version}.crate.tar.gz")); // don't download/extract if we already have done so if !krate_file_path.is_file() { // create a file path to download and write the crate data into let mut krate_dest = fs::File::create(&krate_file_path).unwrap(); let mut krate_req = get(&url).unwrap().into_reader(); // copy the crate into the file io::copy(&mut krate_req, &mut krate_dest).unwrap(); // unzip the tarball let ungz_tar = flate2::read::GzDecoder::new(fs::File::open(&krate_file_path).unwrap()); // extract the tar archive let mut archive = tar::Archive::new(ungz_tar); archive.unpack(&extract_dir).expect("Failed to extract!"); } // crate is extracted, return a new Krate object which contains the path to the extracted // sources that clippy can check Crate { version: version.clone(), name: name.clone(), path: extract_dir.join(format!("{name}-{version}/")), options: options.clone(), base_url: file_link.clone(), } }, CrateSource::Git { url, commit } => { let repo_path = { let mut repo_path = PathBuf::from(LINTCHECK_SOURCES); // add a -git suffix in case we have the same crate from crates.io and a git repo repo_path.push(format!("{name}-git")); repo_path }; // clone the repo if we have not done so if !repo_path.is_dir() { println!("Cloning {url} and checking out {commit}"); if !Command::new("git") .arg("clone") .arg(url) .arg(&repo_path) .status() .expect("Failed to clone git repo!") .success() { eprintln!("Failed to clone {url} into {}", repo_path.display()); } } // check out the commit/branch/whatever if !Command::new("git") .args(["-c", "advice.detachedHead=false"]) .arg("checkout") .arg(commit) .current_dir(&repo_path) .status() .expect("Failed to check out commit") .success() { eprintln!("Failed to checkout {commit} of repo at {}", repo_path.display()); } Crate { version: commit.clone(), name: name.clone(), path: repo_path, options: options.clone(), base_url: file_link.clone(), } }, CrateSource::Path { path } => { fn is_cache_dir(entry: &DirEntry) -> bool { fs::read(entry.path().join("CACHEDIR.TAG")) .map(|x| x.starts_with(b"Signature: 8a477f597d28d172789f06886806bc55")) .unwrap_or(false) } // copy path into the dest_crate_root but skip directories that contain a CACHEDIR.TAG file. // The target/ directory contains a CACHEDIR.TAG file so it is the most commonly skipped directory // as a result of this filter. let dest_crate_root = PathBuf::from(LINTCHECK_SOURCES).join(name); if dest_crate_root.exists() { println!("Deleting existing directory at {dest_crate_root:?}"); fs::remove_dir_all(&dest_crate_root).unwrap(); } println!("Copying {path:?} to {dest_crate_root:?}"); for entry in WalkDir::new(path).into_iter().filter_entry(|e| !is_cache_dir(e)) { let entry = entry.unwrap(); let entry_path = entry.path(); let relative_entry_path = entry_path.strip_prefix(path).unwrap(); let dest_path = dest_crate_root.join(relative_entry_path); let metadata = entry_path.symlink_metadata().unwrap(); if metadata.is_dir() { fs::create_dir(dest_path).unwrap(); } else if metadata.is_file() { fs::copy(entry_path, dest_path).unwrap(); } } Crate { version: String::from("local"), name: name.clone(), path: dest_crate_root, options: options.clone(), base_url: file_link.clone(), } }, } } } /// Create necessary directories to run the lintcheck tool. /// /// # Panics /// /// This function panics if creating one of the dirs fails. fn create_dirs(krate_download_dir: &Path, extract_dir: &Path) { fs::create_dir("target/lintcheck/").unwrap_or_else(|err| { assert_eq!( err.kind(), ErrorKind::AlreadyExists, "cannot create lintcheck target dir" ); }); fs::create_dir(krate_download_dir).unwrap_or_else(|err| { assert_eq!(err.kind(), ErrorKind::AlreadyExists, "cannot create crate download dir"); }); fs::create_dir(extract_dir).unwrap_or_else(|err| { assert_eq!( err.kind(), ErrorKind::AlreadyExists, "cannot create crate extraction dir" ); }); }