diff options
Diffstat (limited to 'src/tools/rust-analyzer/xtask')
18 files changed, 4142 insertions, 0 deletions
diff --git a/src/tools/rust-analyzer/xtask/Cargo.toml b/src/tools/rust-analyzer/xtask/Cargo.toml new file mode 100644 index 00000000000..a83d32e4141 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "xtask" +version = "0.1.0" +publish = false +license = "MIT OR Apache-2.0" +edition = "2021" +rust-version.workspace = true + +[dependencies] +anyhow.workspace = true +flate2 = "1.0.24" +write-json = "0.1.2" +xshell.workspace = true +xflags = "0.3.0" +time = { version = "0.3", default-features = false } +zip = { version = "0.6", default-features = false, features = ["deflate", "time"] } +stdx.workspace = true +proc-macro2 = "1.0.47" +quote = "1.0.20" +ungrammar = "1.16.1" +itertools.workspace = true +# Avoid adding more dependencies to this crate + +[lints] +workspace = true diff --git a/src/tools/rust-analyzer/xtask/src/codegen.rs b/src/tools/rust-analyzer/xtask/src/codegen.rs new file mode 100644 index 00000000000..7dc1b40783c --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/codegen.rs @@ -0,0 +1,220 @@ +use std::{ + fmt, fs, mem, + path::{Path, PathBuf}, +}; + +use xshell::{cmd, Shell}; + +use crate::{flags, project_root}; + +pub(crate) mod assists_doc_tests; +pub(crate) mod diagnostics_docs; +mod grammar; +mod lints; + +impl flags::Codegen { + pub(crate) fn run(self, _sh: &Shell) -> anyhow::Result<()> { + match self.codegen_type.unwrap_or_default() { + flags::CodegenType::All => { + diagnostics_docs::generate(self.check); + assists_doc_tests::generate(self.check); + // lints::generate(self.check) Updating clones the rust repo, so don't run it unless + // explicitly asked for + } + flags::CodegenType::Grammar => grammar::generate(self.check), + flags::CodegenType::AssistsDocTests => assists_doc_tests::generate(self.check), + flags::CodegenType::DiagnosticsDocs => diagnostics_docs::generate(self.check), + flags::CodegenType::LintDefinitions => lints::generate(self.check), + } + Ok(()) + } +} + +fn list_rust_files(dir: &Path) -> Vec<PathBuf> { + let mut res = list_files(dir); + res.retain(|it| { + it.file_name().unwrap_or_default().to_str().unwrap_or_default().ends_with(".rs") + }); + res +} + +fn list_files(dir: &Path) -> Vec<PathBuf> { + let mut res = Vec::new(); + let mut work = vec![dir.to_path_buf()]; + while let Some(dir) = work.pop() { + for entry in dir.read_dir().unwrap() { + let entry = entry.unwrap(); + let file_type = entry.file_type().unwrap(); + let path = entry.path(); + let is_hidden = + path.file_name().unwrap_or_default().to_str().unwrap_or_default().starts_with('.'); + if !is_hidden { + if file_type.is_dir() { + work.push(path); + } else if file_type.is_file() { + res.push(path); + } + } + } + } + res +} + +#[derive(Clone)] +pub(crate) struct CommentBlock { + pub(crate) id: String, + pub(crate) line: usize, + pub(crate) contents: Vec<String>, + is_doc: bool, +} + +impl CommentBlock { + fn extract(tag: &str, text: &str) -> Vec<CommentBlock> { + assert!(tag.starts_with(char::is_uppercase)); + + let tag = format!("{tag}:"); + let mut blocks = CommentBlock::extract_untagged(text); + blocks.retain_mut(|block| { + let first = block.contents.remove(0); + let Some(id) = first.strip_prefix(&tag) else { + return false; + }; + + if block.is_doc { + panic!("Use plain (non-doc) comments with tags like {tag}:\n {first}"); + } + + block.id = id.trim().to_owned(); + true + }); + blocks + } + + fn extract_untagged(text: &str) -> Vec<CommentBlock> { + let mut res = Vec::new(); + + let lines = text.lines().map(str::trim_start); + + let dummy_block = + CommentBlock { id: String::new(), line: 0, contents: Vec::new(), is_doc: false }; + let mut block = dummy_block.clone(); + for (line_num, line) in lines.enumerate() { + match line.strip_prefix("//") { + Some(mut contents) => { + if let Some('/' | '!') = contents.chars().next() { + contents = &contents[1..]; + block.is_doc = true; + } + if let Some(' ') = contents.chars().next() { + contents = &contents[1..]; + } + block.contents.push(contents.to_owned()); + } + None => { + if !block.contents.is_empty() { + let block = mem::replace(&mut block, dummy_block.clone()); + res.push(block); + } + block.line = line_num + 2; + } + } + } + if !block.contents.is_empty() { + res.push(block); + } + res + } +} + +#[derive(Debug)] +pub(crate) struct Location { + pub(crate) file: PathBuf, + pub(crate) line: usize, +} + +impl fmt::Display for Location { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let path = self.file.strip_prefix(project_root()).unwrap().display().to_string(); + let path = path.replace('\\', "/"); + let name = self.file.file_name().unwrap(); + write!( + f, + "https://github.com/rust-lang/rust-analyzer/blob/master/{}#L{}[{}]", + path, + self.line, + name.to_str().unwrap() + ) + } +} + +fn ensure_rustfmt(sh: &Shell) { + let version = cmd!(sh, "rustup run stable rustfmt --version").read().unwrap_or_default(); + if !version.contains("stable") { + panic!( + "Failed to run rustfmt from toolchain 'stable'. \ + Please run `rustup component add rustfmt --toolchain stable` to install it.", + ); + } +} + +fn reformat(text: String) -> String { + let sh = Shell::new().unwrap(); + ensure_rustfmt(&sh); + let rustfmt_toml = project_root().join("rustfmt.toml"); + let mut stdout = cmd!( + sh, + "rustup run stable rustfmt --config-path {rustfmt_toml} --config fn_single_line=true" + ) + .stdin(text) + .read() + .unwrap(); + if !stdout.ends_with('\n') { + stdout.push('\n'); + } + stdout +} + +fn add_preamble(generator: &'static str, mut text: String) -> String { + let preamble = format!("//! Generated by `{generator}`, do not edit by hand.\n\n"); + text.insert_str(0, &preamble); + text +} + +/// Checks that the `file` has the specified `contents`. If that is not the +/// case, updates the file and then fails the test. +#[allow(clippy::print_stderr)] +fn ensure_file_contents(file: &Path, contents: &str, check: bool) { + if let Ok(old_contents) = fs::read_to_string(file) { + if normalize_newlines(&old_contents) == normalize_newlines(contents) { + // File is already up to date. + return; + } + } + + let display_path = file.strip_prefix(project_root()).unwrap_or(file); + if check { + panic!( + "{} was not up-to-date{}", + file.display(), + if std::env::var("CI").is_ok() { + "\n NOTE: run `cargo codegen` locally and commit the updated files\n" + } else { + "" + } + ); + } else { + eprintln!( + "\n\x1b[31;1merror\x1b[0m: {} was not up-to-date, updating\n", + display_path.display() + ); + + if let Some(parent) = file.parent() { + let _ = fs::create_dir_all(parent); + } + fs::write(file, contents).unwrap(); + } +} + +fn normalize_newlines(s: &str) -> String { + s.replace("\r\n", "\n") +} diff --git a/src/tools/rust-analyzer/xtask/src/codegen/assists_doc_tests.rs b/src/tools/rust-analyzer/xtask/src/codegen/assists_doc_tests.rs new file mode 100644 index 00000000000..b2d89dde765 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/codegen/assists_doc_tests.rs @@ -0,0 +1,197 @@ +//! Generates `assists.md` documentation. + +use std::{fmt, fs, path::Path}; + +use stdx::format_to_acc; + +use crate::{ + codegen::{ + add_preamble, ensure_file_contents, list_rust_files, reformat, CommentBlock, Location, + }, + project_root, +}; + +pub(crate) fn generate(check: bool) { + let assists = Assist::collect(); + + { + // Generate doctests. + + let mut buf = " +use super::check_doc_test; +" + .to_owned(); + for assist in assists.iter() { + for (idx, section) in assist.sections.iter().enumerate() { + let test_id = + if idx == 0 { assist.id.clone() } else { format!("{}_{idx}", &assist.id) }; + let test = format!( + r######" +#[test] +fn doctest_{}() {{ + check_doc_test( + "{}", +r#####" +{}"#####, r#####" +{}"#####) +}} +"######, + &test_id, + &assist.id, + reveal_hash_comments(§ion.before), + reveal_hash_comments(§ion.after) + ); + + buf.push_str(&test) + } + } + let buf = add_preamble("sourcegen_assists_docs", reformat(buf)); + ensure_file_contents( + &project_root().join("crates/ide-assists/src/tests/generated.rs"), + &buf, + check, + ); + } + + { + // Generate assists manual. Note that we do _not_ commit manual to the + // git repo. Instead, `cargo xtask release` runs this test before making + // a release. + + let contents = add_preamble( + "sourcegen_assists_docs", + assists.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n"), + ); + let dst = project_root().join("docs/user/generated_assists.adoc"); + fs::write(dst, contents).unwrap(); + } +} + +#[derive(Debug)] +struct Section { + doc: String, + before: String, + after: String, +} + +#[derive(Debug)] +struct Assist { + id: String, + location: Location, + sections: Vec<Section>, +} + +impl Assist { + fn collect() -> Vec<Assist> { + let handlers_dir = project_root().join("crates/ide-assists/src/handlers"); + + let mut res = Vec::new(); + for path in list_rust_files(&handlers_dir) { + collect_file(&mut res, path.as_path()); + } + res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id)); + return res; + + fn collect_file(acc: &mut Vec<Assist>, path: &Path) { + let text = fs::read_to_string(path).unwrap(); + let comment_blocks = CommentBlock::extract("Assist", &text); + + for block in comment_blocks { + let id = block.id; + assert!( + id.chars().all(|it| it.is_ascii_lowercase() || it == '_'), + "invalid assist id: {id:?}" + ); + let mut lines = block.contents.iter().peekable(); + let location = Location { file: path.to_path_buf(), line: block.line }; + let mut assist = Assist { id, location, sections: Vec::new() }; + + while lines.peek().is_some() { + let doc = take_until(lines.by_ref(), "```").trim().to_owned(); + assert!( + (doc.chars().next().unwrap().is_ascii_uppercase() && doc.ends_with('.')) + || !assist.sections.is_empty(), + "\n\n{}: assist docs should be proper sentences, with capitalization and a full stop at the end.\n\n{}\n\n", + &assist.id, + doc, + ); + + let before = take_until(lines.by_ref(), "```"); + + assert_eq!(lines.next().unwrap().as_str(), "->"); + assert_eq!(lines.next().unwrap().as_str(), "```"); + let after = take_until(lines.by_ref(), "```"); + + assist.sections.push(Section { doc, before, after }); + } + + acc.push(assist) + } + } + + fn take_until<'a>(lines: impl Iterator<Item = &'a String>, marker: &str) -> String { + let mut buf = Vec::new(); + for line in lines { + if line == marker { + break; + } + buf.push(line.clone()); + } + buf.join("\n") + } + } +} + +impl fmt::Display for Assist { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let _ = writeln!( + f, + "[discrete]\n=== `{}` +**Source:** {}", + self.id, self.location, + ); + + for section in &self.sections { + let before = section.before.replace("$0", "┃"); // Unicode pseudo-graphics bar + let after = section.after.replace("$0", "┃"); + let _ = writeln!( + f, + " +{} + +.Before +```rust +{}``` + +.After +```rust +{}```", + section.doc, + hide_hash_comments(&before), + hide_hash_comments(&after) + ); + } + + Ok(()) + } +} + +fn hide_hash_comments(text: &str) -> String { + text.split('\n') // want final newline + .filter(|&it| !(it.starts_with("# ") || it == "#")) + .fold(String::new(), |mut acc, it| format_to_acc!(acc, "{it}\n")) +} + +fn reveal_hash_comments(text: &str) -> String { + text.split('\n') // want final newline + .map(|it| { + if let Some(stripped) = it.strip_prefix("# ") { + stripped + } else if it == "#" { + "" + } else { + it + } + }) + .fold(String::new(), |mut acc, it| format_to_acc!(acc, "{it}\n")) +} diff --git a/src/tools/rust-analyzer/xtask/src/codegen/diagnostics_docs.rs b/src/tools/rust-analyzer/xtask/src/codegen/diagnostics_docs.rs new file mode 100644 index 00000000000..cf30531e7f9 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/codegen/diagnostics_docs.rs @@ -0,0 +1,77 @@ +//! Generates `assists.md` documentation. + +use std::{fmt, fs, io, path::PathBuf}; + +use crate::{ + codegen::{add_preamble, list_rust_files, CommentBlock, Location}, + project_root, +}; + +pub(crate) fn generate(check: bool) { + let diagnostics = Diagnostic::collect().unwrap(); + if !check { + let contents = + diagnostics.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n"); + let contents = add_preamble("sourcegen_diagnostic_docs", contents); + let dst = project_root().join("docs/user/generated_diagnostic.adoc"); + fs::write(dst, contents).unwrap(); + } +} + +#[derive(Debug)] +struct Diagnostic { + id: String, + location: Location, + doc: String, +} + +impl Diagnostic { + fn collect() -> io::Result<Vec<Diagnostic>> { + let handlers_dir = project_root().join("crates/ide-diagnostics/src/handlers"); + + let mut res = Vec::new(); + for path in list_rust_files(&handlers_dir) { + collect_file(&mut res, path)?; + } + res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id)); + return Ok(res); + + fn collect_file(acc: &mut Vec<Diagnostic>, path: PathBuf) -> io::Result<()> { + let text = fs::read_to_string(&path)?; + let comment_blocks = CommentBlock::extract("Diagnostic", &text); + + for block in comment_blocks { + let id = block.id; + if let Err(msg) = is_valid_diagnostic_name(&id) { + panic!("invalid diagnostic name: {id:?}:\n {msg}") + } + let doc = block.contents.join("\n"); + let location = Location { file: path.clone(), line: block.line }; + acc.push(Diagnostic { id, location, doc }) + } + + Ok(()) + } + } +} + +fn is_valid_diagnostic_name(diagnostic: &str) -> Result<(), String> { + let diagnostic = diagnostic.trim(); + if diagnostic.find(char::is_whitespace).is_some() { + return Err("Diagnostic names can't contain whitespace symbols".into()); + } + if diagnostic.chars().any(|c| c.is_ascii_uppercase()) { + return Err("Diagnostic names can't contain uppercase symbols".into()); + } + if diagnostic.chars().any(|c| !c.is_ascii()) { + return Err("Diagnostic can't contain non-ASCII symbols".into()); + } + + Ok(()) +} + +impl fmt::Display for Diagnostic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "=== {}\n**Source:** {}\n{}", self.id, self.location, self.doc) + } +} diff --git a/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs b/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs new file mode 100644 index 00000000000..cc2fadc9750 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs @@ -0,0 +1,875 @@ +//! This module generates AST datatype used by rust-analyzer. +//! +//! Specifically, it generates the `SyntaxKind` enum and a number of newtype +//! wrappers around `SyntaxNode` which implement `syntax::AstNode`. + +#![allow(clippy::disallowed_types)] + +use std::{ + collections::{BTreeSet, HashSet}, + fmt::Write, + fs, +}; + +use itertools::Itertools; +use proc_macro2::{Punct, Spacing}; +use quote::{format_ident, quote}; +use ungrammar::{Grammar, Rule}; + +use crate::{ + codegen::{add_preamble, ensure_file_contents, reformat}, + project_root, +}; + +mod ast_src; +use self::ast_src::{AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC}; + +pub(crate) fn generate(check: bool) { + let syntax_kinds = generate_syntax_kinds(KINDS_SRC); + let syntax_kinds_file = project_root().join("crates/parser/src/syntax_kind/generated.rs"); + ensure_file_contents(syntax_kinds_file.as_path(), &syntax_kinds, check); + + let grammar = fs::read_to_string(project_root().join("crates/syntax/rust.ungram")) + .unwrap() + .parse() + .unwrap(); + let ast = lower(&grammar); + + let ast_tokens = generate_tokens(&ast); + let ast_tokens_file = project_root().join("crates/syntax/src/ast/generated/tokens.rs"); + ensure_file_contents(ast_tokens_file.as_path(), &ast_tokens, check); + + let ast_nodes = generate_nodes(KINDS_SRC, &ast); + let ast_nodes_file = project_root().join("crates/syntax/src/ast/generated/nodes.rs"); + ensure_file_contents(ast_nodes_file.as_path(), &ast_nodes, check); +} + +fn generate_tokens(grammar: &AstSrc) -> String { + let tokens = grammar.tokens.iter().map(|token| { + let name = format_ident!("{}", token); + let kind = format_ident!("{}", to_upper_snake_case(token)); + quote! { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct #name { + pub(crate) syntax: SyntaxToken, + } + impl std::fmt::Display for #name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.syntax, f) + } + } + impl AstToken for #name { + fn can_cast(kind: SyntaxKind) -> bool { kind == #kind } + fn cast(syntax: SyntaxToken) -> Option<Self> { + if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } + } + fn syntax(&self) -> &SyntaxToken { &self.syntax } + } + } + }); + + add_preamble( + "sourcegen_ast", + reformat( + quote! { + use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken}; + #(#tokens)* + } + .to_string(), + ), + ) + .replace("#[derive", "\n#[derive") +} + +fn generate_nodes(kinds: KindsSrc<'_>, grammar: &AstSrc) -> String { + let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar + .nodes + .iter() + .map(|node| { + let name = format_ident!("{}", node.name); + let kind = format_ident!("{}", to_upper_snake_case(&node.name)); + let traits = node + .traits + .iter() + .filter(|trait_name| { + // Loops have two expressions so this might collide, therefore manual impl it + node.name != "ForExpr" && node.name != "WhileExpr" + || trait_name.as_str() != "HasLoopBody" + }) + .map(|trait_name| { + let trait_name = format_ident!("{}", trait_name); + quote!(impl ast::#trait_name for #name {}) + }); + + let methods = node.fields.iter().map(|field| { + let method_name = field.method_name(); + let ty = field.ty(); + + if field.is_many() { + quote! { + pub fn #method_name(&self) -> AstChildren<#ty> { + support::children(&self.syntax) + } + } + } else if let Some(token_kind) = field.token_kind() { + quote! { + pub fn #method_name(&self) -> Option<#ty> { + support::token(&self.syntax, #token_kind) + } + } + } else { + quote! { + pub fn #method_name(&self) -> Option<#ty> { + support::child(&self.syntax) + } + } + } + }); + ( + quote! { + #[pretty_doc_comment_placeholder_workaround] + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct #name { + pub(crate) syntax: SyntaxNode, + } + + #(#traits)* + + impl #name { + #(#methods)* + } + }, + quote! { + impl AstNode for #name { + fn can_cast(kind: SyntaxKind) -> bool { + kind == #kind + } + fn cast(syntax: SyntaxNode) -> Option<Self> { + if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } + } + fn syntax(&self) -> &SyntaxNode { &self.syntax } + } + }, + ) + }) + .unzip(); + + let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar + .enums + .iter() + .map(|en| { + let variants: Vec<_> = + en.variants.iter().map(|var| format_ident!("{}", var)).sorted().collect(); + let name = format_ident!("{}", en.name); + let kinds: Vec<_> = variants + .iter() + .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string()))) + .collect(); + let traits = en.traits.iter().sorted().map(|trait_name| { + let trait_name = format_ident!("{}", trait_name); + quote!(impl ast::#trait_name for #name {}) + }); + + let ast_node = if en.name == "Stmt" { + quote! {} + } else { + quote! { + impl AstNode for #name { + fn can_cast(kind: SyntaxKind) -> bool { + matches!(kind, #(#kinds)|*) + } + fn cast(syntax: SyntaxNode) -> Option<Self> { + let res = match syntax.kind() { + #( + #kinds => #name::#variants(#variants { syntax }), + )* + _ => return None, + }; + Some(res) + } + fn syntax(&self) -> &SyntaxNode { + match self { + #( + #name::#variants(it) => &it.syntax, + )* + } + } + } + } + }; + + ( + quote! { + #[pretty_doc_comment_placeholder_workaround] + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub enum #name { + #(#variants(#variants),)* + } + + #(#traits)* + }, + quote! { + #( + impl From<#variants> for #name { + fn from(node: #variants) -> #name { + #name::#variants(node) + } + } + )* + #ast_node + }, + ) + }) + .unzip(); + let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar + .nodes + .iter() + .flat_map(|node| node.traits.iter().map(move |t| (t, node))) + .into_group_map() + .into_iter() + .sorted_by_key(|(name, _)| *name) + .map(|(trait_name, nodes)| { + let name = format_ident!("Any{}", trait_name); + let trait_name = format_ident!("{}", trait_name); + let kinds: Vec<_> = nodes + .iter() + .map(|name| format_ident!("{}", to_upper_snake_case(&name.name.to_string()))) + .collect(); + + ( + quote! { + #[pretty_doc_comment_placeholder_workaround] + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct #name { + pub(crate) syntax: SyntaxNode, + } + impl ast::#trait_name for #name {} + }, + quote! { + impl #name { + #[inline] + pub fn new<T: ast::#trait_name>(node: T) -> #name { + #name { + syntax: node.syntax().clone() + } + } + } + impl AstNode for #name { + fn can_cast(kind: SyntaxKind) -> bool { + matches!(kind, #(#kinds)|*) + } + fn cast(syntax: SyntaxNode) -> Option<Self> { + Self::can_cast(syntax.kind()).then_some(#name { syntax }) + } + fn syntax(&self) -> &SyntaxNode { + &self.syntax + } + } + }, + ) + }) + .unzip(); + + let enum_names = grammar.enums.iter().map(|it| &it.name); + let node_names = grammar.nodes.iter().map(|it| &it.name); + + let display_impls = + enum_names.chain(node_names.clone()).map(|it| format_ident!("{}", it)).map(|name| { + quote! { + impl std::fmt::Display for #name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self.syntax(), f) + } + } + } + }); + + let defined_nodes: HashSet<_> = node_names.collect(); + + for node in kinds + .nodes + .iter() + .map(|kind| to_pascal_case(kind)) + .filter(|name| !defined_nodes.iter().any(|&it| it == name)) + { + drop(node) + // FIXME: restore this + // eprintln!("Warning: node {} not defined in ast source", node); + } + + let ast = quote! { + #![allow(non_snake_case)] + use crate::{ + SyntaxNode, SyntaxToken, SyntaxKind::{self, *}, + ast::{self, AstNode, AstChildren, support}, + T, + }; + + #(#node_defs)* + #(#enum_defs)* + #(#any_node_defs)* + #(#node_boilerplate_impls)* + #(#enum_boilerplate_impls)* + #(#any_node_boilerplate_impls)* + #(#display_impls)* + }; + + let ast = ast.to_string().replace("T ! [", "T!["); + + let mut res = String::with_capacity(ast.len() * 2); + + let mut docs = + grammar.nodes.iter().map(|it| &it.doc).chain(grammar.enums.iter().map(|it| &it.doc)); + + for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") { + res.push_str(chunk); + if let Some(doc) = docs.next() { + write_doc_comment(doc, &mut res); + } + } + + let res = add_preamble("sourcegen_ast", reformat(res)); + res.replace("#[derive", "\n#[derive") +} + +fn write_doc_comment(contents: &[String], dest: &mut String) { + for line in contents { + writeln!(dest, "///{line}").unwrap(); + } +} + +fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { + let (single_byte_tokens_values, single_byte_tokens): (Vec<_>, Vec<_>) = grammar + .punct + .iter() + .filter(|(token, _name)| token.len() == 1) + .map(|(token, name)| (token.chars().next().unwrap(), format_ident!("{}", name))) + .unzip(); + + let punctuation_values = grammar.punct.iter().map(|(token, _name)| { + if "{}[]()".contains(token) { + let c = token.chars().next().unwrap(); + quote! { #c } + } else { + let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint)); + quote! { #(#cs)* } + } + }); + let punctuation = + grammar.punct.iter().map(|(_token, name)| format_ident!("{}", name)).collect::<Vec<_>>(); + + let x = |&name| match name { + "Self" => format_ident!("SELF_TYPE_KW"), + name => format_ident!("{}_KW", to_upper_snake_case(name)), + }; + let full_keywords_values = grammar.keywords; + let full_keywords = full_keywords_values.iter().map(x); + + let contextual_keywords_values = &grammar.contextual_keywords; + let contextual_keywords = contextual_keywords_values.iter().map(x); + + let all_keywords_values = grammar + .keywords + .iter() + .chain(grammar.contextual_keywords.iter()) + .copied() + .collect::<Vec<_>>(); + let all_keywords_idents = all_keywords_values.iter().map(|kw| format_ident!("{}", kw)); + let all_keywords = all_keywords_values.iter().map(x).collect::<Vec<_>>(); + + let literals = + grammar.literals.iter().map(|name| format_ident!("{}", name)).collect::<Vec<_>>(); + + let tokens = grammar.tokens.iter().map(|name| format_ident!("{}", name)).collect::<Vec<_>>(); + + let nodes = grammar.nodes.iter().map(|name| format_ident!("{}", name)).collect::<Vec<_>>(); + + let ast = quote! { + #![allow(bad_style, missing_docs, unreachable_pub)] + /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`. + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] + #[repr(u16)] + pub enum SyntaxKind { + // Technical SyntaxKinds: they appear temporally during parsing, + // but never end up in the final tree + #[doc(hidden)] + TOMBSTONE, + #[doc(hidden)] + EOF, + #(#punctuation,)* + #(#all_keywords,)* + #(#literals,)* + #(#tokens,)* + #(#nodes,)* + + // Technical kind so that we can cast from u16 safely + #[doc(hidden)] + __LAST, + } + use self::SyntaxKind::*; + + impl SyntaxKind { + pub fn is_keyword(self) -> bool { + matches!(self, #(#all_keywords)|*) + } + + pub fn is_punct(self) -> bool { + + matches!(self, #(#punctuation)|*) + + } + + pub fn is_literal(self) -> bool { + matches!(self, #(#literals)|*) + } + + pub fn from_keyword(ident: &str) -> Option<SyntaxKind> { + let kw = match ident { + #(#full_keywords_values => #full_keywords,)* + _ => return None, + }; + Some(kw) + } + + pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> { + let kw = match ident { + #(#contextual_keywords_values => #contextual_keywords,)* + _ => return None, + }; + Some(kw) + } + + pub fn from_char(c: char) -> Option<SyntaxKind> { + let tok = match c { + #(#single_byte_tokens_values => #single_byte_tokens,)* + _ => return None, + }; + Some(tok) + } + } + + #[macro_export] + macro_rules! T { + #([#punctuation_values] => { $crate::SyntaxKind::#punctuation };)* + #([#all_keywords_idents] => { $crate::SyntaxKind::#all_keywords };)* + [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; + [ident] => { $crate::SyntaxKind::IDENT }; + [shebang] => { $crate::SyntaxKind::SHEBANG }; + } + }; + + add_preamble("sourcegen_ast", reformat(ast.to_string())) +} + +fn to_upper_snake_case(s: &str) -> String { + let mut buf = String::with_capacity(s.len()); + let mut prev = false; + for c in s.chars() { + if c.is_ascii_uppercase() && prev { + buf.push('_') + } + prev = true; + + buf.push(c.to_ascii_uppercase()); + } + buf +} + +fn to_lower_snake_case(s: &str) -> String { + let mut buf = String::with_capacity(s.len()); + let mut prev = false; + for c in s.chars() { + if c.is_ascii_uppercase() && prev { + buf.push('_') + } + prev = true; + + buf.push(c.to_ascii_lowercase()); + } + buf +} + +fn to_pascal_case(s: &str) -> String { + let mut buf = String::with_capacity(s.len()); + let mut prev_is_underscore = true; + for c in s.chars() { + if c == '_' { + prev_is_underscore = true; + } else if prev_is_underscore { + buf.push(c.to_ascii_uppercase()); + prev_is_underscore = false; + } else { + buf.push(c.to_ascii_lowercase()); + } + } + buf +} + +fn pluralize(s: &str) -> String { + format!("{s}s") +} + +impl Field { + fn is_many(&self) -> bool { + matches!(self, Field::Node { cardinality: Cardinality::Many, .. }) + } + fn token_kind(&self) -> Option<proc_macro2::TokenStream> { + match self { + Field::Token(token) => { + let token: proc_macro2::TokenStream = token.parse().unwrap(); + Some(quote! { T![#token] }) + } + _ => None, + } + } + fn method_name(&self) -> proc_macro2::Ident { + match self { + Field::Token(name) => { + let name = match name.as_str() { + ";" => "semicolon", + "->" => "thin_arrow", + "'{'" => "l_curly", + "'}'" => "r_curly", + "'('" => "l_paren", + "')'" => "r_paren", + "'['" => "l_brack", + "']'" => "r_brack", + "<" => "l_angle", + ">" => "r_angle", + "=" => "eq", + "!" => "excl", + "*" => "star", + "&" => "amp", + "-" => "minus", + "_" => "underscore", + "." => "dot", + ".." => "dotdot", + "..." => "dotdotdot", + "..=" => "dotdoteq", + "=>" => "fat_arrow", + "@" => "at", + ":" => "colon", + "::" => "coloncolon", + "#" => "pound", + "?" => "question_mark", + "," => "comma", + "|" => "pipe", + "~" => "tilde", + _ => name, + }; + format_ident!("{}_token", name) + } + Field::Node { name, .. } => { + if name == "type" { + format_ident!("ty") + } else { + format_ident!("{}", name) + } + } + } + } + fn ty(&self) -> proc_macro2::Ident { + match self { + Field::Token(_) => format_ident!("SyntaxToken"), + Field::Node { ty, .. } => format_ident!("{}", ty), + } + } +} + +fn lower(grammar: &Grammar) -> AstSrc { + let mut res = AstSrc { + tokens: + "Whitespace Comment String ByteString CString IntNumber FloatNumber Char Byte Ident" + .split_ascii_whitespace() + .map(|it| it.to_owned()) + .collect::<Vec<_>>(), + ..Default::default() + }; + + let nodes = grammar.iter().collect::<Vec<_>>(); + + for &node in &nodes { + let name = grammar[node].name.clone(); + let rule = &grammar[node].rule; + match lower_enum(grammar, rule) { + Some(variants) => { + let enum_src = AstEnumSrc { doc: Vec::new(), name, traits: Vec::new(), variants }; + res.enums.push(enum_src); + } + None => { + let mut fields = Vec::new(); + lower_rule(&mut fields, grammar, None, rule); + res.nodes.push(AstNodeSrc { doc: Vec::new(), name, traits: Vec::new(), fields }); + } + } + } + + deduplicate_fields(&mut res); + extract_enums(&mut res); + extract_struct_traits(&mut res); + extract_enum_traits(&mut res); + res.nodes.sort_by_key(|it| it.name.clone()); + res.enums.sort_by_key(|it| it.name.clone()); + res.tokens.sort(); + res.nodes.iter_mut().for_each(|it| { + it.traits.sort(); + it.fields.sort_by_key(|it| match it { + Field::Token(name) => (true, name.clone()), + Field::Node { name, .. } => (false, name.clone()), + }); + }); + res.enums.iter_mut().for_each(|it| { + it.traits.sort(); + it.variants.sort(); + }); + res +} + +fn lower_enum(grammar: &Grammar, rule: &Rule) -> Option<Vec<String>> { + let alternatives = match rule { + Rule::Alt(it) => it, + _ => return None, + }; + let mut variants = Vec::new(); + for alternative in alternatives { + match alternative { + Rule::Node(it) => variants.push(grammar[*it].name.clone()), + Rule::Token(it) if grammar[*it].name == ";" => (), + _ => return None, + } + } + Some(variants) +} + +fn lower_rule(acc: &mut Vec<Field>, grammar: &Grammar, label: Option<&String>, rule: &Rule) { + if lower_separated_list(acc, grammar, label, rule) { + return; + } + + match rule { + Rule::Node(node) => { + let ty = grammar[*node].name.clone(); + let name = label.cloned().unwrap_or_else(|| to_lower_snake_case(&ty)); + let field = Field::Node { name, ty, cardinality: Cardinality::Optional }; + acc.push(field); + } + Rule::Token(token) => { + assert!(label.is_none()); + let mut name = grammar[*token].name.clone(); + if name != "int_number" && name != "string" { + if "[]{}()".contains(&name) { + name = format!("'{name}'"); + } + let field = Field::Token(name); + acc.push(field); + } + } + Rule::Rep(inner) => { + if let Rule::Node(node) = &**inner { + let ty = grammar[*node].name.clone(); + let name = label.cloned().unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty))); + let field = Field::Node { name, ty, cardinality: Cardinality::Many }; + acc.push(field); + return; + } + panic!("unhandled rule: {rule:?}") + } + Rule::Labeled { label: l, rule } => { + assert!(label.is_none()); + let manually_implemented = matches!( + l.as_str(), + "lhs" + | "rhs" + | "then_branch" + | "else_branch" + | "start" + | "end" + | "op" + | "index" + | "base" + | "value" + | "trait" + | "self_ty" + | "iterable" + | "condition" + ); + if manually_implemented { + return; + } + lower_rule(acc, grammar, Some(l), rule); + } + Rule::Seq(rules) | Rule::Alt(rules) => { + for rule in rules { + lower_rule(acc, grammar, label, rule) + } + } + Rule::Opt(rule) => lower_rule(acc, grammar, label, rule), + } +} + +// (T (',' T)* ','?) +fn lower_separated_list( + acc: &mut Vec<Field>, + grammar: &Grammar, + label: Option<&String>, + rule: &Rule, +) -> bool { + let rule = match rule { + Rule::Seq(it) => it, + _ => return false, + }; + let (node, repeat, trailing_sep) = match rule.as_slice() { + [Rule::Node(node), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => { + (node, repeat, Some(trailing_sep)) + } + [Rule::Node(node), Rule::Rep(repeat)] => (node, repeat, None), + _ => return false, + }; + let repeat = match &**repeat { + Rule::Seq(it) => it, + _ => return false, + }; + if !matches!( + repeat.as_slice(), + [comma, Rule::Node(n)] + if trailing_sep.map_or(true, |it| comma == &**it) && n == node + ) { + return false; + } + let ty = grammar[*node].name.clone(); + let name = label.cloned().unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty))); + let field = Field::Node { name, ty, cardinality: Cardinality::Many }; + acc.push(field); + true +} + +fn deduplicate_fields(ast: &mut AstSrc) { + for node in &mut ast.nodes { + let mut i = 0; + 'outer: while i < node.fields.len() { + for j in 0..i { + let f1 = &node.fields[i]; + let f2 = &node.fields[j]; + if f1 == f2 { + node.fields.remove(i); + continue 'outer; + } + } + i += 1; + } + } +} + +fn extract_enums(ast: &mut AstSrc) { + for node in &mut ast.nodes { + for enm in &ast.enums { + let mut to_remove = Vec::new(); + for (i, field) in node.fields.iter().enumerate() { + let ty = field.ty().to_string(); + if enm.variants.iter().any(|it| it == &ty) { + to_remove.push(i); + } + } + if to_remove.len() == enm.variants.len() { + node.remove_field(to_remove); + let ty = enm.name.clone(); + let name = to_lower_snake_case(&ty); + node.fields.push(Field::Node { name, ty, cardinality: Cardinality::Optional }); + } + } + } +} + +fn extract_struct_traits(ast: &mut AstSrc) { + let traits: &[(&str, &[&str])] = &[ + ("HasAttrs", &["attrs"]), + ("HasName", &["name"]), + ("HasVisibility", &["visibility"]), + ("HasGenericParams", &["generic_param_list", "where_clause"]), + ("HasTypeBounds", &["type_bound_list", "colon_token"]), + ("HasModuleItem", &["items"]), + ("HasLoopBody", &["label", "loop_body"]), + ("HasArgList", &["arg_list"]), + ]; + + for node in &mut ast.nodes { + for (name, methods) in traits { + extract_struct_trait(node, name, methods); + } + } + + let nodes_with_doc_comments = [ + "SourceFile", + "Fn", + "Struct", + "Union", + "RecordField", + "TupleField", + "Enum", + "Variant", + "Trait", + "TraitAlias", + "Module", + "Static", + "Const", + "TypeAlias", + "Impl", + "ExternBlock", + "ExternCrate", + "MacroCall", + "MacroRules", + "MacroDef", + "Use", + ]; + + for node in &mut ast.nodes { + if nodes_with_doc_comments.contains(&&*node.name) { + node.traits.push("HasDocComments".into()); + } + } +} + +fn extract_struct_trait(node: &mut AstNodeSrc, trait_name: &str, methods: &[&str]) { + let mut to_remove = Vec::new(); + for (i, field) in node.fields.iter().enumerate() { + let method_name = field.method_name().to_string(); + if methods.iter().any(|&it| it == method_name) { + to_remove.push(i); + } + } + if to_remove.len() == methods.len() { + node.traits.push(trait_name.to_owned()); + node.remove_field(to_remove); + } +} + +fn extract_enum_traits(ast: &mut AstSrc) { + for enm in &mut ast.enums { + if enm.name == "Stmt" { + continue; + } + let nodes = &ast.nodes; + let mut variant_traits = enm + .variants + .iter() + .map(|var| nodes.iter().find(|it| &it.name == var).unwrap()) + .map(|node| node.traits.iter().cloned().collect::<BTreeSet<_>>()); + + let mut enum_traits = match variant_traits.next() { + Some(it) => it, + None => continue, + }; + for traits in variant_traits { + enum_traits = enum_traits.intersection(&traits).cloned().collect(); + } + enm.traits = enum_traits.into_iter().collect(); + } +} + +impl AstNodeSrc { + fn remove_field(&mut self, to_remove: Vec<usize>) { + to_remove.into_iter().rev().for_each(|idx| { + self.fields.remove(idx); + }); + } +} diff --git a/src/tools/rust-analyzer/xtask/src/codegen/grammar/ast_src.rs b/src/tools/rust-analyzer/xtask/src/codegen/grammar/ast_src.rs new file mode 100644 index 00000000000..8221c577892 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/codegen/grammar/ast_src.rs @@ -0,0 +1,273 @@ +//! Defines input for code generation process. + +pub(crate) struct KindsSrc<'a> { + pub(crate) punct: &'a [(&'a str, &'a str)], + pub(crate) keywords: &'a [&'a str], + pub(crate) contextual_keywords: &'a [&'a str], + pub(crate) literals: &'a [&'a str], + pub(crate) tokens: &'a [&'a str], + pub(crate) nodes: &'a [&'a str], +} + +pub(crate) const KINDS_SRC: KindsSrc<'_> = KindsSrc { + punct: &[ + (";", "SEMICOLON"), + (",", "COMMA"), + ("(", "L_PAREN"), + (")", "R_PAREN"), + ("{", "L_CURLY"), + ("}", "R_CURLY"), + ("[", "L_BRACK"), + ("]", "R_BRACK"), + ("<", "L_ANGLE"), + (">", "R_ANGLE"), + ("@", "AT"), + ("#", "POUND"), + ("~", "TILDE"), + ("?", "QUESTION"), + ("$", "DOLLAR"), + ("&", "AMP"), + ("|", "PIPE"), + ("+", "PLUS"), + ("*", "STAR"), + ("/", "SLASH"), + ("^", "CARET"), + ("%", "PERCENT"), + ("_", "UNDERSCORE"), + (".", "DOT"), + ("..", "DOT2"), + ("...", "DOT3"), + ("..=", "DOT2EQ"), + (":", "COLON"), + ("::", "COLON2"), + ("=", "EQ"), + ("==", "EQ2"), + ("=>", "FAT_ARROW"), + ("!", "BANG"), + ("!=", "NEQ"), + ("-", "MINUS"), + ("->", "THIN_ARROW"), + ("<=", "LTEQ"), + (">=", "GTEQ"), + ("+=", "PLUSEQ"), + ("-=", "MINUSEQ"), + ("|=", "PIPEEQ"), + ("&=", "AMPEQ"), + ("^=", "CARETEQ"), + ("/=", "SLASHEQ"), + ("*=", "STAREQ"), + ("%=", "PERCENTEQ"), + ("&&", "AMP2"), + ("||", "PIPE2"), + ("<<", "SHL"), + (">>", "SHR"), + ("<<=", "SHLEQ"), + (">>=", "SHREQ"), + ], + keywords: &[ + "as", "async", "await", "box", "break", "const", "continue", "crate", "do", "dyn", "else", + "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "macro", + "match", "mod", "move", "mut", "pub", "ref", "return", "become", "self", "Self", "static", + "struct", "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", + "yield", + ], + contextual_keywords: &[ + "auto", + "builtin", + "default", + "existential", + "union", + "raw", + "macro_rules", + "yeet", + "offset_of", + "asm", + "format_args", + ], + literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING", "C_STRING"], + tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], + nodes: &[ + "SOURCE_FILE", + "STRUCT", + "UNION", + "ENUM", + "FN", + "RET_TYPE", + "EXTERN_CRATE", + "MODULE", + "USE", + "STATIC", + "CONST", + "TRAIT", + "TRAIT_ALIAS", + "IMPL", + "TYPE_ALIAS", + "MACRO_CALL", + "MACRO_RULES", + "MACRO_ARM", + "TOKEN_TREE", + "MACRO_DEF", + "PAREN_TYPE", + "TUPLE_TYPE", + "MACRO_TYPE", + "NEVER_TYPE", + "PATH_TYPE", + "PTR_TYPE", + "ARRAY_TYPE", + "SLICE_TYPE", + "REF_TYPE", + "INFER_TYPE", + "FN_PTR_TYPE", + "FOR_TYPE", + "IMPL_TRAIT_TYPE", + "DYN_TRAIT_TYPE", + "OR_PAT", + "PAREN_PAT", + "REF_PAT", + "BOX_PAT", + "IDENT_PAT", + "WILDCARD_PAT", + "REST_PAT", + "PATH_PAT", + "RECORD_PAT", + "RECORD_PAT_FIELD_LIST", + "RECORD_PAT_FIELD", + "TUPLE_STRUCT_PAT", + "TUPLE_PAT", + "SLICE_PAT", + "RANGE_PAT", + "LITERAL_PAT", + "MACRO_PAT", + "CONST_BLOCK_PAT", + // atoms + "TUPLE_EXPR", + "ARRAY_EXPR", + "PAREN_EXPR", + "PATH_EXPR", + "CLOSURE_EXPR", + "IF_EXPR", + "WHILE_EXPR", + "LOOP_EXPR", + "FOR_EXPR", + "CONTINUE_EXPR", + "BREAK_EXPR", + "LABEL", + "BLOCK_EXPR", + "STMT_LIST", + "RETURN_EXPR", + "BECOME_EXPR", + "YIELD_EXPR", + "YEET_EXPR", + "LET_EXPR", + "UNDERSCORE_EXPR", + "MACRO_EXPR", + "MATCH_EXPR", + "MATCH_ARM_LIST", + "MATCH_ARM", + "MATCH_GUARD", + "RECORD_EXPR", + "RECORD_EXPR_FIELD_LIST", + "RECORD_EXPR_FIELD", + "OFFSET_OF_EXPR", + "ASM_EXPR", + "FORMAT_ARGS_EXPR", + "FORMAT_ARGS_ARG", + // postfix + "CALL_EXPR", + "INDEX_EXPR", + "METHOD_CALL_EXPR", + "FIELD_EXPR", + "AWAIT_EXPR", + "TRY_EXPR", + "CAST_EXPR", + // unary + "REF_EXPR", + "PREFIX_EXPR", + "RANGE_EXPR", // just weird + "BIN_EXPR", + "EXTERN_BLOCK", + "EXTERN_ITEM_LIST", + "VARIANT", + "RECORD_FIELD_LIST", + "RECORD_FIELD", + "TUPLE_FIELD_LIST", + "TUPLE_FIELD", + "VARIANT_LIST", + "ITEM_LIST", + "ASSOC_ITEM_LIST", + "ATTR", + "META", + "USE_TREE", + "USE_TREE_LIST", + "PATH", + "PATH_SEGMENT", + "LITERAL", + "RENAME", + "VISIBILITY", + "WHERE_CLAUSE", + "WHERE_PRED", + "ABI", + "NAME", + "NAME_REF", + "LET_STMT", + "LET_ELSE", + "EXPR_STMT", + "GENERIC_PARAM_LIST", + "GENERIC_PARAM", + "LIFETIME_PARAM", + "TYPE_PARAM", + "RETURN_TYPE_ARG", + "CONST_PARAM", + "GENERIC_ARG_LIST", + "LIFETIME", + "LIFETIME_ARG", + "TYPE_ARG", + "ASSOC_TYPE_ARG", + "CONST_ARG", + "PARAM_LIST", + "PARAM", + "SELF_PARAM", + "ARG_LIST", + "TYPE_BOUND", + "TYPE_BOUND_LIST", + // macro related + "MACRO_ITEMS", + "MACRO_STMTS", + "MACRO_EAGER_INPUT", + ], +}; + +#[derive(Default, Debug)] +pub(crate) struct AstSrc { + pub(crate) tokens: Vec<String>, + pub(crate) nodes: Vec<AstNodeSrc>, + pub(crate) enums: Vec<AstEnumSrc>, +} + +#[derive(Debug)] +pub(crate) struct AstNodeSrc { + pub(crate) doc: Vec<String>, + pub(crate) name: String, + pub(crate) traits: Vec<String>, + pub(crate) fields: Vec<Field>, +} + +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum Field { + Token(String), + Node { name: String, ty: String, cardinality: Cardinality }, +} + +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum Cardinality { + Optional, + Many, +} + +#[derive(Debug)] +pub(crate) struct AstEnumSrc { + pub(crate) doc: Vec<String>, + pub(crate) name: String, + pub(crate) traits: Vec<String>, + pub(crate) variants: Vec<String>, +} diff --git a/src/tools/rust-analyzer/xtask/src/codegen/lints.rs b/src/tools/rust-analyzer/xtask/src/codegen/lints.rs new file mode 100644 index 00000000000..63abcfc0904 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/codegen/lints.rs @@ -0,0 +1,342 @@ +//! Generates descriptor structures for unstable features from the unstable book +//! and lints from rustc, rustdoc, and clippy. +use std::{borrow::Cow, fs, path::Path}; + +use stdx::format_to; +use xshell::{cmd, Shell}; + +use crate::{ + codegen::{add_preamble, ensure_file_contents, list_files, reformat}, + project_root, +}; + +const DESTINATION: &str = "crates/ide-db/src/generated/lints.rs"; + +/// This clones rustc repo, and so is not worth to keep up-to-date on a constant basis. +pub(crate) fn generate(check: bool) { + let sh = &Shell::new().unwrap(); + + let rust_repo = project_root().join("./target/rust"); + if rust_repo.exists() { + cmd!(sh, "git -C {rust_repo} pull --rebase").run().unwrap(); + } else { + cmd!(sh, "git clone --depth=1 https://github.com/rust-lang/rust {rust_repo}") + .run() + .unwrap(); + } + // need submodules for Cargo to parse the workspace correctly + cmd!( + sh, + "git -C {rust_repo} submodule update --init --recursive --depth=1 -- + compiler library src/tools" + ) + .run() + .unwrap(); + + let mut contents = String::from( + r" +#[derive(Clone)] +pub struct Lint { + pub label: &'static str, + pub description: &'static str, +} + +pub struct LintGroup { + pub lint: Lint, + pub children: &'static [&'static str], +} + +", + ); + + generate_lint_descriptor(sh, &mut contents); + contents.push('\n'); + + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned()); + let unstable_book = project_root().join("./target/unstable-book-gen"); + cmd!( + sh, + "{cargo} run --manifest-path {rust_repo}/src/tools/unstable-book-gen/Cargo.toml -- + {rust_repo}/library {rust_repo}/compiler {rust_repo}/src {unstable_book}" + ) + .run() + .unwrap(); + generate_feature_descriptor(&mut contents, &unstable_book.join("src")); + contents.push('\n'); + + let lints_json = project_root().join("./target/clippy_lints.json"); + cmd!( + sh, + "curl https://rust-lang.github.io/rust-clippy/master/lints.json --output {lints_json}" + ) + .run() + .unwrap(); + generate_descriptor_clippy(&mut contents, &lints_json); + + let contents = add_preamble("sourcegen_lints", reformat(contents)); + + let destination = project_root().join(DESTINATION); + ensure_file_contents(destination.as_path(), &contents, check); +} + +/// Parses the output of `rustdoc -Whelp` and prints `Lint` and `LintGroup` constants into `buf`. +/// +/// As of writing, the output of `rustc -Whelp` (not rustdoc) has the following format: +/// +/// ```text +/// Lint checks provided by rustc: +/// +/// name default meaning +/// ---- ------- ------- +/// +/// ... +/// +/// Lint groups provided by rustc: +/// +/// name sub-lints +/// ---- --------- +/// +/// ... +/// ``` +/// +/// `rustdoc -Whelp` (and any other custom `rustc` driver) adds another two +/// tables after the `rustc` ones, with a different title but the same format. +fn generate_lint_descriptor(sh: &Shell, buf: &mut String) { + let stdout = cmd!(sh, "rustdoc -Whelp").read().unwrap(); + let lints_pat = "---- ------- -------\n"; + let lint_groups_pat = "---- ---------\n"; + let lints = find_and_slice(&stdout, lints_pat); + let lint_groups = find_and_slice(lints, lint_groups_pat); + let lints_rustdoc = find_and_slice(lint_groups, lints_pat); + let lint_groups_rustdoc = find_and_slice(lints_rustdoc, lint_groups_pat); + + buf.push_str(r#"pub const DEFAULT_LINTS: &[Lint] = &["#); + buf.push('\n'); + + let lints = lints.lines().take_while(|l| !l.is_empty()).map(|line| { + let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap(); + let (_default_level, description) = rest.trim().split_once(char::is_whitespace).unwrap(); + (name.trim(), Cow::Borrowed(description.trim()), vec![]) + }); + let lint_groups = lint_groups.lines().take_while(|l| !l.is_empty()).map(|line| { + let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap(); + ( + name.trim(), + format!("lint group for: {}", lints.trim()).into(), + lints + .split_ascii_whitespace() + .map(|s| s.trim().trim_matches(',').replace('-', "_")) + .collect(), + ) + }); + + let mut lints = lints.chain(lint_groups).collect::<Vec<_>>(); + lints.sort_by(|(ident, ..), (ident2, ..)| ident.cmp(ident2)); + + for (name, description, ..) in &lints { + push_lint_completion(buf, &name.replace('-', "_"), description); + } + buf.push_str("];\n\n"); + + buf.push_str(r#"pub const DEFAULT_LINT_GROUPS: &[LintGroup] = &["#); + for (name, description, children) in &lints { + if !children.is_empty() { + // HACK: warnings is emitted with a general description, not with its members + if name == &"warnings" { + push_lint_group(buf, name, description, &Vec::new()); + continue; + } + push_lint_group(buf, &name.replace('-', "_"), description, children); + } + } + buf.push('\n'); + buf.push_str("];\n"); + + // rustdoc + + buf.push('\n'); + buf.push_str(r#"pub const RUSTDOC_LINTS: &[Lint] = &["#); + buf.push('\n'); + + let lints_rustdoc = lints_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| { + let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap(); + let (_default_level, description) = rest.trim().split_once(char::is_whitespace).unwrap(); + (name.trim(), Cow::Borrowed(description.trim()), vec![]) + }); + let lint_groups_rustdoc = + lint_groups_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| { + let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap(); + ( + name.trim(), + format!("lint group for: {}", lints.trim()).into(), + lints + .split_ascii_whitespace() + .map(|s| s.trim().trim_matches(',').replace('-', "_")) + .collect(), + ) + }); + + let mut lints_rustdoc = lints_rustdoc.chain(lint_groups_rustdoc).collect::<Vec<_>>(); + lints_rustdoc.sort_by(|(ident, ..), (ident2, ..)| ident.cmp(ident2)); + + for (name, description, ..) in &lints_rustdoc { + push_lint_completion(buf, &name.replace('-', "_"), description) + } + buf.push_str("];\n\n"); + + buf.push_str(r#"pub const RUSTDOC_LINT_GROUPS: &[LintGroup] = &["#); + for (name, description, children) in &lints_rustdoc { + if !children.is_empty() { + push_lint_group(buf, &name.replace('-', "_"), description, children); + } + } + buf.push('\n'); + buf.push_str("];\n"); +} + +#[track_caller] +fn find_and_slice<'a>(i: &'a str, p: &str) -> &'a str { + let idx = i.find(p).unwrap(); + &i[idx + p.len()..] +} + +/// Parses the unstable book `src_dir` and prints a constant with the list of +/// unstable features into `buf`. +/// +/// It does this by looking for all `.md` files in the `language-features` and +/// `library-features` directories, and using the file name as the feature +/// name, and the file contents as the feature description. +fn generate_feature_descriptor(buf: &mut String, src_dir: &Path) { + let mut features = ["language-features", "library-features"] + .into_iter() + .flat_map(|it| list_files(&src_dir.join(it))) + // Get all `.md` files + .filter(|path| path.extension() == Some("md".as_ref())) + .map(|path| { + let feature_ident = path.file_stem().unwrap().to_str().unwrap().replace('-', "_"); + let doc = fs::read_to_string(path).unwrap(); + (feature_ident, doc) + }) + .collect::<Vec<_>>(); + features.sort_by(|(feature_ident, _), (feature_ident2, _)| feature_ident.cmp(feature_ident2)); + + buf.push_str(r#"pub const FEATURES: &[Lint] = &["#); + for (feature_ident, doc) in features.into_iter() { + push_lint_completion(buf, &feature_ident, &doc) + } + buf.push('\n'); + buf.push_str("];\n"); +} + +#[derive(Default)] +struct ClippyLint { + help: String, + id: String, +} + +fn unescape(s: &str) -> String { + s.replace(r#"\""#, "").replace(r#"\n"#, "\n").replace(r#"\r"#, "") +} + +#[allow(clippy::print_stderr)] +fn generate_descriptor_clippy(buf: &mut String, path: &Path) { + let file_content = std::fs::read_to_string(path).unwrap(); + let mut clippy_lints: Vec<ClippyLint> = Vec::new(); + let mut clippy_groups: std::collections::BTreeMap<String, Vec<String>> = Default::default(); + + for line in file_content.lines().map(str::trim) { + if let Some(line) = line.strip_prefix(r#""id": ""#) { + let clippy_lint = ClippyLint { + id: line.strip_suffix(r#"","#).expect("should be suffixed by comma").into(), + help: String::new(), + }; + clippy_lints.push(clippy_lint) + } else if let Some(line) = line.strip_prefix(r#""group": ""#) { + if let Some(group) = line.strip_suffix("\",") { + clippy_groups + .entry(group.to_owned()) + .or_default() + .push(clippy_lints.last().unwrap().id.clone()); + } + } else if let Some(line) = line.strip_prefix(r#""docs": ""#) { + let header = "### What it does"; + let line = match line.find(header) { + Some(idx) => &line[idx + header.len()..], + None => { + let id = &clippy_lints.last().unwrap().id; + // these just don't have the common header + let allowed = ["allow_attributes", "read_line_without_trim"]; + if allowed.contains(&id.as_str()) { + line + } else { + eprintln!("\nunexpected clippy prefix for {id}, line={line:?}\n",); + continue; + } + } + }; + // Only take the description, any more than this is a lot of additional data we would embed into the exe + // which seems unnecessary + let up_to = line.find(r#"###"#).expect("no second section found?"); + let line = &line[..up_to]; + + let clippy_lint = clippy_lints.last_mut().expect("clippy lint must already exist"); + clippy_lint.help = unescape(line).trim().to_owned(); + } + } + clippy_lints.sort_by(|lint, lint2| lint.id.cmp(&lint2.id)); + + buf.push_str(r#"pub const CLIPPY_LINTS: &[Lint] = &["#); + buf.push('\n'); + for clippy_lint in clippy_lints.into_iter() { + let lint_ident = format!("clippy::{}", clippy_lint.id); + let doc = clippy_lint.help; + push_lint_completion(buf, &lint_ident, &doc); + } + buf.push_str("];\n"); + + buf.push_str(r#"pub const CLIPPY_LINT_GROUPS: &[LintGroup] = &["#); + for (id, children) in clippy_groups { + let children = children.iter().map(|id| format!("clippy::{id}")).collect::<Vec<_>>(); + if !children.is_empty() { + let lint_ident = format!("clippy::{id}"); + let description = format!("lint group for: {}", children.join(", ")); + push_lint_group(buf, &lint_ident, &description, &children); + } + } + buf.push('\n'); + buf.push_str("];\n"); +} + +fn push_lint_completion(buf: &mut String, label: &str, description: &str) { + format_to!( + buf, + r###" Lint {{ + label: "{}", + description: r##"{}"##, + }},"###, + label, + description, + ); +} + +fn push_lint_group(buf: &mut String, label: &str, description: &str, children: &[String]) { + buf.push_str( + r###" LintGroup { + lint: + "###, + ); + + push_lint_completion(buf, label, description); + + let children = format!( + "&[{}]", + children.iter().map(|it| format!("\"{it}\"")).collect::<Vec<_>>().join(", ") + ); + format_to!( + buf, + r###" + children: {}, + }},"###, + children, + ); +} diff --git a/src/tools/rust-analyzer/xtask/src/dist.rs b/src/tools/rust-analyzer/xtask/src/dist.rs new file mode 100644 index 00000000000..2fe9db98cf2 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/dist.rs @@ -0,0 +1,222 @@ +use std::{ + env, + fs::File, + io::{self, BufWriter}, + path::{Path, PathBuf}, +}; + +use flate2::{write::GzEncoder, Compression}; +use time::OffsetDateTime; +use xshell::{cmd, Shell}; +use zip::{write::FileOptions, DateTime, ZipWriter}; + +use crate::{ + date_iso, + flags::{self, Malloc}, + project_root, +}; + +const VERSION_STABLE: &str = "0.3"; +const VERSION_NIGHTLY: &str = "0.4"; +const VERSION_DEV: &str = "0.5"; // keep this one in sync with `package.json` + +impl flags::Dist { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + let stable = sh.var("GITHUB_REF").unwrap_or_default().as_str() == "refs/heads/release"; + + let project_root = project_root(); + let target = Target::get(&project_root); + let allocator = self.allocator(); + let dist = project_root.join("dist"); + sh.remove_path(&dist)?; + sh.create_dir(&dist)?; + + if let Some(patch_version) = self.client_patch_version { + let version = if stable { + format!("{VERSION_STABLE}.{patch_version}") + } else { + // A hack to make VS Code prefer nightly over stable. + format!("{VERSION_NIGHTLY}.{patch_version}") + }; + dist_server(sh, &format!("{version}-standalone"), &target, allocator)?; + let release_tag = if stable { date_iso(sh)? } else { "nightly".to_owned() }; + dist_client(sh, &version, &release_tag, &target)?; + } else { + dist_server(sh, "0.0.0-standalone", &target, allocator)?; + } + Ok(()) + } +} + +fn dist_client( + sh: &Shell, + version: &str, + release_tag: &str, + target: &Target, +) -> anyhow::Result<()> { + let bundle_path = Path::new("editors").join("code").join("server"); + sh.create_dir(&bundle_path)?; + sh.copy_file(&target.server_path, &bundle_path)?; + if let Some(symbols_path) = &target.symbols_path { + sh.copy_file(symbols_path, &bundle_path)?; + } + + let _d = sh.push_dir("./editors/code"); + + let mut patch = Patch::new(sh, "./package.json")?; + patch + .replace( + &format!(r#""version": "{VERSION_DEV}.0-dev""#), + &format!(r#""version": "{version}""#), + ) + .replace(r#""releaseTag": null"#, &format!(r#""releaseTag": "{release_tag}""#)) + .replace(r#""$generated-start": {},"#, "") + .replace(",\n \"$generated-end\": {}", "") + .replace(r#""enabledApiProposals": [],"#, r#""#); + patch.commit(sh)?; + + Ok(()) +} + +fn dist_server( + sh: &Shell, + release: &str, + target: &Target, + allocator: Malloc, +) -> anyhow::Result<()> { + let _e = sh.push_env("CFG_RELEASE", release); + let _e = sh.push_env("CARGO_PROFILE_RELEASE_LTO", "thin"); + + // Uncomment to enable debug info for releases. Note that: + // * debug info is split on windows and macs, so it does nothing for those platforms, + // * on Linux, this blows up the binary size from 8MB to 43MB, which is unreasonable. + // let _e = sh.push_env("CARGO_PROFILE_RELEASE_DEBUG", "1"); + + if target.name.contains("-linux-") { + env::set_var("CC", "clang"); + } + + let target_name = &target.name; + let features = allocator.to_features(); + cmd!(sh, "cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --target {target_name} {features...} --release").run()?; + + let dst = Path::new("dist").join(&target.artifact_name); + gzip(&target.server_path, &dst.with_extension("gz"))?; + if target_name.contains("-windows-") { + zip(&target.server_path, target.symbols_path.as_ref(), &dst.with_extension("zip"))?; + } + + Ok(()) +} + +fn gzip(src_path: &Path, dest_path: &Path) -> anyhow::Result<()> { + let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best()); + let mut input = io::BufReader::new(File::open(src_path)?); + io::copy(&mut input, &mut encoder)?; + encoder.finish()?; + Ok(()) +} + +fn zip(src_path: &Path, symbols_path: Option<&PathBuf>, dest_path: &Path) -> anyhow::Result<()> { + let file = File::create(dest_path)?; + let mut writer = ZipWriter::new(BufWriter::new(file)); + writer.start_file( + src_path.file_name().unwrap().to_str().unwrap(), + FileOptions::default() + .last_modified_time( + DateTime::try_from(OffsetDateTime::from(std::fs::metadata(src_path)?.modified()?)) + .unwrap(), + ) + .unix_permissions(0o755) + .compression_method(zip::CompressionMethod::Deflated) + .compression_level(Some(9)), + )?; + let mut input = io::BufReader::new(File::open(src_path)?); + io::copy(&mut input, &mut writer)?; + if let Some(symbols_path) = symbols_path { + writer.start_file( + symbols_path.file_name().unwrap().to_str().unwrap(), + FileOptions::default() + .last_modified_time( + DateTime::try_from(OffsetDateTime::from( + std::fs::metadata(src_path)?.modified()?, + )) + .unwrap(), + ) + .compression_method(zip::CompressionMethod::Deflated) + .compression_level(Some(9)), + )?; + let mut input = io::BufReader::new(File::open(symbols_path)?); + io::copy(&mut input, &mut writer)?; + } + writer.finish()?; + Ok(()) +} + +struct Target { + name: String, + server_path: PathBuf, + symbols_path: Option<PathBuf>, + artifact_name: String, +} + +impl Target { + fn get(project_root: &Path) -> Self { + let name = match env::var("RA_TARGET") { + Ok(target) => target, + _ => { + if cfg!(target_os = "linux") { + "x86_64-unknown-linux-gnu".to_owned() + } else if cfg!(target_os = "windows") { + "x86_64-pc-windows-msvc".to_owned() + } else if cfg!(target_os = "macos") { + "x86_64-apple-darwin".to_owned() + } else { + panic!("Unsupported OS, maybe try setting RA_TARGET") + } + } + }; + let out_path = project_root.join("target").join(&name).join("release"); + let (exe_suffix, symbols_path) = if name.contains("-windows-") { + (".exe".into(), Some(out_path.join("rust_analyzer.pdb"))) + } else { + (String::new(), None) + }; + let server_path = out_path.join(format!("rust-analyzer{exe_suffix}")); + let artifact_name = format!("rust-analyzer-{name}{exe_suffix}"); + Self { name, server_path, symbols_path, artifact_name } + } +} + +struct Patch { + path: PathBuf, + original_contents: String, + contents: String, +} + +impl Patch { + fn new(sh: &Shell, path: impl Into<PathBuf>) -> anyhow::Result<Patch> { + let path = path.into(); + let contents = sh.read_file(&path)?; + Ok(Patch { path, original_contents: contents.clone(), contents }) + } + + fn replace(&mut self, from: &str, to: &str) -> &mut Patch { + assert!(self.contents.contains(from)); + self.contents = self.contents.replace(from, to); + self + } + + fn commit(&self, sh: &Shell) -> anyhow::Result<()> { + sh.write_file(&self.path, &self.contents)?; + Ok(()) + } +} + +impl Drop for Patch { + fn drop(&mut self) { + // FIXME: find a way to bring this back + let _ = &self.original_contents; + // write_file(&self.path, &self.original_contents).unwrap(); + } +} diff --git a/src/tools/rust-analyzer/xtask/src/flags.rs b/src/tools/rust-analyzer/xtask/src/flags.rs new file mode 100644 index 00000000000..90665459208 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/flags.rs @@ -0,0 +1,273 @@ +#![allow(unreachable_pub)] + +use std::str::FromStr; + +use crate::install::{ClientOpt, ServerOpt}; + +xflags::xflags! { + src "./src/flags.rs" + + /// Run custom build command. + cmd xtask { + + /// Install rust-analyzer server or editor plugin. + cmd install { + /// Install only VS Code plugin. + optional --client + /// One of 'code', 'code-exploration', 'code-insiders', 'codium', or 'code-oss'. + optional --code-bin name: String + + /// Install only the language server. + optional --server + /// Use mimalloc allocator for server + optional --mimalloc + /// Use jemalloc allocator for server + optional --jemalloc + /// build in release with debug info set to 2 + optional --dev-rel + } + + cmd fuzz-tests {} + + cmd release { + optional --dry-run + } + cmd promote { + optional --dry-run + } + cmd dist { + /// Use mimalloc allocator for server + optional --mimalloc + /// Use jemalloc allocator for server + optional --jemalloc + optional --client-patch-version version: String + } + /// Read a changelog AsciiDoc file and update the GitHub Releases entry in Markdown. + cmd publish-release-notes { + /// Only run conversion and show the result. + optional --dry-run + /// Target changelog file. + required changelog: String + } + cmd metrics { + optional measurement_type: MeasurementType + } + /// Builds a benchmark version of rust-analyzer and puts it into `./target`. + cmd bb { + required suffix: String + } + + cmd codegen { + optional codegen_type: CodegenType + optional --check + } + } +} + +// generated start +// The following code is generated by `xflags` macro. +// Run `env UPDATE_XFLAGS=1 cargo build` to regenerate. +#[derive(Debug)] +pub struct Xtask { + pub subcommand: XtaskCmd, +} + +#[derive(Debug)] +pub enum XtaskCmd { + Install(Install), + FuzzTests(FuzzTests), + Release(Release), + Promote(Promote), + Dist(Dist), + PublishReleaseNotes(PublishReleaseNotes), + Metrics(Metrics), + Bb(Bb), + Codegen(Codegen), +} + +#[derive(Debug)] +pub struct Install { + pub client: bool, + pub code_bin: Option<String>, + pub server: bool, + pub mimalloc: bool, + pub jemalloc: bool, + pub dev_rel: bool, +} + +#[derive(Debug)] +pub struct FuzzTests; + +#[derive(Debug)] +pub struct Release { + pub dry_run: bool, +} + +#[derive(Debug)] +pub struct Promote { + pub dry_run: bool, +} + +#[derive(Debug)] +pub struct Dist { + pub mimalloc: bool, + pub jemalloc: bool, + pub client_patch_version: Option<String>, +} + +#[derive(Debug)] +pub struct PublishReleaseNotes { + pub changelog: String, + + pub dry_run: bool, +} + +#[derive(Debug)] +pub struct Metrics { + pub measurement_type: Option<MeasurementType>, +} + +#[derive(Debug)] +pub struct Bb { + pub suffix: String, +} + +#[derive(Debug)] +pub struct Codegen { + pub codegen_type: Option<CodegenType>, + + pub check: bool, +} + +impl Xtask { + #[allow(dead_code)] + pub fn from_env_or_exit() -> Self { + Self::from_env_or_exit_() + } + + #[allow(dead_code)] + pub fn from_env() -> xflags::Result<Self> { + Self::from_env_() + } + + #[allow(dead_code)] + pub fn from_vec(args: Vec<std::ffi::OsString>) -> xflags::Result<Self> { + Self::from_vec_(args) + } +} +// generated end + +#[derive(Debug, Default)] +pub enum CodegenType { + #[default] + All, + Grammar, + AssistsDocTests, + DiagnosticsDocs, + LintDefinitions, +} + +impl FromStr for CodegenType { + type Err = String; + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "all" => Ok(Self::All), + "grammar" => Ok(Self::Grammar), + "assists-doc-tests" => Ok(Self::AssistsDocTests), + "diagnostics-docs" => Ok(Self::DiagnosticsDocs), + "lints-definitions" => Ok(Self::LintDefinitions), + _ => Err("Invalid option".to_owned()), + } + } +} + +#[derive(Debug)] +pub enum MeasurementType { + Build, + RustcTests, + AnalyzeSelf, + AnalyzeRipgrep, + AnalyzeWebRender, + AnalyzeDiesel, + AnalyzeHyper, +} + +impl FromStr for MeasurementType { + type Err = String; + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "build" => Ok(Self::Build), + "rustc_tests" => Ok(Self::RustcTests), + "self" => Ok(Self::AnalyzeSelf), + "ripgrep-13.0.0" => Ok(Self::AnalyzeRipgrep), + "webrender-2022" => Ok(Self::AnalyzeWebRender), + "diesel-1.4.8" => Ok(Self::AnalyzeDiesel), + "hyper-0.14.18" => Ok(Self::AnalyzeHyper), + _ => Err("Invalid option".to_owned()), + } + } +} +impl AsRef<str> for MeasurementType { + fn as_ref(&self) -> &str { + match self { + Self::Build => "build", + Self::RustcTests => "rustc_tests", + Self::AnalyzeSelf => "self", + Self::AnalyzeRipgrep => "ripgrep-13.0.0", + Self::AnalyzeWebRender => "webrender-2022", + Self::AnalyzeDiesel => "diesel-1.4.8", + Self::AnalyzeHyper => "hyper-0.14.18", + } + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Malloc { + System, + Mimalloc, + Jemalloc, +} + +impl Malloc { + pub(crate) fn to_features(self) -> &'static [&'static str] { + match self { + Malloc::System => &[][..], + Malloc::Mimalloc => &["--features", "mimalloc"], + Malloc::Jemalloc => &["--features", "jemalloc"], + } + } +} + +impl Install { + pub(crate) fn server(&self) -> Option<ServerOpt> { + if self.client && !self.server { + return None; + } + let malloc = if self.mimalloc { + Malloc::Mimalloc + } else if self.jemalloc { + Malloc::Jemalloc + } else { + Malloc::System + }; + Some(ServerOpt { malloc, dev_rel: self.dev_rel }) + } + pub(crate) fn client(&self) -> Option<ClientOpt> { + if !self.client && self.server { + return None; + } + Some(ClientOpt { code_bin: self.code_bin.clone() }) + } +} + +impl Dist { + pub(crate) fn allocator(&self) -> Malloc { + if self.mimalloc { + Malloc::Mimalloc + } else if self.jemalloc { + Malloc::Jemalloc + } else { + Malloc::System + } + } +} diff --git a/src/tools/rust-analyzer/xtask/src/install.rs b/src/tools/rust-analyzer/xtask/src/install.rs new file mode 100644 index 00000000000..72e612f9e1d --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/install.rs @@ -0,0 +1,134 @@ +//! Installs rust-analyzer language server and/or editor plugin. + +use std::{env, path::PathBuf, str}; + +use anyhow::{bail, format_err, Context}; +use xshell::{cmd, Shell}; + +use crate::flags::{self, Malloc}; + +impl flags::Install { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + if cfg!(target_os = "macos") { + fix_path_for_mac(sh).context("Fix path for mac")?; + } + if let Some(server) = self.server() { + install_server(sh, server).context("install server")?; + } + if let Some(client) = self.client() { + install_client(sh, client).context("install client")?; + } + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct ClientOpt { + pub(crate) code_bin: Option<String>, +} + +const VS_CODES: &[&str] = &["code", "code-exploration", "code-insiders", "codium", "code-oss"]; + +pub(crate) struct ServerOpt { + pub(crate) malloc: Malloc, + pub(crate) dev_rel: bool, +} + +fn fix_path_for_mac(sh: &Shell) -> anyhow::Result<()> { + let mut vscode_path: Vec<PathBuf> = { + const COMMON_APP_PATH: &str = + r"/Applications/Visual Studio Code.app/Contents/Resources/app/bin"; + const ROOT_DIR: &str = ""; + let home_dir = sh.var("HOME").map_err(|err| { + format_err!("Failed getting HOME from environment with error: {}.", err) + })?; + + [ROOT_DIR, &home_dir] + .into_iter() + .map(|dir| dir.to_owned() + COMMON_APP_PATH) + .map(PathBuf::from) + .filter(|path| path.exists()) + .collect() + }; + + if !vscode_path.is_empty() { + let vars = sh.var_os("PATH").context("Could not get PATH variable from env.")?; + + let mut paths = env::split_paths(&vars).collect::<Vec<_>>(); + paths.append(&mut vscode_path); + let new_paths = env::join_paths(paths).context("build env PATH")?; + sh.set_var("PATH", new_paths); + } + + Ok(()) +} + +fn install_client(sh: &Shell, client_opt: ClientOpt) -> anyhow::Result<()> { + let _dir = sh.push_dir("./editors/code"); + + // Package extension. + if cfg!(unix) { + cmd!(sh, "npm --version").run().context("`npm` is required to build the VS Code plugin")?; + cmd!(sh, "npm ci").run()?; + + cmd!(sh, "npm run package --scripts-prepend-node-path").run()?; + } else { + cmd!(sh, "cmd.exe /c npm --version") + .run() + .context("`npm` is required to build the VS Code plugin")?; + cmd!(sh, "cmd.exe /c npm ci").run()?; + + cmd!(sh, "cmd.exe /c npm run package").run()?; + }; + + // Find the appropriate VS Code binary. + let lifetime_extender; + let candidates: &[&str] = match client_opt.code_bin.as_deref() { + Some(it) => { + lifetime_extender = [it]; + &lifetime_extender[..] + } + None => VS_CODES, + }; + let code = candidates + .iter() + .copied() + .find(|&bin| { + if cfg!(unix) { + cmd!(sh, "{bin} --version").read().is_ok() + } else { + cmd!(sh, "cmd.exe /c {bin}.cmd --version").read().is_ok() + } + }) + .ok_or_else(|| { + format_err!("Can't execute `{} --version`. Perhaps it is not in $PATH?", candidates[0]) + })?; + + // Install & verify. + let installed_extensions = if cfg!(unix) { + cmd!(sh, "{code} --install-extension rust-analyzer.vsix --force").run()?; + cmd!(sh, "{code} --list-extensions").read()? + } else { + cmd!(sh, "cmd.exe /c {code}.cmd --install-extension rust-analyzer.vsix --force").run()?; + cmd!(sh, "cmd.exe /c {code}.cmd --list-extensions").read()? + }; + + if !installed_extensions.contains("rust-analyzer") { + bail!( + "Could not install the Visual Studio Code extension. \ + Please make sure you have at least NodeJS 16.x together with the latest version of VS Code installed and try again. \ + Note that installing via xtask install does not work for VS Code Remote, instead you’ll need to install the .vsix manually." + ); + } + + Ok(()) +} + +fn install_server(sh: &Shell, opts: ServerOpt) -> anyhow::Result<()> { + let features = opts.malloc.to_features(); + let profile = if opts.dev_rel { "dev-rel" } else { "release" }; + + let cmd = cmd!(sh, "cargo install --path crates/rust-analyzer --profile={profile} --locked --force --features force-always-assert {features...}"); + cmd.run()?; + Ok(()) +} diff --git a/src/tools/rust-analyzer/xtask/src/main.rs b/src/tools/rust-analyzer/xtask/src/main.rs new file mode 100644 index 00000000000..9418675a348 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/main.rs @@ -0,0 +1,87 @@ +//! See <https://github.com/matklad/cargo-xtask/>. +//! +//! This binary defines various auxiliary build commands, which are not +//! expressible with just `cargo`. Notably, it provides tests via `cargo test -p xtask` +//! for code generation and `cargo xtask install` for installation of +//! rust-analyzer server and client. +//! +//! This binary is integrated into the `cargo` command line by using an alias in +//! `.cargo/config`. + +#![warn(rust_2018_idioms, unused_lifetimes)] +#![allow(clippy::print_stderr, clippy::print_stdout)] + +mod flags; + +mod codegen; +mod dist; +mod install; +mod metrics; +mod publish; +mod release; + +use anyhow::bail; +use std::{env, path::PathBuf}; +use xshell::{cmd, Shell}; + +fn main() -> anyhow::Result<()> { + let flags = flags::Xtask::from_env_or_exit(); + + let sh = &Shell::new()?; + sh.change_dir(project_root()); + + match flags.subcommand { + flags::XtaskCmd::Install(cmd) => cmd.run(sh), + flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh), + flags::XtaskCmd::Release(cmd) => cmd.run(sh), + flags::XtaskCmd::Promote(cmd) => cmd.run(sh), + flags::XtaskCmd::Dist(cmd) => cmd.run(sh), + flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh), + flags::XtaskCmd::Metrics(cmd) => cmd.run(sh), + flags::XtaskCmd::Codegen(cmd) => cmd.run(sh), + flags::XtaskCmd::Bb(cmd) => { + { + let _d = sh.push_dir("./crates/rust-analyzer"); + cmd!(sh, "cargo build --release --features jemalloc").run()?; + } + sh.copy_file( + "./target/release/rust-analyzer", + format!("./target/rust-analyzer-{}", cmd.suffix), + )?; + Ok(()) + } + } +} + +/// Returns the path to the root directory of `rust-analyzer` project. +fn project_root() -> PathBuf { + let dir = + env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned()); + PathBuf::from(dir).parent().unwrap().to_owned() +} + +fn run_fuzzer(sh: &Shell) -> anyhow::Result<()> { + let _d = sh.push_dir("./crates/syntax"); + let _e = sh.push_env("RUSTUP_TOOLCHAIN", "nightly"); + if cmd!(sh, "cargo fuzz --help").read().is_err() { + cmd!(sh, "cargo install cargo-fuzz").run()?; + }; + + // Expecting nightly rustc + let out = cmd!(sh, "rustc --version").read()?; + if !out.contains("nightly") { + bail!("fuzz tests require nightly rustc") + } + + cmd!(sh, "cargo fuzz run parser").run()?; + Ok(()) +} + +fn date_iso(sh: &Shell) -> anyhow::Result<String> { + let res = cmd!(sh, "date -u +%Y-%m-%d").read()?; + Ok(res) +} + +fn is_release_tag(tag: &str) -> bool { + tag.len() == "2020-02-24".len() && tag.starts_with(|c: char| c.is_ascii_digit()) +} diff --git a/src/tools/rust-analyzer/xtask/src/metrics.rs b/src/tools/rust-analyzer/xtask/src/metrics.rs new file mode 100644 index 00000000000..285abb9efcb --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/metrics.rs @@ -0,0 +1,222 @@ +use std::{ + collections::BTreeMap, + fs, + io::Write as _, + path::Path, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{bail, format_err}; +use xshell::{cmd, Shell}; + +use crate::flags::{self, MeasurementType}; + +type Unit = String; + +impl flags::Metrics { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + let mut metrics = Metrics::new(sh)?; + if !Path::new("./target/rustc-perf").exists() { + sh.create_dir("./target/rustc-perf")?; + cmd!(sh, "git clone https://github.com/rust-lang/rustc-perf.git ./target/rustc-perf") + .run()?; + } + { + let _d = sh.push_dir("./target/rustc-perf"); + let revision = &metrics.perf_revision; + cmd!(sh, "git reset --hard {revision}").run()?; + } + + let _env = sh.push_env("RA_METRICS", "1"); + + let name = match &self.measurement_type { + Some(ms) => { + let name = ms.as_ref(); + match ms { + MeasurementType::Build => { + metrics.measure_build(sh)?; + } + MeasurementType::RustcTests => { + metrics.measure_rustc_tests(sh)?; + } + MeasurementType::AnalyzeSelf => { + metrics.measure_analysis_stats_self(sh)?; + } + MeasurementType::AnalyzeRipgrep + | MeasurementType::AnalyzeWebRender + | MeasurementType::AnalyzeDiesel + | MeasurementType::AnalyzeHyper => { + metrics.measure_analysis_stats(sh, name)?; + } + }; + name + } + None => { + metrics.measure_build(sh)?; + metrics.measure_rustc_tests(sh)?; + metrics.measure_analysis_stats_self(sh)?; + metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeRipgrep.as_ref())?; + metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeWebRender.as_ref())?; + metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeDiesel.as_ref())?; + metrics.measure_analysis_stats(sh, MeasurementType::AnalyzeHyper.as_ref())?; + "all" + } + }; + + let mut file = + fs::File::options().write(true).create(true).open(format!("target/{}.json", name))?; + writeln!(file, "{}", metrics.json())?; + eprintln!("{metrics:#?}"); + Ok(()) + } +} + +impl Metrics { + fn measure_build(&mut self, sh: &Shell) -> anyhow::Result<()> { + eprintln!("\nMeasuring build"); + cmd!(sh, "cargo fetch").run()?; + + let time = Instant::now(); + cmd!(sh, "cargo build --release --package rust-analyzer --bin rust-analyzer").run()?; + let time = time.elapsed(); + self.report("build", time.as_millis() as u64, "ms".into()); + Ok(()) + } + + fn measure_rustc_tests(&mut self, sh: &Shell) -> anyhow::Result<()> { + eprintln!("\nMeasuring rustc tests"); + + cmd!( + sh, + "git clone --depth=1 --branch 1.76.0 https://github.com/rust-lang/rust.git --single-branch" + ) + .run()?; + + let output = cmd!(sh, "./target/release/rust-analyzer rustc-tests ./rust").read()?; + for (metric, value, unit) in parse_metrics(&output) { + self.report(metric, value, unit.into()); + } + Ok(()) + } + + fn measure_analysis_stats_self(&mut self, sh: &Shell) -> anyhow::Result<()> { + self.measure_analysis_stats_path(sh, "self", ".") + } + fn measure_analysis_stats(&mut self, sh: &Shell, bench: &str) -> anyhow::Result<()> { + self.measure_analysis_stats_path( + sh, + bench, + &format!("./target/rustc-perf/collector/compile-benchmarks/{bench}"), + ) + } + fn measure_analysis_stats_path( + &mut self, + sh: &Shell, + name: &str, + path: &str, + ) -> anyhow::Result<()> { + assert!(Path::new(path).exists(), "unable to find bench in {path}"); + eprintln!("\nMeasuring analysis-stats/{name}"); + let output = cmd!( + sh, + "./target/release/rust-analyzer -q analysis-stats {path} --query-sysroot-metadata" + ) + .read()?; + for (metric, value, unit) in parse_metrics(&output) { + self.report(&format!("analysis-stats/{name}/{metric}"), value, unit.into()); + } + Ok(()) + } +} + +fn parse_metrics(output: &str) -> Vec<(&str, u64, &str)> { + output + .lines() + .filter_map(|it| { + let entry = it.split(':').collect::<Vec<_>>(); + match entry.as_slice() { + ["METRIC", name, value, unit] => Some((*name, value.parse().unwrap(), *unit)), + _ => None, + } + }) + .collect() +} + +#[derive(Debug)] +struct Metrics { + host: Host, + timestamp: SystemTime, + revision: String, + perf_revision: String, + metrics: BTreeMap<String, (u64, Unit)>, +} + +#[derive(Debug)] +struct Host { + os: String, + cpu: String, + mem: String, +} + +impl Metrics { + fn new(sh: &Shell) -> anyhow::Result<Metrics> { + let host = Host::new(sh)?; + let timestamp = SystemTime::now(); + let revision = cmd!(sh, "git rev-parse HEAD").read()?; + let perf_revision = "a584462e145a0c04760fd9391daefb4f6bd13a99".into(); + Ok(Metrics { host, timestamp, revision, perf_revision, metrics: BTreeMap::new() }) + } + + fn report(&mut self, name: &str, value: u64, unit: Unit) { + self.metrics.insert(name.into(), (value, unit)); + } + + fn json(&self) -> String { + let mut buf = String::new(); + self.to_json(write_json::object(&mut buf)); + buf + } + + fn to_json(&self, mut obj: write_json::Object<'_>) { + self.host.to_json(obj.object("host")); + let timestamp = self.timestamp.duration_since(UNIX_EPOCH).unwrap(); + obj.number("timestamp", timestamp.as_secs() as f64); + obj.string("revision", &self.revision); + obj.string("perf_revision", &self.perf_revision); + let mut metrics = obj.object("metrics"); + for (k, (value, unit)) in &self.metrics { + metrics.array(k).number(*value as f64).string(unit); + } + } +} + +impl Host { + fn new(sh: &Shell) -> anyhow::Result<Host> { + if cfg!(not(target_os = "linux")) { + bail!("can only collect metrics on Linux "); + } + + let os = read_field(sh, "/etc/os-release", "PRETTY_NAME=")?.trim_matches('"').to_owned(); + + let cpu = read_field(sh, "/proc/cpuinfo", "model name")? + .trim_start_matches(':') + .trim() + .to_owned(); + + let mem = read_field(sh, "/proc/meminfo", "MemTotal:")?; + + return Ok(Host { os, cpu, mem }); + + fn read_field(sh: &Shell, path: &str, field: &str) -> anyhow::Result<String> { + let text = sh.read_file(path)?; + + text.lines() + .find_map(|it| it.strip_prefix(field)) + .map(|it| it.trim().to_owned()) + .ok_or_else(|| format_err!("can't parse {}", path)) + } + } + fn to_json(&self, mut obj: write_json::Object<'_>) { + obj.string("os", &self.os).string("cpu", &self.cpu).string("mem", &self.mem); + } +} diff --git a/src/tools/rust-analyzer/xtask/src/publish.rs b/src/tools/rust-analyzer/xtask/src/publish.rs new file mode 100644 index 00000000000..7faae9b20c4 --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/publish.rs @@ -0,0 +1,110 @@ +mod notes; + +use crate::flags; +use anyhow::bail; +use std::env; +use xshell::{cmd, Shell}; + +impl flags::PublishReleaseNotes { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + let asciidoc = sh.read_file(&self.changelog)?; + let mut markdown = notes::convert_asciidoc_to_markdown(std::io::Cursor::new(&asciidoc))?; + let file_name = check_file_name(self.changelog)?; + let tag_name = &file_name[0..10]; + let original_changelog_url = create_original_changelog_url(&file_name); + let additional_paragraph = + format!("\nSee also the [changelog post]({original_changelog_url})."); + markdown.push_str(&additional_paragraph); + if self.dry_run { + println!("{markdown}"); + } else { + update_release(sh, tag_name, &markdown)?; + } + Ok(()) + } +} + +fn check_file_name<P: AsRef<std::path::Path>>(path: P) -> anyhow::Result<String> { + let file_name = path + .as_ref() + .file_name() + .ok_or_else(|| anyhow::format_err!("file name is not specified as `changelog`"))? + .to_string_lossy(); + + let mut chars = file_name.chars(); + if file_name.len() >= 10 + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap() == '-' + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap() == '-' + && chars.next().unwrap().is_ascii_digit() + && chars.next().unwrap().is_ascii_digit() + { + Ok(file_name.to_string()) + } else { + bail!("unexpected file name format; no date information prefixed") + } +} + +fn create_original_changelog_url(file_name: &str) -> String { + let year = &file_name[0..4]; + let month = &file_name[5..7]; + let day = &file_name[8..10]; + let mut stem = &file_name[11..]; + if let Some(stripped) = stem.strip_suffix(".adoc") { + stem = stripped; + } + format!("https://rust-analyzer.github.io/thisweek/{year}/{month}/{day}/{stem}.html") +} + +fn update_release(sh: &Shell, tag_name: &str, release_notes: &str) -> anyhow::Result<()> { + let token = match env::var("GITHUB_TOKEN") { + Ok(token) => token, + Err(_) => bail!("Please obtain a personal access token from https://github.com/settings/tokens and set the `GITHUB_TOKEN` environment variable."), + }; + let accept = "Accept: application/vnd.github+json"; + let authorization = format!("Authorization: Bearer {token}"); + let api_version = "X-GitHub-Api-Version: 2022-11-28"; + let release_url = "https://api.github.com/repos/rust-lang/rust-analyzer/releases"; + + let release_json = cmd!( + sh, + "curl -sf -H {accept} -H {authorization} -H {api_version} {release_url}/tags/{tag_name}" + ) + .read()?; + let release_id = cmd!(sh, "jq .id").stdin(release_json).read()?; + + let mut patch = String::new(); + // note: the GitHub API doesn't update the target commit if the tag already exists + write_json::object(&mut patch) + .string("tag_name", tag_name) + .string("target_commitish", "master") + .string("name", tag_name) + .string("body", release_notes) + .bool("draft", false) + .bool("prerelease", false); + let _ = cmd!( + sh, + "curl -sf -X PATCH -H {accept} -H {authorization} -H {api_version} {release_url}/{release_id} -d {patch}" + ) + .read()?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn original_changelog_url_creation() { + let input = "2019-07-24-changelog-0.adoc"; + let actual = create_original_changelog_url(input); + let expected = "https://rust-analyzer.github.io/thisweek/2019/07/24/changelog-0.html"; + assert_eq!(actual, expected); + } +} diff --git a/src/tools/rust-analyzer/xtask/src/publish/notes.rs b/src/tools/rust-analyzer/xtask/src/publish/notes.rs new file mode 100644 index 00000000000..c30267295bf --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/publish/notes.rs @@ -0,0 +1,631 @@ +use anyhow::{anyhow, bail}; +use std::{ + borrow::Cow, + io::{BufRead, Lines}, + iter::Peekable, +}; + +const LISTING_DELIMITER: &str = "----"; +const IMAGE_BLOCK_PREFIX: &str = "image::"; +const VIDEO_BLOCK_PREFIX: &str = "video::"; + +struct Converter<'a, 'b, R: BufRead> { + iter: &'a mut Peekable<Lines<R>>, + output: &'b mut String, +} + +impl<'a, 'b, R: BufRead> Converter<'a, 'b, R> { + fn new(iter: &'a mut Peekable<Lines<R>>, output: &'b mut String) -> Self { + Self { iter, output } + } + + fn process(&mut self) -> anyhow::Result<()> { + self.process_document_header()?; + self.skip_blank_lines()?; + self.output.push('\n'); + + loop { + let line = self.iter.peek().unwrap().as_deref().map_err(|e| anyhow!("{e}"))?; + if get_title(line).is_some() { + let line = self.iter.next().unwrap().unwrap(); + let (level, title) = get_title(&line).unwrap(); + self.write_title(level, title); + } else if get_list_item(line).is_some() { + self.process_list()?; + } else if line.starts_with('[') { + self.process_source_code_block(0)?; + } else if line.starts_with(LISTING_DELIMITER) { + self.process_listing_block(None, 0)?; + } else if line.starts_with('.') { + self.process_block_with_title(0)?; + } else if line.starts_with(IMAGE_BLOCK_PREFIX) { + self.process_image_block(None, 0)?; + } else if line.starts_with(VIDEO_BLOCK_PREFIX) { + self.process_video_block(None, 0)?; + } else { + self.process_paragraph(0, |line| line.is_empty())?; + } + + self.skip_blank_lines()?; + if self.iter.peek().is_none() { + break; + } + self.output.push('\n'); + } + Ok(()) + } + + fn process_document_header(&mut self) -> anyhow::Result<()> { + self.process_document_title()?; + + while let Some(line) = self.iter.next() { + let line = line?; + if line.is_empty() { + break; + } + if !line.starts_with(':') { + self.write_line(&line, 0) + } + } + + Ok(()) + } + + fn process_document_title(&mut self) -> anyhow::Result<()> { + if let Some(Ok(line)) = self.iter.next() { + if let Some((level, title)) = get_title(&line) { + let title = process_inline_macros(title)?; + if level == 1 { + self.write_title(level, &title); + return Ok(()); + } + } + } + bail!("document title not found") + } + + fn process_list(&mut self) -> anyhow::Result<()> { + let mut nesting = ListNesting::new(); + while let Some(line) = self.iter.peek() { + let line = line.as_deref().map_err(|e| anyhow!("{e}"))?; + + if get_list_item(line).is_some() { + let line = self.iter.next().unwrap()?; + let line = process_inline_macros(&line)?; + let (marker, item) = get_list_item(&line).unwrap(); + nesting.set_current(marker); + self.write_list_item(item, &nesting); + self.process_paragraph(nesting.indent(), |line| { + line.is_empty() || get_list_item(line).is_some() || line == "+" + })?; + } else if line == "+" { + let _ = self.iter.next().unwrap()?; + let line = self + .iter + .peek() + .ok_or_else(|| anyhow!("list continuation unexpectedly terminated"))?; + let line = line.as_deref().map_err(|e| anyhow!("{e}"))?; + + let indent = nesting.indent(); + if line.starts_with('[') { + self.write_line("", 0); + self.process_source_code_block(indent)?; + } else if line.starts_with(LISTING_DELIMITER) { + self.write_line("", 0); + self.process_listing_block(None, indent)?; + } else if line.starts_with('.') { + self.write_line("", 0); + self.process_block_with_title(indent)?; + } else if line.starts_with(IMAGE_BLOCK_PREFIX) { + self.write_line("", 0); + self.process_image_block(None, indent)?; + } else if line.starts_with(VIDEO_BLOCK_PREFIX) { + self.write_line("", 0); + self.process_video_block(None, indent)?; + } else { + self.write_line("", 0); + let current = nesting.current().unwrap(); + self.process_paragraph(indent, |line| { + line.is_empty() + || get_list_item(line).filter(|(m, _)| m == current).is_some() + || line == "+" + })?; + } + } else { + break; + } + self.skip_blank_lines()?; + } + + Ok(()) + } + + fn process_source_code_block(&mut self, level: usize) -> anyhow::Result<()> { + if let Some(Ok(line)) = self.iter.next() { + if let Some(styles) = line.strip_prefix("[source").and_then(|s| s.strip_suffix(']')) { + let mut styles = styles.split(','); + if !styles.next().unwrap().is_empty() { + bail!("not a source code block"); + } + let language = styles.next(); + return self.process_listing_block(language, level); + } + } + bail!("not a source code block") + } + + fn process_listing_block(&mut self, style: Option<&str>, level: usize) -> anyhow::Result<()> { + if let Some(Ok(line)) = self.iter.next() { + if line == LISTING_DELIMITER { + self.write_indent(level); + self.output.push_str("```"); + if let Some(style) = style { + self.output.push_str(style); + } + self.output.push('\n'); + while let Some(line) = self.iter.next() { + let line = line?; + if line == LISTING_DELIMITER { + self.write_line("```", level); + return Ok(()); + } else { + self.write_line(&line, level); + } + } + bail!("listing block is not terminated") + } + } + bail!("not a listing block") + } + + fn process_block_with_title(&mut self, level: usize) -> anyhow::Result<()> { + if let Some(Ok(line)) = self.iter.next() { + let title = + line.strip_prefix('.').ok_or_else(|| anyhow!("extraction of the title failed"))?; + + let line = self + .iter + .peek() + .ok_or_else(|| anyhow!("target block for the title is not found"))?; + let line = line.as_deref().map_err(|e| anyhow!("{e}"))?; + if line.starts_with(IMAGE_BLOCK_PREFIX) { + return self.process_image_block(Some(title), level); + } else if line.starts_with(VIDEO_BLOCK_PREFIX) { + return self.process_video_block(Some(title), level); + } else { + bail!("title for that block type is not supported"); + } + } + bail!("not a title") + } + + fn process_image_block(&mut self, caption: Option<&str>, level: usize) -> anyhow::Result<()> { + if let Some(Ok(line)) = self.iter.next() { + if let Some((url, attrs)) = parse_media_block(&line, IMAGE_BLOCK_PREFIX) { + let alt = if let Some(stripped) = + attrs.strip_prefix('"').and_then(|s| s.strip_suffix('"')) + { + stripped + } else { + attrs + }; + if let Some(caption) = caption { + self.write_caption_line(caption, level); + } + self.write_indent(level); + self.output.push_str("!["); + self.output.push_str(alt); + self.output.push_str("]("); + self.output.push_str(url); + self.output.push_str(")\n"); + return Ok(()); + } + } + bail!("not a image block") + } + + fn process_video_block(&mut self, caption: Option<&str>, level: usize) -> anyhow::Result<()> { + if let Some(Ok(line)) = self.iter.next() { + if let Some((url, attrs)) = parse_media_block(&line, VIDEO_BLOCK_PREFIX) { + let html_attrs = match attrs { + "options=loop" => "controls loop", + r#"options="autoplay,loop""# => "autoplay controls loop", + _ => bail!("unsupported video syntax"), + }; + if let Some(caption) = caption { + self.write_caption_line(caption, level); + } + self.write_indent(level); + self.output.push_str(r#"<video src=""#); + self.output.push_str(url); + self.output.push_str(r#"" "#); + self.output.push_str(html_attrs); + self.output.push_str(">Your browser does not support the video tag.</video>\n"); + return Ok(()); + } + } + bail!("not a video block") + } + + fn process_paragraph<P>(&mut self, level: usize, predicate: P) -> anyhow::Result<()> + where + P: Fn(&str) -> bool, + { + while let Some(line) = self.iter.peek() { + let line = line.as_deref().map_err(|e| anyhow!("{e}"))?; + if predicate(line) { + break; + } + + self.write_indent(level); + let line = self.iter.next().unwrap()?; + let line = line.trim_start(); + let line = process_inline_macros(line)?; + if let Some(stripped) = line.strip_suffix('+') { + self.output.push_str(stripped); + self.output.push('\\'); + } else { + self.output.push_str(&line); + } + self.output.push('\n'); + } + + Ok(()) + } + + fn skip_blank_lines(&mut self) -> anyhow::Result<()> { + while let Some(line) = self.iter.peek() { + if !line.as_deref().unwrap().is_empty() { + break; + } + self.iter.next().unwrap()?; + } + Ok(()) + } + + fn write_title(&mut self, indent: usize, title: &str) { + for _ in 0..indent { + self.output.push('#'); + } + self.output.push(' '); + self.output.push_str(title); + self.output.push('\n'); + } + + fn write_list_item(&mut self, item: &str, nesting: &ListNesting) { + let (marker, indent) = nesting.marker(); + self.write_indent(indent); + self.output.push_str(marker); + self.output.push_str(item); + self.output.push('\n'); + } + + fn write_caption_line(&mut self, caption: &str, indent: usize) { + self.write_indent(indent); + self.output.push('_'); + self.output.push_str(caption); + self.output.push_str("_\\\n"); + } + + fn write_indent(&mut self, indent: usize) { + for _ in 0..indent { + self.output.push(' '); + } + } + + fn write_line(&mut self, line: &str, indent: usize) { + self.write_indent(indent); + self.output.push_str(line); + self.output.push('\n'); + } +} + +pub(crate) fn convert_asciidoc_to_markdown<R>(input: R) -> anyhow::Result<String> +where + R: BufRead, +{ + let mut output = String::new(); + let mut iter = input.lines().peekable(); + + let mut converter = Converter::new(&mut iter, &mut output); + converter.process()?; + + Ok(output) +} + +fn get_title(line: &str) -> Option<(usize, &str)> { + strip_prefix_symbol(line, '=') +} + +fn get_list_item(line: &str) -> Option<(ListMarker, &str)> { + const HYPHEN_MARKER: &str = "- "; + if let Some(text) = line.strip_prefix(HYPHEN_MARKER) { + Some((ListMarker::Hyphen, text)) + } else if let Some((count, text)) = strip_prefix_symbol(line, '*') { + Some((ListMarker::Asterisk(count), text)) + } else if let Some((count, text)) = strip_prefix_symbol(line, '.') { + Some((ListMarker::Dot(count), text)) + } else { + None + } +} + +fn strip_prefix_symbol(line: &str, symbol: char) -> Option<(usize, &str)> { + let mut iter = line.chars(); + if iter.next()? != symbol { + return None; + } + let mut count = 1; + loop { + match iter.next() { + Some(ch) if ch == symbol => { + count += 1; + } + Some(' ') => { + break; + } + _ => return None, + } + } + Some((count, iter.as_str())) +} + +fn parse_media_block<'a>(line: &'a str, prefix: &str) -> Option<(&'a str, &'a str)> { + if let Some(line) = line.strip_prefix(prefix) { + if let Some((url, rest)) = line.split_once('[') { + if let Some(attrs) = rest.strip_suffix(']') { + return Some((url, attrs)); + } + } + } + None +} + +#[derive(Debug)] +struct ListNesting(Vec<ListMarker>); + +impl ListNesting { + fn new() -> Self { + Self(Vec::<ListMarker>::with_capacity(6)) + } + + fn current(&mut self) -> Option<&ListMarker> { + self.0.last() + } + + fn set_current(&mut self, marker: ListMarker) { + let Self(markers) = self; + if let Some(index) = markers.iter().position(|m| *m == marker) { + markers.truncate(index + 1); + } else { + markers.push(marker); + } + } + + fn indent(&self) -> usize { + self.0.iter().map(|m| m.in_markdown().len()).sum() + } + + fn marker(&self) -> (&str, usize) { + let Self(markers) = self; + let indent = markers.iter().take(markers.len() - 1).map(|m| m.in_markdown().len()).sum(); + let marker = match markers.last() { + None => "", + Some(marker) => marker.in_markdown(), + }; + (marker, indent) + } +} + +#[derive(Debug, PartialEq, Eq)] +enum ListMarker { + Asterisk(usize), + Hyphen, + Dot(usize), +} + +impl ListMarker { + fn in_markdown(&self) -> &str { + match self { + ListMarker::Asterisk(_) => "- ", + ListMarker::Hyphen => "- ", + ListMarker::Dot(_) => "1. ", + } + } +} + +fn process_inline_macros(line: &str) -> anyhow::Result<Cow<'_, str>> { + let mut chars = line.char_indices(); + loop { + let (start, end, a_macro) = match get_next_line_component(&mut chars) { + Component::None => break, + Component::Text => continue, + Component::Macro(s, e, m) => (s, e, m), + }; + let mut src = line.chars(); + let mut processed = String::new(); + for _ in 0..start { + processed.push(src.next().unwrap()); + } + processed.push_str(a_macro.process()?.as_str()); + for _ in start..end { + let _ = src.next().unwrap(); + } + let mut pos = end; + + loop { + let (start, end, a_macro) = match get_next_line_component(&mut chars) { + Component::None => break, + Component::Text => continue, + Component::Macro(s, e, m) => (s, e, m), + }; + for _ in pos..start { + processed.push(src.next().unwrap()); + } + processed.push_str(a_macro.process()?.as_str()); + for _ in start..end { + let _ = src.next().unwrap(); + } + pos = end; + } + for ch in src { + processed.push(ch); + } + return Ok(Cow::Owned(processed)); + } + Ok(Cow::Borrowed(line)) +} + +fn get_next_line_component(chars: &mut std::str::CharIndices<'_>) -> Component { + let (start, mut macro_name) = match chars.next() { + None => return Component::None, + Some((_, ch)) if ch == ' ' || !ch.is_ascii() => return Component::Text, + Some((pos, ch)) => (pos, String::from(ch)), + }; + loop { + match chars.next() { + None => return Component::None, + Some((_, ch)) if ch == ' ' || !ch.is_ascii() => return Component::Text, + Some((_, ':')) => break, + Some((_, ch)) => macro_name.push(ch), + } + } + + let mut macro_target = String::new(); + loop { + match chars.next() { + None => return Component::None, + Some((_, ' ')) => return Component::Text, + Some((_, '[')) => break, + Some((_, ch)) => macro_target.push(ch), + } + } + + let mut attr_value = String::new(); + let end = loop { + match chars.next() { + None => return Component::None, + Some((pos, ']')) => break pos + 1, + Some((_, ch)) => attr_value.push(ch), + } + }; + + Component::Macro(start, end, Macro::new(macro_name, macro_target, attr_value)) +} + +enum Component { + None, + Text, + Macro(usize, usize, Macro), +} + +struct Macro { + name: String, + target: String, + attrs: String, +} + +impl Macro { + fn new(name: String, target: String, attrs: String) -> Self { + Self { name, target, attrs } + } + + fn process(&self) -> anyhow::Result<String> { + let name = &self.name; + let text = match name.as_str() { + "https" => { + let url = &self.target; + let anchor_text = &self.attrs; + format!("[{anchor_text}](https:{url})") + } + "image" => { + let url = &self.target; + let alt = &self.attrs; + format!("") + } + "kbd" => { + let keys = self.attrs.split('+').map(|k| Cow::Owned(format!("<kbd>{k}</kbd>"))); + keys.collect::<Vec<_>>().join("+") + } + "pr" => { + let pr = &self.target; + let url = format!("https://github.com/rust-analyzer/rust-analyzer/pull/{pr}"); + format!("[`#{pr}`]({url})") + } + "commit" => { + let hash = &self.target; + let short = &hash[0..7]; + let url = format!("https://github.com/rust-analyzer/rust-analyzer/commit/{hash}"); + format!("[`{short}`]({url})") + } + "release" => { + let date = &self.target; + let url = format!("https://github.com/rust-analyzer/rust-analyzer/releases/{date}"); + format!("[`{date}`]({url})") + } + _ => bail!("macro not supported: {name}"), + }; + Ok(text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::read_to_string; + + #[test] + fn test_asciidoc_to_markdown_conversion() { + let input = read_to_string("test_data/input.adoc").unwrap(); + let expected = read_to_string("test_data/expected.md").unwrap(); + let actual = convert_asciidoc_to_markdown(std::io::Cursor::new(&input)).unwrap(); + + assert_eq!(actual, expected); + } + + macro_rules! test_inline_macro_processing { + ($(( + $name:ident, + $input:expr, + $expected:expr + ),)*) => ($( + #[test] + fn $name() { + let input = $input; + let actual = process_inline_macros(&input).unwrap(); + let expected = $expected; + assert_eq!(actual, expected) + } + )*); + } + + test_inline_macro_processing! { + (inline_macro_processing_for_empty_line, "", ""), + (inline_macro_processing_for_line_with_no_macro, "foo bar", "foo bar"), + ( + inline_macro_processing_for_macro_in_line_start, + "kbd::[Ctrl+T] foo", + "<kbd>Ctrl</kbd>+<kbd>T</kbd> foo" + ), + ( + inline_macro_processing_for_macro_in_line_end, + "foo kbd::[Ctrl+T]", + "foo <kbd>Ctrl</kbd>+<kbd>T</kbd>" + ), + ( + inline_macro_processing_for_macro_in_the_middle_of_line, + "foo kbd::[Ctrl+T] foo", + "foo <kbd>Ctrl</kbd>+<kbd>T</kbd> foo" + ), + ( + inline_macro_processing_for_several_macros, + "foo kbd::[Ctrl+T] foo kbd::[Enter] foo", + "foo <kbd>Ctrl</kbd>+<kbd>T</kbd> foo <kbd>Enter</kbd> foo" + ), + ( + inline_macro_processing_for_several_macros_without_text_in_between, + "foo kbd::[Ctrl+T]kbd::[Enter] foo", + "foo <kbd>Ctrl</kbd>+<kbd>T</kbd><kbd>Enter</kbd> foo" + ), + } +} diff --git a/src/tools/rust-analyzer/xtask/src/release.rs b/src/tools/rust-analyzer/xtask/src/release.rs new file mode 100644 index 00000000000..1a5e6dfb4cc --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/release.rs @@ -0,0 +1,96 @@ +mod changelog; + +use xshell::{cmd, Shell}; + +use crate::{codegen, date_iso, flags, is_release_tag, project_root}; + +impl flags::Release { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + if !self.dry_run { + cmd!(sh, "git switch release").run()?; + cmd!(sh, "git fetch upstream --tags --force").run()?; + cmd!(sh, "git reset --hard tags/nightly").run()?; + // The `release` branch sometimes has a couple of cherry-picked + // commits for patch releases. If that's the case, just overwrite + // it. As we are setting `release` branch to an up-to-date `nightly` + // tag, this shouldn't be problematic in general. + // + // Note that, as we tag releases, we don't worry about "losing" + // commits -- they'll be kept alive by the tag. More generally, we + // don't care about historic releases all that much, it's fine even + // to delete old tags. + cmd!(sh, "git push --force").run()?; + } + + // Generates bits of manual.adoc. + codegen::diagnostics_docs::generate(false); + codegen::assists_doc_tests::generate(false); + + let website_root = project_root().join("../rust-analyzer.github.io"); + { + let _dir = sh.push_dir(&website_root); + cmd!(sh, "git switch src").run()?; + cmd!(sh, "git pull").run()?; + } + let changelog_dir = website_root.join("./thisweek/_posts"); + + let today = date_iso(sh)?; + let commit = cmd!(sh, "git rev-parse HEAD").read()?; + let changelog_n = sh + .read_dir(changelog_dir.as_path())? + .into_iter() + .filter_map(|p| p.file_stem().map(|s| s.to_string_lossy().to_string())) + .filter_map(|s| s.splitn(5, '-').last().map(|n| n.replace('-', "."))) + .filter_map(|s| s.parse::<f32>().ok()) + .map(|n| 1 + n.floor() as usize) + .max() + .unwrap_or_default(); + + for adoc in [ + "manual.adoc", + "generated_assists.adoc", + "generated_config.adoc", + "generated_diagnostic.adoc", + "generated_features.adoc", + ] { + let src = project_root().join("./docs/user/").join(adoc); + let dst = website_root.join(adoc); + + let contents = sh.read_file(src)?; + sh.write_file(dst, contents)?; + } + + let tags = cmd!(sh, "git tag --list").read()?; + let prev_tag = tags.lines().filter(|line| is_release_tag(line)).last().unwrap(); + + let contents = changelog::get_changelog(sh, changelog_n, &commit, prev_tag, &today)?; + let path = changelog_dir.join(format!("{today}-changelog-{changelog_n}.adoc")); + sh.write_file(path, contents)?; + + Ok(()) + } +} + +impl flags::Promote { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + let _dir = sh.push_dir("../rust-rust-analyzer"); + cmd!(sh, "git switch master").run()?; + cmd!(sh, "git fetch upstream").run()?; + cmd!(sh, "git reset --hard upstream/master").run()?; + + let date = date_iso(sh)?; + let branch = format!("rust-analyzer-{date}"); + cmd!(sh, "git switch -c {branch}").run()?; + cmd!(sh, "git subtree pull -m ':arrow_up: rust-analyzer' -P src/tools/rust-analyzer rust-analyzer release").run()?; + + if !self.dry_run { + cmd!(sh, "git push -u origin {branch}").run()?; + cmd!( + sh, + "xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost" + ) + .run()?; + } + Ok(()) + } +} diff --git a/src/tools/rust-analyzer/xtask/src/release/changelog.rs b/src/tools/rust-analyzer/xtask/src/release/changelog.rs new file mode 100644 index 00000000000..086a4d463ea --- /dev/null +++ b/src/tools/rust-analyzer/xtask/src/release/changelog.rs @@ -0,0 +1,187 @@ +use std::fmt::Write; +use std::{env, iter}; + +use anyhow::bail; +use xshell::{cmd, Shell}; + +pub(crate) fn get_changelog( + sh: &Shell, + changelog_n: usize, + commit: &str, + prev_tag: &str, + today: &str, +) -> anyhow::Result<String> { + let token = match env::var("GITHUB_TOKEN") { + Ok(token) => token, + Err(_) => bail!("Please obtain a personal access token from https://github.com/settings/tokens and set the `GITHUB_TOKEN` environment variable."), + }; + + let git_log = cmd!(sh, "git log {prev_tag}..HEAD --reverse").read()?; + let mut features = String::new(); + let mut fixes = String::new(); + let mut internal = String::new(); + let mut others = String::new(); + for line in git_log.lines() { + let line = line.trim_start(); + if let Some(pr_num) = parse_pr_number(line) { + let accept = "Accept: application/vnd.github.v3+json"; + let authorization = format!("Authorization: token {token}"); + let pr_url = "https://api.github.com/repos/rust-lang/rust-analyzer/issues"; + + // we don't use an HTTPS client or JSON parser to keep the build times low + let pr = pr_num.to_string(); + let cmd = &cmd!(sh, "curl --fail -s -H {accept} -H {authorization} {pr_url}/{pr}"); + let pr_json = match cmd.read() { + Ok(pr_json) => pr_json, + Err(e) => { + // most likely a rust-lang/rust PR + eprintln!("Cannot get info for #{pr}: {e}"); + continue; + } + }; + + let pr_title = cmd!(sh, "jq .title").stdin(&pr_json).read()?; + let pr_title = unescape(&pr_title[1..pr_title.len() - 1]); + let pr_comment = cmd!(sh, "jq .body").stdin(pr_json).read()?; + + let cmd = + &cmd!(sh, "curl --fail -s -H {accept} -H {authorization} {pr_url}/{pr}/comments"); + let pr_info = match cmd.read() { + Ok(comments_json) => { + let pr_comments = cmd!(sh, "jq .[].body").stdin(comments_json).read()?; + + iter::once(pr_comment.as_str()) + .chain(pr_comments.lines()) + .rev() + .find_map(|it| { + let it = unescape(&it[1..it.len() - 1]); + it.lines().find_map(parse_changelog_line) + }) + .into_iter() + .next() + } + Err(e) => { + eprintln!("Cannot get comments for #{pr}: {e}"); + None + } + }; + + let pr_info = pr_info.unwrap_or_else(|| parse_title_line(&pr_title)); + let s = match pr_info.kind { + PrKind::Feature => &mut features, + PrKind::Fix => &mut fixes, + PrKind::Internal => &mut internal, + PrKind::Other => &mut others, + PrKind::Skip => continue, + }; + writeln!(s, "* pr:{pr_num}[] {}", pr_info.message.as_deref().unwrap_or(&pr_title)) + .unwrap(); + } + } + + let contents = format!( + "\ += Changelog #{changelog_n} +:sectanchors: +:experimental: +:page-layout: post + +Commit: commit:{commit}[] + +Release: release:{today}[] (`TBD`) + +== New Features + +{features} + +== Fixes + +{fixes} + +== Internal Improvements + +{internal} + +== Others + +{others} +" + ); + Ok(contents) +} + +#[derive(Clone, Copy)] +enum PrKind { + Feature, + Fix, + Internal, + Other, + Skip, +} + +struct PrInfo { + message: Option<String>, + kind: PrKind, +} + +fn unescape(s: &str) -> String { + s.replace(r#"\""#, "").replace(r#"\n"#, "\n").replace(r#"\r"#, "") +} + +fn parse_pr_number(s: &str) -> Option<u32> { + const BORS_PREFIX: &str = "Merge #"; + const HOMU_PREFIX: &str = "Auto merge of #"; + if let Some(s) = s.strip_prefix(BORS_PREFIX) { + s.parse().ok() + } else if let Some(s) = s.strip_prefix(HOMU_PREFIX) { + if let Some(space) = s.find(' ') { + s[..space].parse().ok() + } else { + None + } + } else { + None + } +} + +fn parse_changelog_line(s: &str) -> Option<PrInfo> { + let parts = s.splitn(3, ' ').collect::<Vec<_>>(); + if parts.len() < 2 || parts[0] != "changelog" { + return None; + } + let message = parts.get(2).map(|it| it.to_string()); + let kind = match parts[1].trim_end_matches(':') { + "feature" => PrKind::Feature, + "fix" => PrKind::Fix, + "internal" => PrKind::Internal, + "skip" => PrKind::Skip, + _ => { + let kind = PrKind::Other; + let message = format!("{} {}", parts[1], message.unwrap_or_default()); + return Some(PrInfo { kind, message: Some(message) }); + } + }; + let res = PrInfo { message, kind }; + Some(res) +} + +fn parse_title_line(s: &str) -> PrInfo { + let lower = s.to_ascii_lowercase(); + const PREFIXES: [(&str, PrKind); 5] = [ + ("feat: ", PrKind::Feature), + ("feature: ", PrKind::Feature), + ("fix: ", PrKind::Fix), + ("internal: ", PrKind::Internal), + ("minor: ", PrKind::Skip), + ]; + + for (prefix, kind) in PREFIXES { + if lower.starts_with(prefix) { + let message = match &kind { + PrKind::Skip => None, + _ => Some(s[prefix.len()..].to_string()), + }; + return PrInfo { message, kind }; + } + } + PrInfo { kind: PrKind::Other, message: Some(s.to_owned()) } +} diff --git a/src/tools/rust-analyzer/xtask/test_data/expected.md b/src/tools/rust-analyzer/xtask/test_data/expected.md new file mode 100644 index 00000000000..19c940c67bd --- /dev/null +++ b/src/tools/rust-analyzer/xtask/test_data/expected.md @@ -0,0 +1,81 @@ +# Changelog #256 + +Hello! + +Commit: [`0123456`](https://github.com/rust-analyzer/rust-analyzer/commit/0123456789abcdef0123456789abcdef01234567) \ +Release: [`2022-01-01`](https://github.com/rust-analyzer/rust-analyzer/releases/2022-01-01) + +## New Features + +- **BREAKING** [`#1111`](https://github.com/rust-analyzer/rust-analyzer/pull/1111) shortcut <kbd>ctrl</kbd>+<kbd>r</kbd> + - hyphen-prefixed list item +- nested list item + - `foo` -> `foofoo` + - `bar` -> `barbar` +- listing in the secondary level + 1. install + 1. add to config + + ```json + {"foo":"bar"} + ``` +- list item with continuation + +  + +  + + <video src="https://example.com/movie.mp4" controls loop>Your browser does not support the video tag.</video> + + <video src="https://example.com/movie.mp4" autoplay controls loop>Your browser does not support the video tag.</video> + + _Image_\ +  + + _Video_\ + <video src="https://example.com/movie.mp4" controls loop>Your browser does not support the video tag.</video> + + ```bash + rustup update nightly + ``` + + ``` + This is a plain listing. + ``` +- single line item followed by empty lines +- multiline list + item followed by empty lines +- multiline list + item with indent +- multiline list + item not followed by empty lines +- multiline list + item followed by different marker + - foo + - bar +- multiline list + item followed by list continuation + + paragraph + paragraph + +## Another Section + +- foo bar baz +- list item with an inline image +  + +The highlight of the month is probably [`#1111`](https://github.com/rust-analyzer/rust-analyzer/pull/1111). +See [online manual](https://example.com/manual) for more information. + +```bash +rustup update nightly +``` + +``` +rustup update nightly +``` + +``` +This is a plain listing. +``` diff --git a/src/tools/rust-analyzer/xtask/test_data/input.adoc b/src/tools/rust-analyzer/xtask/test_data/input.adoc new file mode 100644 index 00000000000..105bd8df0db --- /dev/null +++ b/src/tools/rust-analyzer/xtask/test_data/input.adoc @@ -0,0 +1,90 @@ += Changelog #256 +:sectanchors: +:page-layout: post + +Hello! + +Commit: commit:0123456789abcdef0123456789abcdef01234567[] + +Release: release:2022-01-01[] + +== New Features + +* **BREAKING** pr:1111[] shortcut kbd:[ctrl+r] +- hyphen-prefixed list item +* nested list item +** `foo` -> `foofoo` +** `bar` -> `barbar` +* listing in the secondary level +. install +. add to config ++ +[source,json] +---- +{"foo":"bar"} +---- +* list item with continuation ++ +image::https://example.com/animation.gif[] ++ +image::https://example.com/animation.gif["alt text"] ++ +video::https://example.com/movie.mp4[options=loop] ++ +video::https://example.com/movie.mp4[options="autoplay,loop"] ++ +.Image +image::https://example.com/animation.gif[] ++ +.Video +video::https://example.com/movie.mp4[options=loop] ++ +[source,bash] +---- +rustup update nightly +---- ++ +---- +This is a plain listing. +---- +* single line item followed by empty lines + +* multiline list +item followed by empty lines + +* multiline list + item with indent + +* multiline list +item not followed by empty lines +* multiline list +item followed by different marker +** foo +** bar +* multiline list +item followed by list continuation ++ +paragraph +paragraph + +== Another Section + +* foo bar baz +* list item with an inline image + image:https://example.com/animation.gif[] + +The highlight of the month is probably pr:1111[]. +See https://example.com/manual[online manual] for more information. + +[source,bash] +---- +rustup update nightly +---- + +[source] +---- +rustup update nightly +---- + +---- +This is a plain listing. +---- |