diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Cargo.lock | 4 | ||||
| -rw-r--r-- | crates/ide/src/fixture.rs | 19 | ||||
| -rw-r--r-- | crates/ide/src/lib.rs | 2 | ||||
| -rw-r--r-- | crates/ide/src/static_index.rs | 256 | ||||
| -rw-r--r-- | crates/rust-analyzer/Cargo.toml | 2 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/bin/main.rs | 1 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli.rs | 1 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/analysis_stats.rs | 2 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/flags.rs | 10 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/lsif.rs | 268 |
11 files changed, 562 insertions, 5 deletions
diff --git a/.gitignore b/.gitignore index 7e097c01585..f3e3cab1d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ generated_assists.adoc generated_features.adoc generated_diagnostic.adoc .DS_Store +/out/ +/dump.lsif diff --git a/Cargo.lock b/Cargo.lock index 9da1f56deaf..47286277018 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -862,9 +862,9 @@ dependencies = [ [[package]] name = "lsp-types" -version = "0.89.2" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852e0dedfd52cc32325598b2631e0eba31b7b708959676a9f837042f276b09a2" +checksum = "a7404037aab080771c90b0a499836d9d8a10336ecd07badf969567b65c6d51a1" dependencies = [ "bitflags", "serde", diff --git a/crates/ide/src/fixture.rs b/crates/ide/src/fixture.rs index 700f4dc9550..2ea6f6a9ab1 100644 --- a/crates/ide/src/fixture.rs +++ b/crates/ide/src/fixture.rs @@ -66,3 +66,22 @@ pub(crate) fn annotations(ra_fixture: &str) -> (Analysis, FilePosition, Vec<(Fil .collect(); (host.analysis(), FilePosition { file_id, offset }, annotations) } + +/// Creates analysis from a multi-file fixture with annonations without $0 +pub(crate) fn annotations_without_marker(ra_fixture: &str) -> (Analysis, Vec<(FileRange, String)>) { + let mut host = AnalysisHost::default(); + let change_fixture = ChangeFixture::parse(ra_fixture); + host.db.set_enable_proc_attr_macros(true); + host.db.apply_change(change_fixture.change); + + let annotations = change_fixture + .files + .iter() + .flat_map(|&file_id| { + let file_text = host.analysis().file_text(file_id).unwrap(); + let annotations = extract_annotations(&file_text); + annotations.into_iter().map(move |(range, data)| (FileRange { file_id, range }, data)) + }) + .collect(); + (host.analysis(), annotations) +} diff --git a/crates/ide/src/lib.rs b/crates/ide/src/lib.rs index 21872c81d13..dbfa99bdf24 100644 --- a/crates/ide/src/lib.rs +++ b/crates/ide/src/lib.rs @@ -46,6 +46,7 @@ mod references; mod rename; mod runnables; mod ssr; +mod static_index; mod status; mod syntax_highlighting; mod syntax_tree; @@ -86,6 +87,7 @@ pub use crate::{ references::ReferenceSearchResult, rename::RenameError, runnables::{Runnable, RunnableKind, TestId}, + static_index::{StaticIndex, StaticIndexedFile, TokenId, TokenStaticData}, syntax_highlighting::{ tags::{Highlight, HlMod, HlMods, HlOperator, HlPunct, HlTag}, HlRange, diff --git a/crates/ide/src/static_index.rs b/crates/ide/src/static_index.rs new file mode 100644 index 00000000000..aa62e2eae5a --- /dev/null +++ b/crates/ide/src/static_index.rs @@ -0,0 +1,256 @@ +//! This module provides `StaticIndex` which is used for powering +//! read-only code browsers and emitting LSIF + +use std::collections::HashMap; + +use hir::Semantics; +use hir::{db::HirDatabase, Crate, Module}; +use ide_db::base_db::{FileId, FileRange, SourceDatabaseExt}; +use ide_db::defs::Definition; +use ide_db::RootDatabase; +use rustc_hash::FxHashSet; +use syntax::{AstNode, SyntaxKind::*, T}; +use syntax::{SyntaxToken, TextRange}; + +use crate::display::TryToNav; +use crate::hover::hover_for_definition; +use crate::{Analysis, Fold, HoverConfig, HoverDocFormat, HoverResult}; + +/// A static representation of fully analyzed source code. +/// +/// The intended use-case is powering read-only code browsers and emitting LSIF +pub struct StaticIndex<'a> { + pub files: Vec<StaticIndexedFile>, + pub tokens: TokenStore, + analysis: &'a Analysis, + db: &'a RootDatabase, + def_map: HashMap<Definition, TokenId>, +} + +pub struct ReferenceData { + pub range: FileRange, + pub is_definition: bool, +} + +pub struct TokenStaticData { + pub hover: Option<HoverResult>, + pub definition: Option<FileRange>, + pub references: Vec<ReferenceData>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct TokenId(usize); + +#[derive(Default)] +pub struct TokenStore(Vec<TokenStaticData>); + +impl TokenStore { + pub fn insert(&mut self, data: TokenStaticData) -> TokenId { + let id = TokenId(self.0.len()); + self.0.push(data); + id + } + + pub fn get_mut(&mut self, id: TokenId) -> Option<&mut TokenStaticData> { + self.0.get_mut(id.0) + } + + pub fn get(&self, id: TokenId) -> Option<&TokenStaticData> { + self.0.get(id.0) + } + + pub fn iter(self) -> impl Iterator<Item = (TokenId, TokenStaticData)> { + self.0.into_iter().enumerate().map(|(i, x)| (TokenId(i), x)) + } +} + +pub struct StaticIndexedFile { + pub file_id: FileId, + pub folds: Vec<Fold>, + pub tokens: Vec<(TextRange, TokenId)>, +} + +fn all_modules(db: &dyn HirDatabase) -> Vec<Module> { + let mut worklist: Vec<_> = + Crate::all(db).into_iter().map(|krate| krate.root_module(db)).collect(); + let mut modules = Vec::new(); + + while let Some(module) = worklist.pop() { + modules.push(module); + worklist.extend(module.children(db)); + } + + modules +} + +impl StaticIndex<'_> { + fn add_file(&mut self, file_id: FileId) { + let folds = self.analysis.folding_ranges(file_id).unwrap(); + // hovers + let sema = hir::Semantics::new(self.db); + let tokens_or_nodes = sema.parse(file_id).syntax().clone(); + let tokens = tokens_or_nodes.descendants_with_tokens().filter_map(|x| match x { + syntax::NodeOrToken::Node(_) => None, + syntax::NodeOrToken::Token(x) => Some(x), + }); + let hover_config = + HoverConfig { links_in_hover: true, documentation: Some(HoverDocFormat::Markdown) }; + let tokens = tokens.filter(|token| match token.kind() { + IDENT | INT_NUMBER | LIFETIME_IDENT | T![self] | T![super] | T![crate] => true, + _ => false, + }); + let mut result = StaticIndexedFile { file_id, folds, tokens: vec![] }; + for token in tokens { + let range = token.text_range(); + let node = token.parent().unwrap(); + let def = if let Some(x) = get_definition(&sema, token.clone()) { + x + } else { + continue; + }; + let id = if let Some(x) = self.def_map.get(&def) { + *x + } else { + let x = self.tokens.insert(TokenStaticData { + hover: hover_for_definition(&sema, file_id, def, &node, &hover_config), + definition: def + .try_to_nav(self.db) + .map(|x| FileRange { file_id: x.file_id, range: x.focus_or_full_range() }), + references: vec![], + }); + self.def_map.insert(def, x); + x + }; + let token = self.tokens.get_mut(id).unwrap(); + token.references.push(ReferenceData { + range: FileRange { range, file_id }, + is_definition: if let Some(x) = def.try_to_nav(self.db) { + x.file_id == file_id && x.focus_or_full_range() == range + } else { + false + }, + }); + result.tokens.push((range, id)); + } + self.files.push(result); + } + + pub fn compute<'a>(db: &'a RootDatabase, analysis: &'a Analysis) -> StaticIndex<'a> { + let work = all_modules(db).into_iter().filter(|module| { + let file_id = module.definition_source(db).file_id.original_file(db); + let source_root = db.file_source_root(file_id); + let source_root = db.source_root(source_root); + !source_root.is_library + }); + let mut this = StaticIndex { + files: vec![], + tokens: Default::default(), + analysis, + db, + def_map: Default::default(), + }; + let mut visited_files = FxHashSet::default(); + for module in work { + let file_id = module.definition_source(db).file_id.original_file(db); + if visited_files.contains(&file_id) { + continue; + } + this.add_file(file_id); + // mark the file + visited_files.insert(file_id); + } + this + } +} + +fn get_definition(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Option<Definition> { + for token in sema.descend_into_macros_many(token) { + let def = Definition::from_token(&sema, &token); + if let [x] = def.as_slice() { + return Some(*x); + } else { + continue; + }; + } + None +} + +#[cfg(test)] +mod tests { + use crate::{fixture, StaticIndex}; + use ide_db::base_db::FileRange; + use std::collections::HashSet; + + fn check_all_ranges(ra_fixture: &str) { + let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture); + let s = StaticIndex::compute(&*analysis.db, &analysis); + let mut range_set: HashSet<_> = ranges.iter().map(|x| x.0).collect(); + for f in s.files { + for (range, _) in f.tokens { + let x = FileRange { file_id: f.file_id, range }; + if !range_set.contains(&x) { + panic!("additional range {:?}", x); + } + range_set.remove(&x); + } + } + if !range_set.is_empty() { + panic!("unfound ranges {:?}", range_set); + } + } + + fn check_definitions(ra_fixture: &str) { + let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture); + let s = StaticIndex::compute(&*analysis.db, &analysis); + let mut range_set: HashSet<_> = ranges.iter().map(|x| x.0).collect(); + for (_, t) in s.tokens.iter() { + if let Some(x) = t.definition { + if !range_set.contains(&x) { + panic!("additional definition {:?}", x); + } + range_set.remove(&x); + } + } + if !range_set.is_empty() { + panic!("unfound definitions {:?}", range_set); + } + } + + #[test] + fn struct_and_enum() { + check_all_ranges( + r#" +struct Foo; + //^^^ +enum E { X(Foo) } + //^ ^ ^^^ +"#, + ); + check_definitions( + r#" +struct Foo; + //^^^ +enum E { X(Foo) } + //^ ^ +"#, + ); + } + + #[test] + fn derives() { + check_all_ranges( + r#" +#[rustc_builtin_macro] +pub macro Copy {} + //^^^^ +#[rustc_builtin_macro] +pub macro derive {} + //^^^^^^ +#[derive(Copy)] +//^^^^^^ ^^^^ +struct Hello(i32); + //^^^^^ ^^^ +"#, + ); + } +} diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index 1dfa23414de..29a3fbb67e0 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -22,7 +22,7 @@ crossbeam-channel = "0.5.0" dissimilar = "1.0.2" itertools = "0.10.0" jod-thread = "0.1.0" -lsp-types = { version = "0.89.0", features = ["proposed"] } +lsp-types = { version = "0.90.0", features = ["proposed"] } parking_lot = "0.11.0" xflags = "0.2.1" oorandom = "11.1.2" diff --git a/crates/rust-analyzer/src/bin/main.rs b/crates/rust-analyzer/src/bin/main.rs index 2390bee8243..74c041020be 100644 --- a/crates/rust-analyzer/src/bin/main.rs +++ b/crates/rust-analyzer/src/bin/main.rs @@ -87,6 +87,7 @@ fn try_main() -> Result<()> { flags::RustAnalyzerCmd::Diagnostics(cmd) => cmd.run()?, flags::RustAnalyzerCmd::Ssr(cmd) => cmd.run()?, flags::RustAnalyzerCmd::Search(cmd) => cmd.run()?, + flags::RustAnalyzerCmd::Lsif(cmd) => cmd.run()?, } Ok(()) } diff --git a/crates/rust-analyzer/src/cli.rs b/crates/rust-analyzer/src/cli.rs index efd8a2aa9fc..6ccdaa86dd6 100644 --- a/crates/rust-analyzer/src/cli.rs +++ b/crates/rust-analyzer/src/cli.rs @@ -8,6 +8,7 @@ mod highlight; mod analysis_stats; mod diagnostics; mod ssr; +mod lsif; mod progress_report; diff --git a/crates/rust-analyzer/src/cli/analysis_stats.rs b/crates/rust-analyzer/src/cli/analysis_stats.rs index a2dc842e75b..55a542c3c16 100644 --- a/crates/rust-analyzer/src/cli/analysis_stats.rs +++ b/crates/rust-analyzer/src/cli/analysis_stats.rs @@ -367,8 +367,6 @@ fn expr_syntax_range( ) -> Option<(VfsPath, LineCol, LineCol)> { let src = sm.expr_syntax(expr_id); if let Ok(src) = src { - // FIXME: it might be nice to have a function (on Analysis?) that goes from Source<T> -> (LineCol, LineCol) directly - // But also, we should just turn the type mismatches into diagnostics and provide these let root = db.parse_or_expand(src.file_id).unwrap(); let node = src.map(|e| e.to_node(&root).syntax().clone()); let original_range = node.as_ref().original_file_range(db); diff --git a/crates/rust-analyzer/src/cli/flags.rs b/crates/rust-analyzer/src/cli/flags.rs index e2e250143c0..b759d912c96 100644 --- a/crates/rust-analyzer/src/cli/flags.rs +++ b/crates/rust-analyzer/src/cli/flags.rs @@ -102,6 +102,10 @@ xflags::xflags! { } cmd proc-macro {} + + cmd lsif + required path: PathBuf + {} } } @@ -129,6 +133,7 @@ pub enum RustAnalyzerCmd { Ssr(Ssr), Search(Search), ProcMacro(ProcMacro), + Lsif(Lsif), } #[derive(Debug)] @@ -190,6 +195,11 @@ pub struct Search { #[derive(Debug)] pub struct ProcMacro; +#[derive(Debug)] +pub struct Lsif { + pub path: PathBuf, +} + impl RustAnalyzer { pub const HELP: &'static str = Self::HELP_; diff --git a/crates/rust-analyzer/src/cli/lsif.rs b/crates/rust-analyzer/src/cli/lsif.rs new file mode 100644 index 00000000000..a2354431460 --- /dev/null +++ b/crates/rust-analyzer/src/cli/lsif.rs @@ -0,0 +1,268 @@ +//! LSIF (language server index format) generator + +use std::collections::HashMap; +use std::env; +use std::time::Instant; + +use ide::{ + Analysis, FileId, FileRange, RootDatabase, StaticIndex, StaticIndexedFile, TokenId, + TokenStaticData, +}; +use ide_db::LineIndexDatabase; + +use ide_db::base_db::salsa::{self, ParallelDatabase}; +use lsp_types::{self, lsif}; +use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace}; +use vfs::{AbsPathBuf, Vfs}; + +use crate::cli::{ + flags, + load_cargo::{load_workspace, LoadCargoConfig}, + Result, +}; +use crate::line_index::{LineEndings, LineIndex, OffsetEncoding}; +use crate::to_proto; + +/// Need to wrap Snapshot to provide `Clone` impl for `map_with` +struct Snap<DB>(DB); +impl<DB: ParallelDatabase> Clone for Snap<salsa::Snapshot<DB>> { + fn clone(&self) -> Snap<salsa::Snapshot<DB>> { + Snap(self.0.snapshot()) + } +} + +struct LsifManager<'a> { + count: i32, + token_map: HashMap<TokenId, Id>, + range_map: HashMap<FileRange, Id>, + file_map: HashMap<FileId, Id>, + analysis: &'a Analysis, + db: &'a RootDatabase, + vfs: &'a Vfs, +} + +#[derive(Clone, Copy)] +struct Id(i32); + +impl From<Id> for lsp_types::NumberOrString { + fn from(Id(x): Id) -> Self { + lsp_types::NumberOrString::Number(x) + } +} + +impl LsifManager<'_> { + fn new<'a>(analysis: &'a Analysis, db: &'a RootDatabase, vfs: &'a Vfs) -> LsifManager<'a> { + LsifManager { + count: 0, + token_map: HashMap::default(), + range_map: HashMap::default(), + file_map: HashMap::default(), + analysis, + db, + vfs, + } + } + + fn add(&mut self, data: lsif::Element) -> Id { + let id = Id(self.count); + self.emit(&serde_json::to_string(&lsif::Entry { id: id.into(), data }).unwrap()); + self.count += 1; + id + } + + fn add_vertex(&mut self, vertex: lsif::Vertex) -> Id { + self.add(lsif::Element::Vertex(vertex)) + } + + fn add_edge(&mut self, edge: lsif::Edge) -> Id { + self.add(lsif::Element::Edge(edge)) + } + + // FIXME: support file in addition to stdout here + fn emit(&self, data: &str) { + println!("{}", data); + } + + fn get_token_id(&mut self, id: TokenId) -> Id { + if let Some(x) = self.token_map.get(&id) { + return *x; + } + let result_set_id = self.add_vertex(lsif::Vertex::ResultSet(lsif::ResultSet { key: None })); + self.token_map.insert(id, result_set_id); + result_set_id + } + + fn get_range_id(&mut self, id: FileRange) -> Id { + if let Some(x) = self.range_map.get(&id) { + return *x; + } + let file_id = id.file_id; + let doc_id = self.get_file_id(file_id); + let line_index = self.db.line_index(file_id); + let line_index = LineIndex { + index: line_index.clone(), + encoding: OffsetEncoding::Utf16, + endings: LineEndings::Unix, + }; + let range_id = self.add_vertex(lsif::Vertex::Range { + range: to_proto::range(&line_index, id.range), + tag: None, + }); + self.add_edge(lsif::Edge::Contains(lsif::EdgeDataMultiIn { + in_vs: vec![range_id.into()], + out_v: doc_id.into(), + })); + range_id + } + + fn get_file_id(&mut self, id: FileId) -> Id { + if let Some(x) = self.file_map.get(&id) { + return *x; + } + let path = self.vfs.file_path(id); + let path = path.as_path().unwrap(); + let doc_id = self.add_vertex(lsif::Vertex::Document(lsif::Document { + language_id: "rust".to_string(), + uri: lsp_types::Url::from_file_path(path).unwrap(), + })); + self.file_map.insert(id, doc_id); + doc_id + } + + fn add_token(&mut self, id: TokenId, token: TokenStaticData) { + let result_set_id = self.get_token_id(id); + if let Some(hover) = token.hover { + let hover_id = self.add_vertex(lsif::Vertex::HoverResult { + result: lsp_types::Hover { + contents: lsp_types::HoverContents::Markup(to_proto::markup_content( + hover.markup, + )), + range: None, + }, + }); + self.add_edge(lsif::Edge::Hover(lsif::EdgeData { + in_v: hover_id.into(), + out_v: result_set_id.into(), + })); + } + if let Some(def) = token.definition { + let result_id = self.add_vertex(lsif::Vertex::DefinitionResult); + let def_vertex = self.get_range_id(def); + self.add_edge(lsif::Edge::Item(lsif::Item { + document: (*self.file_map.get(&def.file_id).unwrap()).into(), + property: None, + edge_data: lsif::EdgeDataMultiIn { + in_vs: vec![def_vertex.into()], + out_v: result_id.into(), + }, + })); + self.add_edge(lsif::Edge::Definition(lsif::EdgeData { + in_v: result_id.into(), + out_v: result_set_id.into(), + })); + } + if !token.references.is_empty() { + let result_id = self.add_vertex(lsif::Vertex::ReferenceResult); + self.add_edge(lsif::Edge::References(lsif::EdgeData { + in_v: result_id.into(), + out_v: result_set_id.into(), + })); + for x in token.references { + let vertex = *self.range_map.get(&x.range).unwrap(); + self.add_edge(lsif::Edge::Item(lsif::Item { + document: (*self.file_map.get(&x.range.file_id).unwrap()).into(), + property: Some(if x.is_definition { + lsif::ItemKind::Definitions + } else { + lsif::ItemKind::References + }), + edge_data: lsif::EdgeDataMultiIn { + in_vs: vec![vertex.into()], + out_v: result_id.into(), + }, + })); + } + } + } + + fn add_file(&mut self, file: StaticIndexedFile) { + let StaticIndexedFile { file_id, tokens, folds } = file; + let doc_id = self.get_file_id(file_id); + let text = self.analysis.file_text(file_id).unwrap(); + let line_index = self.db.line_index(file_id); + let line_index = LineIndex { + index: line_index.clone(), + encoding: OffsetEncoding::Utf16, + endings: LineEndings::Unix, + }; + let result = folds + .into_iter() + .map(|it| to_proto::folding_range(&*text, &line_index, false, it)) + .collect(); + let folding_id = self.add_vertex(lsif::Vertex::FoldingRangeResult { result }); + self.add_edge(lsif::Edge::FoldingRange(lsif::EdgeData { + in_v: folding_id.into(), + out_v: doc_id.into(), + })); + let tokens_id = tokens + .into_iter() + .map(|(range, id)| { + let range_id = self.add_vertex(lsif::Vertex::Range { + range: to_proto::range(&line_index, range), + tag: None, + }); + self.range_map.insert(FileRange { file_id, range }, range_id); + let result_set_id = self.get_token_id(id); + self.add_edge(lsif::Edge::Next(lsif::EdgeData { + in_v: result_set_id.into(), + out_v: range_id.into(), + })); + range_id.into() + }) + .collect(); + self.add_edge(lsif::Edge::Contains(lsif::EdgeDataMultiIn { + in_vs: tokens_id, + out_v: doc_id.into(), + })); + } +} + +impl flags::Lsif { + pub fn run(self) -> Result<()> { + eprintln!("Generating LSIF started..."); + let now = Instant::now(); + let cargo_config = CargoConfig::default(); + let no_progress = &|_| (); + let load_cargo_config = LoadCargoConfig { + load_out_dirs_from_check: true, + with_proc_macro: true, + prefill_caches: false, + }; + let path = AbsPathBuf::assert(env::current_dir()?.join(&self.path)); + let manifest = ProjectManifest::discover_single(&path)?; + + let workspace = ProjectWorkspace::load(manifest, &cargo_config, no_progress)?; + + let (host, vfs, _proc_macro) = load_workspace(workspace, &load_cargo_config)?; + let db = host.raw_database(); + let analysis = host.analysis(); + + let si = StaticIndex::compute(db, &analysis); + + let mut lsif = LsifManager::new(&analysis, db, &vfs); + lsif.add_vertex(lsif::Vertex::MetaData(lsif::MetaData { + version: String::from("0.5.0"), + project_root: lsp_types::Url::from_file_path(path).unwrap(), + position_encoding: lsif::Encoding::Utf16, + tool_info: None, + })); + for file in si.files { + lsif.add_file(file); + } + for (id, token) in si.tokens.iter() { + lsif.add_token(id, token); + } + eprintln!("Generating LSIF finished in {:?}", now.elapsed()); + Ok(()) + } +} |