Auto merge of #17900 - darichey:exclude-vendored-libraries, r=davidbarsky

Add scip/lsif flag to exclude vendored libaries

#17809 changed StaticIndex to include vendored libraries. This PR adds a flag to disable that behavior.

At work, our monorepo has too many rust targets to index all at once, so we split them up into several shards. Since all of our libraries are vendored, if rust-analyzer includes them, sharding no longer has much benefit, because every shard will have to index the entire transitive dependency graphs of all of its targets. We get around the issue presented in #17809 because some other shard will index the libraries directly.

5 files changed, 95 insertions, 19 deletions

diff --git a/src/tools/rust-analyzer/crates/ide/src/lib.rs b/src/tools/rust-analyzer/crates/ide/src/lib.rs
index eff4bc3d376..ba0aaae19c9 100644
--- a/src/tools/rust-analyzer/crates/ide/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/lib.rs
@@ -104,7 +104,9 @@ pub use crate::{
     rename::RenameError,
     runnables::{Runnable, RunnableKind, TestId},
     signature_help::SignatureHelp,
-    static_index::{StaticIndex, StaticIndexedFile, TokenId, TokenStaticData},
+    static_index::{
+        StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, VendoredLibrariesConfig,
+    },
     syntax_highlighting::{
         tags::{Highlight, HlMod, HlMods, HlOperator, HlPunct, HlTag},
         HighlightConfig, HlRange,
diff --git a/src/tools/rust-analyzer/crates/ide/src/static_index.rs b/src/tools/rust-analyzer/crates/ide/src/static_index.rs
index 0f8d03c8d55..1cbe8c62a81 100644
--- a/src/tools/rust-analyzer/crates/ide/src/static_index.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/static_index.rs
@@ -124,6 +124,11 @@ fn documentation_for_definition(
     )
 }
 
+pub enum VendoredLibrariesConfig<'a> {
+    Included { workspace_root: &'a VfsPath },
+    Excluded,
+}
+
 impl StaticIndex<'_> {
     fn add_file(&mut self, file_id: FileId) {
         let current_crate = crates_for(self.db, file_id).pop().map(Into::into);
@@ -240,15 +245,22 @@ impl StaticIndex<'_> {
         self.files.push(result);
     }
 
-    pub fn compute<'a>(analysis: &'a Analysis, workspace_root: &VfsPath) -> StaticIndex<'a> {
+    pub fn compute<'a>(
+        analysis: &'a Analysis,
+        vendored_libs_config: VendoredLibrariesConfig<'_>,
+    ) -> StaticIndex<'a> {
         let db = &*analysis.db;
         let work = all_modules(db).into_iter().filter(|module| {
             let file_id = module.definition_source_file_id(db).original_file(db);
             let source_root = db.file_source_root(file_id.into());
             let source_root = db.source_root(source_root);
-            let is_vendored = source_root
-                .path_for_file(&file_id.into())
-                .is_some_and(|module_path| module_path.starts_with(workspace_root));
+            let is_vendored = match vendored_libs_config {
+                VendoredLibrariesConfig::Included { workspace_root } => source_root
+                    .path_for_file(&file_id.into())
+                    .is_some_and(|module_path| module_path.starts_with(workspace_root)),
+                VendoredLibrariesConfig::Excluded => false,
+            };
+
             !source_root.is_library || is_vendored
         });
         let mut this = StaticIndex {
@@ -278,10 +290,11 @@ mod tests {
     use ide_db::{base_db::VfsPath, FileRange, FxHashSet};
     use syntax::TextSize;
 
-    fn check_all_ranges(ra_fixture: &str) {
+    use super::VendoredLibrariesConfig;
+
+    fn check_all_ranges(ra_fixture: &str, vendored_libs_config: VendoredLibrariesConfig<'_>) {
         let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture);
-        let s =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let s = StaticIndex::compute(&analysis, vendored_libs_config);
         let mut range_set: FxHashSet<_> = ranges.iter().map(|it| it.0).collect();
         for f in s.files {
             for (range, _) in f.tokens {
@@ -298,10 +311,9 @@ mod tests {
     }
 
     #[track_caller]
-    fn check_definitions(ra_fixture: &str) {
+    fn check_definitions(ra_fixture: &str, vendored_libs_config: VendoredLibrariesConfig<'_>) {
         let (analysis, ranges) = fixture::annotations_without_marker(ra_fixture);
-        let s =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let s = StaticIndex::compute(&analysis, vendored_libs_config);
         let mut range_set: FxHashSet<_> = ranges.iter().map(|it| it.0).collect();
         for (_, t) in s.tokens.iter() {
             if let Some(t) = t.definition {
@@ -329,6 +341,9 @@ struct Foo;
 enum E { X(Foo) }
    //^   ^ ^^^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
         check_definitions(
             r#"
@@ -337,6 +352,9 @@ struct Foo;
 enum E { X(Foo) }
    //^   ^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 
@@ -359,6 +377,9 @@ pub func() {
 
 }
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 
@@ -377,10 +398,31 @@ struct ExternalLibrary(i32);
 struct VendoredLibrary(i32);
      //^^^^^^^^^^^^^^^ ^^^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 
     #[test]
+    fn vendored_crate_excluded() {
+        check_all_ranges(
+            r#"
+//- /workspace/main.rs crate:main deps:external,vendored
+struct Main(i32);
+     //^^^^ ^^^
+
+//- /external/lib.rs new_source_root:library crate:external@0.1.0,https://a.b/foo.git library
+struct ExternalLibrary(i32);
+
+//- /workspace/vendored/lib.rs new_source_root:library crate:vendored@0.1.0,https://a.b/bar.git library
+struct VendoredLibrary(i32);
+"#,
+            VendoredLibrariesConfig::Excluded,
+        )
+    }
+
+    #[test]
     fn derives() {
         check_all_ranges(
             r#"
@@ -394,6 +436,9 @@ pub macro Copy {}
 struct Hello(i32);
      //^^^^^ ^^^
 "#,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
         );
     }
 }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
index 2a3e74c680b..16d90de661a 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
@@ -138,6 +138,9 @@ xflags::xflags! {
 
         cmd lsif {
             required path: PathBuf
+
+            /// Exclude code from vendored libraries from the resulting index.
+            optional --exclude-vendored-libraries
         }
 
         cmd scip {
@@ -148,6 +151,9 @@ xflags::xflags! {
 
             /// A path to an json configuration file that can be used to customize cargo behavior.
             optional --config-path config_path: PathBuf
+
+            /// Exclude code from vendored libraries from the resulting index.
+            optional --exclude-vendored-libraries
         }
     }
 }
@@ -259,6 +265,8 @@ pub struct Search {
 #[derive(Debug)]
 pub struct Lsif {
     pub path: PathBuf,
+
+    pub exclude_vendored_libraries: bool,
 }
 
 #[derive(Debug)]
@@ -267,6 +275,7 @@ pub struct Scip {
 
     pub output: Option<PathBuf>,
     pub config_path: Option<PathBuf>,
+    pub exclude_vendored_libraries: bool,
 }
 
 impl RustAnalyzer {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
index 016f0edc2dd..89fe712ced0 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/lsif.rs
@@ -5,7 +5,7 @@ use std::time::Instant;
 
 use ide::{
     Analysis, AnalysisHost, FileId, FileRange, MonikerKind, PackageInformation, RootDatabase,
-    StaticIndex, StaticIndexedFile, TokenId, TokenStaticData,
+    StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, VendoredLibrariesConfig,
 };
 use ide_db::{line_index::WideEncoding, LineIndexDatabase};
 use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
@@ -296,7 +296,13 @@ impl flags::Lsif {
         let db = host.raw_database();
         let analysis = host.analysis();
 
-        let si = StaticIndex::compute(&analysis, &path.clone().into());
+        let vendored_libs_config = if self.exclude_vendored_libraries {
+            VendoredLibrariesConfig::Excluded
+        } else {
+            VendoredLibrariesConfig::Included { workspace_root: &path.clone().into() }
+        };
+
+        let si = StaticIndex::compute(&analysis, vendored_libs_config);
 
         let mut lsif = LsifManager::new(&analysis, db, &vfs);
         lsif.add_vertex(lsif::Vertex::MetaData(lsif::MetaData {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
index 2fc0ef8d4da..34b20851dd6 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/scip.rs
@@ -4,7 +4,7 @@ use std::{path::PathBuf, time::Instant};
 
 use ide::{
     AnalysisHost, LineCol, MonikerDescriptorKind, MonikerResult, StaticIndex, StaticIndexedFile,
-    SymbolInformationKind, TextRange, TokenId,
+    SymbolInformationKind, TextRange, TokenId, VendoredLibrariesConfig,
 };
 use ide_db::LineIndexDatabase;
 use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
@@ -63,7 +63,13 @@ impl flags::Scip {
         let db = host.raw_database();
         let analysis = host.analysis();
 
-        let si = StaticIndex::compute(&analysis, &root.clone().into());
+        let vendored_libs_config = if self.exclude_vendored_libraries {
+            VendoredLibrariesConfig::Excluded
+        } else {
+            VendoredLibrariesConfig::Included { workspace_root: &root.clone().into() }
+        };
+
+        let si = StaticIndex::compute(&analysis, vendored_libs_config);
 
         let metadata = scip_types::Metadata {
             version: scip_types::ProtocolVersion::UnspecifiedProtocolVersion.into(),
@@ -352,8 +358,12 @@ mod test {
         let (host, position) = position(ra_fixture);
 
         let analysis = host.analysis();
-        let si =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let si = StaticIndex::compute(
+            &analysis,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
+        );
 
         let FilePosition { file_id, offset } = position;
 
@@ -617,8 +627,12 @@ pub mod example_mod {
         host.raw_database_mut().apply_change(change_fixture.change);
 
         let analysis = host.analysis();
-        let si =
-            StaticIndex::compute(&analysis, &VfsPath::new_virtual_path("/workspace".to_owned()));
+        let si = StaticIndex::compute(
+            &analysis,
+            VendoredLibrariesConfig::Included {
+                workspace_root: &VfsPath::new_virtual_path("/workspace".to_owned()),
+            },
+        );
 
         let file = si.files.first().unwrap();
         let (_, token_id) = file.tokens.first().unwrap();