diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/ci/docker/host-x86_64/mingw-check/Dockerfile | 2 | ||||
| -rw-r--r-- | src/librustdoc/html/render/mod.rs | 33 | ||||
| -rw-r--r-- | src/librustdoc/html/render/search_index.rs | 101 | ||||
| -rw-r--r-- | src/librustdoc/html/render/write_shared.rs | 31 | ||||
| -rw-r--r-- | src/librustdoc/html/static/.eslintrc.js | 2 | ||||
| -rw-r--r-- | src/librustdoc/html/static/js/main.js | 28 | ||||
| -rw-r--r-- | src/librustdoc/html/static/js/search.js | 348 | ||||
| -rw-r--r-- | src/librustdoc/html/static/js/storage.js | 4 | ||||
| -rw-r--r-- | src/tools/rustdoc-js/.eslintrc.js | 2 | ||||
| -rw-r--r-- | src/tools/rustdoc-js/tester.js | 104 |
10 files changed, 427 insertions, 228 deletions
diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index 30d3a52d82b..de8db8ee034 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -56,7 +56,7 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \ /scripts/validate-error-codes.sh && \ reuse --include-submodules lint && \ # Runs checks to ensure that there are no ES5 issues in our JS code. - es-check es6 ../src/librustdoc/html/static/js/*.js && \ + es-check es8 ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/tools/rustdoc-js/.eslintrc.js ../src/tools/rustdoc-js/tester.js && \ eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs index 6c5040414bc..c1a7593c26f 100644 --- a/src/librustdoc/html/render/mod.rs +++ b/src/librustdoc/html/render/mod.rs @@ -184,40 +184,15 @@ pub(crate) enum RenderTypeId { impl RenderTypeId { pub fn write_to_string(&self, string: &mut String) { - // (sign, value) - let (sign, id): (bool, u32) = match &self { + let id: i32 = match &self { // 0 is a sentinel, everything else is one-indexed // concrete type - RenderTypeId::Index(idx) if *idx >= 0 => (false, (idx + 1isize).try_into().unwrap()), + RenderTypeId::Index(idx) if *idx >= 0 => (idx + 1isize).try_into().unwrap(), // generic type parameter - RenderTypeId::Index(idx) => (true, (-*idx).try_into().unwrap()), + RenderTypeId::Index(idx) => (*idx).try_into().unwrap(), _ => panic!("must convert render types to indexes before serializing"), }; - // zig-zag encoding - let value: u32 = (id << 1) | (if sign { 1 } else { 0 }); - // Self-terminating hex use capital letters for everything but the - // least significant digit, which is lowercase. For example, decimal 17 - // would be `` Aa `` if zig-zag encoding weren't used. - // - // Zig-zag encoding, however, stores the sign bit as the last bit. - // This means, in the last hexit, 1 is actually `c`, -1 is `b` - // (`a` is the imaginary -0), and, because all the bits are shifted - // by one, `` A` `` is actually 8 and `` Aa `` is -8. - // - // https://rust-lang.github.io/rustc-dev-guide/rustdoc-internals/search.html - // describes the encoding in more detail. - let mut shift: u32 = 28; - let mut mask: u32 = 0xF0_00_00_00; - while shift < 32 { - let hexit = (value & mask) >> shift; - if hexit != 0 || shift == 0 { - let hex = - char::try_from(if shift == 0 { '`' } else { '@' } as u32 + hexit).unwrap(); - string.push(hex); - } - shift = shift.wrapping_sub(4); - mask = mask >> 4; - } + search_index::write_vlqhex_to_string(id, string); } } diff --git a/src/librustdoc/html/render/search_index.rs b/src/librustdoc/html/render/search_index.rs index f153a908329..34a4a89aa7b 100644 --- a/src/librustdoc/html/render/search_index.rs +++ b/src/librustdoc/html/render/search_index.rs @@ -17,12 +17,25 @@ use crate::html::format::join_with_double_colon; use crate::html::markdown::short_markdown_summary; use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, RenderTypeId}; +/// The serialized search description sharded version +/// +/// The `index` is a JSON-encoded list of names and other information. +/// +/// The desc has newlined descriptions, split up by size into 1MiB shards. +/// For example, `(4, "foo\nbar\nbaz\nquux")`. +pub(crate) struct SerializedSearchIndex { + pub(crate) index: String, + pub(crate) desc: Vec<(usize, String)>, +} + +const DESC_INDEX_SHARD_LEN: usize = 1024 * 1024; + /// Builds the search index from the collected metadata pub(crate) fn build_index<'tcx>( krate: &clean::Crate, cache: &mut Cache, tcx: TyCtxt<'tcx>, -) -> String { +) -> SerializedSearchIndex { let mut itemid_to_pathid = FxHashMap::default(); let mut primitives = FxHashMap::default(); let mut associated_types = FxHashMap::default(); @@ -318,7 +331,6 @@ pub(crate) fn build_index<'tcx>( .collect::<Vec<_>>(); struct CrateData<'a> { - doc: String, items: Vec<&'a IndexItem>, paths: Vec<(ItemType, Vec<Symbol>)>, // The String is alias name and the vec is the list of the elements with this alias. @@ -327,6 +339,9 @@ pub(crate) fn build_index<'tcx>( aliases: &'a BTreeMap<String, Vec<usize>>, // Used when a type has more than one impl with an associated item with the same name. associated_item_disambiguators: &'a Vec<(usize, String)>, + // A list of shard lengths encoded as vlqhex. See the comment in write_vlqhex_to_string + // for information on the format. + descindex: String, } struct Paths { @@ -408,7 +423,6 @@ pub(crate) fn build_index<'tcx>( let mut names = Vec::with_capacity(self.items.len()); let mut types = String::with_capacity(self.items.len()); let mut full_paths = Vec::with_capacity(self.items.len()); - let mut descriptions = Vec::with_capacity(self.items.len()); let mut parents = Vec::with_capacity(self.items.len()); let mut functions = String::with_capacity(self.items.len()); let mut deprecated = Vec::with_capacity(self.items.len()); @@ -431,7 +445,6 @@ pub(crate) fn build_index<'tcx>( parents.push(item.parent_idx.map(|x| x + 1).unwrap_or(0)); names.push(item.name.as_str()); - descriptions.push(&item.desc); if !item.path.is_empty() { full_paths.push((index, &item.path)); @@ -454,14 +467,12 @@ pub(crate) fn build_index<'tcx>( let has_aliases = !self.aliases.is_empty(); let mut crate_data = serializer.serialize_struct("CrateData", if has_aliases { 9 } else { 8 })?; - crate_data.serialize_field("doc", &self.doc)?; crate_data.serialize_field("t", &types)?; crate_data.serialize_field("n", &names)?; - // Serialize as an array of item indices and full paths crate_data.serialize_field("q", &full_paths)?; - crate_data.serialize_field("d", &descriptions)?; crate_data.serialize_field("i", &parents)?; crate_data.serialize_field("f", &functions)?; + crate_data.serialize_field("D", &self.descindex)?; crate_data.serialize_field("c", &deprecated)?; crate_data.serialize_field("p", &paths)?; crate_data.serialize_field("b", &self.associated_item_disambiguators)?; @@ -472,16 +483,46 @@ pub(crate) fn build_index<'tcx>( } } - // Collect the index into a string - format!( + let desc = { + let mut result = Vec::new(); + let mut set = String::new(); + let mut len: usize = 0; + for desc in std::iter::once(&crate_doc).chain(crate_items.iter().map(|item| &item.desc)) { + if set.len() >= DESC_INDEX_SHARD_LEN { + result.push((len, std::mem::replace(&mut set, String::new()))); + len = 0; + } else if len != 0 { + set.push('\n'); + } + set.push_str(&desc); + len += 1; + } + result.push((len, std::mem::replace(&mut set, String::new()))); + result + }; + + let descindex = { + let mut descindex = String::with_capacity(desc.len() * 4); + for &(len, _) in desc.iter() { + write_vlqhex_to_string(len.try_into().unwrap(), &mut descindex); + } + descindex + }; + + assert_eq!(crate_items.len() + 1, desc.iter().map(|(len, _)| *len).sum::<usize>()); + + // The index, which is actually used to search, is JSON + // It uses `JSON.parse(..)` to actually load, since JSON + // parses faster than the full JavaScript syntax. + let index = format!( r#"["{}",{}]"#, krate.name(tcx), serde_json::to_string(&CrateData { - doc: crate_doc, items: crate_items, paths: crate_paths, aliases: &aliases, associated_item_disambiguators: &associated_item_disambiguators, + descindex, }) .expect("failed serde conversion") // All these `replace` calls are because we have to go through JS string for JSON content. @@ -489,7 +530,45 @@ pub(crate) fn build_index<'tcx>( .replace('\'', r"\'") // We need to escape double quotes for the JSON. .replace("\\\"", "\\\\\"") - ) + ); + SerializedSearchIndex { index, desc } +} + +pub(crate) fn write_vlqhex_to_string(n: i32, string: &mut String) { + let (sign, magnitude): (bool, u32) = + if n >= 0 { (false, n.try_into().unwrap()) } else { (true, (-n).try_into().unwrap()) }; + // zig-zag encoding + let value: u32 = (magnitude << 1) | (if sign { 1 } else { 0 }); + // Self-terminating hex use capital letters for everything but the + // least significant digit, which is lowercase. For example, decimal 17 + // would be `` Aa `` if zig-zag encoding weren't used. + // + // Zig-zag encoding, however, stores the sign bit as the last bit. + // This means, in the last hexit, 1 is actually `c`, -1 is `b` + // (`a` is the imaginary -0), and, because all the bits are shifted + // by one, `` A` `` is actually 8 and `` Aa `` is -8. + // + // https://rust-lang.github.io/rustc-dev-guide/rustdoc-internals/search.html + // describes the encoding in more detail. + let mut shift: u32 = 28; + let mut mask: u32 = 0xF0_00_00_00; + // first skip leading zeroes + while shift < 32 { + let hexit = (value & mask) >> shift; + if hexit != 0 || shift == 0 { + break; + } + shift = shift.wrapping_sub(4); + mask = mask >> 4; + } + // now write the rest + while shift < 32 { + let hexit = (value & mask) >> shift; + let hex = char::try_from(if shift == 0 { '`' } else { '@' } as u32 + hexit).unwrap(); + string.push(hex); + shift = shift.wrapping_sub(4); + mask = mask >> 4; + } } pub(crate) fn get_function_type_for_search<'tcx>( diff --git a/src/librustdoc/html/render/write_shared.rs b/src/librustdoc/html/render/write_shared.rs index fbd45b2b48e..c806bf1cc66 100644 --- a/src/librustdoc/html/render/write_shared.rs +++ b/src/librustdoc/html/render/write_shared.rs @@ -24,6 +24,7 @@ use crate::formats::cache::Cache; use crate::formats::item_type::ItemType; use crate::formats::Impl; use crate::html::format::Buffer; +use crate::html::render::search_index::SerializedSearchIndex; use crate::html::render::{AssocItemLink, ImplRenderingParameters}; use crate::html::{layout, static_files}; use crate::visit::DocVisitor; @@ -46,7 +47,7 @@ use crate::{try_err, try_none}; pub(super) fn write_shared( cx: &mut Context<'_>, krate: &Crate, - search_index: String, + search_index: SerializedSearchIndex, options: &RenderOptions, ) -> Result<(), Error> { // Write out the shared files. Note that these are shared among all rustdoc @@ -312,7 +313,7 @@ pub(super) fn write_shared( let dst = cx.dst.join(&format!("search-index{}.js", cx.shared.resource_suffix)); let (mut all_indexes, mut krates) = try_err!(collect_json(&dst, krate.name(cx.tcx()).as_str()), &dst); - all_indexes.push(search_index); + all_indexes.push(search_index.index); krates.push(krate.name(cx.tcx()).to_string()); krates.sort(); @@ -335,6 +336,32 @@ else if (window.initSearch) window.initSearch(searchIndex); Ok(v.into_bytes()) })?; + let search_desc_dir = cx.dst.join(format!("search.desc/{krate}", krate = krate.name(cx.tcx()))); + if Path::new(&search_desc_dir).exists() { + try_err!(std::fs::remove_dir_all(&search_desc_dir), &search_desc_dir); + } + try_err!(std::fs::create_dir_all(&search_desc_dir), &search_desc_dir); + let kratename = krate.name(cx.tcx()).to_string(); + for (i, (_, data)) in search_index.desc.into_iter().enumerate() { + let output_filename = static_files::suffix_path( + &format!("{kratename}-desc-{i}-.js"), + &cx.shared.resource_suffix, + ); + let path = search_desc_dir.join(output_filename); + try_err!( + std::fs::write( + &path, + &format!( + r##"searchState.loadedDescShard({kratename}, {i}, {data})"##, + kratename = serde_json::to_string(&kratename).unwrap(), + data = serde_json::to_string(&data).unwrap(), + ) + .into_bytes() + ), + &path + ); + } + write_invocation_specific("crates.js", &|| { let krates = krates.iter().map(|k| format!("\"{k}\"")).join(","); Ok(format!("window.ALL_CRATES = [{krates}];").into_bytes()) diff --git a/src/librustdoc/html/static/.eslintrc.js b/src/librustdoc/html/static/.eslintrc.js index 1a34530c2d1..a1e9cc6dfa1 100644 --- a/src/librustdoc/html/static/.eslintrc.js +++ b/src/librustdoc/html/static/.eslintrc.js @@ -5,7 +5,7 @@ module.exports = { }, "extends": "eslint:recommended", "parserOptions": { - "ecmaVersion": 2015, + "ecmaVersion": 8, "sourceType": "module" }, "rules": { diff --git a/src/librustdoc/html/static/js/main.js b/src/librustdoc/html/static/js/main.js index b9a769a7c6d..2c1330614b8 100644 --- a/src/librustdoc/html/static/js/main.js +++ b/src/librustdoc/html/static/js/main.js @@ -329,6 +329,26 @@ function preLoadCss(cssUrl) { search.innerHTML = "<h3 class=\"search-loading\">" + searchState.loadingText + "</h3>"; searchState.showResults(search); }, + descShards: new Map(), + loadDesc: async function({descShard, descIndex}) { + if (descShard.promise === null) { + descShard.promise = new Promise((resolve, reject) => { + descShard.resolve = resolve; + const ds = descShard; + const fname = `${ds.crate}-desc-${ds.shard}-`; + const url = resourcePath( + `search.desc/${descShard.crate}/${fname}`, + ".js", + ); + loadScript(url, reject); + }); + } + const list = await descShard.promise; + return list[descIndex]; + }, + loadedDescShard: function (crate, shard, data) { + this.descShards.get(crate)[shard].resolve(data.split("\n")); + }, }; const toggleAllDocsId = "toggle-all-docs"; @@ -381,7 +401,7 @@ function preLoadCss(cssUrl) { window.location.replace("#" + item.id); }, 0); } - } + }, ); } } @@ -585,7 +605,7 @@ function preLoadCss(cssUrl) { const script = document .querySelector("script[data-ignore-extern-crates]"); const ignoreExternCrates = new Set( - (script ? script.getAttribute("data-ignore-extern-crates") : "").split(",") + (script ? script.getAttribute("data-ignore-extern-crates") : "").split(","), ); for (const lib of libs) { if (lib === window.currentCrate || ignoreExternCrates.has(lib)) { @@ -1098,7 +1118,7 @@ function preLoadCss(cssUrl) { } else { wrapper.style.setProperty( "--popover-arrow-offset", - (wrapperPos.right - pos.right + 4) + "px" + (wrapperPos.right - pos.right + 4) + "px", ); } wrapper.style.visibility = ""; @@ -1680,7 +1700,7 @@ href="https://doc.rust-lang.org/${channel}/rustdoc/read-documentation/search.htm pendingSidebarResizingFrame = false; document.documentElement.style.setProperty( "--resizing-sidebar-width", - desiredSidebarSize + "px" + desiredSidebarSize + "px", ); }, 100); } diff --git a/src/librustdoc/html/static/js/search.js b/src/librustdoc/html/static/js/search.js index cbfa503f260..2732d6b1543 100644 --- a/src/librustdoc/html/static/js/search.js +++ b/src/librustdoc/html/static/js/search.js @@ -206,14 +206,14 @@ const editDistanceState = { // insertion this.current[j - 1] + 1, // substitution - this.prev[j - 1] + substitutionCost + this.prev[j - 1] + substitutionCost, ); if ((i > 1) && (j > 1) && (a[aIdx] === b[bIdx - 1]) && (a[aIdx - 1] === b[bIdx])) { // transposition this.current[j] = Math.min( this.current[j], - this.prevPrev[j - 2] + 1 + this.prevPrev[j - 2] + 1, ); } } @@ -856,8 +856,8 @@ function initSearch(rawSearchIndex) { parserState, parserState.userQuery.slice(start, end), generics, - isInGenerics - ) + isInGenerics, + ), ); } } @@ -1295,7 +1295,7 @@ function initSearch(rawSearchIndex) { * * @return {ResultsTable} */ - function execQuery(parsedQuery, filterCrates, currentCrate) { + async function execQuery(parsedQuery, filterCrates, currentCrate) { const results_others = new Map(), results_in_args = new Map(), results_returned = new Map(); @@ -1326,6 +1326,7 @@ function initSearch(rawSearchIndex) { duplicates.add(obj.fullPath); obj.href = res[1]; + obj.desc = result.desc; out.push(obj); if (out.length >= MAX_RESULTS) { break; @@ -1342,9 +1343,9 @@ function initSearch(rawSearchIndex) { * @param {Results} results * @param {boolean} isType * @param {string} preferredCrate - * @returns {[ResultObject]} + * @returns {Promise<[ResultObject]>} */ - function sortResults(results, isType, preferredCrate) { + async function sortResults(results, isType, preferredCrate) { const userQuery = parsedQuery.userQuery; const result_list = []; for (const result of results.values()) { @@ -1352,6 +1353,12 @@ function initSearch(rawSearchIndex) { result.word = searchIndex[result.id].word; result_list.push(result); } + for (const result of result_list) { + result.desc = searchState.loadDesc(result.item); + } + for (const result of result_list) { + result.desc = await result.desc; + } result_list.sort((aaa, bbb) => { let a, b; @@ -1422,8 +1429,8 @@ function initSearch(rawSearchIndex) { } // sort by description (no description goes later) - a = (aaa.item.desc === ""); - b = (bbb.item.desc === ""); + a = (aaa.desc === ""); + b = (bbb.desc === ""); if (a !== b) { return a - b; } @@ -1477,7 +1484,7 @@ function initSearch(rawSearchIndex) { whereClause, mgensIn, solutionCb, - unboxingDepth + unboxingDepth, ) { if (unboxingDepth >= UNBOXING_LIMIT) { return false; @@ -1524,7 +1531,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + 1 + unboxingDepth + 1, )) { continue; } @@ -1541,7 +1548,7 @@ function initSearch(rawSearchIndex) { whereClause, mgensScratch, solutionCb, - unboxingDepth + 1 + unboxingDepth + 1, )) { return true; } @@ -1551,7 +1558,7 @@ function initSearch(rawSearchIndex) { whereClause, mgens ? new Map(mgens) : null, solutionCb, - unboxingDepth + 1 + unboxingDepth + 1, )) { return true; } @@ -1625,7 +1632,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgensScratch, - unboxingDepth + unboxingDepth, ); if (!solution) { return false; @@ -1638,7 +1645,7 @@ function initSearch(rawSearchIndex) { whereClause, simplifiedMgens, solutionCb, - unboxingDepth + unboxingDepth, ); if (passesUnification) { return true; @@ -1646,7 +1653,7 @@ function initSearch(rawSearchIndex) { } return false; }, - unboxingDepth + unboxingDepth, ); if (passesUnification) { return true; @@ -1663,7 +1670,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + 1 + unboxingDepth + 1, )) { continue; } @@ -1689,7 +1696,7 @@ function initSearch(rawSearchIndex) { whereClause, mgensScratch, solutionCb, - unboxingDepth + 1 + unboxingDepth + 1, ); if (passesUnification) { return true; @@ -1820,7 +1827,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgensIn, - unboxingDepth + unboxingDepth, ) { if (fnType.bindings.size < queryElem.bindings.size) { return false; @@ -1849,7 +1856,7 @@ function initSearch(rawSearchIndex) { // possible solutions return false; }, - unboxingDepth + unboxingDepth, ); return newSolutions; }); @@ -1887,7 +1894,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + unboxingDepth, ) { if (unboxingDepth >= UNBOXING_LIMIT) { return false; @@ -1914,7 +1921,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgensTmp, - unboxingDepth + unboxingDepth, ); } else if (fnType.generics.length > 0 || fnType.bindings.size > 0) { const simplifiedGenerics = [ @@ -1926,7 +1933,7 @@ function initSearch(rawSearchIndex) { queryElem, whereClause, mgens, - unboxingDepth + unboxingDepth, ); } return false; @@ -1975,7 +1982,7 @@ function initSearch(rawSearchIndex) { elem, whereClause, mgens, - unboxingDepth + 1 + unboxingDepth + 1, ); } if (row.id > 0 && elem.id > 0 && elem.pathWithoutLast.length === 0 && @@ -1989,7 +1996,7 @@ function initSearch(rawSearchIndex) { elem, whereClause, mgens, - unboxingDepth + unboxingDepth, ); } } @@ -2007,7 +2014,7 @@ function initSearch(rawSearchIndex) { return 0; } const maxPathEditDistance = Math.floor( - contains.reduce((acc, next) => acc + next.length, 0) / 3 + contains.reduce((acc, next) => acc + next.length, 0) / 3, ); let ret_dist = maxPathEditDistance + 1; const path = ty.path.split("::"); @@ -2066,7 +2073,8 @@ function initSearch(rawSearchIndex) { crate: item.crate, name: item.name, path: item.path, - desc: item.desc, + descShard: item.descShard, + descIndex: item.descIndex, ty: item.ty, parent: item.parent, type: item.type, @@ -2192,7 +2200,7 @@ function initSearch(rawSearchIndex) { results_others, results_in_args, results_returned, - maxEditDistance + maxEditDistance, ) { if (!row || (filterCrates !== null && row.crate !== filterCrates)) { return; @@ -2204,7 +2212,7 @@ function initSearch(rawSearchIndex) { // atoms in the function not present in the query const tfpDist = compareTypeFingerprints( fullId, - parsedQuery.typeFingerprint + parsedQuery.typeFingerprint, ); if (tfpDist !== null) { const in_args = row.type && row.type.inputs @@ -2276,7 +2284,7 @@ function initSearch(rawSearchIndex) { const tfpDist = compareTypeFingerprints( row.id, - parsedQuery.typeFingerprint + parsedQuery.typeFingerprint, ); if (tfpDist === null) { return; @@ -2298,10 +2306,10 @@ function initSearch(rawSearchIndex) { row.type.where_clause, mgens, null, - 0 // unboxing depth + 0, // unboxing depth ); }, - 0 // unboxing depth + 0, // unboxing depth )) { return; } @@ -2419,7 +2427,7 @@ function initSearch(rawSearchIndex) { } return [typeNameIdMap.get(name).id, constraints]; - }) + }), ); } @@ -2446,7 +2454,7 @@ function initSearch(rawSearchIndex) { results_others, results_in_args, results_returned, - maxEditDistance + maxEditDistance, ); } } @@ -2478,9 +2486,9 @@ function initSearch(rawSearchIndex) { } const ret = createQueryResults( - sortResults(results_in_args, true, currentCrate), - sortResults(results_returned, true, currentCrate), - sortResults(results_others, false, currentCrate), + await sortResults(results_in_args, true, currentCrate), + await sortResults(results_returned, true, currentCrate), + await sortResults(results_others, false, currentCrate), parsedQuery); handleAliases(ret, parsedQuery.original.replace(/"/g, ""), filterCrates, currentCrate); if (parsedQuery.error !== null && ret.others.length !== 0) { @@ -2581,14 +2589,14 @@ function initSearch(rawSearchIndex) { * @param {ParsedQuery} query * @param {boolean} display - True if this is the active tab */ - function addTab(array, query, display) { + async function addTab(array, query, display) { const extraClass = display ? " active" : ""; const output = document.createElement("div"); if (array.length > 0) { output.className = "search-results " + extraClass; - array.forEach(item => { + for (const item of array) { const name = item.name; const type = itemTypes[item.ty]; const longType = longItemTypes[item.ty]; @@ -2624,7 +2632,7 @@ ${item.displayPath}<span class="${type}">${name}</span>\ link.appendChild(description); output.appendChild(link); - }); + } } else if (query.error === null) { output.className = "search-failed" + extraClass; output.innerHTML = "No results :(<br/>" + @@ -2666,7 +2674,7 @@ ${item.displayPath}<span class="${type}">${name}</span>\ * @param {boolean} go_to_first * @param {string} filterCrates */ - function showResults(results, go_to_first, filterCrates) { + async function showResults(results, go_to_first, filterCrates) { const search = searchState.outputElement(); if (go_to_first || (results.others.length === 1 && getSettingValue("go-to-only-result") === "true") @@ -2699,9 +2707,9 @@ ${item.displayPath}<span class="${type}">${name}</span>\ currentResults = results.query.userQuery; - const ret_others = addTab(results.others, results.query, true); - const ret_in_args = addTab(results.in_args, results.query, false); - const ret_returned = addTab(results.returned, results.query, false); + const ret_others = await addTab(results.others, results.query, true); + const ret_in_args = await addTab(results.in_args, results.query, false); + const ret_returned = await addTab(results.returned, results.query, false); // Navigate to the relevant tab if the current tab is empty, like in case users search // for "-> String". If they had selected another tab previously, they have to click on @@ -2822,7 +2830,7 @@ ${item.displayPath}<span class="${type}">${name}</span>\ * and display the results. * @param {boolean} [forced] */ - function search(forced) { + async function search(forced) { const query = parseQuery(searchState.input.value.trim()); let filterCrates = getFilterCrates(); @@ -2850,8 +2858,8 @@ ${item.displayPath}<span class="${type}">${name}</span>\ // recent search query is added to the browser history. updateSearchHistory(buildUrl(query.original, filterCrates)); - showResults( - execQuery(query, filterCrates, window.currentCrate), + await showResults( + await execQuery(query, filterCrates, window.currentCrate), params.go_to_first, filterCrates); } @@ -2920,7 +2928,7 @@ ${item.displayPath}<span class="${type}">${name}</span>\ pathIndex = type[PATH_INDEX_DATA]; generics = buildItemSearchTypeAll( type[GENERICS_DATA], - lowercasePaths + lowercasePaths, ); if (type.length > BINDINGS_DATA && type[BINDINGS_DATA].length > 0) { bindings = new Map(type[BINDINGS_DATA].map(binding => { @@ -3030,101 +3038,49 @@ ${item.displayPath}<span class="${type}">${name}</span>\ * The raw function search type format is generated using serde in * librustdoc/html/render/mod.rs: IndexItemFunctionType::write_to_string * - * @param {{ - * string: string, - * offset: number, - * backrefQueue: FunctionSearchType[] - * }} itemFunctionDecoder * @param {Array<{name: string, ty: number}>} lowercasePaths - * @param {Map<string, integer>} * * @return {null|FunctionSearchType} */ - function buildFunctionSearchType(itemFunctionDecoder, lowercasePaths) { - const c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - itemFunctionDecoder.offset += 1; - const [zero, ua, la, ob, cb] = ["0", "@", "`", "{", "}"].map(c => c.charCodeAt(0)); - // `` ` `` is used as a sentinel because it's fewer bytes than `null`, and decodes to zero - // `0` is a backref - if (c === la) { - return null; - } - // sixteen characters after "0" are backref - if (c >= zero && c < ua) { - return itemFunctionDecoder.backrefQueue[c - zero]; - } - if (c !== ob) { - throw ["Unexpected ", c, " in function: expected ", "{", "; this is a bug"]; - } - // call after consuming `{` - function decodeList() { - let c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - const ret = []; - while (c !== cb) { - ret.push(decode()); - c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - } - itemFunctionDecoder.offset += 1; // eat cb - return ret; - } - // consumes and returns a list or integer - function decode() { - let n = 0; - let c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - if (c === ob) { - itemFunctionDecoder.offset += 1; - return decodeList(); - } - while (c < la) { - n = (n << 4) | (c & 0xF); - itemFunctionDecoder.offset += 1; - c = itemFunctionDecoder.string.charCodeAt(itemFunctionDecoder.offset); - } - // last character >= la - n = (n << 4) | (c & 0xF); - const [sign, value] = [n & 1, n >> 1]; - itemFunctionDecoder.offset += 1; - return sign ? -value : value; - } - const functionSearchType = decodeList(); - const INPUTS_DATA = 0; - const OUTPUT_DATA = 1; - let inputs, output; - if (typeof functionSearchType[INPUTS_DATA] === "number") { - inputs = [buildItemSearchType(functionSearchType[INPUTS_DATA], lowercasePaths)]; - } else { - inputs = buildItemSearchTypeAll( - functionSearchType[INPUTS_DATA], - lowercasePaths - ); - } - if (functionSearchType.length > 1) { - if (typeof functionSearchType[OUTPUT_DATA] === "number") { - output = [buildItemSearchType(functionSearchType[OUTPUT_DATA], lowercasePaths)]; + function buildFunctionSearchTypeCallback(lowercasePaths) { + return functionSearchType => { + if (functionSearchType === 0) { + return null; + } + const INPUTS_DATA = 0; + const OUTPUT_DATA = 1; + let inputs, output; + if (typeof functionSearchType[INPUTS_DATA] === "number") { + inputs = [buildItemSearchType(functionSearchType[INPUTS_DATA], lowercasePaths)]; } else { - output = buildItemSearchTypeAll( - functionSearchType[OUTPUT_DATA], - lowercasePaths + inputs = buildItemSearchTypeAll( + functionSearchType[INPUTS_DATA], + lowercasePaths, ); } - } else { - output = []; - } - const where_clause = []; - const l = functionSearchType.length; - for (let i = 2; i < l; ++i) { - where_clause.push(typeof functionSearchType[i] === "number" - ? [buildItemSearchType(functionSearchType[i], lowercasePaths)] - : buildItemSearchTypeAll(functionSearchType[i], lowercasePaths)); - } - const ret = { - inputs, output, where_clause, + if (functionSearchType.length > 1) { + if (typeof functionSearchType[OUTPUT_DATA] === "number") { + output = [buildItemSearchType(functionSearchType[OUTPUT_DATA], lowercasePaths)]; + } else { + output = buildItemSearchTypeAll( + functionSearchType[OUTPUT_DATA], + lowercasePaths, + ); + } + } else { + output = []; + } + const where_clause = []; + const l = functionSearchType.length; + for (let i = 2; i < l; ++i) { + where_clause.push(typeof functionSearchType[i] === "number" + ? [buildItemSearchType(functionSearchType[i], lowercasePaths)] + : buildItemSearchTypeAll(functionSearchType[i], lowercasePaths)); + } + return { + inputs, output, where_clause, + }; }; - itemFunctionDecoder.backrefQueue.unshift(ret); - if (itemFunctionDecoder.backrefQueue.length > 16) { - itemFunctionDecoder.backrefQueue.pop(); - } - return ret; } /** @@ -3245,6 +3201,68 @@ ${item.displayPath}<span class="${type}">${name}</span>\ return functionTypeFingerprint[(fullId * 4) + 3]; } + class VlqHexDecoder { + constructor(string, cons) { + this.string = string; + this.cons = cons; + this.offset = 0; + this.backrefQueue = []; + } + // call after consuming `{` + decodeList() { + const cb = "}".charCodeAt(0); + let c = this.string.charCodeAt(this.offset); + const ret = []; + while (c !== cb) { + ret.push(this.decode()); + c = this.string.charCodeAt(this.offset); + } + this.offset += 1; // eat cb + return ret; + } + // consumes and returns a list or integer + decode() { + const [ob, la] = ["{", "`"].map(c => c.charCodeAt(0)); + let n = 0; + let c = this.string.charCodeAt(this.offset); + if (c === ob) { + this.offset += 1; + return this.decodeList(); + } + while (c < la) { + n = (n << 4) | (c & 0xF); + this.offset += 1; + c = this.string.charCodeAt(this.offset); + } + // last character >= la + n = (n << 4) | (c & 0xF); + const [sign, value] = [n & 1, n >> 1]; + this.offset += 1; + return sign ? -value : value; + } + next() { + const c = this.string.charCodeAt(this.offset); + const [zero, ua, la] = ["0", "@", "`"].map(c => c.charCodeAt(0)); + // sixteen characters after "0" are backref + if (c >= zero && c < ua) { + this.offset += 1; + return this.backrefQueue[c - zero]; + } + // special exception: 0 doesn't use backref encoding + // it's already one character, and it's always nullish + if (c === la) { + this.offset += 1; + return this.cons(0); + } + const result = this.cons(this.decode()); + this.backrefQueue.unshift(result); + if (this.backrefQueue.length > 16) { + this.backrefQueue.pop(); + } + return result; + } + } + /** * Convert raw search index into in-memory search index. * @@ -3271,18 +3289,32 @@ ${item.displayPath}<span class="${type}">${name}</span>\ id = 0; for (const [crate, crateCorpus] of rawSearchIndex) { + // a string representing the lengths of each description shard + // a string representing the list of function types + const itemDescShardDecoder = new VlqHexDecoder(crateCorpus.D, noop => noop); + let descShard = { + crate, + shard: 0, + start: 0, + len: itemDescShardDecoder.next(), + promise: null, + resolve: null, + }; + const descShardList = [ descShard ]; + // This object should have exactly the same set of fields as the "row" // object defined below. Your JavaScript runtime will thank you. // https://mathiasbynens.be/notes/shapes-ics const crateRow = { - crate: crate, + crate, ty: 3, // == ExternCrate name: crate, path: "", - desc: crateCorpus.doc, + descShard, + descIndex: 0, parent: undefined, type: null, - id: id, + id, word: crate, normalizedName: crate.indexOf("_") === -1 ? crate : crate.replace(/_/g, ""), deprecated: null, @@ -3302,16 +3334,8 @@ ${item.displayPath}<span class="${type}">${name}</span>\ // i.e. if indices 4 and 11 are present, but 5-10 and 12-13 are not present, // 5-10 will fall back to the path for 4 and 12-13 will fall back to the path for 11 const itemPaths = new Map(crateCorpus.q); - // an array of (String) descriptions - const itemDescs = crateCorpus.d; // an array of (Number) the parent path index + 1 to `paths`, or 0 if none const itemParentIdxs = crateCorpus.i; - // a string representing the list of function types - const itemFunctionDecoder = { - string: crateCorpus.f, - offset: 0, - backrefQueue: [], - }; // an array of (Number) indices for the deprecated items const deprecatedItems = new Set(crateCorpus.c); // an array of (Number) indices for the deprecated items @@ -3326,6 +3350,12 @@ ${item.displayPath}<span class="${type}">${name}</span>\ // an array of [{name: String, ty: Number}] const lowercasePaths = []; + // a string representing the list of function types + const itemFunctionDecoder = new VlqHexDecoder( + crateCorpus.f, + buildFunctionSearchTypeCallback(lowercasePaths), + ); + // convert `rawPaths` entries into object form // generate normalizedPaths for function search mode let len = paths.length; @@ -3353,13 +3383,26 @@ ${item.displayPath}<span class="${type}">${name}</span>\ // faster analysis operations lastPath = ""; len = itemTypes.length; + let descIndex = 1; for (let i = 0; i < len; ++i) { + if (descIndex >= descShard.len) { + descShard = { + crate, + shard: descShard.shard + 1, + start: descShard.start + descShard.len, + len: itemDescShardDecoder.next(), + promise: null, + resolve: null, + }; + descIndex = 0; + descShardList.push(descShard); + } let word = ""; if (typeof itemNames[i] === "string") { word = itemNames[i].toLowerCase(); } const path = itemPaths.has(i) ? itemPaths.get(i) : lastPath; - const type = buildFunctionSearchType(itemFunctionDecoder, lowercasePaths); + const type = itemFunctionDecoder.next(); if (type !== null) { if (type) { const fp = functionTypeFingerprint.subarray(id * 4, (id + 1) * 4); @@ -3380,14 +3423,15 @@ ${item.displayPath}<span class="${type}">${name}</span>\ // This object should have exactly the same set of fields as the "crateRow" // object defined above. const row = { - crate: crate, + crate, ty: itemTypes.charCodeAt(i) - charA, name: itemNames[i], - path: path, - desc: itemDescs[i], + path, + descShard, + descIndex, parent: itemParentIdxs[i] > 0 ? paths[itemParentIdxs[i] - 1] : undefined, type, - id: id, + id, word, normalizedName: word.indexOf("_") === -1 ? word : word.replace(/_/g, ""), deprecated: deprecatedItems.has(i), @@ -3396,6 +3440,7 @@ ${item.displayPath}<span class="${type}">${name}</span>\ id += 1; searchIndex.push(row); lastPath = row.path; + descIndex += 1; } if (aliases) { @@ -3419,6 +3464,7 @@ ${item.displayPath}<span class="${type}">${name}</span>\ } } currentIndex += itemTypes.length; + searchState.descShards.set(crate, descShardList); } // Drop the (rather large) hash table used for reusing function items TYPES_POOL = new Map(); diff --git a/src/librustdoc/html/static/js/storage.js b/src/librustdoc/html/static/js/storage.js index bda7b3c647e..73c543567c0 100644 --- a/src/librustdoc/html/static/js/storage.js +++ b/src/librustdoc/html/static/js/storage.js @@ -211,14 +211,14 @@ function updateSidebarWidth() { if (desktopSidebarWidth && desktopSidebarWidth !== "null") { document.documentElement.style.setProperty( "--desktop-sidebar-width", - desktopSidebarWidth + "px" + desktopSidebarWidth + "px", ); } const srcSidebarWidth = getSettingValue("src-sidebar-width"); if (srcSidebarWidth && srcSidebarWidth !== "null") { document.documentElement.style.setProperty( "--src-sidebar-width", - srcSidebarWidth + "px" + srcSidebarWidth + "px", ); } } diff --git a/src/tools/rustdoc-js/.eslintrc.js b/src/tools/rustdoc-js/.eslintrc.js index 4ab3a315733..b9d0e251c24 100644 --- a/src/tools/rustdoc-js/.eslintrc.js +++ b/src/tools/rustdoc-js/.eslintrc.js @@ -6,7 +6,7 @@ module.exports = { }, "extends": "eslint:recommended", "parserOptions": { - "ecmaVersion": 2015, + "ecmaVersion": 8, "sourceType": "module" }, "rules": { diff --git a/src/tools/rustdoc-js/tester.js b/src/tools/rustdoc-js/tester.js index 86881ef362e..ae4121cc03f 100644 --- a/src/tools/rustdoc-js/tester.js +++ b/src/tools/rustdoc-js/tester.js @@ -1,3 +1,4 @@ +/* global globalThis */ const fs = require("fs"); const path = require("path"); @@ -133,7 +134,7 @@ function valueCheck(fullPath, expected, result, error_text, queryName) { expected_value, result.get(key), error_text, - queryName + queryName, ); } else { error_text.push(`${queryName}==> EXPECTED has extra key in map from field ` + @@ -212,11 +213,11 @@ function runParser(query, expected, parseQuery, queryName) { return error_text; } -function runSearch(query, expected, doSearch, loadedFile, queryName) { +async function runSearch(query, expected, doSearch, loadedFile, queryName) { const ignore_order = loadedFile.ignore_order; const exact_check = loadedFile.exact_check; - const results = doSearch(query, loadedFile.FILTER_CRATE); + const results = await doSearch(query, loadedFile.FILTER_CRATE); const error_text = []; for (const key in expected) { @@ -238,7 +239,7 @@ function runSearch(query, expected, doSearch, loadedFile, queryName) { } let prev_pos = -1; - entry.forEach((elem, index) => { + for (const [index, elem] of entry.entries()) { const entry_pos = lookForEntry(elem, results[key]); if (entry_pos === -1) { error_text.push(queryName + "==> Result not found in '" + key + "': '" + @@ -260,13 +261,13 @@ function runSearch(query, expected, doSearch, loadedFile, queryName) { } else { prev_pos = entry_pos; } - }); + } } return error_text; } -function runCorrections(query, corrections, getCorrections, loadedFile) { - const qc = getCorrections(query, loadedFile.FILTER_CRATE); +async function runCorrections(query, corrections, getCorrections, loadedFile) { + const qc = await getCorrections(query, loadedFile.FILTER_CRATE); const error_text = []; if (corrections === null) { @@ -299,18 +300,27 @@ function checkResult(error_text, loadedFile, displaySuccess) { return 1; } -function runCheckInner(callback, loadedFile, entry, getCorrections, extra) { +async function runCheckInner(callback, loadedFile, entry, getCorrections, extra) { if (typeof entry.query !== "string") { console.log("FAILED"); console.log("==> Missing `query` field"); return false; } - let error_text = callback(entry.query, entry, extra ? "[ query `" + entry.query + "`]" : ""); + let error_text = await callback( + entry.query, + entry, + extra ? "[ query `" + entry.query + "`]" : "", + ); if (checkResult(error_text, loadedFile, false) !== 0) { return false; } if (entry.correction !== undefined) { - error_text = runCorrections(entry.query, entry.correction, getCorrections, loadedFile); + error_text = await runCorrections( + entry.query, + entry.correction, + getCorrections, + loadedFile, + ); if (checkResult(error_text, loadedFile, false) !== 0) { return false; } @@ -318,16 +328,16 @@ function runCheckInner(callback, loadedFile, entry, getCorrections, extra) { return true; } -function runCheck(loadedFile, key, getCorrections, callback) { +async function runCheck(loadedFile, key, getCorrections, callback) { const expected = loadedFile[key]; if (Array.isArray(expected)) { for (const entry of expected) { - if (!runCheckInner(callback, loadedFile, entry, getCorrections, true)) { + if (!await runCheckInner(callback, loadedFile, entry, getCorrections, true)) { return 1; } } - } else if (!runCheckInner(callback, loadedFile, expected, getCorrections, false)) { + } else if (!await runCheckInner(callback, loadedFile, expected, getCorrections, false)) { return 1; } console.log("OK"); @@ -338,7 +348,7 @@ function hasCheck(content, checkName) { return content.startsWith(`const ${checkName}`) || content.includes(`\nconst ${checkName}`); } -function runChecks(testFile, doSearch, parseQuery, getCorrections) { +async function runChecks(testFile, doSearch, parseQuery, getCorrections) { let checkExpected = false; let checkParsed = false; let testFileContent = readFile(testFile); @@ -367,12 +377,12 @@ function runChecks(testFile, doSearch, parseQuery, getCorrections) { let res = 0; if (checkExpected) { - res += runCheck(loadedFile, "EXPECTED", getCorrections, (query, expected, text) => { + res += await runCheck(loadedFile, "EXPECTED", getCorrections, (query, expected, text) => { return runSearch(query, expected, doSearch, loadedFile, text); }); } if (checkParsed) { - res += runCheck(loadedFile, "PARSED", getCorrections, (query, expected, text) => { + res += await runCheck(loadedFile, "PARSED", getCorrections, (query, expected, text) => { return runParser(query, expected, parseQuery, text); }); } @@ -393,6 +403,35 @@ function loadSearchJS(doc_folder, resource_suffix) { const searchIndexJs = path.join(doc_folder, "search-index" + resource_suffix + ".js"); const searchIndex = require(searchIndexJs); + globalThis.searchState = { + descShards: new Map(), + loadDesc: async function({descShard, descIndex}) { + if (descShard.promise === null) { + descShard.promise = new Promise((resolve, reject) => { + descShard.resolve = resolve; + const ds = descShard; + const fname = `${ds.crate}-desc-${ds.shard}-${resource_suffix}.js`; + fs.readFile( + `${doc_folder}/search.desc/${descShard.crate}/${fname}`, + (err, data) => { + if (err) { + reject(err); + } else { + eval(data.toString("utf8")); + } + }, + ); + }); + } + const list = await descShard.promise; + return list[descIndex]; + }, + loadedDescShard: function (crate, shard, data) { + //console.log(this.descShards); + this.descShards.get(crate)[shard].resolve(data.split("\n")); + }, + }; + const staticFiles = path.join(doc_folder, "static.files"); const searchJs = fs.readdirSync(staticFiles).find(f => f.match(/search.*\.js$/)); const searchModule = require(path.join(staticFiles, searchJs)); @@ -474,7 +513,7 @@ function parseOptions(args) { return null; } -function main(argv) { +async function main(argv) { const opts = parseOptions(argv.slice(2)); if (opts === null) { return 1; @@ -482,7 +521,7 @@ function main(argv) { const parseAndSearch = loadSearchJS( opts["doc_folder"], - opts["resource_suffix"] + opts["resource_suffix"], ); let errors = 0; @@ -494,21 +533,34 @@ function main(argv) { }; if (opts["test_file"].length !== 0) { - opts["test_file"].forEach(file => { + for (const file of opts["test_file"]) { process.stdout.write(`Testing ${file} ... `); - errors += runChecks(file, doSearch, parseAndSearch.parseQuery, getCorrections); - }); + errors += await runChecks(file, doSearch, parseAndSearch.parseQuery, getCorrections); + } } else if (opts["test_folder"].length !== 0) { - fs.readdirSync(opts["test_folder"]).forEach(file => { + for (const file of fs.readdirSync(opts["test_folder"])) { if (!file.endsWith(".js")) { - return; + continue; } process.stdout.write(`Testing ${file} ... `); - errors += runChecks(path.join(opts["test_folder"], file), doSearch, + errors += await runChecks(path.join(opts["test_folder"], file), doSearch, parseAndSearch.parseQuery, getCorrections); - }); + } } return errors > 0 ? 1 : 0; } -process.exit(main(process.argv)); +main(process.argv).catch(e => { + console.log(e); + process.exit(1); +}).then(x => process.exit(x)); + +process.on("beforeExit", () => { + console.log("process did not complete"); + process.exit(1); +}); + +/*process.on("uncaughtException", (err) => { + console.log(`Uncaught Exception: ${err.message}`); + process.exit(1); +});*/ |