Add updated support for example-analyzer

Move rendering of examples into

Finalize design

Cleanup, rename found -> scraped

Softer yellow

Clean up dead code

Document scrape_examples

More simplification and documentation

Remove extra css

Test

5 files changed, 407 insertions, 2 deletions

diff --git a/src/librustdoc/html/render/context.rs b/src/librustdoc/html/render/context.rs
index b99d2fe5aa0..49bf760c29c 100644
--- a/src/librustdoc/html/render/context.rs
+++ b/src/librustdoc/html/render/context.rs
@@ -124,6 +124,7 @@ crate struct SharedContext<'tcx> {
     crate span_correspondance_map: FxHashMap<rustc_span::Span, LinkFromSrc>,
     /// The [`Cache`] used during rendering.
     crate cache: Cache,
+    pub(super) repository_url: Option<String>,
 }
 
 impl SharedContext<'_> {
@@ -140,7 +141,11 @@ impl SharedContext<'_> {
     /// Returns the `collapsed_doc_value` of the given item if this is the main crate, otherwise
     /// returns the `doc_value`.
     crate fn maybe_collapsed_doc_value<'a>(&self, item: &'a clean::Item) -> Option<String> {
-        if self.collapsed { item.collapsed_doc_value() } else { item.doc_value() }
+        if self.collapsed {
+            item.collapsed_doc_value()
+        } else {
+            item.doc_value()
+        }
     }
 
     crate fn edition(&self) -> Edition {
@@ -389,6 +394,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
             generate_redirect_map,
             show_type_layout,
             generate_link_to_definition,
+            repository_url,
             ..
         } = options;
 
@@ -480,6 +486,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
             templates,
             span_correspondance_map: matches,
             cache,
+            repository_url,
         };
 
         // Add the default themes to the `Vec` of stylepaths
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index 11682afdf89..0fb7723b68b 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -39,6 +39,7 @@ crate use span_map::{collect_spans_and_sources, LinkFromSrc};
 use std::collections::VecDeque;
 use std::default::Default;
 use std::fmt;
+use std::fs;
 use std::path::PathBuf;
 use std::str;
 use std::string::ToString;
@@ -68,6 +69,8 @@ use crate::html::format::{
     print_generic_bounds, print_where_clause, Buffer, HrefError, PrintWithSpace,
 };
 use crate::html::markdown::{HeadingOffset, Markdown, MarkdownHtml, MarkdownSummaryLine};
+use crate::html::sources;
+use crate::scrape_examples::FnCallLocations;
 
 /// A pair of name and its optional document.
 crate type NameDoc = (String, Option<String>);
@@ -584,6 +587,13 @@ fn document_full_inner(
             render_markdown(w, cx, &s, item.links(cx), heading_offset);
         }
     }
+
+    match &*item.kind {
+        clean::ItemKind::FunctionItem(f) | clean::ItemKind::MethodItem(f, _) => {
+            render_call_locations(w, cx, &f.call_locations);
+        }
+        _ => {}
+    }
 }
 
 /// Add extra information about an item such as:
@@ -2440,3 +2450,88 @@ fn collect_paths_for_type(first_ty: clean::Type, cache: &Cache) -> Vec<String> {
     }
     out
 }
+
+fn render_call_locations(
+    w: &mut Buffer,
+    cx: &Context<'_>,
+    call_locations: &Option<FnCallLocations>,
+) {
+    let call_locations = match call_locations.as_ref() {
+        Some(call_locations) => call_locations,
+        None => {
+            return;
+        }
+    };
+
+    let filtered_locations: Vec<_> = call_locations
+        .iter()
+        .filter_map(|(file, locs)| {
+            // TODO(wcrichto): file I/O should be cached
+            let mut contents = match fs::read_to_string(&file) {
+                Ok(contents) => contents,
+                Err(e) => {
+                    eprintln!("Failed to read file {}", e);
+                    return None;
+                }
+            };
+
+            // Remove the utf-8 BOM if any
+            if contents.starts_with('\u{feff}') {
+                contents.drain(..3);
+            }
+
+            Some((file, contents, locs))
+        })
+        .collect();
+
+    let n_examples = filtered_locations.len();
+    if n_examples == 0 {
+        return;
+    }
+
+    let id = cx.id_map.borrow_mut().derive("scraped-examples");
+    write!(
+        w,
+        r##"<div class="docblock scraped-example-list">
+          <h1 id="scraped-examples" class="small-section-header">
+             <a href="#{}">Uses found in <code>examples/</code></a>
+          </h1>"##,
+        id
+    );
+
+    let write_example = |w: &mut Buffer, (file, contents, locs): (&String, String, _)| {
+        let ex_title = match cx.shared.repository_url.as_ref() {
+            Some(url) => format!(
+                r#"<a href="{url}/{file}" target="_blank">{file}</a>"#,
+                file = file,
+                url = url
+            ),
+            None => file.clone(),
+        };
+        let edition = cx.shared.edition();
+        write!(
+            w,
+            r#"<div class="scraped-example" data-code="{code}" data-locs="{locations}">
+           <strong>{title}</strong>
+           <div class="code-wrapper">"#,
+            code = contents.replace("\"", "&quot;"),
+            locations = serde_json::to_string(&locs).unwrap(),
+            title = ex_title,
+        );
+        write!(w, r#"<span class="prev">&pr;</span> <span class="next">&sc;</span>"#);
+        write!(w, r#"<span class="expand">&varr;</span>"#);
+        sources::print_src(w, &contents, edition);
+        write!(w, "</div></div>");
+    };
+
+    let mut it = filtered_locations.into_iter();
+    write_example(w, it.next().unwrap());
+
+    if n_examples > 1 {
+        write!(w, r#"<div class="more-scraped-examples hidden">"#);
+        it.for_each(|ex| write_example(w, ex));
+        write!(w, "</div>");
+    }
+
+    write!(w, "</div>");
+}
diff --git a/src/librustdoc/html/sources.rs b/src/librustdoc/html/sources.rs
index 71c64231a21..d6dead15205 100644
--- a/src/librustdoc/html/sources.rs
+++ b/src/librustdoc/html/sources.rs
@@ -243,7 +243,7 @@ where
 
 /// Wrapper struct to render the source code of a file. This will do things like
 /// adding line numbers to the left-hand side.
-fn print_src(
+crate fn print_src(
     buf: &mut Buffer,
     s: &str,
     edition: Edition,
diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index 5d33681847a..ca8db4530f3 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -1970,3 +1970,110 @@ details.undocumented[open] > summary::before {
 		margin-left: 12px;
 	}
 }
+
+/* This part is for the new "examples" components */
+
+.scraped-example:not(.expanded) .code-wrapper pre.line-numbers, .scraped-example:not(.expanded) .code-wrapper .example-wrap pre.rust {
+	overflow: hidden;
+	height: 240px;
+}
+
+.scraped-example .code-wrapper .prev {
+	position: absolute;
+	top: 0.25em;
+	right: 2.25em;
+	z-index: 100;
+	cursor: pointer;
+}
+
+.scraped-example .code-wrapper .next {
+	position: absolute;
+	top: 0.25em;
+	right: 1.25em;
+	z-index: 100;
+	cursor: pointer;
+}
+
+.scraped-example .code-wrapper .expand {
+	position: absolute;
+	top: 0.25em;
+	right: 0.25em;
+	z-index: 100;
+	cursor: pointer;
+}
+
+.scraped-example .code-wrapper {
+	position: relative;
+	display: flex;
+	flex-direction: row;
+	flex-wrap: wrap;
+	width: 100%;
+}
+
+.scraped-example:not(.expanded) .code-wrapper:before {
+	content: " ";
+	width: 100%;
+	height: 20px;
+	position: absolute;
+	z-index: 100;
+	top: 0;
+	background: linear-gradient(to bottom, rgba(255, 255, 255, 1), rgba(255, 255, 255, 0));
+}
+
+.scraped-example:not(.expanded) .code-wrapper:after {
+	content: " ";
+	width: 100%;
+	height: 20px;
+	position: absolute;
+	z-index: 100;
+	bottom: 0;
+	background: linear-gradient(to top, rgba(255, 255, 255, 1), rgba(255, 255, 255, 0));
+}
+
+.scraped-example:not(.expanded) .code-wrapper {
+	overflow: hidden;
+	height: 240px;
+}
+
+.scraped-example .code-wrapper .line-numbers {
+	margin: 0;
+	padding: 14px 0;
+}
+
+.scraped-example .code-wrapper .line-numbers span {
+	padding: 0 14px;
+}
+
+.scraped-example .code-wrapper .example-wrap {
+	flex: 1;
+	overflow-x: auto;
+	overflow-y: hidden;
+	margin-bottom: 0;
+}
+
+.scraped-example .code-wrapper .example-wrap pre.rust {
+	overflow-x: inherit;
+	width: inherit;
+	overflow-y: hidden;
+}
+
+.scraped-example .line-numbers span.highlight {
+	background: #f6fdb0;
+}
+
+.scraped-example .example-wrap .rust span.highlight {
+	background: #f6fdb0;
+}
+
+.more-scraped-examples {
+	padding-left: 10px;
+	border-left: 1px solid #ccc;
+}
+
+.toggle-examples .collapse-toggle {
+	position: relative;
+}
+
+.toggle-examples a {
+	color: #999 !important; // FIXME(wcrichto): why is important needed
+}
diff --git a/src/librustdoc/html/static/js/main.js b/src/librustdoc/html/static/js/main.js
index e396fd9d288..5ac00ff244a 100644
--- a/src/librustdoc/html/static/js/main.js
+++ b/src/librustdoc/html/static/js/main.js
@@ -979,6 +979,202 @@ function hideThemeButtonState() {
     onHashChange(null);
     window.addEventListener("hashchange", onHashChange);
     searchState.setup();
+
+    /////// EXAMPLE ANALYZER
+
+    // Merge the full set of [from, to] offsets into a minimal set of non-overlapping
+    // [from, to] offsets.
+    // NB: This is such a archetypal software engineering interview question that
+    // I can't believe I actually had to write it. Yes, it's O(N) in the input length --
+    // but it does assume a sorted input!
+    function distinctRegions(locs) {
+        var start = -1;
+        var end = -1;
+        var output = [];
+        for (var i = 0; i < locs.length; i++) {
+            var loc = locs[i];
+            if (loc[0] > end) {
+                if (end > 0) {
+                    output.push([start, end]);
+                }
+                start = loc[0];
+                end = loc[1];
+            } else {
+                end = Math.max(end, loc[1]);
+            }
+        }
+        if (end > 0) {
+            output.push([start, end]);
+        }
+        return output;
+    }
+
+    function convertLocsStartsToLineOffsets(code, locs) {
+        locs = distinctRegions(locs.slice(0).sort(function (a, b) {
+            return a[0] === b[0] ? a[1] - b[1] : a[0] - b[0];
+        })); // sort by start; use end if start is equal.
+        var codeLines = code.split("\n");
+        var lineIndex = 0;
+        var totalOffset = 0;
+        var output = [];
+
+        while (locs.length > 0 && lineIndex < codeLines.length) {
+            var lineLength = codeLines[lineIndex].length + 1; // +1 here and later is due to omitted \n
+            while (locs.length > 0 && totalOffset + lineLength > locs[0][0]) {
+                var endIndex = lineIndex;
+                var charsRemaining = locs[0][1] - totalOffset;
+                while (endIndex < codeLines.length && charsRemaining > codeLines[endIndex].length + 1) {
+                    charsRemaining -= codeLines[endIndex].length + 1;
+                    endIndex += 1;
+                }
+                output.push({
+                    from: [lineIndex, locs[0][0] - totalOffset],
+                    to: [endIndex, charsRemaining]
+                });
+                locs.shift();
+            }
+            lineIndex++;
+            totalOffset += lineLength;
+        }
+        return output;
+    }
+
+    // inserts str into html, *but* calculates idx by eliding anything in html that's not in raw.
+    // ideally this would work by walking the element tree...but this is good enough for now.
+    function insertStrAtRawIndex(raw, html, idx, str) {
+        if (idx > raw.length) {
+            return html;
+        }
+        if (idx == raw.length) {
+            return html + str;
+        }
+        var rawIdx = 0;
+        var htmlIdx = 0;
+        while (rawIdx < idx && rawIdx < raw.length) {
+            while (raw[rawIdx] !== html[htmlIdx] && htmlIdx < html.length) {
+                htmlIdx++;
+            }
+            rawIdx++;
+            htmlIdx++;
+        }
+        return html.substring(0, htmlIdx) + str + html.substr(htmlIdx);
+    }
+
+    // Scroll code block to put the given code location in the middle of the viewer
+    function scrollToLoc(elt, loc) {
+        var wrapper = elt.querySelector(".code-wrapper");
+        var halfHeight = wrapper.offsetHeight / 2;
+        var lines = elt.querySelector('.line-numbers');
+        var offsetMid = (lines.children[loc.from[0]].offsetTop + lines.children[loc.to[0]].offsetTop) / 2;
+        var scrollOffset = offsetMid - halfHeight;
+        lines.scrollTo(0, scrollOffset);
+        elt.querySelector(".rust").scrollTo(0, scrollOffset);
+    }
+
+    function updateScrapedExample(example) {
+        var code = example.attributes.getNamedItem("data-code").textContent;
+        var codeLines = code.split("\n");
+        var locs = JSON.parse(example.attributes.getNamedItem("data-locs").textContent);
+        locs = convertLocsStartsToLineOffsets(code, locs);
+
+        // Add call-site highlights to code listings
+        var litParent = example.querySelector('.example-wrap pre.rust');
+        var litHtml = litParent.innerHTML.split("\n");
+        onEach(locs, function (loc) {
+            for (var i = loc.from[0]; i < loc.to[0] + 1; i++) {
+                addClass(example.querySelector('.line-numbers').children[i], "highlight");
+            }
+            litHtml[loc.to[0]] = insertStrAtRawIndex(
+                codeLines[loc.to[0]],
+                litHtml[loc.to[0]],
+                loc.to[1],
+                "</span>");
+            litHtml[loc.from[0]] = insertStrAtRawIndex(
+                codeLines[loc.from[0]],
+                litHtml[loc.from[0]],
+                loc.from[1],
+                '<span class="highlight" data-loc="' + JSON.stringify(loc).replace(/"/g, "&quot;") + '">');
+        }, true); // do this backwards to avoid shifting later offsets
+        litParent.innerHTML = litHtml.join('\n');
+
+        // Toggle through list of examples in a given file
+        var locIndex = 0;
+        if (locs.length > 1) {
+            example.querySelector('.prev')
+                .addEventListener('click', function () {
+                    locIndex = (locIndex - 1 + locs.length) % locs.length;
+                    scrollToLoc(example, locs[locIndex]);
+                });
+            example.querySelector('.next')
+                .addEventListener('click', function () {
+                    locIndex = (locIndex + 1) % locs.length;
+                    scrollToLoc(example, locs[locIndex]);
+                });
+        } else {
+            example.querySelector('.prev').remove();
+            example.querySelector('.next').remove();
+        }
+
+        // Show full code on expansion
+        example.querySelector('.expand').addEventListener('click', function () {
+            if (hasClass(example, "expanded")) {
+                removeClass(example, "expanded");
+                scrollToLoc(example, locs[0]);
+            } else {
+                addClass(example, "expanded");
+            }
+        });
+
+        // Start with the first example in view
+        scrollToLoc(example, locs[0]);
+    }
+
+    function updateScrapedExamples() {
+        onEach(document.getElementsByClassName('scraped-example-list'), function (exampleSet) {
+            updateScrapedExample(exampleSet.querySelector(".small-section-header + .scraped-example"));
+        });
+
+        onEach(document.getElementsByClassName("more-scraped-examples"), function (more) {
+            var toggle = createSimpleToggle(true);
+            var label = "More examples";
+            var wrapper = createToggle(toggle, label, 14, "toggle-examples", false);
+            more.parentNode.insertBefore(wrapper, more);
+            var examples_init = false;
+
+            // Show additional examples on click
+            wrapper.onclick = function () {
+                if (hasClass(this, "collapsed")) {
+                    removeClass(this, "collapsed");
+                    onEachLazy(this.parentNode.getElementsByClassName("hidden"), function (x) {
+                        if (hasClass(x, "content") === false) {
+                            removeClass(x, "hidden");
+                            addClass(x, "x")
+                        }
+                    }, true);
+                    this.querySelector('.toggle-label').innerHTML = "Hide examples";
+                    this.querySelector('.inner').innerHTML = labelForToggleButton(false);
+                    if (!examples_init) {
+                        examples_init = true;
+                        onEach(more.getElementsByClassName('scraped-example'), updateScrapedExample);
+                    }
+                } else {
+                    addClass(this, "collapsed");
+                    onEachLazy(this.parentNode.getElementsByClassName("x"), function (x) {
+                        if (hasClass(x, "content") === false) {
+                            addClass(x, "hidden");
+                            removeClass(x, "x")
+                        }
+                    }, true);
+                    this.querySelector('.toggle-label').innerHTML = label;
+                    this.querySelector('.inner').innerHTML = labelForToggleButton(true);
+                }
+            };
+        });
+    }
+
+    var start = Date.now();
+    updateScrapedExamples();
+    console.log("updated examples took", Date.now() - start, "ms");
 }());
 
 (function () {