Add updated support for example-analyzer

Move rendering of examples into

Finalize design

Cleanup, rename found -> scraped

Softer yellow

Clean up dead code

Document scrape_examples

More simplification and documentation

Remove extra css

Test

12 files changed, 609 insertions, 21 deletions

diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs
index 4a888b22332..09622d721f7 100644
--- a/src/librustdoc/clean/inline.rs
+++ b/src/librustdoc/clean/inline.rs
@@ -235,6 +235,7 @@ fn build_external_function(cx: &mut DocContext<'_>, did: DefId) -> clean::Functi
         decl,
         generics,
         header: hir::FnHeader { unsafety: sig.unsafety(), abi: sig.abi(), constness, asyncness },
+        call_locations: None,
     }
 }
 
diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs
index a55d85f5841..e2b1ff4547b 100644
--- a/src/librustdoc/clean/mod.rs
+++ b/src/librustdoc/clean/mod.rs
@@ -801,7 +801,10 @@ impl<'a> Clean<Function> for (&'a hir::FnSig<'a>, &'a hir::Generics<'a>, hir::Bo
     fn clean(&self, cx: &mut DocContext<'_>) -> Function {
         let (generics, decl) =
             enter_impl_trait(cx, |cx| (self.1.clean(cx), (&*self.0.decl, self.2).clean(cx)));
-        Function { decl, generics, header: self.0.header }
+        let mut function = Function { decl, generics, header: self.0.header, call_locations: None };
+        let def_id = self.2.hir_id.owner.to_def_id();
+        function.load_call_locations(def_id, cx);
+        function
     }
 }
 
@@ -933,12 +936,14 @@ impl Clean<Item> for hir::TraitItem<'_> {
                     let (generics, decl) = enter_impl_trait(cx, |cx| {
                         (self.generics.clean(cx), (&*sig.decl, &names[..]).clean(cx))
                     });
-                    let mut t = Function { header: sig.header, decl, generics };
+                    let mut t =
+                        Function { header: sig.header, decl, generics, call_locations: None };
                     if t.header.constness == hir::Constness::Const
                         && is_unstable_const_fn(cx.tcx, local_did).is_some()
                     {
                         t.header.constness = hir::Constness::NotConst;
                     }
+                    t.load_call_locations(self.def_id.to_def_id(), cx);
                     TyMethodItem(t)
                 }
                 hir::TraitItemKind::Type(ref bounds, ref default) => {
@@ -1057,21 +1062,21 @@ impl Clean<Item> for ty::AssocItem {
                         ty::ImplContainer(_) => Some(self.defaultness),
                         ty::TraitContainer(_) => None,
                     };
-                    MethodItem(
-                        Function {
-                            generics,
-                            decl,
-                            header: hir::FnHeader {
-                                unsafety: sig.unsafety(),
-                                abi: sig.abi(),
-                                constness,
-                                asyncness,
-                            },
+                    let mut function = Function {
+                        generics,
+                        decl,
+                        header: hir::FnHeader {
+                            unsafety: sig.unsafety(),
+                            abi: sig.abi(),
+                            constness,
+                            asyncness,
                         },
-                        defaultness,
-                    )
+                        call_locations: None,
+                    };
+                    function.load_call_locations(self.def_id, cx);
+                    MethodItem(function, defaultness)
                 } else {
-                    TyMethodItem(Function {
+                    let mut function = Function {
                         generics,
                         decl,
                         header: hir::FnHeader {
@@ -1080,7 +1085,10 @@ impl Clean<Item> for ty::AssocItem {
                             constness: hir::Constness::NotConst,
                             asyncness: hir::IsAsync::NotAsync,
                         },
-                    })
+                        call_locations: None,
+                    };
+                    function.load_call_locations(self.def_id, cx);
+                    TyMethodItem(function)
                 }
             }
             ty::AssocKind::Type => {
@@ -2098,6 +2106,7 @@ impl Clean<Item> for (&hir::ForeignItem<'_>, Option<Symbol>) {
                             constness: hir::Constness::NotConst,
                             asyncness: hir::IsAsync::NotAsync,
                         },
+                        call_locations: None,
                     })
                 }
                 hir::ForeignItemKind::Static(ref ty, mutability) => {
diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs
index 0e78fe7aec3..2e0be44d932 100644
--- a/src/librustdoc/clean/types.rs
+++ b/src/librustdoc/clean/types.rs
@@ -42,6 +42,7 @@ use crate::formats::cache::Cache;
 use crate::formats::item_type::ItemType;
 use crate::html::render::cache::ExternalLocation;
 use crate::html::render::Context;
+use crate::scrape_examples::FnCallLocations;
 
 use self::FnRetTy::*;
 use self::ItemKind::*;
@@ -1254,6 +1255,17 @@ crate struct Function {
     crate decl: FnDecl,
     crate generics: Generics,
     crate header: hir::FnHeader,
+    crate call_locations: Option<FnCallLocations>,
+}
+
+impl Function {
+    crate fn load_call_locations(&mut self, def_id: hir::def_id::DefId, cx: &DocContext<'_>) {
+        if let Some(call_locations) = cx.render_options.call_locations.as_ref() {
+            let key = cx.tcx.def_path(def_id).to_string_no_crate_verbose();
+            self.call_locations = call_locations.get(&key).cloned();
+            debug!("call_locations: {} -- {:?}", key, self.call_locations);
+        }
+    }
 }
 
 #[derive(Clone, PartialEq, Eq, Debug, Hash)]
diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs
index ac440a39515..f34f773ea56 100644
--- a/src/librustdoc/config.rs
+++ b/src/librustdoc/config.rs
@@ -25,6 +25,7 @@ use crate::html::render::StylePath;
 use crate::html::static_files;
 use crate::opts;
 use crate::passes::{self, Condition, DefaultPassOption};
+use crate::scrape_examples::AllCallLocations;
 use crate::theme;
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
@@ -158,6 +159,8 @@ crate struct Options {
     crate json_unused_externs: bool,
     /// Whether to skip capturing stdout and stderr of tests.
     crate nocapture: bool,
+
+    crate scrape_examples: Vec<String>,
 }
 
 impl fmt::Debug for Options {
@@ -280,6 +283,8 @@ crate struct RenderOptions {
     crate emit: Vec<EmitType>,
     /// If `true`, HTML source pages will generate links for items to their definition.
     crate generate_link_to_definition: bool,
+    crate call_locations: Option<AllCallLocations>,
+    crate repository_url: Option<String>,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -671,6 +676,9 @@ impl Options {
             return Err(1);
         }
 
+        let repository_url = matches.opt_str("repository-url");
+        let scrape_examples = matches.opt_strs("scrape-examples");
+
         let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format);
 
         Ok(Options {
@@ -737,10 +745,13 @@ impl Options {
                 ),
                 emit,
                 generate_link_to_definition,
+                call_locations: None,
+                repository_url,
             },
             crate_name,
             output_format,
             json_unused_externs,
+            scrape_examples,
         })
     }
 
diff --git a/src/librustdoc/html/render/context.rs b/src/librustdoc/html/render/context.rs
index b99d2fe5aa0..49bf760c29c 100644
--- a/src/librustdoc/html/render/context.rs
+++ b/src/librustdoc/html/render/context.rs
@@ -124,6 +124,7 @@ crate struct SharedContext<'tcx> {
     crate span_correspondance_map: FxHashMap<rustc_span::Span, LinkFromSrc>,
     /// The [`Cache`] used during rendering.
     crate cache: Cache,
+    pub(super) repository_url: Option<String>,
 }
 
 impl SharedContext<'_> {
@@ -140,7 +141,11 @@ impl SharedContext<'_> {
     /// Returns the `collapsed_doc_value` of the given item if this is the main crate, otherwise
     /// returns the `doc_value`.
     crate fn maybe_collapsed_doc_value<'a>(&self, item: &'a clean::Item) -> Option<String> {
-        if self.collapsed { item.collapsed_doc_value() } else { item.doc_value() }
+        if self.collapsed {
+            item.collapsed_doc_value()
+        } else {
+            item.doc_value()
+        }
     }
 
     crate fn edition(&self) -> Edition {
@@ -389,6 +394,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
             generate_redirect_map,
             show_type_layout,
             generate_link_to_definition,
+            repository_url,
             ..
         } = options;
 
@@ -480,6 +486,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
             templates,
             span_correspondance_map: matches,
             cache,
+            repository_url,
         };
 
         // Add the default themes to the `Vec` of stylepaths
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index 11682afdf89..0fb7723b68b 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -39,6 +39,7 @@ crate use span_map::{collect_spans_and_sources, LinkFromSrc};
 use std::collections::VecDeque;
 use std::default::Default;
 use std::fmt;
+use std::fs;
 use std::path::PathBuf;
 use std::str;
 use std::string::ToString;
@@ -68,6 +69,8 @@ use crate::html::format::{
     print_generic_bounds, print_where_clause, Buffer, HrefError, PrintWithSpace,
 };
 use crate::html::markdown::{HeadingOffset, Markdown, MarkdownHtml, MarkdownSummaryLine};
+use crate::html::sources;
+use crate::scrape_examples::FnCallLocations;
 
 /// A pair of name and its optional document.
 crate type NameDoc = (String, Option<String>);
@@ -584,6 +587,13 @@ fn document_full_inner(
             render_markdown(w, cx, &s, item.links(cx), heading_offset);
         }
     }
+
+    match &*item.kind {
+        clean::ItemKind::FunctionItem(f) | clean::ItemKind::MethodItem(f, _) => {
+            render_call_locations(w, cx, &f.call_locations);
+        }
+        _ => {}
+    }
 }
 
 /// Add extra information about an item such as:
@@ -2440,3 +2450,88 @@ fn collect_paths_for_type(first_ty: clean::Type, cache: &Cache) -> Vec<String> {
     }
     out
 }
+
+fn render_call_locations(
+    w: &mut Buffer,
+    cx: &Context<'_>,
+    call_locations: &Option<FnCallLocations>,
+) {
+    let call_locations = match call_locations.as_ref() {
+        Some(call_locations) => call_locations,
+        None => {
+            return;
+        }
+    };
+
+    let filtered_locations: Vec<_> = call_locations
+        .iter()
+        .filter_map(|(file, locs)| {
+            // TODO(wcrichto): file I/O should be cached
+            let mut contents = match fs::read_to_string(&file) {
+                Ok(contents) => contents,
+                Err(e) => {
+                    eprintln!("Failed to read file {}", e);
+                    return None;
+                }
+            };
+
+            // Remove the utf-8 BOM if any
+            if contents.starts_with('\u{feff}') {
+                contents.drain(..3);
+            }
+
+            Some((file, contents, locs))
+        })
+        .collect();
+
+    let n_examples = filtered_locations.len();
+    if n_examples == 0 {
+        return;
+    }
+
+    let id = cx.id_map.borrow_mut().derive("scraped-examples");
+    write!(
+        w,
+        r##"<div class="docblock scraped-example-list">
+          <h1 id="scraped-examples" class="small-section-header">
+             <a href="#{}">Uses found in <code>examples/</code></a>
+          </h1>"##,
+        id
+    );
+
+    let write_example = |w: &mut Buffer, (file, contents, locs): (&String, String, _)| {
+        let ex_title = match cx.shared.repository_url.as_ref() {
+            Some(url) => format!(
+                r#"<a href="{url}/{file}" target="_blank">{file}</a>"#,
+                file = file,
+                url = url
+            ),
+            None => file.clone(),
+        };
+        let edition = cx.shared.edition();
+        write!(
+            w,
+            r#"<div class="scraped-example" data-code="{code}" data-locs="{locations}">
+           <strong>{title}</strong>
+           <div class="code-wrapper">"#,
+            code = contents.replace("\"", "&quot;"),
+            locations = serde_json::to_string(&locs).unwrap(),
+            title = ex_title,
+        );
+        write!(w, r#"<span class="prev">&pr;</span> <span class="next">&sc;</span>"#);
+        write!(w, r#"<span class="expand">&varr;</span>"#);
+        sources::print_src(w, &contents, edition);
+        write!(w, "</div></div>");
+    };
+
+    let mut it = filtered_locations.into_iter();
+    write_example(w, it.next().unwrap());
+
+    if n_examples > 1 {
+        write!(w, r#"<div class="more-scraped-examples hidden">"#);
+        it.for_each(|ex| write_example(w, ex));
+        write!(w, "</div>");
+    }
+
+    write!(w, "</div>");
+}
diff --git a/src/librustdoc/html/sources.rs b/src/librustdoc/html/sources.rs
index 71c64231a21..d6dead15205 100644
--- a/src/librustdoc/html/sources.rs
+++ b/src/librustdoc/html/sources.rs
@@ -243,7 +243,7 @@ where
 
 /// Wrapper struct to render the source code of a file. This will do things like
 /// adding line numbers to the left-hand side.
-fn print_src(
+crate fn print_src(
     buf: &mut Buffer,
     s: &str,
     edition: Edition,
diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index 5d33681847a..ca8db4530f3 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -1970,3 +1970,110 @@ details.undocumented[open] > summary::before {
 		margin-left: 12px;
 	}
 }
+
+/* This part is for the new "examples" components */
+
+.scraped-example:not(.expanded) .code-wrapper pre.line-numbers, .scraped-example:not(.expanded) .code-wrapper .example-wrap pre.rust {
+	overflow: hidden;
+	height: 240px;
+}
+
+.scraped-example .code-wrapper .prev {
+	position: absolute;
+	top: 0.25em;
+	right: 2.25em;
+	z-index: 100;
+	cursor: pointer;
+}
+
+.scraped-example .code-wrapper .next {
+	position: absolute;
+	top: 0.25em;
+	right: 1.25em;
+	z-index: 100;
+	cursor: pointer;
+}
+
+.scraped-example .code-wrapper .expand {
+	position: absolute;
+	top: 0.25em;
+	right: 0.25em;
+	z-index: 100;
+	cursor: pointer;
+}
+
+.scraped-example .code-wrapper {
+	position: relative;
+	display: flex;
+	flex-direction: row;
+	flex-wrap: wrap;
+	width: 100%;
+}
+
+.scraped-example:not(.expanded) .code-wrapper:before {
+	content: " ";
+	width: 100%;
+	height: 20px;
+	position: absolute;
+	z-index: 100;
+	top: 0;
+	background: linear-gradient(to bottom, rgba(255, 255, 255, 1), rgba(255, 255, 255, 0));
+}
+
+.scraped-example:not(.expanded) .code-wrapper:after {
+	content: " ";
+	width: 100%;
+	height: 20px;
+	position: absolute;
+	z-index: 100;
+	bottom: 0;
+	background: linear-gradient(to top, rgba(255, 255, 255, 1), rgba(255, 255, 255, 0));
+}
+
+.scraped-example:not(.expanded) .code-wrapper {
+	overflow: hidden;
+	height: 240px;
+}
+
+.scraped-example .code-wrapper .line-numbers {
+	margin: 0;
+	padding: 14px 0;
+}
+
+.scraped-example .code-wrapper .line-numbers span {
+	padding: 0 14px;
+}
+
+.scraped-example .code-wrapper .example-wrap {
+	flex: 1;
+	overflow-x: auto;
+	overflow-y: hidden;
+	margin-bottom: 0;
+}
+
+.scraped-example .code-wrapper .example-wrap pre.rust {
+	overflow-x: inherit;
+	width: inherit;
+	overflow-y: hidden;
+}
+
+.scraped-example .line-numbers span.highlight {
+	background: #f6fdb0;
+}
+
+.scraped-example .example-wrap .rust span.highlight {
+	background: #f6fdb0;
+}
+
+.more-scraped-examples {
+	padding-left: 10px;
+	border-left: 1px solid #ccc;
+}
+
+.toggle-examples .collapse-toggle {
+	position: relative;
+}
+
+.toggle-examples a {
+	color: #999 !important; // FIXME(wcrichto): why is important needed
+}
diff --git a/src/librustdoc/html/static/js/main.js b/src/librustdoc/html/static/js/main.js
index e396fd9d288..5ac00ff244a 100644
--- a/src/librustdoc/html/static/js/main.js
+++ b/src/librustdoc/html/static/js/main.js
@@ -979,6 +979,202 @@ function hideThemeButtonState() {
     onHashChange(null);
     window.addEventListener("hashchange", onHashChange);
     searchState.setup();
+
+    /////// EXAMPLE ANALYZER
+
+    // Merge the full set of [from, to] offsets into a minimal set of non-overlapping
+    // [from, to] offsets.
+    // NB: This is such a archetypal software engineering interview question that
+    // I can't believe I actually had to write it. Yes, it's O(N) in the input length --
+    // but it does assume a sorted input!
+    function distinctRegions(locs) {
+        var start = -1;
+        var end = -1;
+        var output = [];
+        for (var i = 0; i < locs.length; i++) {
+            var loc = locs[i];
+            if (loc[0] > end) {
+                if (end > 0) {
+                    output.push([start, end]);
+                }
+                start = loc[0];
+                end = loc[1];
+            } else {
+                end = Math.max(end, loc[1]);
+            }
+        }
+        if (end > 0) {
+            output.push([start, end]);
+        }
+        return output;
+    }
+
+    function convertLocsStartsToLineOffsets(code, locs) {
+        locs = distinctRegions(locs.slice(0).sort(function (a, b) {
+            return a[0] === b[0] ? a[1] - b[1] : a[0] - b[0];
+        })); // sort by start; use end if start is equal.
+        var codeLines = code.split("\n");
+        var lineIndex = 0;
+        var totalOffset = 0;
+        var output = [];
+
+        while (locs.length > 0 && lineIndex < codeLines.length) {
+            var lineLength = codeLines[lineIndex].length + 1; // +1 here and later is due to omitted \n
+            while (locs.length > 0 && totalOffset + lineLength > locs[0][0]) {
+                var endIndex = lineIndex;
+                var charsRemaining = locs[0][1] - totalOffset;
+                while (endIndex < codeLines.length && charsRemaining > codeLines[endIndex].length + 1) {
+                    charsRemaining -= codeLines[endIndex].length + 1;
+                    endIndex += 1;
+                }
+                output.push({
+                    from: [lineIndex, locs[0][0] - totalOffset],
+                    to: [endIndex, charsRemaining]
+                });
+                locs.shift();
+            }
+            lineIndex++;
+            totalOffset += lineLength;
+        }
+        return output;
+    }
+
+    // inserts str into html, *but* calculates idx by eliding anything in html that's not in raw.
+    // ideally this would work by walking the element tree...but this is good enough for now.
+    function insertStrAtRawIndex(raw, html, idx, str) {
+        if (idx > raw.length) {
+            return html;
+        }
+        if (idx == raw.length) {
+            return html + str;
+        }
+        var rawIdx = 0;
+        var htmlIdx = 0;
+        while (rawIdx < idx && rawIdx < raw.length) {
+            while (raw[rawIdx] !== html[htmlIdx] && htmlIdx < html.length) {
+                htmlIdx++;
+            }
+            rawIdx++;
+            htmlIdx++;
+        }
+        return html.substring(0, htmlIdx) + str + html.substr(htmlIdx);
+    }
+
+    // Scroll code block to put the given code location in the middle of the viewer
+    function scrollToLoc(elt, loc) {
+        var wrapper = elt.querySelector(".code-wrapper");
+        var halfHeight = wrapper.offsetHeight / 2;
+        var lines = elt.querySelector('.line-numbers');
+        var offsetMid = (lines.children[loc.from[0]].offsetTop + lines.children[loc.to[0]].offsetTop) / 2;
+        var scrollOffset = offsetMid - halfHeight;
+        lines.scrollTo(0, scrollOffset);
+        elt.querySelector(".rust").scrollTo(0, scrollOffset);
+    }
+
+    function updateScrapedExample(example) {
+        var code = example.attributes.getNamedItem("data-code").textContent;
+        var codeLines = code.split("\n");
+        var locs = JSON.parse(example.attributes.getNamedItem("data-locs").textContent);
+        locs = convertLocsStartsToLineOffsets(code, locs);
+
+        // Add call-site highlights to code listings
+        var litParent = example.querySelector('.example-wrap pre.rust');
+        var litHtml = litParent.innerHTML.split("\n");
+        onEach(locs, function (loc) {
+            for (var i = loc.from[0]; i < loc.to[0] + 1; i++) {
+                addClass(example.querySelector('.line-numbers').children[i], "highlight");
+            }
+            litHtml[loc.to[0]] = insertStrAtRawIndex(
+                codeLines[loc.to[0]],
+                litHtml[loc.to[0]],
+                loc.to[1],
+                "</span>");
+            litHtml[loc.from[0]] = insertStrAtRawIndex(
+                codeLines[loc.from[0]],
+                litHtml[loc.from[0]],
+                loc.from[1],
+                '<span class="highlight" data-loc="' + JSON.stringify(loc).replace(/"/g, "&quot;") + '">');
+        }, true); // do this backwards to avoid shifting later offsets
+        litParent.innerHTML = litHtml.join('\n');
+
+        // Toggle through list of examples in a given file
+        var locIndex = 0;
+        if (locs.length > 1) {
+            example.querySelector('.prev')
+                .addEventListener('click', function () {
+                    locIndex = (locIndex - 1 + locs.length) % locs.length;
+                    scrollToLoc(example, locs[locIndex]);
+                });
+            example.querySelector('.next')
+                .addEventListener('click', function () {
+                    locIndex = (locIndex + 1) % locs.length;
+                    scrollToLoc(example, locs[locIndex]);
+                });
+        } else {
+            example.querySelector('.prev').remove();
+            example.querySelector('.next').remove();
+        }
+
+        // Show full code on expansion
+        example.querySelector('.expand').addEventListener('click', function () {
+            if (hasClass(example, "expanded")) {
+                removeClass(example, "expanded");
+                scrollToLoc(example, locs[0]);
+            } else {
+                addClass(example, "expanded");
+            }
+        });
+
+        // Start with the first example in view
+        scrollToLoc(example, locs[0]);
+    }
+
+    function updateScrapedExamples() {
+        onEach(document.getElementsByClassName('scraped-example-list'), function (exampleSet) {
+            updateScrapedExample(exampleSet.querySelector(".small-section-header + .scraped-example"));
+        });
+
+        onEach(document.getElementsByClassName("more-scraped-examples"), function (more) {
+            var toggle = createSimpleToggle(true);
+            var label = "More examples";
+            var wrapper = createToggle(toggle, label, 14, "toggle-examples", false);
+            more.parentNode.insertBefore(wrapper, more);
+            var examples_init = false;
+
+            // Show additional examples on click
+            wrapper.onclick = function () {
+                if (hasClass(this, "collapsed")) {
+                    removeClass(this, "collapsed");
+                    onEachLazy(this.parentNode.getElementsByClassName("hidden"), function (x) {
+                        if (hasClass(x, "content") === false) {
+                            removeClass(x, "hidden");
+                            addClass(x, "x")
+                        }
+                    }, true);
+                    this.querySelector('.toggle-label').innerHTML = "Hide examples";
+                    this.querySelector('.inner').innerHTML = labelForToggleButton(false);
+                    if (!examples_init) {
+                        examples_init = true;
+                        onEach(more.getElementsByClassName('scraped-example'), updateScrapedExample);
+                    }
+                } else {
+                    addClass(this, "collapsed");
+                    onEachLazy(this.parentNode.getElementsByClassName("x"), function (x) {
+                        if (hasClass(x, "content") === false) {
+                            addClass(x, "hidden");
+                            removeClass(x, "x")
+                        }
+                    }, true);
+                    this.querySelector('.toggle-label').innerHTML = label;
+                    this.querySelector('.inner').innerHTML = labelForToggleButton(true);
+                }
+            };
+        });
+    }
+
+    var start = Date.now();
+    updateScrapedExamples();
+    console.log("updated examples took", Date.now() - start, "ms");
 }());
 
 (function () {
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index 731fc4ff3ce..866514d7c9b 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -289,7 +289,7 @@ crate fn from_fn_header(header: &rustc_hir::FnHeader) -> HashSet<Qualifiers> {
 
 impl FromWithTcx<clean::Function> for Function {
     fn from_tcx(function: clean::Function, tcx: TyCtxt<'_>) -> Self {
-        let clean::Function { decl, generics, header } = function;
+        let clean::Function { decl, generics, header, call_locations: _ } = function;
         Function {
             decl: decl.into_tcx(tcx),
             generics: generics.into_tcx(tcx),
@@ -530,7 +530,7 @@ crate fn from_function_method(
     has_body: bool,
     tcx: TyCtxt<'_>,
 ) -> Method {
-    let clean::Function { header, decl, generics } = function;
+    let clean::Function { header, decl, generics, call_locations: _ } = function;
     Method {
         decl: decl.into_tcx(tcx),
         generics: generics.into_tcx(tcx),
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index efc8e31498a..df0e309c94a 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -119,6 +119,7 @@ mod json;
 crate mod lint;
 mod markdown;
 mod passes;
+mod scrape_examples;
 mod theme;
 mod visit_ast;
 mod visit_lib;
@@ -618,6 +619,8 @@ fn opts() -> Vec<RustcOptGroup> {
                 "Make the identifiers in the HTML source code pages navigable",
             )
         }),
+        unstable("scrape-examples", |o| o.optmulti("", "scrape-examples", "", "")),
+        unstable("repository-url", |o| o.optopt("", "repository-url", "", "TODO")),
     ]
 }
 
@@ -697,7 +700,7 @@ fn run_renderer<'tcx, T: formats::FormatRenderer<'tcx>>(
     }
 }
 
-fn main_options(options: config::Options) -> MainResult {
+fn main_options(mut options: config::Options) -> MainResult {
     let diag = core::new_handler(options.error_format, None, &options.debugging_opts);
 
     match (options.should_test, options.markdown_input()) {
@@ -712,6 +715,15 @@ fn main_options(options: config::Options) -> MainResult {
         (false, false) => {}
     }
 
+    if options.scrape_examples.len() > 0 {
+        if let Some(crate_name) = &options.crate_name {
+            options.render_options.call_locations =
+                Some(scrape_examples::scrape(&options.scrape_examples, crate_name)?);
+        } else {
+            // raise an error?
+        }
+    }
+
     // need to move these items separately because we lose them by the time the closure is called,
     // but we can't create the Handler ahead of time because it's not Send
     let show_coverage = options.show_coverage;
diff --git a/src/librustdoc/scrape_examples.rs b/src/librustdoc/scrape_examples.rs
new file mode 100644
index 00000000000..da076f87a9c
--- /dev/null
+++ b/src/librustdoc/scrape_examples.rs
@@ -0,0 +1,138 @@
+//! This module analyzes provided crates to find examples of uses for items in the
+//! current crate being documented.
+
+use rayon::prelude::*;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::{
+    self as hir,
+    intravisit::{self, Visitor},
+};
+use rustc_interface::interface;
+use rustc_middle::hir::map::Map;
+use rustc_middle::ty::{TyCtxt, TyKind};
+use rustc_span::symbol::Symbol;
+
+crate type FnCallLocations = FxHashMap<String, Vec<(usize, usize)>>;
+crate type AllCallLocations = FxHashMap<String, FnCallLocations>;
+
+/// Visitor for traversing a crate and finding instances of function calls.
+struct FindCalls<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    map: Map<'tcx>,
+
+    /// Workspace-relative path to the root of the crate. Used to remember
+    /// which example a particular call came from.
+    file_name: String,
+
+    /// Name of the crate being documented, to filter out calls to irrelevant
+    /// functions.
+    krate: Symbol,
+
+    /// Data structure to accumulate call sites across all examples.
+    calls: &'a mut AllCallLocations,
+}
+
+impl<'a, 'tcx> Visitor<'tcx> for FindCalls<'a, 'tcx>
+where
+    'tcx: 'a,
+{
+    type Map = Map<'tcx>;
+
+    fn nested_visit_map(&mut self) -> intravisit::NestedVisitorMap<Self::Map> {
+        intravisit::NestedVisitorMap::OnlyBodies(self.map)
+    }
+
+    fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) {
+        intravisit::walk_expr(self, ex);
+
+        // Get type of function if expression is a function call
+        let types = self.tcx.typeck(ex.hir_id.owner);
+        let (ty, span) = match ex.kind {
+            hir::ExprKind::Call(f, _) => (types.node_type(f.hir_id), ex.span),
+            hir::ExprKind::MethodCall(_, _, _, span) => {
+                let types = self.tcx.typeck(ex.hir_id.owner);
+                let def_id = types.type_dependent_def_id(ex.hir_id).unwrap();
+                (self.tcx.type_of(def_id), span)
+            }
+            _ => {
+                return;
+            }
+        };
+
+        // Save call site if the function resovles to a concrete definition
+        if let TyKind::FnDef(def_id, _) = ty.kind() {
+            if self.tcx.crate_name(def_id.krate) == self.krate {
+                let key = self.tcx.def_path(*def_id).to_string_no_crate_verbose();
+                let entries = self.calls.entry(key).or_insert_with(FxHashMap::default);
+                entries
+                    .entry(self.file_name.clone())
+                    .or_insert_with(Vec::new)
+                    .push((span.lo().0 as usize, span.hi().0 as usize));
+            }
+        }
+    }
+}
+
+struct Callbacks {
+    calls: AllCallLocations,
+    krate: String,
+    file_name: String,
+}
+
+impl rustc_driver::Callbacks for Callbacks {
+    fn after_analysis<'tcx>(
+        &mut self,
+        _compiler: &rustc_interface::interface::Compiler,
+        queries: &'tcx rustc_interface::Queries<'tcx>,
+    ) -> rustc_driver::Compilation {
+        queries.global_ctxt().unwrap().take().enter(|tcx| {
+            let mut finder = FindCalls {
+                calls: &mut self.calls,
+                tcx,
+                map: tcx.hir(),
+                file_name: self.file_name.clone(),
+                krate: Symbol::intern(&self.krate),
+            };
+            tcx.hir().krate().visit_all_item_likes(&mut finder.as_deep_visitor());
+        });
+
+        rustc_driver::Compilation::Stop
+    }
+}
+
+/// Executes rustc on each example and collects call locations into a single structure.
+///
+/// # Arguments:
+/// * `examples` is an array of invocations to rustc, generated by Cargo.
+/// * `krate` is the name of the crate being documented.
+pub fn scrape(examples: &[String], krate: &str) -> interface::Result<AllCallLocations> {
+    // Scrape each crate in parallel
+    // TODO(wcrichto): do we need optional support for no rayon?
+    let maps = examples
+        .par_iter()
+        .map(|example| {
+            // TODO(wcrichto): is there a more robust way to get arguments than split(" ")?
+            let mut args = example.split(" ").map(|s| s.to_owned()).collect::<Vec<_>>();
+            let file_name = args[0].clone();
+            args.insert(0, "_".to_string());
+
+            // TODO(wcrichto): is there any setup / cleanup that needs to be performed
+            // here upon the invocation of rustc_driver?
+            debug!("Scraping examples from krate {} with args:\n{:?}", krate, args);
+            let mut callbacks =
+                Callbacks { calls: FxHashMap::default(), file_name, krate: krate.to_string() };
+            rustc_driver::RunCompiler::new(&args, &mut callbacks).run()?;
+            Ok(callbacks.calls)
+        })
+        .collect::<interface::Result<Vec<_>>>()?;
+
+    // Merge the call locations into a single result
+    let mut all_map = FxHashMap::default();
+    for map in maps {
+        for (function, calls) in map.into_iter() {
+            all_map.entry(function).or_insert_with(FxHashMap::default).extend(calls.into_iter());
+        }
+    }
+
+    Ok(all_map)
+}