Rollup merge of #70172 - eddyb:retokenize-external-src, r=petrochenkov

parse/lexer: support `StringReader::retokenize` called on external files.

This ~~should theoretically~~ fixes #69933, ~~but I'm not sure what the best way to test it is~~.
**EDIT**: see https://github.com/rust-lang/rust/issues/69933#issuecomment-602019598.

r? @petrochenkov cc @Xanewok @staktrace

2 files changed, 15 insertions, 7 deletions

diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs
index f7fb704fcbc..6c0b2c40c76 100644
--- a/src/librustc_parse/lexer/mod.rs
+++ b/src/librustc_parse/lexer/mod.rs
@@ -46,12 +46,20 @@ impl<'a> StringReader<'a> {
         source_file: Lrc<rustc_span::SourceFile>,
         override_span: Option<Span>,
     ) -> Self {
-        if source_file.src.is_none() {
+        // Make sure external source is loaded first, before accessing it.
+        // While this can't show up during normal parsing, `retokenize` may
+        // be called with a source file from an external crate.
+        sess.source_map().ensure_source_file_source_present(source_file.clone());
+
+        // FIXME(eddyb) use `Lrc<str>` or similar to avoid cloning the `String`.
+        let src = if let Some(src) = &source_file.src {
+            src.clone()
+        } else if let Some(src) = source_file.external_src.borrow().get_source() {
+            src.clone()
+        } else {
             sess.span_diagnostic
                 .bug(&format!("cannot lex `source_file` without source: {}", source_file.name));
-        }
-
-        let src = (*source_file.src.as_ref().unwrap()).clone();
+        };
 
         StringReader {
             sess,
diff --git a/src/librustc_span/lib.rs b/src/librustc_span/lib.rs
index dbc180114f1..28864737072 100644
--- a/src/librustc_span/lib.rs
+++ b/src/librustc_span/lib.rs
@@ -856,7 +856,7 @@ pub enum ExternalSource {
 #[derive(PartialEq, Eq, Clone, Debug)]
 pub enum ExternalSourceKind {
     /// The external source has been loaded already.
-    Present(String),
+    Present(Lrc<String>),
     /// No attempt has been made to load the external source.
     AbsentOk,
     /// A failed attempt has been made to load the external source.
@@ -872,7 +872,7 @@ impl ExternalSource {
         }
     }
 
-    pub fn get_source(&self) -> Option<&str> {
+    pub fn get_source(&self) -> Option<&Lrc<String>> {
         match self {
             ExternalSource::Foreign { kind: ExternalSourceKind::Present(ref src), .. } => Some(src),
             _ => None,
@@ -1138,7 +1138,7 @@ impl SourceFile {
                     hasher.write(src.as_bytes());
 
                     if hasher.finish::<u128>() == self.src_hash {
-                        *src_kind = ExternalSourceKind::Present(src);
+                        *src_kind = ExternalSourceKind::Present(Lrc::new(src));
                         return true;
                     }
                 } else {