From 88130f1796e98837dc6f58aea9b9a0f49b85f426 Mon Sep 17 00:00:00 2001 From: Matthew Russo Date: Tue, 30 Oct 2018 10:11:24 -0400 Subject: updates all Filename variants to take a fingerprint --- src/libsyntax/parse/token.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 4a5f3e240da..04a791fbcb9 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -545,7 +545,8 @@ impl Token { let tokens_for_real = nt.1.force(|| { // FIXME(#43081): Avoid this pretty-print + reparse hack let source = pprust::token_to_string(self); - parse_stream_from_source_str(FileName::MacroExpansion, source, sess, Some(span)) + let filename = FileName::macro_expansion_source_code(&source); + parse_stream_from_source_str(filename, source, sess, Some(span)) }); // During early phases of the compiler the AST could get modified @@ -781,10 +782,12 @@ fn prepend_attrs(sess: &ParseSess, assert_eq!(attr.style, ast::AttrStyle::Outer, "inner attributes should prevent cached tokens from existing"); + let source = pprust::attr_to_string(attr); + let macro_filename = FileName::macro_expansion_source_code(&source); if attr.is_sugared_doc { let stream = parse_stream_from_source_str( - FileName::MacroExpansion, - pprust::attr_to_string(attr), + macro_filename, + source, sess, Some(span), ); @@ -805,8 +808,8 @@ fn prepend_attrs(sess: &ParseSess, // should eventually be removed. } else { let stream = parse_stream_from_source_str( - FileName::MacroExpansion, - pprust::path_to_string(&attr.path), + macro_filename, + source, sess, Some(span), ); -- cgit 1.4.1-3-g733a5 From f0f8aa9e05726bfbc065fa2504daf3232f29bc03 Mon Sep 17 00:00:00 2001 From: Matthew Russo Date: Tue, 4 Dec 2018 15:18:03 -0500 Subject: adds DocTest filename variant, refactors doctest_offset out of source_map, fixes remaining test failures --- src/librustc/ich/impls_syntax.rs | 3 +- src/librustc_errors/emitter.rs | 7 ++-- src/librustc_errors/lib.rs | 2 +- src/librustdoc/test.rs | 18 +++++----- src/libsyntax/ext/source_util.rs | 1 + src/libsyntax/parse/lexer/mod.rs | 2 +- src/libsyntax/parse/mod.rs | 10 +++--- src/libsyntax/source_map.rs | 42 ++++++++---------------- src/libsyntax_ext/proc_macro_server.rs | 2 +- src/libsyntax_pos/lib.rs | 12 +++++-- src/test/run-make-fulldeps/issue-19371/foo.rs | 3 +- src/test/rustdoc-ui/failed-doctest-output.stdout | 4 +-- 12 files changed, 52 insertions(+), 54 deletions(-) (limited to 'src/libsyntax/parse') diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index dbf0147928a..439c6e42e1a 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -423,7 +423,8 @@ impl_stable_hash_for!(enum ::syntax_pos::FileName { ProcMacroSourceCode(s), CliCrateAttr(s), CfgSpec(s), - Custom(s) + Custom(s), + DocTest(pb, line), }); impl<'a> HashStable> for SourceFile { diff --git a/src/librustc_errors/emitter.rs b/src/librustc_errors/emitter.rs index 7e69e98071d..ab2ab25c91a 100644 --- a/src/librustc_errors/emitter.rs +++ b/src/librustc_errors/emitter.rs @@ -1044,7 +1044,7 @@ impl EmitterWriter { buffer.append(buffer_msg_line_offset, &format!("{}:{}:{}", loc.file.name, - sm.doctest_offset_line(loc.line), + sm.doctest_offset_line(&loc.file.name, loc.line), loc.col.0 + 1), Style::LineAndColumn); for _ in 0..max_line_num_len { @@ -1054,7 +1054,7 @@ impl EmitterWriter { buffer.prepend(0, &format!("{}:{}:{}: ", loc.file.name, - sm.doctest_offset_line(loc.line), + sm.doctest_offset_line(&loc.file.name, loc.line), loc.col.0 + 1), Style::LineAndColumn); } @@ -1075,7 +1075,8 @@ impl EmitterWriter { }; format!("{}:{}{}", annotated_file.file.name, - sm.doctest_offset_line(first_line.line_index), + sm.doctest_offset_line( + &annotated_file.file.name, first_line.line_index), col) } else { annotated_file.file.name.to_string() diff --git a/src/librustc_errors/lib.rs b/src/librustc_errors/lib.rs index 0fb77a7a3ab..b6528cbe2c8 100644 --- a/src/librustc_errors/lib.rs +++ b/src/librustc_errors/lib.rs @@ -130,7 +130,7 @@ pub trait SourceMapper { fn merge_spans(&self, sp_lhs: Span, sp_rhs: Span) -> Option; fn call_span_if_macro(&self, sp: Span) -> Span; fn ensure_source_file_source_present(&self, source_file: Lrc) -> bool; - fn doctest_offset_line(&self, line: usize) -> usize; + fn doctest_offset_line(&self, file: &FileName, line: usize) -> usize; } impl CodeSuggestion { diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index d9bab91fd0c..74583196818 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -197,8 +197,14 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize, let (test, line_offset) = make_test(test, Some(cratename), as_test_harness, opts); // FIXME(#44940): if doctests ever support path remapping, then this filename // needs to be the result of SourceMap::span_to_unmapped_path + + let path = match filename { + FileName::Real(path) => path.clone(), + _ => PathBuf::from(r"doctest.rs"), + }; + let input = config::Input::Str { - name: filename.to_owned(), + name: FileName::DocTest(path, line as isize - line_offset as isize), input: test, }; let outputs = OutputTypes::new(&[(OutputType::Exe, None)]); @@ -252,9 +258,7 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize, let _bomb = Bomb(data.clone(), old.unwrap_or(box io::stdout())); let (libdir, outdir, compile_result) = driver::spawn_thread_pool(sessopts, |sessopts| { - let source_map = Lrc::new(SourceMap::new_doctest( - sessopts.file_path_mapping(), filename.clone(), line as isize - line_offset as isize - )); + let source_map = Lrc::new(SourceMap::new(sessopts.file_path_mapping())); let emitter = errors::emitter::EmitterWriter::new(box Sink(data.clone()), Some(source_map.clone()), false, @@ -401,7 +405,7 @@ pub fn make_test(s: &str, use errors::emitter::EmitterWriter; use errors::Handler; - let filename = FileName::Anon; + let filename = FileName::anon_source_code(s); let source = crates + &everything_else; // any errors in parsing should also appear when the doctest is compiled for real, so just @@ -411,8 +415,6 @@ pub fn make_test(s: &str, let handler = Handler::with_emitter(false, false, box emitter); let sess = ParseSess::with_span_handler(handler, cm); - debug!("about to parse: \n{}", source); - let mut found_main = false; let mut found_extern_crate = cratename.is_none(); @@ -487,8 +489,6 @@ pub fn make_test(s: &str, prog.push_str("\n}"); } - info!("final test program: {}", prog); - (prog, line_offset) } diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs index 654be85862f..75e25083d03 100644 --- a/src/libsyntax/ext/source_util.rs +++ b/src/libsyntax/ext/source_util.rs @@ -204,6 +204,7 @@ fn res_rel_file(cx: &mut ExtCtxt, sp: syntax_pos::Span, arg: String) -> PathBuf let callsite = sp.source_callsite(); let mut path = match cx.source_map().span_to_unmapped_path(callsite) { FileName::Real(path) => path, + FileName::DocTest(path, _) => path, other => panic!("cannot resolve relative path in non-file source `{}`", other), }; path.pop(); diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index c90c62c13f9..5c3e11c8c19 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1898,7 +1898,7 @@ mod tests { sess: &'a ParseSess, teststr: String) -> StringReader<'a> { - let sf = sm.new_source_file(PathBuf::from("zebra.rs").into(), teststr); + let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); StringReader::new(sess, sf, None) } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index ac972f20f94..a229ccfa265 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -977,23 +977,25 @@ mod tests { with_globals(|| { let sess = ParseSess::new(FilePathMapping::empty()); - let name = FileName::Custom("source".to_string()); + let name_1 = FileName::Custom("crlf_source_1".to_string()); let source = "/// doc comment\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name.clone(), source, &sess) + let item = parse_item_from_source_str(name_1, source, &sess) .unwrap().unwrap(); let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap(); assert_eq!(doc, "/// doc comment"); + let name_2 = FileName::Custom("crlf_source_2".to_string()); let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name.clone(), source, &sess) + let item = parse_item_from_source_str(name_2, source, &sess) .unwrap().unwrap(); let docs = item.attrs.iter().filter(|a| a.path == "doc") .map(|a| a.value_str().unwrap().to_string()).collect::>(); let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; assert_eq!(&docs[..], b); + let name_3 = FileName::Custom("clrf_source_3".to_string()); let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name, source, &sess).unwrap().unwrap(); + let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap(); assert_eq!(doc, "/** doc comment\n * with CRLF */"); }); diff --git a/src/libsyntax/source_map.rs b/src/libsyntax/source_map.rs index e22d4f8f133..ee60d2dd615 100644 --- a/src/libsyntax/source_map.rs +++ b/src/libsyntax/source_map.rs @@ -144,9 +144,6 @@ pub struct SourceMap { // This is used to apply the file path remapping as specified via // --remap-path-prefix to all SourceFiles allocated within this SourceMap. path_mapping: FilePathMapping, - /// In case we are in a doctest, replace all file names with the PathBuf, - /// and add the given offsets to the line info - doctest_offset: Option<(FileName, isize)>, } impl SourceMap { @@ -155,19 +152,9 @@ impl SourceMap { files: Default::default(), file_loader: Box::new(RealFileLoader), path_mapping, - doctest_offset: None, } } - pub fn new_doctest(path_mapping: FilePathMapping, - file: FileName, line: isize) -> SourceMap { - SourceMap { - doctest_offset: Some((file, line)), - ..SourceMap::new(path_mapping) - } - - } - pub fn with_file_loader(file_loader: Box, path_mapping: FilePathMapping) -> SourceMap { @@ -175,7 +162,6 @@ impl SourceMap { files: Default::default(), file_loader: file_loader, path_mapping, - doctest_offset: None, } } @@ -189,11 +175,7 @@ impl SourceMap { pub fn load_file(&self, path: &Path) -> io::Result> { let src = self.file_loader.read_file(path)?; - let filename = if let Some((ref name, _)) = self.doctest_offset { - name.clone() - } else { - path.to_owned().into() - }; + let filename = path.to_owned().into(); Ok(self.new_source_file(filename, src)) } @@ -328,15 +310,17 @@ impl SourceMap { } // If there is a doctest_offset, apply it to the line - pub fn doctest_offset_line(&self, mut orig: usize) -> usize { - if let Some((_, line)) = self.doctest_offset { - if line >= 0 { - orig = orig + line as usize; - } else { - orig = orig - (-line) as usize; - } + pub fn doctest_offset_line(&self, file: &FileName, orig: usize) -> usize { + return match file { + FileName::DocTest(_, offset) => { + return if *offset >= 0 { + orig + *offset as usize + } else { + orig - (-(*offset)) as usize + } + }, + _ => orig } - orig } /// Lookup source information about a BytePos @@ -1001,8 +985,8 @@ impl SourceMapper for SourceMap { } ) } - fn doctest_offset_line(&self, line: usize) -> usize { - self.doctest_offset_line(line) + fn doctest_offset_line(&self, file: &FileName, line: usize) -> usize { + self.doctest_offset_line(file, line) } } diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs index 4babc2e612f..6c7da589a42 100644 --- a/src/libsyntax_ext/proc_macro_server.rs +++ b/src/libsyntax_ext/proc_macro_server.rs @@ -402,7 +402,7 @@ impl server::TokenStream for Rustc<'_> { } fn from_str(&mut self, src: &str) -> Self::TokenStream { parse::parse_stream_from_source_str( - FileName::ProcMacroSourceCode, + FileName::proc_macro_source_code(src.clone()), src.to_string(), self.sess, Some(self.call_site), diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs index ff3c1cf9f47..4d42b85ea75 100644 --- a/src/libsyntax_pos/lib.rs +++ b/src/libsyntax_pos/lib.rs @@ -103,6 +103,7 @@ pub enum FileName { CliCrateAttr(u64), /// Custom sources for explicit parser calls from plugins and drivers Custom(String), + DocTest(PathBuf, isize), } impl std::fmt::Display for FileName { @@ -119,6 +120,7 @@ impl std::fmt::Display for FileName { CfgSpec(_) => write!(fmt, ""), CliCrateAttr(_) => write!(fmt, ""), Custom(ref s) => write!(fmt, "<{}>", s), + DocTest(ref path, _) => write!(fmt, "{}", path.display()), } } } @@ -142,7 +144,8 @@ impl FileName { CfgSpec(_) | CliCrateAttr(_) | Custom(_) | - QuoteExpansion(_) => false, + QuoteExpansion(_) | + DocTest(_, _) => false, } } @@ -156,7 +159,8 @@ impl FileName { CfgSpec(_) | CliCrateAttr(_) | Custom(_) | - QuoteExpansion(_) => false, + QuoteExpansion(_) | + DocTest(_, _) => false, Macros(_) => true, } } @@ -196,6 +200,10 @@ impl FileName { src.hash(&mut hasher); FileName::CliCrateAttr(hasher.finish()) } + + pub fn doc_test_source_code(path: PathBuf, line: isize) -> FileName{ + FileName::DocTest(path, line) + } } /// Spans represent a region of code, used for error reporting. Positions in spans diff --git a/src/test/run-make-fulldeps/issue-19371/foo.rs b/src/test/run-make-fulldeps/issue-19371/foo.rs index 4dfecb33c14..c342e1c4835 100644 --- a/src/test/run-make-fulldeps/issue-19371/foo.rs +++ b/src/test/run-make-fulldeps/issue-19371/foo.rs @@ -72,7 +72,8 @@ fn compile(code: String, output: PathBuf, sysroot: PathBuf) { driver::spawn_thread_pool(opts, |opts| { let (sess, cstore, codegen_backend) = basic_sess(opts); let control = CompileController::basic(); - let input = Input::Str { name: FileName::Anon, input: code }; + let name = FileName::anon_source_code(&code); + let input = Input::Str { name, input: code }; let _ = compile_input( codegen_backend, &sess, diff --git a/src/test/rustdoc-ui/failed-doctest-output.stdout b/src/test/rustdoc-ui/failed-doctest-output.stdout index cab7bda6c67..52e0cdca950 100644 --- a/src/test/rustdoc-ui/failed-doctest-output.stdout +++ b/src/test/rustdoc-ui/failed-doctest-output.stdout @@ -12,7 +12,7 @@ error[E0425]: cannot find value `no` in this scope 3 | no | ^^ not found in this scope -thread '$DIR/failed-doctest-output.rs - OtherStruct (line 27)' panicked at 'couldn't compile the test', src/librustdoc/test.rs:323:13 +thread '$DIR/failed-doctest-output.rs - OtherStruct (line 27)' panicked at 'couldn't compile the test', src/librustdoc/test.rs:327:13 note: Run with `RUST_BACKTRACE=1` for a backtrace. ---- $DIR/failed-doctest-output.rs - SomeStruct (line 21) stdout ---- @@ -21,7 +21,7 @@ thread '$DIR/failed-doctest-output.rs - SomeStruct (line 21)' panicked at 'test thread 'main' panicked at 'oh no', $DIR/failed-doctest-output.rs:3:1 note: Run with `RUST_BACKTRACE=1` for a backtrace. -', src/librustdoc/test.rs:358:17 +', src/librustdoc/test.rs:362:17 failures: -- cgit 1.4.1-3-g733a5