diff options
| author | Noah Lev <camelidcamel@gmail.com> | 2024-05-31 00:31:26 -0700 |
|---|---|---|
| committer | Guillaume Gomez <guillaume.gomez@huawei.com> | 2024-06-07 17:48:47 +0200 |
| commit | 85499ebf13d0c3c731ac5d69f02ed8e7eb2735bb (patch) | |
| tree | e67218a143f231cf8eda150ba8d1b4c6b4e296c8 | |
| parent | 16db1a1bd0ca3f82a470363b8e6f95a8d03f63c5 (diff) | |
| download | rust-85499ebf13d0c3c731ac5d69f02ed8e7eb2735bb.tar.gz rust-85499ebf13d0c3c731ac5d69f02ed8e7eb2735bb.zip | |
Separate doctest collection from running
| -rw-r--r-- | src/librustdoc/doctest.rs | 182 | ||||
| -rw-r--r-- | src/librustdoc/doctest/markdown.rs | 110 | ||||
| -rw-r--r-- | src/librustdoc/doctest/rust.rs | 133 |
3 files changed, 237 insertions, 188 deletions
diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 22b5fa0951a..8088b57dd76 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -8,7 +8,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::sync::Lrc; use rustc_errors::emitter::stderr_destination; use rustc_errors::{ColorConfig, ErrorGuaranteed, FatalError}; -use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE}; +use rustc_hir::def_id::LOCAL_CRATE; use rustc_hir::CRATE_HIR_ID; use rustc_interface::interface; use rustc_parse::new_parser_from_source_str; @@ -19,10 +19,9 @@ use rustc_session::parse::ParseSess; use rustc_span::edition::Edition; use rustc_span::source_map::SourceMap; use rustc_span::symbol::sym; -use rustc_span::{BytePos, FileName, Pos, Span, DUMMY_SP}; +use rustc_span::FileName; use rustc_target::spec::{Target, TargetTriple}; -use std::env; use std::fs::File; use std::io::{self, Write}; use std::panic; @@ -38,7 +37,8 @@ use crate::config::Options as RustdocOptions; use crate::html::markdown::{ErrorCodes, Ignore, LangString}; use crate::lint::init_lints; -use self::rust::HirCollector; +use self::markdown::MdDoctest; +use self::rust::{HirCollector, RustDoctest}; /// Options that apply to all doctests in a crate or Markdown file (for `rustdoc foo.md`). #[derive(Clone, Default)] @@ -182,29 +182,19 @@ pub(crate) fn run( let mut collector = Collector::new( tcx.crate_name(LOCAL_CRATE).to_string(), options, - false, opts, - Some(compiler.sess.psess.clone_source_map()), - None, - enable_per_target_ignores, file_path, ); - let mut hir_collector = HirCollector { - sess: &compiler.sess, - collector: &mut collector, - map: tcx.hir(), - codes: ErrorCodes::from( - compiler.sess.opts.unstable_features.is_nightly_build(), - ), + let hir_collector = HirCollector::new( + &compiler.sess, + tcx.hir(), + ErrorCodes::from(compiler.sess.opts.unstable_features.is_nightly_build()), + enable_per_target_ignores, tcx, - }; - hir_collector.visit_testable( - "".to_string(), - CRATE_DEF_ID, - tcx.hir().span(CRATE_HIR_ID), - |this| tcx.hir().walk_toplevel_module(this), ); + let tests = hir_collector.collect_crate(); + tests.into_iter().for_each(|t| collector.add_test(ScrapedDoctest::Rust(t))); collector }); @@ -985,6 +975,12 @@ impl IndividualTestOptions { } } +/// A doctest scraped from the code, ready to be turned into a runnable test. +enum ScrapedDoctest { + Rust(RustDoctest), + Markdown(MdDoctest), +} + pub(crate) trait DoctestVisitor { fn visit_test(&mut self, test: String, config: LangString, line: usize); fn get_line(&self) -> usize { @@ -996,36 +992,9 @@ pub(crate) trait DoctestVisitor { pub(crate) struct Collector { pub(crate) tests: Vec<test::TestDescAndFn>, - // The name of the test displayed to the user, separated by `::`. - // - // In tests from Rust source, this is the path to the item - // e.g., `["std", "vec", "Vec", "push"]`. - // - // In tests from a markdown file, this is the titles of all headers (h1~h6) - // of the sections that contain the code block, e.g., if the markdown file is - // written as: - // - // ``````markdown - // # Title - // - // ## Subtitle - // - // ```rust - // assert!(true); - // ``` - // `````` - // - // the `names` vector of that test will be `["Title", "Subtitle"]`. - names: Vec<String>, - rustdoc_options: RustdocOptions, - use_headers: bool, - enable_per_target_ignores: bool, crate_name: String, opts: GlobalTestOptions, - position: Span, - source_map: Option<Lrc<SourceMap>>, - filename: Option<PathBuf>, visited_tests: FxHashMap<(String, usize), usize>, unused_extern_reports: Arc<Mutex<Vec<UnusedExterns>>>, compiling_test_count: AtomicUsize, @@ -1036,24 +1005,14 @@ impl Collector { pub(crate) fn new( crate_name: String, rustdoc_options: RustdocOptions, - use_headers: bool, opts: GlobalTestOptions, - source_map: Option<Lrc<SourceMap>>, - filename: Option<PathBuf>, - enable_per_target_ignores: bool, arg_file: PathBuf, ) -> Collector { Collector { tests: Vec::new(), - names: Vec::new(), rustdoc_options, - use_headers, - enable_per_target_ignores, crate_name, opts, - position: DUMMY_SP, - source_map, - filename, visited_tests: FxHashMap::default(), unused_extern_reports: Default::default(), compiling_test_count: AtomicUsize::new(0), @@ -1061,8 +1020,8 @@ impl Collector { } } - fn generate_name(&self, line: usize, filename: &FileName) -> String { - let mut item_path = self.names.join("::"); + fn generate_name(&self, filename: &FileName, line: usize, logical_path: &[String]) -> String { + let mut item_path = logical_path.join("::"); item_path.retain(|c| c != ' '); if !item_path.is_empty() { item_path.push(' '); @@ -1070,40 +1029,24 @@ impl Collector { format!("{} - {item_path}(line {line})", filename.prefer_local()) } - pub(crate) fn set_position(&mut self, position: Span) { - self.position = position; - } - - fn get_filename(&self) -> FileName { - if let Some(ref source_map) = self.source_map { - let filename = source_map.span_to_filename(self.position); - if let FileName::Real(ref filename) = filename - && let Ok(cur_dir) = env::current_dir() - && let Some(local_path) = filename.local_path() - && let Ok(path) = local_path.strip_prefix(&cur_dir) - { - return path.to_owned().into(); + fn add_test(&mut self, test: ScrapedDoctest) { + let (filename, line, logical_path, langstr, text) = match test { + ScrapedDoctest::Rust(RustDoctest { filename, line, logical_path, langstr, text }) => { + (filename, line, logical_path, langstr, text) } - filename - } else if let Some(ref filename) = self.filename { - filename.clone().into() - } else { - FileName::Custom("input".to_owned()) - } - } -} + ScrapedDoctest::Markdown(MdDoctest { filename, line, logical_path, langstr, text }) => { + (filename, line, logical_path, langstr, text) + } + }; -impl DoctestVisitor for Collector { - fn visit_test(&mut self, test: String, config: LangString, line: usize) { - let filename = self.get_filename(); - let name = self.generate_name(line, &filename); + let name = self.generate_name(&filename, line, &logical_path); let crate_name = self.crate_name.clone(); let opts = self.opts.clone(); - let edition = config.edition.unwrap_or(self.rustdoc_options.edition); + let edition = langstr.edition.unwrap_or(self.rustdoc_options.edition); let target_str = self.rustdoc_options.target.to_string(); let unused_externs = self.unused_extern_reports.clone(); - let no_run = config.no_run || self.rustdoc_options.no_run; - if !config.compile_fail { + let no_run = langstr.no_run || self.rustdoc_options.no_run; + if !langstr.compile_fail { self.compiling_test_count.fetch_add(1, Ordering::SeqCst); } @@ -1140,11 +1083,11 @@ impl DoctestVisitor for Collector { let rustdoc_test_options = IndividualTestOptions::new(&self.rustdoc_options, &self.arg_file, test_id); - debug!("creating test {name}: {test}"); + debug!("creating test {name}: {text}"); self.tests.push(test::TestDescAndFn { desc: test::TestDesc { name: test::DynTestName(name), - ignore: match config.ignore { + ignore: match langstr.ignore { Ignore::All => true, Ignore::None => false, Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)), @@ -1157,7 +1100,7 @@ impl DoctestVisitor for Collector { end_col: 0, // compiler failures are test failures should_panic: test::ShouldPanic::No, - compile_fail: config.compile_fail, + compile_fail: langstr.compile_fail, no_run, test_type: test::TestType::DocTest, }, @@ -1166,11 +1109,11 @@ impl DoctestVisitor for Collector { unused_externs.lock().unwrap().push(uext); }; let res = run_test( - &test, + &text, &crate_name, line, rustdoc_test_options, - config, + langstr, no_run, &opts, edition, @@ -1233,59 +1176,6 @@ impl DoctestVisitor for Collector { })), }); } - - fn get_line(&self) -> usize { - if let Some(ref source_map) = self.source_map { - let line = self.position.lo().to_usize(); - let line = source_map.lookup_char_pos(BytePos(line as u32)).line; - if line > 0 { line - 1 } else { line } - } else { - 0 - } - } - - fn visit_header(&mut self, name: &str, level: u32) { - if self.use_headers { - // We use these headings as test names, so it's good if - // they're valid identifiers. - let name = name - .chars() - .enumerate() - .map(|(i, c)| { - if (i == 0 && rustc_lexer::is_id_start(c)) - || (i != 0 && rustc_lexer::is_id_continue(c)) - { - c - } else { - '_' - } - }) - .collect::<String>(); - - // Here we try to efficiently assemble the header titles into the - // test name in the form of `h1::h2::h3::h4::h5::h6`. - // - // Suppose that originally `self.names` contains `[h1, h2, h3]`... - let level = level as usize; - if level <= self.names.len() { - // ... Consider `level == 2`. All headers in the lower levels - // are irrelevant in this new level. So we should reset - // `self.names` to contain headers until <h2>, and replace that - // slot with the new name: `[h1, name]`. - self.names.truncate(level); - self.names[level - 1] = name; - } else { - // ... On the other hand, consider `level == 5`. This means we - // need to extend `self.names` to contain five headers. We fill - // in the missing level (<h4>) with `_`. Thus `self.names` will - // become `[h1, h2, h3, "_", name]`. - if level - 1 > self.names.len() { - self.names.resize(level - 1, "_".to_owned()); - } - self.names.push(name); - } - } - } } #[cfg(test)] // used in tests diff --git a/src/librustdoc/doctest/markdown.rs b/src/librustdoc/doctest/markdown.rs index 13239b9a517..6f2769b7681 100644 --- a/src/librustdoc/doctest/markdown.rs +++ b/src/librustdoc/doctest/markdown.rs @@ -2,12 +2,84 @@ use std::fs::read_to_string; -use rustc_span::DUMMY_SP; +use rustc_span::FileName; use tempfile::tempdir; -use super::{generate_args_file, Collector, GlobalTestOptions}; +use super::{generate_args_file, Collector, DoctestVisitor, GlobalTestOptions, ScrapedDoctest}; use crate::config::Options; -use crate::html::markdown::{find_testable_code, ErrorCodes}; +use crate::html::markdown::{find_testable_code, ErrorCodes, LangString}; + +pub(super) struct MdDoctest { + pub(super) filename: FileName, + pub(super) line: usize, + pub(super) logical_path: Vec<String>, + pub(super) langstr: LangString, + pub(super) text: String, +} + +struct MdCollector { + tests: Vec<MdDoctest>, + cur_path: Vec<String>, + filename: FileName, +} + +impl DoctestVisitor for MdCollector { + fn visit_test(&mut self, test: String, config: LangString, line: usize) { + let filename = self.filename.clone(); + self.tests.push(MdDoctest { + filename, + line, + logical_path: self.cur_path.clone(), + langstr: config, + text: test, + }); + } + + fn get_line(&self) -> usize { + 0 + } + + fn visit_header(&mut self, name: &str, level: u32) { + // We use these headings as test names, so it's good if + // they're valid identifiers. + let name = name + .chars() + .enumerate() + .map(|(i, c)| { + if (i == 0 && rustc_lexer::is_id_start(c)) + || (i != 0 && rustc_lexer::is_id_continue(c)) + { + c + } else { + '_' + } + }) + .collect::<String>(); + + // Here we try to efficiently assemble the header titles into the + // test name in the form of `h1::h2::h3::h4::h5::h6`. + // + // Suppose that originally `self.cur_path` contains `[h1, h2, h3]`... + let level = level as usize; + if level <= self.cur_path.len() { + // ... Consider `level == 2`. All headers in the lower levels + // are irrelevant in this new level. So we should reset + // `self.names` to contain headers until <h2>, and replace that + // slot with the new name: `[h1, name]`. + self.cur_path.truncate(level); + self.cur_path[level - 1] = name; + } else { + // ... On the other hand, consider `level == 5`. This means we + // need to extend `self.names` to contain five headers. We fill + // in the missing level (<h4>) with `_`. Thus `self.names` will + // become `[h1, h2, h3, "_", name]`. + if level - 1 > self.cur_path.len() { + self.cur_path.resize(level - 1, "_".to_owned()); + } + self.cur_path.push(name); + } + } +} /// Runs any tests/code examples in the markdown file `input`. pub(crate) fn test(options: Options) -> Result<(), String> { @@ -27,21 +99,29 @@ pub(crate) fn test(options: Options) -> Result<(), String> { let file_path = temp_dir.path().join("rustdoc-cfgs"); generate_args_file(&file_path, &options)?; - let mut collector = Collector::new( - options.input.filestem().to_string(), - options.clone(), - true, - opts, - None, - options.input.opt_path().map(ToOwned::to_owned), - options.enable_per_target_ignores, - file_path, - ); - collector.set_position(DUMMY_SP); + let mut md_collector = MdCollector { + tests: vec![], + cur_path: vec![], + filename: options + .input + .opt_path() + .map(ToOwned::to_owned) + .map(FileName::from) + .unwrap_or(FileName::Custom("input".to_owned())), + }; let codes = ErrorCodes::from(options.unstable_features.is_nightly_build()); - find_testable_code(&input_str, &mut collector, codes, options.enable_per_target_ignores, None); + find_testable_code( + &input_str, + &mut md_collector, + codes, + options.enable_per_target_ignores, + None, + ); + let mut collector = + Collector::new(options.input.filestem().to_string(), options.clone(), opts, file_path); + md_collector.tests.into_iter().for_each(|t| collector.add_test(ScrapedDoctest::Markdown(t))); crate::doctest::run_tests(options.test_args, options.nocapture, collector.tests); Ok(()) } diff --git a/src/librustdoc/doctest/rust.rs b/src/librustdoc/doctest/rust.rs index e4527de77a0..9e62ed34a58 100644 --- a/src/librustdoc/doctest/rust.rs +++ b/src/librustdoc/doctest/rust.rs @@ -1,29 +1,108 @@ //! Doctest functionality used only for doctests in `.rs` source files. -use rustc_data_structures::fx::FxHashSet; -use rustc_hir::def_id::LocalDefId; -use rustc_hir::{self as hir, intravisit}; +use std::env; + +use rustc_data_structures::{fx::FxHashSet, sync::Lrc}; +use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID}; +use rustc_hir::{self as hir, intravisit, CRATE_HIR_ID}; use rustc_middle::hir::map::Map; use rustc_middle::hir::nested_filter; use rustc_middle::ty::TyCtxt; use rustc_resolve::rustdoc::span_of_fragments; use rustc_session::Session; -use rustc_span::{Span, DUMMY_SP}; +use rustc_span::source_map::SourceMap; +use rustc_span::{BytePos, FileName, Pos, Span, DUMMY_SP}; -use super::Collector; +use super::DoctestVisitor; use crate::clean::{types::AttributesExt, Attributes}; -use crate::html::markdown::{self, ErrorCodes}; - -pub(super) struct HirCollector<'a, 'hir, 'tcx> { - pub(super) sess: &'a Session, - pub(super) collector: &'a mut Collector, - pub(super) map: Map<'hir>, - pub(super) codes: ErrorCodes, - pub(super) tcx: TyCtxt<'tcx>, +use crate::html::markdown::{self, ErrorCodes, LangString}; + +pub(super) struct RustDoctest { + pub(super) filename: FileName, + pub(super) line: usize, + pub(super) logical_path: Vec<String>, + pub(super) langstr: LangString, + pub(super) text: String, +} + +struct RustCollector { + source_map: Lrc<SourceMap>, + tests: Vec<RustDoctest>, + cur_path: Vec<String>, + position: Span, +} + +impl RustCollector { + fn get_filename(&self) -> FileName { + let filename = self.source_map.span_to_filename(self.position); + if let FileName::Real(ref filename) = filename + && let Ok(cur_dir) = env::current_dir() + && let Some(local_path) = filename.local_path() + && let Ok(path) = local_path.strip_prefix(&cur_dir) + { + return path.to_owned().into(); + } + filename + } +} + +impl DoctestVisitor for RustCollector { + fn visit_test(&mut self, test: String, config: LangString, line: usize) { + self.tests.push(RustDoctest { + filename: self.get_filename(), + line, + logical_path: self.cur_path.clone(), + langstr: config, + text: test, + }); + } + + fn get_line(&self) -> usize { + let line = self.position.lo().to_usize(); + let line = self.source_map.lookup_char_pos(BytePos(line as u32)).line; + if line > 0 { line - 1 } else { line } + } + + fn visit_header(&mut self, _name: &str, _level: u32) {} +} + +pub(super) struct HirCollector<'a, 'tcx> { + sess: &'a Session, + map: Map<'tcx>, + codes: ErrorCodes, + tcx: TyCtxt<'tcx>, + enable_per_target_ignores: bool, + collector: RustCollector, +} + +impl<'a, 'tcx> HirCollector<'a, 'tcx> { + pub fn new( + sess: &'a Session, + map: Map<'tcx>, + codes: ErrorCodes, + enable_per_target_ignores: bool, + tcx: TyCtxt<'tcx>, + ) -> Self { + let collector = RustCollector { + source_map: sess.psess.clone_source_map(), + cur_path: vec![], + position: DUMMY_SP, + tests: vec![], + }; + Self { sess, map, codes, enable_per_target_ignores, tcx, collector } + } + + pub fn collect_crate(mut self) -> Vec<RustDoctest> { + let tcx = self.tcx; + self.visit_testable("".to_string(), CRATE_DEF_ID, tcx.hir().span(CRATE_HIR_ID), |this| { + tcx.hir().walk_toplevel_module(this) + }); + self.collector.tests + } } -impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> { - pub(super) fn visit_testable<F: FnOnce(&mut Self)>( +impl<'a, 'tcx> HirCollector<'a, 'tcx> { + fn visit_testable<F: FnOnce(&mut Self)>( &mut self, name: String, def_id: LocalDefId, @@ -39,7 +118,7 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> { let has_name = !name.is_empty(); if has_name { - self.collector.names.push(name); + self.collector.cur_path.push(name); } // The collapse-docs pass won't combine sugared/raw doc attributes, or included files with @@ -52,12 +131,12 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> { .find(|attr| attr.doc_str().is_some()) .map(|attr| attr.span.ctxt().outer_expn().expansion_cause().unwrap_or(attr.span)) .unwrap_or(DUMMY_SP); - self.collector.set_position(span); + self.collector.position = span; markdown::find_testable_code( &doc, - self.collector, + &mut self.collector, self.codes, - self.collector.enable_per_target_ignores, + self.enable_per_target_ignores, Some(&crate::html::markdown::ExtraInfo::new( self.tcx, def_id.to_def_id(), @@ -69,19 +148,19 @@ impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> { nested(self); if has_name { - self.collector.names.pop(); + self.collector.cur_path.pop(); } } } -impl<'a, 'hir, 'tcx> intravisit::Visitor<'hir> for HirCollector<'a, 'hir, 'tcx> { +impl<'a, 'tcx> intravisit::Visitor<'tcx> for HirCollector<'a, 'tcx> { type NestedFilter = nested_filter::All; fn nested_visit_map(&mut self) -> Self::Map { self.map } - fn visit_item(&mut self, item: &'hir hir::Item<'_>) { + fn visit_item(&mut self, item: &'tcx hir::Item<'_>) { let name = match &item.kind { hir::ItemKind::Impl(impl_) => { rustc_hir_pretty::id_to_string(&self.map, impl_.self_ty.hir_id) @@ -94,31 +173,31 @@ impl<'a, 'hir, 'tcx> intravisit::Visitor<'hir> for HirCollector<'a, 'hir, 'tcx> }); } - fn visit_trait_item(&mut self, item: &'hir hir::TraitItem<'_>) { + fn visit_trait_item(&mut self, item: &'tcx hir::TraitItem<'_>) { self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| { intravisit::walk_trait_item(this, item); }); } - fn visit_impl_item(&mut self, item: &'hir hir::ImplItem<'_>) { + fn visit_impl_item(&mut self, item: &'tcx hir::ImplItem<'_>) { self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| { intravisit::walk_impl_item(this, item); }); } - fn visit_foreign_item(&mut self, item: &'hir hir::ForeignItem<'_>) { + fn visit_foreign_item(&mut self, item: &'tcx hir::ForeignItem<'_>) { self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| { intravisit::walk_foreign_item(this, item); }); } - fn visit_variant(&mut self, v: &'hir hir::Variant<'_>) { + fn visit_variant(&mut self, v: &'tcx hir::Variant<'_>) { self.visit_testable(v.ident.to_string(), v.def_id, v.span, |this| { intravisit::walk_variant(this, v); }); } - fn visit_field_def(&mut self, f: &'hir hir::FieldDef<'_>) { + fn visit_field_def(&mut self, f: &'tcx hir::FieldDef<'_>) { self.visit_testable(f.ident.to_string(), f.def_id, f.span, |this| { intravisit::walk_field_def(this, f); }); |
