diff options
Diffstat (limited to 'src/libtest/lib.rs')
| -rw-r--r-- | src/libtest/lib.rs | 1577 |
1 files changed, 1577 insertions, 0 deletions
diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs new file mode 100644 index 00000000000..226dd75d740 --- /dev/null +++ b/src/libtest/lib.rs @@ -0,0 +1,1577 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// Support code for rustc's built in test runner generator. Currently, +// none of this is meant for users. It is intended to support the +// simplest interface possible for representing and running tests +// while providing a base that other test frameworks may build off of. + +#[crate_id = "test#0.10-pre"]; +#[comment = "Rust internal test library only used by rustc"]; +#[license = "MIT/ASL2"]; +#[crate_type = "rlib"]; +#[crate_type = "dylib"]; + +#[feature(asm)]; + +extern crate collections; +extern crate extra; +extern crate getopts; +extern crate serialize; +extern crate term; + +use collections::TreeMap; +use extra::json::ToJson; +use extra::json; +use extra::stats::Stats; +use extra::stats; +use extra::time::precise_time_ns; +use getopts::{OptGroup, optflag, optopt}; +use serialize::Decodable; +use term::Terminal; +use term::color::{Color, RED, YELLOW, GREEN, CYAN}; + +use std::cmp; +use std::io; +use std::io::{File, PortReader, ChanWriter}; +use std::io::stdio::StdWriter; +use std::str; +use std::task; +use std::to_str::ToStr; +use std::f64; +use std::os; + +// to be used by rustc to compile tests in libtest +pub mod test { + pub use {BenchHarness, TestName, TestResult, TestDesc, + TestDescAndFn, TestOpts, TrFailed, TrIgnored, TrOk, + Metric, MetricMap, MetricAdded, MetricRemoved, + MetricChange, Improvement, Regression, LikelyNoise, + StaticTestFn, StaticTestName, DynTestName, DynTestFn, + run_test, test_main, test_main_static, filter_tests, + parse_opts}; +} + +// The name of a test. By convention this follows the rules for rust +// paths; i.e. it should be a series of identifiers separated by double +// colons. This way if some test runner wants to arrange the tests +// hierarchically it may. + +#[deriving(Clone)] +pub enum TestName { + StaticTestName(&'static str), + DynTestName(~str) +} +impl ToStr for TestName { + fn to_str(&self) -> ~str { + match (*self).clone() { + StaticTestName(s) => s.to_str(), + DynTestName(s) => s.to_str() + } + } +} + +#[deriving(Clone)] +enum NamePadding { PadNone, PadOnLeft, PadOnRight } + +impl TestDesc { + fn padded_name(&self, column_count: uint, align: NamePadding) -> ~str { + use std::num::Saturating; + let name = self.name.to_str(); + let fill = column_count.saturating_sub(name.len()); + let pad = " ".repeat(fill); + match align { + PadNone => name, + PadOnLeft => pad.append(name), + PadOnRight => name.append(pad), + } + } +} + +/// Represents a benchmark function. +pub trait TDynBenchFn { + fn run(&self, harness: &mut BenchHarness); +} + +// A function that runs a test. If the function returns successfully, +// the test succeeds; if the function fails then the test fails. We +// may need to come up with a more clever definition of test in order +// to support isolation of tests into tasks. +pub enum TestFn { + StaticTestFn(extern fn()), + StaticBenchFn(extern fn(&mut BenchHarness)), + StaticMetricFn(proc(&mut MetricMap)), + DynTestFn(proc()), + DynMetricFn(proc(&mut MetricMap)), + DynBenchFn(~TDynBenchFn) +} + +impl TestFn { + fn padding(&self) -> NamePadding { + match self { + &StaticTestFn(..) => PadNone, + &StaticBenchFn(..) => PadOnRight, + &StaticMetricFn(..) => PadOnRight, + &DynTestFn(..) => PadNone, + &DynMetricFn(..) => PadOnRight, + &DynBenchFn(..) => PadOnRight, + } + } +} + +// Structure passed to BenchFns +pub struct BenchHarness { + priv iterations: u64, + priv ns_start: u64, + priv ns_end: u64, + bytes: u64 +} + +// The definition of a single test. A test runner will run a list of +// these. +#[deriving(Clone)] +pub struct TestDesc { + name: TestName, + ignore: bool, + should_fail: bool +} + +pub struct TestDescAndFn { + desc: TestDesc, + testfn: TestFn, +} + +#[deriving(Clone, Encodable, Decodable, Eq)] +pub struct Metric { + priv value: f64, + priv noise: f64 +} + +impl Metric { + pub fn new(value: f64, noise: f64) -> Metric { + Metric {value: value, noise: noise} + } +} + +#[deriving(Eq)] +pub struct MetricMap(TreeMap<~str,Metric>); + +impl Clone for MetricMap { + fn clone(&self) -> MetricMap { + let MetricMap(ref map) = *self; + MetricMap(map.clone()) + } +} + +/// Analysis of a single change in metric +#[deriving(Eq)] +pub enum MetricChange { + LikelyNoise, + MetricAdded, + MetricRemoved, + Improvement(f64), + Regression(f64) +} + +pub type MetricDiff = TreeMap<~str,MetricChange>; + +// The default console test runner. It accepts the command line +// arguments and a vector of test_descs. +pub fn test_main(args: &[~str], tests: ~[TestDescAndFn]) { + let opts = + match parse_opts(args) { + Some(Ok(o)) => o, + Some(Err(msg)) => fail!("{}", msg), + None => return + }; + match run_tests_console(&opts, tests) { + Ok(true) => {} + Ok(false) => fail!("Some tests failed"), + Err(e) => fail!("io error when running tests: {}", e), + } +} + +// A variant optimized for invocation with a static test vector. +// This will fail (intentionally) when fed any dynamic tests, because +// it is copying the static values out into a dynamic vector and cannot +// copy dynamic values. It is doing this because from this point on +// a ~[TestDescAndFn] is used in order to effect ownership-transfer +// semantics into parallel test runners, which in turn requires a ~[] +// rather than a &[]. +pub fn test_main_static(args: &[~str], tests: &[TestDescAndFn]) { + let owned_tests = tests.map(|t| { + match t.testfn { + StaticTestFn(f) => + TestDescAndFn { testfn: StaticTestFn(f), desc: t.desc.clone() }, + + StaticBenchFn(f) => + TestDescAndFn { testfn: StaticBenchFn(f), desc: t.desc.clone() }, + + _ => { + fail!("non-static tests passed to test::test_main_static"); + } + } + }); + test_main(args, owned_tests) +} + +pub struct TestOpts { + filter: Option<~str>, + run_ignored: bool, + run_tests: bool, + run_benchmarks: bool, + ratchet_metrics: Option<Path>, + ratchet_noise_percent: Option<f64>, + save_metrics: Option<Path>, + test_shard: Option<(uint,uint)>, + logfile: Option<Path> +} + +/// Result of parsing the options. +pub type OptRes = Result<TestOpts, ~str>; + +fn optgroups() -> ~[getopts::OptGroup] { + ~[getopts::optflag("", "ignored", "Run ignored tests"), + getopts::optflag("", "test", "Run tests and not benchmarks"), + getopts::optflag("", "bench", "Run benchmarks instead of tests"), + getopts::optflag("h", "help", "Display this message (longer with --help)"), + getopts::optopt("", "save-metrics", "Location to save bench metrics", + "PATH"), + getopts::optopt("", "ratchet-metrics", + "Location to load and save metrics from. The metrics \ + loaded are cause benchmarks to fail if they run too \ + slowly", "PATH"), + getopts::optopt("", "ratchet-noise-percent", + "Tests within N% of the recorded metrics will be \ + considered as passing", "PERCENTAGE"), + getopts::optopt("", "logfile", "Write logs to the specified file instead \ + of stdout", "PATH"), + getopts::optopt("", "test-shard", "run shard A, of B shards, worth of the testsuite", + "A.B")] +} + +fn usage(binary: &str, helpstr: &str) { + let message = format!("Usage: {} [OPTIONS] [FILTER]", binary); + println!("{}", getopts::usage(message, optgroups())); + println!(""); + if helpstr == "help" { + println!("{}", "\ +The FILTER is matched against the name of all tests to run, and if any tests +have a substring match, only those tests are run. + +By default, all tests are run in parallel. This can be altered with the +RUST_TEST_TASKS environment variable when running tests (set it to 1). + +Test Attributes: + + #[test] - Indicates a function is a test to be run. This function + takes no arguments. + #[bench] - Indicates a function is a benchmark to be run. This + function takes one argument (test::BenchHarness). + #[should_fail] - This function (also labeled with #[test]) will only pass if + the code causes a failure (an assertion failure or fail!) + #[ignore] - When applied to a function which is already attributed as a + test, then the test runner will ignore these tests during + normal test runs. Running with --ignored will run these + tests. This may also be written as #[ignore(cfg(...))] to + ignore the test on certain configurations."); + } +} + +// Parses command line arguments into test options +pub fn parse_opts(args: &[~str]) -> Option<OptRes> { + let args_ = args.tail(); + let matches = + match getopts::getopts(args_, optgroups()) { + Ok(m) => m, + Err(f) => return Some(Err(f.to_err_msg())) + }; + + if matches.opt_present("h") { usage(args[0], "h"); return None; } + if matches.opt_present("help") { usage(args[0], "help"); return None; } + + let filter = + if matches.free.len() > 0 { + Some((matches).free[0].clone()) + } else { + None + }; + + let run_ignored = matches.opt_present("ignored"); + + let logfile = matches.opt_str("logfile"); + let logfile = logfile.map(|s| Path::new(s)); + + let run_benchmarks = matches.opt_present("bench"); + let run_tests = ! run_benchmarks || + matches.opt_present("test"); + + let ratchet_metrics = matches.opt_str("ratchet-metrics"); + let ratchet_metrics = ratchet_metrics.map(|s| Path::new(s)); + + let ratchet_noise_percent = matches.opt_str("ratchet-noise-percent"); + let ratchet_noise_percent = ratchet_noise_percent.map(|s| from_str::<f64>(s).unwrap()); + + let save_metrics = matches.opt_str("save-metrics"); + let save_metrics = save_metrics.map(|s| Path::new(s)); + + let test_shard = matches.opt_str("test-shard"); + let test_shard = opt_shard(test_shard); + + let test_opts = TestOpts { + filter: filter, + run_ignored: run_ignored, + run_tests: run_tests, + run_benchmarks: run_benchmarks, + ratchet_metrics: ratchet_metrics, + ratchet_noise_percent: ratchet_noise_percent, + save_metrics: save_metrics, + test_shard: test_shard, + logfile: logfile + }; + + Some(Ok(test_opts)) +} + +pub fn opt_shard(maybestr: Option<~str>) -> Option<(uint,uint)> { + match maybestr { + None => None, + Some(s) => { + let vector = s.split('.').to_owned_vec(); + if vector.len() == 2 { + match (from_str::<uint>(vector[0]), + from_str::<uint>(vector[1])) { + (Some(a), Some(b)) => Some((a, b)), + _ => None + } + } else { + None + } + } + } +} + + +#[deriving(Clone, Eq)] +pub struct BenchSamples { + priv ns_iter_summ: stats::Summary, + priv mb_s: uint +} + +#[deriving(Clone, Eq)] +pub enum TestResult { + TrOk, + TrFailed, + TrIgnored, + TrMetrics(MetricMap), + TrBench(BenchSamples), +} + +enum OutputLocation<T> { + Pretty(term::Terminal<T>), + Raw(T), +} + +struct ConsoleTestState<T> { + log_out: Option<File>, + out: OutputLocation<T>, + use_color: bool, + total: uint, + passed: uint, + failed: uint, + ignored: uint, + measured: uint, + metrics: MetricMap, + failures: ~[(TestDesc, ~[u8])], + max_name_len: uint, // number of columns to fill when aligning names +} + +impl<T: Writer> ConsoleTestState<T> { + pub fn new(opts: &TestOpts, + _: Option<T>) -> io::IoResult<ConsoleTestState<StdWriter>> { + let log_out = match opts.logfile { + Some(ref path) => Some(if_ok!(File::create(path))), + None => None + }; + let out = match term::Terminal::new(io::stdout()) { + Err(_) => Raw(io::stdout()), + Ok(t) => Pretty(t) + }; + Ok(ConsoleTestState { + out: out, + log_out: log_out, + use_color: use_color(), + total: 0u, + passed: 0u, + failed: 0u, + ignored: 0u, + measured: 0u, + metrics: MetricMap::new(), + failures: ~[], + max_name_len: 0u, + }) + } + + pub fn write_ok(&mut self) -> io::IoResult<()> { + self.write_pretty("ok", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::IoResult<()> { + self.write_pretty("FAILED", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::IoResult<()> { + self.write_pretty("ignored", term::color::YELLOW) + } + + pub fn write_metric(&mut self) -> io::IoResult<()> { + self.write_pretty("metric", term::color::CYAN) + } + + pub fn write_bench(&mut self) -> io::IoResult<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_added(&mut self) -> io::IoResult<()> { + self.write_pretty("added", term::color::GREEN) + } + + pub fn write_improved(&mut self) -> io::IoResult<()> { + self.write_pretty("improved", term::color::GREEN) + } + + pub fn write_removed(&mut self) -> io::IoResult<()> { + self.write_pretty("removed", term::color::YELLOW) + } + + pub fn write_regressed(&mut self) -> io::IoResult<()> { + self.write_pretty("regressed", term::color::RED) + } + + pub fn write_pretty(&mut self, + word: &str, + color: term::color::Color) -> io::IoResult<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + if_ok!(term.fg(color)); + } + if_ok!(term.write(word.as_bytes())); + if self.use_color { + if_ok!(term.reset()); + } + Ok(()) + } + Raw(ref mut stdout) => stdout.write(word.as_bytes()) + } + } + + pub fn write_plain(&mut self, s: &str) -> io::IoResult<()> { + match self.out { + Pretty(ref mut term) => term.write(s.as_bytes()), + Raw(ref mut stdout) => stdout.write(s.as_bytes()) + } + } + + pub fn write_run_start(&mut self, len: uint) -> io::IoResult<()> { + self.total = len; + let noun = if len != 1 { &"tests" } else { &"test" }; + self.write_plain(format!("\nrunning {} {}\n", len, noun)) + } + + pub fn write_test_start(&mut self, test: &TestDesc, + align: NamePadding) -> io::IoResult<()> { + let name = test.padded_name(self.max_name_len, align); + self.write_plain(format!("test {} ... ", name)) + } + + pub fn write_result(&mut self, result: &TestResult) -> io::IoResult<()> { + if_ok!(match *result { + TrOk => self.write_ok(), + TrFailed => self.write_failed(), + TrIgnored => self.write_ignored(), + TrMetrics(ref mm) => { + if_ok!(self.write_metric()); + self.write_plain(format!(": {}", fmt_metrics(mm))) + } + TrBench(ref bs) => { + if_ok!(self.write_bench()); + self.write_plain(format!(": {}", fmt_bench_samples(bs))) + } + }); + self.write_plain("\n") + } + + pub fn write_log(&mut self, test: &TestDesc, + result: &TestResult) -> io::IoResult<()> { + match self.log_out { + None => Ok(()), + Some(ref mut o) => { + let s = format!("{} {}\n", match *result { + TrOk => ~"ok", + TrFailed => ~"failed", + TrIgnored => ~"ignored", + TrMetrics(ref mm) => fmt_metrics(mm), + TrBench(ref bs) => fmt_bench_samples(bs) + }, test.name.to_str()); + o.write(s.as_bytes()) + } + } + } + + pub fn write_failures(&mut self) -> io::IoResult<()> { + if_ok!(self.write_plain("\nfailures:\n")); + let mut failures = ~[]; + let mut fail_out = ~""; + for &(ref f, ref stdout) in self.failures.iter() { + failures.push(f.name.to_str()); + if stdout.len() > 0 { + fail_out.push_str(format!("---- {} stdout ----\n\t", + f.name.to_str())); + let output = str::from_utf8_lossy(*stdout); + fail_out.push_str(output.as_slice().replace("\n", "\n\t")); + fail_out.push_str("\n"); + } + } + if fail_out.len() > 0 { + if_ok!(self.write_plain("\n")); + if_ok!(self.write_plain(fail_out)); + } + + if_ok!(self.write_plain("\nfailures:\n")); + failures.sort(); + for name in failures.iter() { + if_ok!(self.write_plain(format!(" {}\n", name.to_str()))); + } + Ok(()) + } + + pub fn write_metric_diff(&mut self, diff: &MetricDiff) -> io::IoResult<()> { + let mut noise = 0; + let mut improved = 0; + let mut regressed = 0; + let mut added = 0; + let mut removed = 0; + + for (k, v) in diff.iter() { + match *v { + LikelyNoise => noise += 1, + MetricAdded => { + added += 1; + if_ok!(self.write_added()); + if_ok!(self.write_plain(format!(": {}\n", *k))); + } + MetricRemoved => { + removed += 1; + if_ok!(self.write_removed()); + if_ok!(self.write_plain(format!(": {}\n", *k))); + } + Improvement(pct) => { + improved += 1; + if_ok!(self.write_plain(format!(": {}", *k))); + if_ok!(self.write_improved()); + if_ok!(self.write_plain(format!(" by {:.2f}%\n", pct as f64))); + } + Regression(pct) => { + regressed += 1; + if_ok!(self.write_plain(format!(": {}", *k))); + if_ok!(self.write_regressed()); + if_ok!(self.write_plain(format!(" by {:.2f}%\n", pct as f64))); + } + } + } + if_ok!(self.write_plain(format!("result of ratchet: {} metrics added, \ + {} removed, {} improved, {} regressed, \ + {} noise\n", + added, removed, improved, regressed, + noise))); + if regressed == 0 { + if_ok!(self.write_plain("updated ratchet file\n")); + } else { + if_ok!(self.write_plain("left ratchet file untouched\n")); + } + Ok(()) + } + + pub fn write_run_finish(&mut self, + ratchet_metrics: &Option<Path>, + ratchet_pct: Option<f64>) -> io::IoResult<bool> { + assert!(self.passed + self.failed + self.ignored + self.measured == self.total); + + let ratchet_success = match *ratchet_metrics { + None => true, + Some(ref pth) => { + if_ok!(self.write_plain(format!("\nusing metrics ratcher: {}\n", + pth.display()))); + match ratchet_pct { + None => (), + Some(pct) => + if_ok!(self.write_plain(format!("with noise-tolerance \ + forced to: {}%\n", + pct))) + } + let (diff, ok) = self.metrics.ratchet(pth, ratchet_pct); + if_ok!(self.write_metric_diff(&diff)); + ok + } + }; + + let test_success = self.failed == 0u; + if !test_success { + if_ok!(self.write_failures()); + } + + let success = ratchet_success && test_success; + + if_ok!(self.write_plain("\ntest result: ")); + if success { + // There's no parallelism at this point so it's safe to use color + if_ok!(self.write_ok()); + } else { + if_ok!(self.write_failed()); + } + let s = format!(". {} passed; {} failed; {} ignored; {} measured\n\n", + self.passed, self.failed, self.ignored, self.measured); + if_ok!(self.write_plain(s)); + return Ok(success); + } +} + +pub fn fmt_metrics(mm: &MetricMap) -> ~str { + let MetricMap(ref mm) = *mm; + let v : ~[~str] = mm.iter() + .map(|(k,v)| format!("{}: {} (+/- {})", + *k, + v.value as f64, + v.noise as f64)) + .collect(); + v.connect(", ") +} + +pub fn fmt_bench_samples(bs: &BenchSamples) -> ~str { + if bs.mb_s != 0 { + format!("{:>9} ns/iter (+/- {}) = {} MB/s", + bs.ns_iter_summ.median as uint, + (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as uint, + bs.mb_s) + } else { + format!("{:>9} ns/iter (+/- {})", + bs.ns_iter_summ.median as uint, + (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as uint) + } +} + +// A simple console test runner +pub fn run_tests_console(opts: &TestOpts, + tests: ~[TestDescAndFn]) -> io::IoResult<bool> { + fn callback<T: Writer>(event: &TestEvent, + st: &mut ConsoleTestState<T>) -> io::IoResult<()> { + debug!("callback(event={:?})", event); + match (*event).clone() { + TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()), + TeWait(ref test, padding) => st.write_test_start(test, padding), + TeResult(test, result, stdout) => { + if_ok!(st.write_log(&test, &result)); + if_ok!(st.write_result(&result)); + match result { + TrOk => st.passed += 1, + TrIgnored => st.ignored += 1, + TrMetrics(mm) => { + let tname = test.name.to_str(); + let MetricMap(mm) = mm; + for (k,v) in mm.iter() { + st.metrics.insert_metric(tname + "." + *k, + v.value, v.noise); + } + st.measured += 1 + } + TrBench(bs) => { + st.metrics.insert_metric(test.name.to_str(), + bs.ns_iter_summ.median, + bs.ns_iter_summ.max - bs.ns_iter_summ.min); + st.measured += 1 + } + TrFailed => { + st.failed += 1; + st.failures.push((test, stdout)); + } + } + Ok(()) + } + } + } + let mut st = if_ok!(ConsoleTestState::new(opts, None::<StdWriter>)); + fn len_if_padded(t: &TestDescAndFn) -> uint { + match t.testfn.padding() { + PadNone => 0u, + PadOnLeft | PadOnRight => t.desc.name.to_str().len(), + } + } + match tests.iter().max_by(|t|len_if_padded(*t)) { + Some(t) => { + let n = t.desc.name.to_str(); + debug!("Setting max_name_len from: {}", n); + st.max_name_len = n.len(); + }, + None => {} + } + if_ok!(run_tests(opts, tests, |x| callback(&x, &mut st))); + match opts.save_metrics { + None => (), + Some(ref pth) => { + if_ok!(st.metrics.save(pth)); + if_ok!(st.write_plain(format!("\nmetrics saved to: {}", + pth.display()))); + } + } + return st.write_run_finish(&opts.ratchet_metrics, opts.ratchet_noise_percent); +} + +#[test] +fn should_sort_failures_before_printing_them() { + use std::io::MemWriter; + use std::str; + + let test_a = TestDesc { + name: StaticTestName("a"), + ignore: false, + should_fail: false + }; + + let test_b = TestDesc { + name: StaticTestName("b"), + ignore: false, + should_fail: false + }; + + let mut st = ConsoleTestState { + log_out: None, + out: Raw(MemWriter::new()), + use_color: false, + total: 0u, + passed: 0u, + failed: 0u, + ignored: 0u, + measured: 0u, + max_name_len: 10u, + metrics: MetricMap::new(), + failures: ~[(test_b, ~[]), (test_a, ~[])] + }; + + st.write_failures().unwrap(); + let s = match st.out { + Raw(ref m) => str::from_utf8_lossy(m.get_ref()), + Pretty(_) => unreachable!() + }; + + let apos = s.as_slice().find_str("a").unwrap(); + let bpos = s.as_slice().find_str("b").unwrap(); + assert!(apos < bpos); +} + +fn use_color() -> bool { return get_concurrency() == 1; } + +#[deriving(Clone)] +enum TestEvent { + TeFiltered(~[TestDesc]), + TeWait(TestDesc, NamePadding), + TeResult(TestDesc, TestResult, ~[u8] /* stdout */), +} + +pub type MonitorMsg = (TestDesc, TestResult, ~[u8] /* stdout */); + +fn run_tests(opts: &TestOpts, + tests: ~[TestDescAndFn], + callback: |e: TestEvent| -> io::IoResult<()>) -> io::IoResult<()> { + let filtered_tests = filter_tests(opts, tests); + let filtered_descs = filtered_tests.map(|t| t.desc.clone()); + + if_ok!(callback(TeFiltered(filtered_descs))); + + let (filtered_tests, filtered_benchs_and_metrics) = + filtered_tests.partition(|e| { + match e.testfn { + StaticTestFn(_) | DynTestFn(_) => true, + _ => false + } + }); + + // It's tempting to just spawn all the tests at once, but since we have + // many tests that run in other processes we would be making a big mess. + let concurrency = get_concurrency(); + debug!("using {} test tasks", concurrency); + + let mut remaining = filtered_tests; + remaining.reverse(); + let mut pending = 0; + + let (p, ch) = Chan::<MonitorMsg>::new(); + + while pending > 0 || !remaining.is_empty() { + while pending < concurrency && !remaining.is_empty() { + let test = remaining.pop().unwrap(); + if concurrency == 1 { + // We are doing one test at a time so we can print the name + // of the test before we run it. Useful for debugging tests + // that hang forever. + if_ok!(callback(TeWait(test.desc.clone(), test.testfn.padding()))); + } + run_test(!opts.run_tests, test, ch.clone()); + pending += 1; + } + + let (desc, result, stdout) = p.recv(); + if concurrency != 1 { + if_ok!(callback(TeWait(desc.clone(), PadNone))); + } + if_ok!(callback(TeResult(desc, result, stdout))); + pending -= 1; + } + + // All benchmarks run at the end, in serial. + // (this includes metric fns) + for b in filtered_benchs_and_metrics.move_iter() { + if_ok!(callback(TeWait(b.desc.clone(), b.testfn.padding()))); + run_test(!opts.run_benchmarks, b, ch.clone()); + let (test, result, stdout) = p.recv(); + if_ok!(callback(TeResult(test, result, stdout))); + } + Ok(()) +} + +fn get_concurrency() -> uint { + use std::rt; + match os::getenv("RUST_TEST_TASKS") { + Some(s) => { + let opt_n: Option<uint> = FromStr::from_str(s); + match opt_n { + Some(n) if n > 0 => n, + _ => fail!("RUST_TEST_TASKS is `{}`, should be a positive integer.", s) + } + } + None => { + rt::default_sched_threads() + } + } +} + +pub fn filter_tests( + opts: &TestOpts, + tests: ~[TestDescAndFn]) -> ~[TestDescAndFn] +{ + let mut filtered = tests; + + // Remove tests that don't match the test filter + filtered = if opts.filter.is_none() { + filtered + } else { + let filter_str = match opts.filter { + Some(ref f) => (*f).clone(), + None => ~"" + }; + + fn filter_fn(test: TestDescAndFn, filter_str: &str) -> + Option<TestDescAndFn> { + if test.desc.name.to_str().contains(filter_str) { + return Some(test); + } else { + return None; + } + } + + filtered.move_iter().filter_map(|x| filter_fn(x, filter_str)).collect() + }; + + // Maybe pull out the ignored test and unignore them + filtered = if !opts.run_ignored { + filtered + } else { + fn filter(test: TestDescAndFn) -> Option<TestDescAndFn> { + if test.desc.ignore { + let TestDescAndFn {desc, testfn} = test; + Some(TestDescAndFn { + desc: TestDesc {ignore: false, ..desc}, + testfn: testfn + }) + } else { + None + } + }; + filtered.move_iter().filter_map(|x| filter(x)).collect() + }; + + // Sort the tests alphabetically + filtered.sort_by(|t1, t2| t1.desc.name.to_str().cmp(&t2.desc.name.to_str())); + + // Shard the remaining tests, if sharding requested. + match opts.test_shard { + None => filtered, + Some((a,b)) => + filtered.move_iter().enumerate() + .filter(|&(i,_)| i % b == a) + .map(|(_,t)| t) + .to_owned_vec() + } +} + +pub fn run_test(force_ignore: bool, + test: TestDescAndFn, + monitor_ch: Chan<MonitorMsg>) { + + let TestDescAndFn {desc, testfn} = test; + + if force_ignore || desc.ignore { + monitor_ch.send((desc, TrIgnored, ~[])); + return; + } + + fn run_test_inner(desc: TestDesc, + monitor_ch: Chan<MonitorMsg>, + testfn: proc()) { + spawn(proc() { + let (p, c) = Chan::new(); + let mut reader = PortReader::new(p); + let stdout = ChanWriter::new(c.clone()); + let stderr = ChanWriter::new(c); + let mut task = task::task().named(match desc.name { + DynTestName(ref name) => name.clone().into_maybe_owned(), + StaticTestName(name) => name.into_maybe_owned(), + }); + task.opts.stdout = Some(~stdout as ~Writer); + task.opts.stderr = Some(~stderr as ~Writer); + let result_future = task.future_result(); + task.spawn(testfn); + + let stdout = reader.read_to_end().unwrap(); + let task_result = result_future.recv(); + let test_result = calc_result(&desc, task_result.is_ok()); + monitor_ch.send((desc.clone(), test_result, stdout)); + }) + } + + match testfn { + DynBenchFn(bencher) => { + let bs = ::bench::benchmark(|harness| bencher.run(harness)); + monitor_ch.send((desc, TrBench(bs), ~[])); + return; + } + StaticBenchFn(benchfn) => { + let bs = ::bench::benchmark(|harness| benchfn(harness)); + monitor_ch.send((desc, TrBench(bs), ~[])); + return; + } + DynMetricFn(f) => { + let mut mm = MetricMap::new(); + f(&mut mm); + monitor_ch.send((desc, TrMetrics(mm), ~[])); + return; + } + StaticMetricFn(f) => { + let mut mm = MetricMap::new(); + f(&mut mm); + monitor_ch.send((desc, TrMetrics(mm), ~[])); + return; + } + DynTestFn(f) => run_test_inner(desc, monitor_ch, f), + StaticTestFn(f) => run_test_inner(desc, monitor_ch, proc() f()) + } +} + +fn calc_result(desc: &TestDesc, task_succeeded: bool) -> TestResult { + if task_succeeded { + if desc.should_fail { TrFailed } + else { TrOk } + } else { + if desc.should_fail { TrOk } + else { TrFailed } + } +} + + +impl ToJson for Metric { + fn to_json(&self) -> json::Json { + let mut map = ~TreeMap::new(); + map.insert(~"value", json::Number(self.value)); + map.insert(~"noise", json::Number(self.noise)); + json::Object(map) + } +} + +impl MetricMap { + + pub fn new() -> MetricMap { + MetricMap(TreeMap::new()) + } + + /// Load MetricDiff from a file. + /// + /// # Failure + /// + /// This function will fail if the path does not exist or the path does not + /// contain a valid metric map. + pub fn load(p: &Path) -> MetricMap { + assert!(p.exists()); + let mut f = File::open(p).unwrap(); + let value = json::from_reader(&mut f as &mut io::Reader).unwrap(); + let mut decoder = json::Decoder::new(value); + MetricMap(Decodable::decode(&mut decoder)) + } + + /// Write MetricDiff to a file. + pub fn save(&self, p: &Path) -> io::IoResult<()> { + let mut file = if_ok!(File::create(p)); + let MetricMap(ref map) = *self; + map.to_json().to_pretty_writer(&mut file) + } + + /// Compare against another MetricMap. Optionally compare all + /// measurements in the maps using the provided `noise_pct` as a + /// percentage of each value to consider noise. If `None`, each + /// measurement's noise threshold is independently chosen as the + /// maximum of that measurement's recorded noise quantity in either + /// map. + pub fn compare_to_old(&self, old: &MetricMap, + noise_pct: Option<f64>) -> MetricDiff { + let mut diff : MetricDiff = TreeMap::new(); + let MetricMap(ref selfmap) = *self; + let MetricMap(ref old) = *old; + for (k, vold) in old.iter() { + let r = match selfmap.find(k) { + None => MetricRemoved, + Some(v) => { + let delta = (v.value - vold.value); + let noise = match noise_pct { + None => f64::max(vold.noise.abs(), v.noise.abs()), + Some(pct) => vold.value * pct / 100.0 + }; + if delta.abs() <= noise { + LikelyNoise + } else { + let pct = delta.abs() / cmp::max(vold.value, f64::EPSILON) * 100.0; + if vold.noise < 0.0 { + // When 'noise' is negative, it means we want + // to see deltas that go up over time, and can + // only tolerate slight negative movement. + if delta < 0.0 { + Regression(pct) + } else { + Improvement(pct) + } + } else { + // When 'noise' is positive, it means we want + // to see deltas that go down over time, and + // can only tolerate slight positive movements. + if delta < 0.0 { + Improvement(pct) + } else { + Regression(pct) + } + } + } + } + }; + diff.insert((*k).clone(), r); + } + let MetricMap(ref map) = *self; + for (k, _) in map.iter() { + if !diff.contains_key(k) { + diff.insert((*k).clone(), MetricAdded); + } + } + diff + } + + /// Insert a named `value` (+/- `noise`) metric into the map. The value + /// must be non-negative. The `noise` indicates the uncertainty of the + /// metric, which doubles as the "noise range" of acceptable + /// pairwise-regressions on this named value, when comparing from one + /// metric to the next using `compare_to_old`. + /// + /// If `noise` is positive, then it means this metric is of a value + /// you want to see grow smaller, so a change larger than `noise` in the + /// positive direction represents a regression. + /// + /// If `noise` is negative, then it means this metric is of a value + /// you want to see grow larger, so a change larger than `noise` in the + /// negative direction represents a regression. + pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) { + let m = Metric { + value: value, + noise: noise + }; + let MetricMap(ref mut map) = *self; + map.insert(name.to_owned(), m); + } + + /// Attempt to "ratchet" an external metric file. This involves loading + /// metrics from a metric file (if it exists), comparing against + /// the metrics in `self` using `compare_to_old`, and rewriting the + /// file to contain the metrics in `self` if none of the + /// `MetricChange`s are `Regression`. Returns the diff as well + /// as a boolean indicating whether the ratchet succeeded. + pub fn ratchet(&self, p: &Path, pct: Option<f64>) -> (MetricDiff, bool) { + let old = if p.exists() { + MetricMap::load(p) + } else { + MetricMap::new() + }; + + let diff : MetricDiff = self.compare_to_old(&old, pct); + let ok = diff.iter().all(|(_, v)| { + match *v { + Regression(_) => false, + _ => true + } + }); + + if ok { + debug!("rewriting file '{:?}' with updated metrics", p); + self.save(p).unwrap(); + } + return (diff, ok) + } +} + + +// Benchmarking + +/// A function that is opaque to the optimizer, to allow benchmarks to +/// pretend to use outputs to assist in avoiding dead-code +/// elimination. +/// +/// This function is a no-op, and does not even read from `dummy`. +pub fn black_box<T>(dummy: T) { + // we need to "use" the argument in some way LLVM can't + // introspect. + unsafe {asm!("" : : "r"(&dummy))} +} + + +impl BenchHarness { + /// Callback for benchmark functions to run in their body. + pub fn iter<T>(&mut self, inner: || -> T) { + self.ns_start = precise_time_ns(); + let k = self.iterations; + for _ in range(0u64, k) { + black_box(inner()); + } + self.ns_end = precise_time_ns(); + } + + pub fn ns_elapsed(&mut self) -> u64 { + if self.ns_start == 0 || self.ns_end == 0 { + 0 + } else { + self.ns_end - self.ns_start + } + } + + pub fn ns_per_iter(&mut self) -> u64 { + if self.iterations == 0 { + 0 + } else { + self.ns_elapsed() / cmp::max(self.iterations, 1) + } + } + + pub fn bench_n(&mut self, n: u64, f: |&mut BenchHarness|) { + self.iterations = n; + debug!("running benchmark for {} iterations", + n as uint); + f(self); + } + + // This is a more statistics-driven benchmark algorithm + pub fn auto_bench(&mut self, f: |&mut BenchHarness|) -> stats::Summary { + + // Initial bench run to get ballpark figure. + let mut n = 1_u64; + self.bench_n(n, |x| f(x)); + + // Try to estimate iter count for 1ms falling back to 1m + // iterations if first run took < 1ns. + if self.ns_per_iter() == 0 { + n = 1_000_000; + } else { + n = 1_000_000 / cmp::max(self.ns_per_iter(), 1); + } + // if the first run took more than 1ms we don't want to just + // be left doing 0 iterations on every loop. The unfortunate + // side effect of not being able to do as many runs is + // automatically handled by the statistical analysis below + // (i.e. larger error bars). + if n == 0 { n = 1; } + + debug!("Initial run took {} ns, iter count that takes 1ms estimated as {}", + self.ns_per_iter(), n); + + let mut total_run = 0; + let samples : &mut [f64] = [0.0_f64, ..50]; + loop { + let loop_start = precise_time_ns(); + + for p in samples.mut_iter() { + self.bench_n(n, |x| f(x)); + *p = self.ns_per_iter() as f64; + }; + + stats::winsorize(samples, 5.0); + let summ = stats::Summary::new(samples); + + for p in samples.mut_iter() { + self.bench_n(5 * n, |x| f(x)); + *p = self.ns_per_iter() as f64; + }; + + stats::winsorize(samples, 5.0); + let summ5 = stats::Summary::new(samples); + + debug!("{} samples, median {}, MAD={}, MADP={}", + samples.len(), + summ.median as f64, + summ.median_abs_dev as f64, + summ.median_abs_dev_pct as f64); + + let now = precise_time_ns(); + let loop_run = now - loop_start; + + // If we've run for 100ms and seem to have converged to a + // stable median. + if loop_run > 100_000_000 && + summ.median_abs_dev_pct < 1.0 && + summ.median - summ5.median < summ5.median_abs_dev { + return summ5; + } + + total_run += loop_run; + // Longest we ever run for is 3s. + if total_run > 3_000_000_000 { + return summ5; + } + + n *= 2; + } + } +} + +pub mod bench { + use std::cmp; + use super::{BenchHarness, BenchSamples}; + + pub fn benchmark(f: |&mut BenchHarness|) -> BenchSamples { + let mut bs = BenchHarness { + iterations: 0, + ns_start: 0, + ns_end: 0, + bytes: 0 + }; + + let ns_iter_summ = bs.auto_bench(f); + + let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); + let iter_s = 1_000_000_000 / ns_iter; + let mb_s = (bs.bytes * iter_s) / 1_000_000; + + BenchSamples { + ns_iter_summ: ns_iter_summ, + mb_s: mb_s as uint + } + } +} + +#[cfg(test)] +mod tests { + use test::{TrFailed, TrIgnored, TrOk, filter_tests, parse_opts, + TestDesc, TestDescAndFn, TestOpts, run_test, + Metric, MetricMap, MetricAdded, MetricRemoved, + Improvement, Regression, LikelyNoise, + StaticTestName, DynTestName, DynTestFn}; + use extra::tempfile::TempDir; + + #[test] + pub fn do_not_run_ignored_tests() { + fn f() { fail!(); } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: true, + should_fail: false + }, + testfn: DynTestFn(proc() f()), + }; + let (p, ch) = Chan::new(); + run_test(false, desc, ch); + let (_, res, _) = p.recv(); + assert!(res != TrOk); + } + + #[test] + pub fn ignored_tests_result_in_ignored() { + fn f() { } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: true, + should_fail: false + }, + testfn: DynTestFn(proc() f()), + }; + let (p, ch) = Chan::new(); + run_test(false, desc, ch); + let (_, res, _) = p.recv(); + assert_eq!(res, TrIgnored); + } + + #[test] + fn test_should_fail() { + fn f() { fail!(); } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_fail: true + }, + testfn: DynTestFn(proc() f()), + }; + let (p, ch) = Chan::new(); + run_test(false, desc, ch); + let (_, res, _) = p.recv(); + assert_eq!(res, TrOk); + } + + #[test] + fn test_should_fail_but_succeeds() { + fn f() { } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_fail: true + }, + testfn: DynTestFn(proc() f()), + }; + let (p, ch) = Chan::new(); + run_test(false, desc, ch); + let (_, res, _) = p.recv(); + assert_eq!(res, TrFailed); + } + + #[test] + fn first_free_arg_should_be_a_filter() { + let args = ~[~"progname", ~"filter"]; + let opts = match parse_opts(args) { + Some(Ok(o)) => o, + _ => fail!("Malformed arg in first_free_arg_should_be_a_filter") + }; + assert!("filter" == opts.filter.clone().unwrap()); + } + + #[test] + fn parse_ignored_flag() { + let args = ~[~"progname", ~"filter", ~"--ignored"]; + let opts = match parse_opts(args) { + Some(Ok(o)) => o, + _ => fail!("Malformed arg in parse_ignored_flag") + }; + assert!((opts.run_ignored)); + } + + #[test] + pub fn filter_for_ignored_option() { + // When we run ignored tests the test filter should filter out all the + // unignored tests and flip the ignore flag on the rest to false + + let opts = TestOpts { + filter: None, + run_ignored: true, + logfile: None, + run_tests: true, + run_benchmarks: false, + ratchet_noise_percent: None, + ratchet_metrics: None, + save_metrics: None, + test_shard: None + }; + + let tests = ~[ + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("1"), + ignore: true, + should_fail: false, + }, + testfn: DynTestFn(proc() {}), + }, + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("2"), + ignore: false, + should_fail: false + }, + testfn: DynTestFn(proc() {}), + }, + ]; + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].desc.name.to_str(), ~"1"); + assert!(filtered[0].desc.ignore == false); + } + + #[test] + pub fn sort_tests() { + let opts = TestOpts { + filter: None, + run_ignored: false, + logfile: None, + run_tests: true, + run_benchmarks: false, + ratchet_noise_percent: None, + ratchet_metrics: None, + save_metrics: None, + test_shard: None + }; + + let names = + ~[~"sha1::test", ~"int::test_to_str", ~"int::test_pow", + ~"test::do_not_run_ignored_tests", + ~"test::ignored_tests_result_in_ignored", + ~"test::first_free_arg_should_be_a_filter", + ~"test::parse_ignored_flag", ~"test::filter_for_ignored_option", + ~"test::sort_tests"]; + let tests = + { + fn testfn() { } + let mut tests = ~[]; + for name in names.iter() { + let test = TestDescAndFn { + desc: TestDesc { + name: DynTestName((*name).clone()), + ignore: false, + should_fail: false + }, + testfn: DynTestFn(testfn), + }; + tests.push(test); + } + tests + }; + let filtered = filter_tests(&opts, tests); + + let expected = + ~[~"int::test_pow", ~"int::test_to_str", ~"sha1::test", + ~"test::do_not_run_ignored_tests", + ~"test::filter_for_ignored_option", + ~"test::first_free_arg_should_be_a_filter", + ~"test::ignored_tests_result_in_ignored", + ~"test::parse_ignored_flag", + ~"test::sort_tests"]; + + for (a, b) in expected.iter().zip(filtered.iter()) { + assert!(*a == b.desc.name.to_str()); + } + } + + #[test] + pub fn test_metricmap_compare() { + let mut m1 = MetricMap::new(); + let mut m2 = MetricMap::new(); + m1.insert_metric("in-both-noise", 1000.0, 200.0); + m2.insert_metric("in-both-noise", 1100.0, 200.0); + + m1.insert_metric("in-first-noise", 1000.0, 2.0); + m2.insert_metric("in-second-noise", 1000.0, 2.0); + + m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0); + m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0); + + m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0); + m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0); + + m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0); + m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0); + + m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0); + m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0); + + let diff1 = m2.compare_to_old(&m1, None); + + assert_eq!(*(diff1.find(&~"in-both-noise").unwrap()), LikelyNoise); + assert_eq!(*(diff1.find(&~"in-first-noise").unwrap()), MetricRemoved); + assert_eq!(*(diff1.find(&~"in-second-noise").unwrap()), MetricAdded); + assert_eq!(*(diff1.find(&~"in-both-want-downwards-but-regressed").unwrap()), + Regression(100.0)); + assert_eq!(*(diff1.find(&~"in-both-want-downwards-and-improved").unwrap()), + Improvement(50.0)); + assert_eq!(*(diff1.find(&~"in-both-want-upwards-but-regressed").unwrap()), + Regression(50.0)); + assert_eq!(*(diff1.find(&~"in-both-want-upwards-and-improved").unwrap()), + Improvement(100.0)); + assert_eq!(diff1.len(), 7); + + let diff2 = m2.compare_to_old(&m1, Some(200.0)); + + assert_eq!(*(diff2.find(&~"in-both-noise").unwrap()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-first-noise").unwrap()), MetricRemoved); + assert_eq!(*(diff2.find(&~"in-second-noise").unwrap()), MetricAdded); + assert_eq!(*(diff2.find(&~"in-both-want-downwards-but-regressed").unwrap()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-both-want-downwards-and-improved").unwrap()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-both-want-upwards-but-regressed").unwrap()), LikelyNoise); + assert_eq!(*(diff2.find(&~"in-both-want-upwards-and-improved").unwrap()), LikelyNoise); + assert_eq!(diff2.len(), 7); + } + + #[test] + pub fn ratchet_test() { + + let dpth = TempDir::new("test-ratchet").expect("missing test for ratchet"); + let pth = dpth.path().join("ratchet.json"); + + let mut m1 = MetricMap::new(); + m1.insert_metric("runtime", 1000.0, 2.0); + m1.insert_metric("throughput", 50.0, 2.0); + + let mut m2 = MetricMap::new(); + m2.insert_metric("runtime", 1100.0, 2.0); + m2.insert_metric("throughput", 50.0, 2.0); + + m1.save(&pth).unwrap(); + + // Ask for a ratchet that should fail to advance. + let (diff1, ok1) = m2.ratchet(&pth, None); + assert_eq!(ok1, false); + assert_eq!(diff1.len(), 2); + assert_eq!(*(diff1.find(&~"runtime").unwrap()), Regression(10.0)); + assert_eq!(*(diff1.find(&~"throughput").unwrap()), LikelyNoise); + + // Check that it was not rewritten. + let m3 = MetricMap::load(&pth); + let MetricMap(m3) = m3; + assert_eq!(m3.len(), 2); + assert_eq!(*(m3.find(&~"runtime").unwrap()), Metric::new(1000.0, 2.0)); + assert_eq!(*(m3.find(&~"throughput").unwrap()), Metric::new(50.0, 2.0)); + + // Ask for a ratchet with an explicit noise-percentage override, + // that should advance. + let (diff2, ok2) = m2.ratchet(&pth, Some(10.0)); + assert_eq!(ok2, true); + assert_eq!(diff2.len(), 2); + assert_eq!(*(diff2.find(&~"runtime").unwrap()), LikelyNoise); + assert_eq!(*(diff2.find(&~"throughput").unwrap()), LikelyNoise); + + // Check that it was rewritten. + let m4 = MetricMap::load(&pth); + let MetricMap(m4) = m4; + assert_eq!(m4.len(), 2); + assert_eq!(*(m4.find(&~"runtime").unwrap()), Metric::new(1100.0, 2.0)); + assert_eq!(*(m4.find(&~"throughput").unwrap()), Metric::new(50.0, 2.0)); + } +} + |