diff options
| author | Jacob Greenfield <xales@naveria.com> | 2019-04-07 01:48:59 -0400 |
|---|---|---|
| committer | Jacob Greenfield <xales@naveria.com> | 2019-04-07 03:16:31 -0400 |
| commit | 28ea249ab526c6b114c4dc9ba311fc62fccb28e2 (patch) | |
| tree | 2bcccba6a6b5fd002891ba57a8db91aab36cba77 /src/libtest | |
| parent | dec0a98c4b392b5fd153ba8b944c496218717813 (diff) | |
| download | rust-28ea249ab526c6b114c4dc9ba311fc62fccb28e2.tar.gz rust-28ea249ab526c6b114c4dc9ba311fc62fccb28e2.zip | |
Revert "Auto merge of #57842 - gnzlbg:extract_libtest, r=gnzlbg"
This reverts commit 3eb4890dfe6db0279fdd3cda19f9643873ae3db9, reversing changes made to 7a4df3b53da369110984a2b57419c05a53e33b38.
Diffstat (limited to 'src/libtest')
| -rw-r--r-- | src/libtest/Cargo.toml | 3 | ||||
| -rw-r--r-- | src/libtest/README.md | 13 | ||||
| -rw-r--r-- | src/libtest/formatters/json.rs | 208 | ||||
| -rw-r--r-- | src/libtest/formatters/mod.rs | 22 | ||||
| -rw-r--r-- | src/libtest/formatters/pretty.rs | 232 | ||||
| -rw-r--r-- | src/libtest/formatters/terse.rs | 235 | ||||
| -rw-r--r-- | src/libtest/lib.rs | 2228 | ||||
| -rw-r--r-- | src/libtest/stats.rs | 922 |
8 files changed, 3823 insertions, 40 deletions
diff --git a/src/libtest/Cargo.toml b/src/libtest/Cargo.toml index 26ac7888184..10bdd6e877c 100644 --- a/src/libtest/Cargo.toml +++ b/src/libtest/Cargo.toml @@ -10,7 +10,8 @@ path = "lib.rs" crate-type = ["dylib", "rlib"] [dependencies] -libtest = { version = "0.0.1" } +getopts = "0.2" +term = { path = "../libterm" } # not actually used but needed to always have proc_macro in the sysroot proc_macro = { path = "../libproc_macro" } diff --git a/src/libtest/README.md b/src/libtest/README.md deleted file mode 100644 index 6d9fe30dcad..00000000000 --- a/src/libtest/README.md +++ /dev/null @@ -1,13 +0,0 @@ -WIP - stable libtest -=== - -The migration of libtest to stable Rust is currently in progress. - -You can find libtest at: https://github.com/rust-lang/libtest . If you need to -make a change: - -* perform the change there, -* do a new crates.io release, and -* send a PR to rust-lang/rust bumping the libtest version. - -The roadmap of the migration is being tracked here: https://github.com/rust-lang/libtest/issues/2 diff --git a/src/libtest/formatters/json.rs b/src/libtest/formatters/json.rs new file mode 100644 index 00000000000..a06497f9862 --- /dev/null +++ b/src/libtest/formatters/json.rs @@ -0,0 +1,208 @@ +use super::*; + +pub(crate) struct JsonFormatter<T> { + out: OutputLocation<T>, +} + +impl<T: Write> JsonFormatter<T> { + pub fn new(out: OutputLocation<T>) -> Self { + Self { out } + } + + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref())?; + self.out.write_all(b"\n") + } + + fn write_event( + &mut self, + ty: &str, + name: &str, + evt: &str, + extra: Option<String>, + ) -> io::Result<()> { + if let Some(extras) = extra { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + ty, name, evt, extras + )) + } else { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + ty, name, evt + )) + } + } +} + +impl<T: Write> OutputFormatter for JsonFormatter<T> { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "suite", "event": "started", "test_count": {} }}"#, + test_count + )) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + desc.name + )) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()> { + match *result { + TrOk => self.write_event("test", desc.name.as_slice(), "ok", None), + + TrFailed => { + let extra_data = if stdout.len() > 0 { + Some(format!( + r#""stdout": "{}""#, + EscapedString(String::from_utf8_lossy(stdout)) + )) + } else { + None + }; + + self.write_event("test", desc.name.as_slice(), "failed", extra_data) + } + + TrFailedMsg(ref m) => self.write_event( + "test", + desc.name.as_slice(), + "failed", + Some(format!(r#""message": "{}""#, EscapedString(m))), + ), + + TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None), + + TrAllowedFail => { + self.write_event("test", desc.name.as_slice(), "allowed_failure", None) + } + + TrBench(ref bs) => { + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + let mbps = if bs.mb_s == 0 { + String::new() + } else { + format!(r#", "mib_per_second": {}"#, bs.mb_s) + }; + + let line = format!( + "{{ \"type\": \"bench\", \ + \"name\": \"{}\", \ + \"median\": {}, \ + \"deviation\": {}{} }}", + desc.name, median, deviation, mbps + ); + + self.write_message(&*line) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + desc.name + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> { + self.write_message(&*format!( + "{{ \"type\": \"suite\", \ + \"event\": \"{}\", \ + \"passed\": {}, \ + \"failed\": {}, \ + \"allowed_fail\": {}, \ + \"ignored\": {}, \ + \"measured\": {}, \ + \"filtered_out\": {} }}", + if state.failed == 0 { "ok" } else { "failed" }, + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ))?; + + Ok(state.failed == 0) + } +} + +/// A formatting utility used to print strings with characters in need of escaping. +/// Base code taken form `libserialize::json::escape_str` +struct EscapedString<S: AsRef<str>>(S); + +impl<S: AsRef<str>> ::std::fmt::Display for EscapedString<S> { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + let mut start = 0; + + for (i, byte) in self.0.as_ref().bytes().enumerate() { + let escaped = match byte { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\x00' => "\\u0000", + b'\x01' => "\\u0001", + b'\x02' => "\\u0002", + b'\x03' => "\\u0003", + b'\x04' => "\\u0004", + b'\x05' => "\\u0005", + b'\x06' => "\\u0006", + b'\x07' => "\\u0007", + b'\x08' => "\\b", + b'\t' => "\\t", + b'\n' => "\\n", + b'\x0b' => "\\u000b", + b'\x0c' => "\\f", + b'\r' => "\\r", + b'\x0e' => "\\u000e", + b'\x0f' => "\\u000f", + b'\x10' => "\\u0010", + b'\x11' => "\\u0011", + b'\x12' => "\\u0012", + b'\x13' => "\\u0013", + b'\x14' => "\\u0014", + b'\x15' => "\\u0015", + b'\x16' => "\\u0016", + b'\x17' => "\\u0017", + b'\x18' => "\\u0018", + b'\x19' => "\\u0019", + b'\x1a' => "\\u001a", + b'\x1b' => "\\u001b", + b'\x1c' => "\\u001c", + b'\x1d' => "\\u001d", + b'\x1e' => "\\u001e", + b'\x1f' => "\\u001f", + b'\x7f' => "\\u007f", + _ => { + continue; + } + }; + + if start < i { + f.write_str(&self.0.as_ref()[start..i])?; + } + + f.write_str(escaped)?; + + start = i + 1; + } + + if start != self.0.as_ref().len() { + f.write_str(&self.0.as_ref()[start..])?; + } + + Ok(()) + } +} diff --git a/src/libtest/formatters/mod.rs b/src/libtest/formatters/mod.rs new file mode 100644 index 00000000000..be5f6a65039 --- /dev/null +++ b/src/libtest/formatters/mod.rs @@ -0,0 +1,22 @@ +use super::*; + +mod pretty; +mod json; +mod terse; + +pub(crate) use self::pretty::PrettyFormatter; +pub(crate) use self::json::JsonFormatter; +pub(crate) use self::terse::TerseFormatter; + +pub(crate) trait OutputFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()>; + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool>; +} diff --git a/src/libtest/formatters/pretty.rs b/src/libtest/formatters/pretty.rs new file mode 100644 index 00000000000..4af00428ca8 --- /dev/null +++ b/src/libtest/formatters/pretty.rs @@ -0,0 +1,232 @@ +use super::*; + +pub(crate) struct PrettyFormatter<T> { + out: OutputLocation<T>, + use_color: bool, + + /// Number of columns to fill when aligning names + max_name_len: usize, + + is_multithreaded: bool, +} + +impl<T: Write> PrettyFormatter<T> { + pub fn new( + out: OutputLocation<T>, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + PrettyFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + } + } + + #[cfg(test)] + pub fn output_location(&self) -> &OutputLocation<T> { + &self.out + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result("ok", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("FAILED", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("ignored", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("FAILED (allowed)", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + self.write_plain("\n") + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_successes(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + + Ok(()) + } +} + +impl<T: Write> OutputFormatter for PrettyFormatter<T> { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // When running tests concurrently, we should not print + // the test's name as the result will be mis-aligned. + // When running the tests serially, we print the name here so + // that the user can see which test hangs. + if !self.is_multithreaded { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> { + if state.options.display_output { + self.write_successes(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, state.failed, state.ignored, state.measured, state.filtered_out + ) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/formatters/terse.rs b/src/libtest/formatters/terse.rs new file mode 100644 index 00000000000..1400fba5d60 --- /dev/null +++ b/src/libtest/formatters/terse.rs @@ -0,0 +1,235 @@ +use super::*; + +pub(crate) struct TerseFormatter<T> { + out: OutputLocation<T>, + use_color: bool, + is_multithreaded: bool, + /// Number of columns to fill when aligning names + max_name_len: usize, + + test_count: usize, + total_test_count: usize, +} + +impl<T: Write> TerseFormatter<T> { + pub fn new( + out: OutputLocation<T>, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + TerseFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + test_count: 0, + total_test_count: 0, // initialized later, when write_run_start is called + } + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result(".", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("F", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("i", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("a", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + // we insert a new line every 100 dots in order to flush the + // screen when dealing with line-buffered output (e.g., piping to + // `stamp` in the rust CI). + let out = format!(" {}/{}\n", self.test_count+1, self.total_test_count); + self.write_plain(&out)?; + } + + self.test_count += 1; + Ok(()) + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain<S: AsRef<str>>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + + Ok(()) + } +} + +impl<T: Write> OutputFormatter for TerseFormatter<T> { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + self.total_test_count = test_count; + let noun = if test_count != 1 { "tests" } else { "test" }; + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // Remnants from old libtest code that used the padding value + // in order to indicate benchmarks. + // When running benchmarks, terse-mode should still print their name as if + // it is the Pretty formatter. + if !self.is_multithreaded && desc.name.padding() == PadOnRight { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> { + if state.options.display_output { + self.write_outputs(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, state.failed, state.ignored, state.measured, state.filtered_out + ) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 5c91c0ec43b..26612964c30 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -8,48 +8,2224 @@ //! //! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details. +// Currently, not much of this is meant for users. It is intended to +// support the simplest interface possible for representing and +// running tests while providing a base that other test frameworks may +// build off of. + +// N.B., this is also specified in this crate's Cargo.toml, but libsyntax contains logic specific to +// this crate, which relies on this attribute (rather than the value of `--crate-name` passed by +// cargo) to detect this crate. + +#![deny(rust_2018_idioms)] #![crate_name = "test"] #![unstable(feature = "test", issue = "27812")] -#![doc(html_root_url = "https://doc.rust-lang.org/nightly/", - test(attr(deny(warnings))))] +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))] #![feature(asm)] +#![feature(fnbox)] +#![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc, rustc_private))] +#![feature(nll)] +#![feature(set_stdio)] +#![feature(panic_unwind)] #![feature(staged_api)] +#![feature(termination_trait_lib)] #![feature(test)] -extern crate libtest; +use getopts; +#[cfg(any(unix, target_os = "cloudabi"))] +extern crate libc; +use term; + +// FIXME(#54291): rustc and/or LLVM don't yet support building with panic-unwind +// on aarch64-pc-windows-msvc, so we don't link libtest against +// libunwind (for the time being), even though it means that +// libtest won't be fully functional on this platform. +// +// See also: https://github.com/rust-lang/rust/issues/54190#issuecomment-422904437 +#[cfg(not(all(windows, target_arch = "aarch64")))] +extern crate panic_unwind; + +pub use self::ColorConfig::*; +use self::NamePadding::*; +use self::OutputLocation::*; +use self::TestEvent::*; +pub use self::TestFn::*; +pub use self::TestName::*; +pub use self::TestResult::*; + +use std::any::Any; +use std::borrow::Cow; +use std::boxed::FnBox; +use std::cmp; +use std::collections::BTreeMap; +use std::env; +use std::fmt; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::path::PathBuf; +use std::process; +use std::process::Termination; +use std::sync::mpsc::{channel, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +const TEST_WARN_TIMEOUT_S: u64 = 60; +const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode + +// to be used by rustc to compile tests in libtest +pub mod test { + pub use crate::{ + assert_test_result, filter_tests, parse_opts, run_test, test_main, test_main_static, + Bencher, DynTestFn, DynTestName, Metric, MetricMap, Options, RunIgnored, ShouldPanic, + StaticBenchFn, StaticTestFn, StaticTestName, TestDesc, TestDescAndFn, TestName, TestOpts, + TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk, + }; +} + +mod formatters; +pub mod stats; + +use crate::formatters::{JsonFormatter, OutputFormatter, PrettyFormatter, TerseFormatter}; + +/// Whether to execute tests concurrently or not +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Concurrent { + Yes, + No, +} + +// The name of a test. By convention this follows the rules for rust +// paths; i.e., it should be a series of identifiers separated by double +// colons. This way if some test runner wants to arrange the tests +// hierarchically it may. + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub enum TestName { + StaticTestName(&'static str), + DynTestName(String), + AlignedTestName(Cow<'static, str>, NamePadding), +} +impl TestName { + fn as_slice(&self) -> &str { + match *self { + StaticTestName(s) => s, + DynTestName(ref s) => s, + AlignedTestName(ref s, _) => &*s, + } + } + + fn padding(&self) -> NamePadding { + match self { + &AlignedTestName(_, p) => p, + _ => PadNone, + } + } + + fn with_padding(&self, padding: NamePadding) -> TestName { + let name = match self { + &TestName::StaticTestName(name) => Cow::Borrowed(name), + &TestName::DynTestName(ref name) => Cow::Owned(name.clone()), + &TestName::AlignedTestName(ref name, _) => name.clone(), + }; + + TestName::AlignedTestName(name, padding) + } +} +impl fmt::Display for TestName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.as_slice(), f) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum NamePadding { + PadNone, + PadOnRight, +} + +impl TestDesc { + fn padded_name(&self, column_count: usize, align: NamePadding) -> String { + let mut name = String::from(self.name.as_slice()); + let fill = column_count.saturating_sub(name.len()); + let pad = " ".repeat(fill); + match align { + PadNone => name, + PadOnRight => { + name.push_str(&pad); + name + } + } + } +} + +/// Represents a benchmark function. +pub trait TDynBenchFn: Send { + fn run(&self, harness: &mut Bencher); +} + +// A function that runs a test. If the function returns successfully, +// the test succeeds; if the function panics then the test fails. We +// may need to come up with a more clever definition of test in order +// to support isolation of tests into threads. +pub enum TestFn { + StaticTestFn(fn()), + StaticBenchFn(fn(&mut Bencher)), + DynTestFn(Box<dyn FnBox() + Send>), + DynBenchFn(Box<dyn TDynBenchFn + 'static>), +} + +impl TestFn { + fn padding(&self) -> NamePadding { + match *self { + StaticTestFn(..) => PadNone, + StaticBenchFn(..) => PadOnRight, + DynTestFn(..) => PadNone, + DynBenchFn(..) => PadOnRight, + } + } +} + +impl fmt::Debug for TestFn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match *self { + StaticTestFn(..) => "StaticTestFn(..)", + StaticBenchFn(..) => "StaticBenchFn(..)", + DynTestFn(..) => "DynTestFn(..)", + DynBenchFn(..) => "DynBenchFn(..)", + }) + } +} + +/// Manager of the benchmarking runs. +/// +/// This is fed into functions marked with `#[bench]` to allow for +/// set-up & tear-down before running a piece of code repeatedly via a +/// call to `iter`. +#[derive(Clone)] +pub struct Bencher { + mode: BenchMode, + summary: Option<stats::Summary>, + pub bytes: u64, +} + +#[derive(Clone, PartialEq, Eq)] +pub enum BenchMode { + Auto, + Single, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum ShouldPanic { + No, + Yes, + YesWithMessage(&'static str), +} + +// The definition of a single test. A test runner will run a list of +// these. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct TestDesc { + pub name: TestName, + pub ignore: bool, + pub should_panic: ShouldPanic, + pub allow_fail: bool, +} + +#[derive(Debug)] +pub struct TestDescAndFn { + pub desc: TestDesc, + pub testfn: TestFn, +} + +#[derive(Clone, PartialEq, Debug, Copy)] +pub struct Metric { + value: f64, + noise: f64, +} + +impl Metric { + pub fn new(value: f64, noise: f64) -> Metric { + Metric { value, noise } + } +} + +/// In case we want to add other options as well, just add them in this struct. +#[derive(Copy, Clone, Debug)] +pub struct Options { + display_output: bool, +} + +impl Options { + pub fn new() -> Options { + Options { + display_output: false, + } + } + + pub fn display_output(mut self, display_output: bool) -> Options { + self.display_output = display_output; + self + } +} + +// The default console test runner. It accepts the command line +// arguments and a vector of test_descs. +pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Options) { + let mut opts = match parse_opts(args) { + Some(Ok(o)) => o, + Some(Err(msg)) => { + eprintln!("error: {}", msg); + process::exit(101); + } + None => return, + }; + + opts.options = options; + if opts.list { + if let Err(e) = list_tests_console(&opts, tests) { + eprintln!("error: io error when listing tests: {:?}", e); + process::exit(101); + } + } else { + match run_tests_console(&opts, tests) { + Ok(true) => {} + Ok(false) => process::exit(101), + Err(e) => { + eprintln!("error: io error when listing tests: {:?}", e); + process::exit(101); + } + } + } +} + +// A variant optimized for invocation with a static test vector. +// This will panic (intentionally) when fed any dynamic tests, because +// it is copying the static values out into a dynamic vector and cannot +// copy dynamic values. It is doing this because from this point on +// a Vec<TestDescAndFn> is used in order to effect ownership-transfer +// semantics into parallel test runners, which in turn requires a Vec<> +// rather than a &[]. +pub fn test_main_static(tests: &[&TestDescAndFn]) { + let args = env::args().collect::<Vec<_>>(); + let owned_tests = tests + .iter() + .map(|t| match t.testfn { + StaticTestFn(f) => TestDescAndFn { + testfn: StaticTestFn(f), + desc: t.desc.clone(), + }, + StaticBenchFn(f) => TestDescAndFn { + testfn: StaticBenchFn(f), + desc: t.desc.clone(), + }, + _ => panic!("non-static tests passed to test::test_main_static"), + }) + .collect(); + test_main(&args, owned_tests, Options::new()) +} + +/// Invoked when unit tests terminate. Should panic if the unit +/// Tests is considered a failure. By default, invokes `report()` +/// and checks for a `0` result. +pub fn assert_test_result<T: Termination>(result: T) { + let code = result.report(); + assert_eq!( + code, 0, + "the test returned a termination value with a non-zero status code ({}) \ + which indicates a failure", + code + ); +} + +#[derive(Copy, Clone, Debug)] +pub enum ColorConfig { + AutoColor, + AlwaysColor, + NeverColor, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum OutputFormat { + Pretty, + Terse, + Json, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RunIgnored { + Yes, + No, + Only, +} + +#[derive(Debug)] +pub struct TestOpts { + pub list: bool, + pub filter: Option<String>, + pub filter_exact: bool, + pub exclude_should_panic: bool, + pub run_ignored: RunIgnored, + pub run_tests: bool, + pub bench_benchmarks: bool, + pub logfile: Option<PathBuf>, + pub nocapture: bool, + pub color: ColorConfig, + pub format: OutputFormat, + pub test_threads: Option<usize>, + pub skip: Vec<String>, + pub options: Options, +} + +impl TestOpts { + #[cfg(test)] + fn new() -> TestOpts { + TestOpts { + list: false, + filter: None, + filter_exact: false, + exclude_should_panic: false, + run_ignored: RunIgnored::No, + run_tests: false, + bench_benchmarks: false, + logfile: None, + nocapture: false, + color: AutoColor, + format: OutputFormat::Pretty, + test_threads: None, + skip: vec![], + options: Options::new(), + } + } +} + +/// Result of parsing the options. +pub type OptRes = Result<TestOpts, String>; + +fn optgroups() -> getopts::Options { + let mut opts = getopts::Options::new(); + opts.optflag("", "include-ignored", "Run ignored and not ignored tests") + .optflag("", "ignored", "Run only ignored tests") + .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic") + .optflag("", "test", "Run tests and not benchmarks") + .optflag("", "bench", "Run benchmarks instead of tests") + .optflag("", "list", "List all tests and benchmarks") + .optflag("h", "help", "Display this message (longer with --help)") + .optopt( + "", + "logfile", + "Write logs to the specified file instead \ + of stdout", + "PATH", + ) + .optflag( + "", + "nocapture", + "don't capture stdout/stderr of each \ + task, allow printing directly", + ) + .optopt( + "", + "test-threads", + "Number of threads used for running tests \ + in parallel", + "n_threads", + ) + .optmulti( + "", + "skip", + "Skip tests whose names contain FILTER (this flag can \ + be used multiple times)", + "FILTER", + ) + .optflag( + "q", + "quiet", + "Display one character per test instead of one line. \ + Alias to --format=terse", + ) + .optflag( + "", + "exact", + "Exactly match filters rather than by substring", + ) + .optopt( + "", + "color", + "Configure coloring of output: + auto = colorize if stdout is a tty and tests are run on serially (default); + always = always colorize output; + never = never colorize output;", + "auto|always|never", + ) + .optopt( + "", + "format", + "Configure formatting of output: + pretty = Print verbose output; + terse = Display one character per test; + json = Output a json document", + "pretty|terse|json", + ) + .optopt( + "Z", + "", + "Enable nightly-only flags: + unstable-options = Allow use of experimental features", + "unstable-options", + ); + return opts; +} + +fn usage(binary: &str, options: &getopts::Options) { + let message = format!("Usage: {} [OPTIONS] [FILTER]", binary); + println!( + r#"{usage} + +The FILTER string is tested against the name of all tests, and only those +tests whose names contain the filter are run. + +By default, all tests are run in parallel. This can be altered with the +--test-threads flag or the RUST_TEST_THREADS environment variable when running +tests (set it to 1). + +All tests have their standard output and standard error captured by default. +This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE +environment variable to a value other than "0". Logging is not captured by default. + +Test Attributes: + + #[test] - Indicates a function is a test to be run. This function + takes no arguments. + #[bench] - Indicates a function is a benchmark to be run. This + function takes one argument (test::Bencher). + #[should_panic] - This function (also labeled with #[test]) will only pass if + the code causes a panic (an assertion failure or panic!) + A message may be provided, which the failure string must + contain: #[should_panic(expected = "foo")]. + #[ignore] - When applied to a function which is already attributed as a + test, then the test runner will ignore these tests during + normal test runs. Running with --ignored or --include-ignored will run + these tests."#, + usage = options.usage(&message) + ); +} + +// FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566 +fn is_nightly() -> bool { + // Whether this is a feature-staged build, i.e., on the beta or stable channel + let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some(); + // Whether we should enable unstable features for bootstrapping + let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok(); + + bootstrap || !disable_unstable_features +} + +// Parses command line arguments into test options +pub fn parse_opts(args: &[String]) -> Option<OptRes> { + let mut allow_unstable = false; + let opts = optgroups(); + let args = args.get(1..).unwrap_or(args); + let matches = match opts.parse(args) { + Ok(m) => m, + Err(f) => return Some(Err(f.to_string())), + }; + + if let Some(opt) = matches.opt_str("Z") { + if !is_nightly() { + return Some(Err( + "the option `Z` is only accepted on the nightly compiler".into(), + )); + } + + match &*opt { + "unstable-options" => { + allow_unstable = true; + } + _ => { + return Some(Err("Unrecognized option to `Z`".into())); + } + } + }; + + if matches.opt_present("h") { + usage(&args[0], &opts); + return None; + } + + let filter = if !matches.free.is_empty() { + Some(matches.free[0].clone()) + } else { + None + }; + + let exclude_should_panic = matches.opt_present("exclude-should-panic"); + if !allow_unstable && exclude_should_panic { + return Some(Err( + "The \"exclude-should-panic\" flag is only accepted on the nightly compiler".into(), + )); + } + + let include_ignored = matches.opt_present("include-ignored"); + if !allow_unstable && include_ignored { + return Some(Err( + "The \"include-ignored\" flag is only accepted on the nightly compiler".into(), + )); + } + + let run_ignored = match (include_ignored, matches.opt_present("ignored")) { + (true, true) => { + return Some(Err( + "the options --include-ignored and --ignored are mutually exclusive".into(), + )); + } + (true, false) => RunIgnored::Yes, + (false, true) => RunIgnored::Only, + (false, false) => RunIgnored::No, + }; + let quiet = matches.opt_present("quiet"); + let exact = matches.opt_present("exact"); + let list = matches.opt_present("list"); + + let logfile = matches.opt_str("logfile"); + let logfile = logfile.map(|s| PathBuf::from(&s)); + + let bench_benchmarks = matches.opt_present("bench"); + let run_tests = !bench_benchmarks || matches.opt_present("test"); + + let mut nocapture = matches.opt_present("nocapture"); + if !nocapture { + nocapture = match env::var("RUST_TEST_NOCAPTURE") { + Ok(val) => &val != "0", + Err(_) => false, + }; + } + + let test_threads = match matches.opt_str("test-threads") { + Some(n_str) => match n_str.parse::<usize>() { + Ok(0) => return Some(Err("argument for --test-threads must not be 0".to_string())), + Ok(n) => Some(n), + Err(e) => { + return Some(Err(format!( + "argument for --test-threads must be a number > 0 \ + (error: {})", + e + ))); + } + }, + None => None, + }; + + let color = match matches.opt_str("color").as_ref().map(|s| &**s) { + Some("auto") | None => AutoColor, + Some("always") => AlwaysColor, + Some("never") => NeverColor, + + Some(v) => { + return Some(Err(format!( + "argument for --color must be auto, always, or never (was \ + {})", + v + ))); + } + }; + + let format = match matches.opt_str("format").as_ref().map(|s| &**s) { + None if quiet => OutputFormat::Terse, + Some("pretty") | None => OutputFormat::Pretty, + Some("terse") => OutputFormat::Terse, + Some("json") => { + if !allow_unstable { + return Some(Err( + "The \"json\" format is only accepted on the nightly compiler".into(), + )); + } + OutputFormat::Json + } + + Some(v) => { + return Some(Err(format!( + "argument for --format must be pretty, terse, or json (was \ + {})", + v + ))); + } + }; + + let test_opts = TestOpts { + list, + filter, + filter_exact: exact, + exclude_should_panic, + run_ignored, + run_tests, + bench_benchmarks, + logfile, + nocapture, + color, + format, + test_threads, + skip: matches.opt_strs("skip"), + options: Options::new(), + }; + + Some(Ok(test_opts)) +} + +#[derive(Clone, PartialEq)] +pub struct BenchSamples { + ns_iter_summ: stats::Summary, + mb_s: usize, +} + +#[derive(Clone, PartialEq)] +pub enum TestResult { + TrOk, + TrFailed, + TrFailedMsg(String), + TrIgnored, + TrAllowedFail, + TrBench(BenchSamples), +} + +unsafe impl Send for TestResult {} + +enum OutputLocation<T> { + Pretty(Box<term::StdoutTerminal>), + Raw(T), +} + +impl<T: Write> Write for OutputLocation<T> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + match *self { + Pretty(ref mut term) => term.write(buf), + Raw(ref mut stdout) => stdout.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match *self { + Pretty(ref mut term) => term.flush(), + Raw(ref mut stdout) => stdout.flush(), + } + } +} + +struct ConsoleTestState { + log_out: Option<File>, + total: usize, + passed: usize, + failed: usize, + ignored: usize, + allowed_fail: usize, + filtered_out: usize, + measured: usize, + metrics: MetricMap, + failures: Vec<(TestDesc, Vec<u8>)>, + not_failures: Vec<(TestDesc, Vec<u8>)>, + options: Options, +} + +impl ConsoleTestState { + pub fn new(opts: &TestOpts) -> io::Result<ConsoleTestState> { + let log_out = match opts.logfile { + Some(ref path) => Some(File::create(path)?), + None => None, + }; + + Ok(ConsoleTestState { + log_out, + total: 0, + passed: 0, + failed: 0, + ignored: 0, + allowed_fail: 0, + filtered_out: 0, + measured: 0, + metrics: MetricMap::new(), + failures: Vec::new(), + not_failures: Vec::new(), + options: opts.options, + }) + } + + pub fn write_log<S: AsRef<str>>(&mut self, msg: S) -> io::Result<()> { + let msg = msg.as_ref(); + match self.log_out { + None => Ok(()), + Some(ref mut o) => o.write_all(msg.as_bytes()), + } + } + + pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> { + self.write_log(format!( + "{} {}\n", + match *result { + TrOk => "ok".to_owned(), + TrFailed => "failed".to_owned(), + TrFailedMsg(ref msg) => format!("failed: {}", msg), + TrIgnored => "ignored".to_owned(), + TrAllowedFail => "failed (allowed)".to_owned(), + TrBench(ref bs) => fmt_bench_samples(bs), + }, + test.name + )) + } + + fn current_test_count(&self) -> usize { + self.passed + self.failed + self.ignored + self.measured + self.allowed_fail + } +} + +// Format a number with thousands separators +fn fmt_thousands_sep(mut n: usize, sep: char) -> String { + use std::fmt::Write; + let mut output = String::new(); + let mut trailing = false; + for &pow in &[9, 6, 3, 0] { + let base = 10_usize.pow(pow); + if pow == 0 || trailing || n / base != 0 { + if !trailing { + output.write_fmt(format_args!("{}", n / base)).unwrap(); + } else { + output.write_fmt(format_args!("{:03}", n / base)).unwrap(); + } + if pow != 0 { + output.push(sep); + } + trailing = true; + } + n %= base; + } + + output +} + +pub fn fmt_bench_samples(bs: &BenchSamples) -> String { + use std::fmt::Write; + let mut output = String::new(); + + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + output + .write_fmt(format_args!( + "{:>11} ns/iter (+/- {})", + fmt_thousands_sep(median, ','), + fmt_thousands_sep(deviation, ',') + )) + .unwrap(); + if bs.mb_s != 0 { + output + .write_fmt(format_args!(" = {} MB/s", bs.mb_s)) + .unwrap(); + } + output +} + +// List the tests to console, and optionally to logfile. Filters are honored. +pub fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> { + let mut output = match term::stdout() { + None => Raw(io::stdout()), + Some(t) => Pretty(t), + }; + + let quiet = opts.format == OutputFormat::Terse; + let mut st = ConsoleTestState::new(opts)?; + + let mut ntest = 0; + let mut nbench = 0; + + for test in filter_tests(&opts, tests) { + use crate::TestFn::*; + + let TestDescAndFn { + desc: TestDesc { name, .. }, + testfn, + } = test; + + let fntype = match testfn { + StaticTestFn(..) | DynTestFn(..) => { + ntest += 1; + "test" + } + StaticBenchFn(..) | DynBenchFn(..) => { + nbench += 1; + "benchmark" + } + }; + + writeln!(output, "{}: {}", name, fntype)?; + st.write_log(format!("{} {}\n", fntype, name))?; + } + + fn plural(count: u32, s: &str) -> String { + match count { + 1 => format!("{} {}", 1, s), + n => format!("{} {}s", n, s), + } + } + + if !quiet { + if ntest != 0 || nbench != 0 { + writeln!(output, "")?; + } + + writeln!( + output, + "{}, {}", + plural(ntest, "test"), + plural(nbench, "benchmark") + )?; + } + + Ok(()) +} + +// A simple console test runner +pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> { + fn callback( + event: &TestEvent, + st: &mut ConsoleTestState, + out: &mut dyn OutputFormatter, + ) -> io::Result<()> { + match (*event).clone() { + TeFiltered(ref filtered_tests) => { + st.total = filtered_tests.len(); + out.write_run_start(filtered_tests.len()) + } + TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), + TeWait(ref test) => out.write_test_start(test), + TeTimeout(ref test) => out.write_timeout(test), + TeResult(test, result, stdout) => { + st.write_log_result(&test, &result)?; + out.write_result(&test, &result, &*stdout)?; + match result { + TrOk => { + st.passed += 1; + st.not_failures.push((test, stdout)); + } + TrIgnored => st.ignored += 1, + TrAllowedFail => st.allowed_fail += 1, + TrBench(bs) => { + st.metrics.insert_metric( + test.name.as_slice(), + bs.ns_iter_summ.median, + bs.ns_iter_summ.max - bs.ns_iter_summ.min, + ); + st.measured += 1 + } + TrFailed => { + st.failed += 1; + st.failures.push((test, stdout)); + } + TrFailedMsg(msg) => { + st.failed += 1; + let mut stdout = stdout; + stdout.extend_from_slice(format!("note: {}", msg).as_bytes()); + st.failures.push((test, stdout)); + } + } + Ok(()) + } + } + } + + let output = match term::stdout() { + None => Raw(io::stdout()), + Some(t) => Pretty(t), + }; + + let max_name_len = tests + .iter() + .max_by_key(|t| len_if_padded(*t)) + .map(|t| t.desc.name.as_slice().len()) + .unwrap_or(0); + + let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1; + + let mut out: Box<dyn OutputFormatter> = match opts.format { + OutputFormat::Pretty => Box::new(PrettyFormatter::new( + output, + use_color(opts), + max_name_len, + is_multithreaded, + )), + OutputFormat::Terse => Box::new(TerseFormatter::new( + output, + use_color(opts), + max_name_len, + is_multithreaded, + )), + OutputFormat::Json => Box::new(JsonFormatter::new(output)), + }; + let mut st = ConsoleTestState::new(opts)?; + fn len_if_padded(t: &TestDescAndFn) -> usize { + match t.testfn.padding() { + PadNone => 0, + PadOnRight => t.desc.name.as_slice().len(), + } + } + + run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?; + + assert!(st.current_test_count() == st.total); + + return out.write_run_finish(&st); +} + +#[test] +fn should_sort_failures_before_printing_them() { + let test_a = TestDesc { + name: StaticTestName("a"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + let test_b = TestDesc { + name: StaticTestName("b"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + let mut out = PrettyFormatter::new(Raw(Vec::new()), false, 10, false); + + let st = ConsoleTestState { + log_out: None, + total: 0, + passed: 0, + failed: 0, + ignored: 0, + allowed_fail: 0, + filtered_out: 0, + measured: 0, + metrics: MetricMap::new(), + failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], + options: Options::new(), + not_failures: Vec::new(), + }; + + out.write_failures(&st).unwrap(); + let s = match out.output_location() { + &Raw(ref m) => String::from_utf8_lossy(&m[..]), + &Pretty(_) => unreachable!(), + }; + + let apos = s.find("a").unwrap(); + let bpos = s.find("b").unwrap(); + assert!(apos < bpos); +} + +fn use_color(opts: &TestOpts) -> bool { + match opts.color { + AutoColor => !opts.nocapture && stdout_isatty(), + AlwaysColor => true, + NeverColor => false, + } +} + +#[cfg(any( + target_os = "cloudabi", + target_os = "redox", + all(target_arch = "wasm32", not(target_os = "emscripten")), + all(target_vendor = "fortanix", target_env = "sgx") +))] +fn stdout_isatty() -> bool { + // FIXME: Implement isatty on Redox and SGX + false +} +#[cfg(unix)] +fn stdout_isatty() -> bool { + unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 } +} +#[cfg(windows)] +fn stdout_isatty() -> bool { + type DWORD = u32; + type BOOL = i32; + type HANDLE = *mut u8; + type LPDWORD = *mut u32; + const STD_OUTPUT_HANDLE: DWORD = -11i32 as DWORD; + extern "system" { + fn GetStdHandle(which: DWORD) -> HANDLE; + fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: LPDWORD) -> BOOL; + } + unsafe { + let handle = GetStdHandle(STD_OUTPUT_HANDLE); + let mut out = 0; + GetConsoleMode(handle, &mut out) != 0 + } +} + +#[derive(Clone)] +pub enum TestEvent { + TeFiltered(Vec<TestDesc>), + TeWait(TestDesc), + TeResult(TestDesc, TestResult, Vec<u8>), + TeTimeout(TestDesc), + TeFilteredOut(usize), +} + +pub type MonitorMsg = (TestDesc, TestResult, Vec<u8>); -// FIXME: we should be more explicit about the exact APIs that we -// export to users. -pub use libtest::{ - assert_test_result, filter_tests, parse_opts, run_test, test_main, test_main_static, - Bencher, DynTestFn, DynTestName, Metric, MetricMap, Options, RunIgnored, ShouldPanic, - StaticBenchFn, StaticTestFn, StaticTestName, TestDesc, TestDescAndFn, TestName, TestOpts, - TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk, stats::Summary -}; +struct Sink(Arc<Mutex<Vec<u8>>>); +impl Write for Sink { + fn write(&mut self, data: &[u8]) -> io::Result<usize> { + Write::write(&mut *self.0.lock().unwrap(), data) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()> +where + F: FnMut(TestEvent) -> io::Result<()>, +{ + use std::collections::{self, HashMap}; + use std::hash::BuildHasherDefault; + use std::sync::mpsc::RecvTimeoutError; + // Use a deterministic hasher + type TestMap = + HashMap<TestDesc, Instant, BuildHasherDefault<collections::hash_map::DefaultHasher>>; + + let tests_len = tests.len(); + + let mut filtered_tests = filter_tests(opts, tests); + if !opts.bench_benchmarks { + filtered_tests = convert_benchmarks_to_tests(filtered_tests); + } + + let filtered_tests = { + let mut filtered_tests = filtered_tests; + for test in filtered_tests.iter_mut() { + test.desc.name = test.desc.name.with_padding(test.testfn.padding()); + } + + filtered_tests + }; + + let filtered_out = tests_len - filtered_tests.len(); + callback(TeFilteredOut(filtered_out))?; + + let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect(); + + callback(TeFiltered(filtered_descs))?; + + let (filtered_tests, filtered_benchs): (Vec<_>, _) = + filtered_tests.into_iter().partition(|e| match e.testfn { + StaticTestFn(_) | DynTestFn(_) => true, + _ => false, + }); + + let concurrency = opts.test_threads.unwrap_or_else(get_concurrency); + + let mut remaining = filtered_tests; + remaining.reverse(); + let mut pending = 0; + + let (tx, rx) = channel::<MonitorMsg>(); + + let mut running_tests: TestMap = HashMap::default(); + + fn get_timed_out_tests(running_tests: &mut TestMap) -> Vec<TestDesc> { + let now = Instant::now(); + let timed_out = running_tests + .iter() + .filter_map(|(desc, timeout)| { + if &now >= timeout { + Some(desc.clone()) + } else { + None + } + }) + .collect(); + for test in &timed_out { + running_tests.remove(test); + } + timed_out + }; + + fn calc_timeout(running_tests: &TestMap) -> Option<Duration> { + running_tests.values().min().map(|next_timeout| { + let now = Instant::now(); + if *next_timeout >= now { + *next_timeout - now + } else { + Duration::new(0, 0) + } + }) + }; + + if concurrency == 1 { + while !remaining.is_empty() { + let test = remaining.pop().unwrap(); + callback(TeWait(test.desc.clone()))?; + run_test(opts, !opts.run_tests, test, tx.clone(), Concurrent::No); + let (test, result, stdout) = rx.recv().unwrap(); + callback(TeResult(test, result, stdout))?; + } + } else { + while pending > 0 || !remaining.is_empty() { + while pending < concurrency && !remaining.is_empty() { + let test = remaining.pop().unwrap(); + let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); + running_tests.insert(test.desc.clone(), timeout); + callback(TeWait(test.desc.clone()))?; //here no pad + run_test(opts, !opts.run_tests, test, tx.clone(), Concurrent::Yes); + pending += 1; + } + + let mut res; + loop { + if let Some(timeout) = calc_timeout(&running_tests) { + res = rx.recv_timeout(timeout); + for test in get_timed_out_tests(&mut running_tests) { + callback(TeTimeout(test))?; + } + if res != Err(RecvTimeoutError::Timeout) { + break; + } + } else { + res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected); + break; + } + } + + let (desc, result, stdout) = res.unwrap(); + running_tests.remove(&desc); + + callback(TeResult(desc, result, stdout))?; + pending -= 1; + } + } + + if opts.bench_benchmarks { + // All benchmarks run at the end, in serial. + for b in filtered_benchs { + callback(TeWait(b.desc.clone()))?; + run_test(opts, false, b, tx.clone(), Concurrent::No); + let (test, result, stdout) = rx.recv().unwrap(); + callback(TeResult(test, result, stdout))?; + } + } + Ok(()) +} + +#[allow(deprecated)] +fn get_concurrency() -> usize { + return match env::var("RUST_TEST_THREADS") { + Ok(s) => { + let opt_n: Option<usize> = s.parse().ok(); + match opt_n { + Some(n) if n > 0 => n, + _ => panic!( + "RUST_TEST_THREADS is `{}`, should be a positive integer.", + s + ), + } + } + Err(..) => num_cpus(), + }; + + #[cfg(windows)] + #[allow(nonstandard_style)] + fn num_cpus() -> usize { + #[repr(C)] + struct SYSTEM_INFO { + wProcessorArchitecture: u16, + wReserved: u16, + dwPageSize: u32, + lpMinimumApplicationAddress: *mut u8, + lpMaximumApplicationAddress: *mut u8, + dwActiveProcessorMask: *mut u8, + dwNumberOfProcessors: u32, + dwProcessorType: u32, + dwAllocationGranularity: u32, + wProcessorLevel: u16, + wProcessorRevision: u16, + } + extern "system" { + fn GetSystemInfo(info: *mut SYSTEM_INFO) -> i32; + } + unsafe { + let mut sysinfo = std::mem::zeroed(); + GetSystemInfo(&mut sysinfo); + sysinfo.dwNumberOfProcessors as usize + } + } + + #[cfg(target_os = "redox")] + fn num_cpus() -> usize { + // FIXME: Implement num_cpus on Redox + 1 + } + + #[cfg(any( + all(target_arch = "wasm32", not(target_os = "emscripten")), + all(target_vendor = "fortanix", target_env = "sgx") + ))] + fn num_cpus() -> usize { + 1 + } + + #[cfg(any( + target_os = "android", + target_os = "cloudabi", + target_os = "emscripten", + target_os = "fuchsia", + target_os = "ios", + target_os = "linux", + target_os = "macos", + target_os = "solaris" + ))] + fn num_cpus() -> usize { + unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize } + } + + #[cfg(any( + target_os = "freebsd", + target_os = "dragonfly", + target_os = "bitrig", + target_os = "netbsd" + ))] + fn num_cpus() -> usize { + use std::ptr; + + let mut cpus: libc::c_uint = 0; + let mut cpus_size = std::mem::size_of_val(&cpus); + + unsafe { + cpus = libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as libc::c_uint; + } + if cpus < 1 { + let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; + unsafe { + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ); + } + if cpus < 1 { + cpus = 1; + } + } + cpus as usize + } + + #[cfg(target_os = "openbsd")] + fn num_cpus() -> usize { + use std::ptr; + + let mut cpus: libc::c_uint = 0; + let mut cpus_size = std::mem::size_of_val(&cpus); + let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; + + unsafe { + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ); + } + if cpus < 1 { + cpus = 1; + } + cpus as usize + } + + #[cfg(target_os = "haiku")] + fn num_cpus() -> usize { + // FIXME: implement + 1 + } + + #[cfg(target_os = "l4re")] + fn num_cpus() -> usize { + // FIXME: implement + 1 + } +} + +pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> { + let mut filtered = tests; + let matches_filter = |test: &TestDescAndFn, filter: &str| { + let test_name = test.desc.name.as_slice(); + + match opts.filter_exact { + true => test_name == filter, + false => test_name.contains(filter), + } + }; + + // Remove tests that don't match the test filter + if let Some(ref filter) = opts.filter { + filtered.retain(|test| matches_filter(test, filter)); + } + + // Skip tests that match any of the skip filters + filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf))); + + // Excludes #[should_panic] tests + if opts.exclude_should_panic { + filtered.retain(|test| test.desc.should_panic == ShouldPanic::No); + } + + // maybe unignore tests + match opts.run_ignored { + RunIgnored::Yes => { + filtered + .iter_mut() + .for_each(|test| test.desc.ignore = false); + } + RunIgnored::Only => { + filtered.retain(|test| test.desc.ignore); + filtered + .iter_mut() + .for_each(|test| test.desc.ignore = false); + } + RunIgnored::No => {} + } + + // Sort the tests alphabetically + filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice())); + + filtered +} + +pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> { + // convert benchmarks to tests, if we're not benchmarking them + tests + .into_iter() + .map(|x| { + let testfn = match x.testfn { + DynBenchFn(bench) => DynTestFn(Box::new(move || { + bench::run_once(|b| __rust_begin_short_backtrace(|| bench.run(b))) + })), + StaticBenchFn(benchfn) => DynTestFn(Box::new(move || { + bench::run_once(|b| __rust_begin_short_backtrace(|| benchfn(b))) + })), + f => f, + }; + TestDescAndFn { + desc: x.desc, + testfn, + } + }) + .collect() +} + +pub fn run_test( + opts: &TestOpts, + force_ignore: bool, + test: TestDescAndFn, + monitor_ch: Sender<MonitorMsg>, + concurrency: Concurrent, +) { + let TestDescAndFn { desc, testfn } = test; + + let ignore_because_panic_abort = cfg!(target_arch = "wasm32") + && !cfg!(target_os = "emscripten") + && desc.should_panic != ShouldPanic::No; + + if force_ignore || desc.ignore || ignore_because_panic_abort { + monitor_ch.send((desc, TrIgnored, Vec::new())).unwrap(); + return; + } + + fn run_test_inner( + desc: TestDesc, + monitor_ch: Sender<MonitorMsg>, + nocapture: bool, + testfn: Box<dyn FnBox() + Send>, + concurrency: Concurrent, + ) { + // Buffer for capturing standard I/O + let data = Arc::new(Mutex::new(Vec::new())); + let data2 = data.clone(); + + let name = desc.name.clone(); + let runtest = move || { + let oldio = if !nocapture { + Some(( + io::set_print(Some(Box::new(Sink(data2.clone())))), + io::set_panic(Some(Box::new(Sink(data2)))), + )) + } else { + None + }; + + let result = catch_unwind(AssertUnwindSafe(testfn)); + + if let Some((printio, panicio)) = oldio { + io::set_print(printio); + io::set_panic(panicio); + }; + + let test_result = calc_result(&desc, result); + let stdout = data.lock().unwrap().to_vec(); + monitor_ch + .send((desc.clone(), test_result, stdout)) + .unwrap(); + }; + + // If the platform is single-threaded we're just going to run + // the test synchronously, regardless of the concurrency + // level. + let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32"); + if concurrency == Concurrent::Yes && supports_threads { + let cfg = thread::Builder::new().name(name.as_slice().to_owned()); + cfg.spawn(runtest).unwrap(); + } else { + runtest(); + } + } + + match testfn { + DynBenchFn(bencher) => { + crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| { + bencher.run(harness) + }); + } + StaticBenchFn(benchfn) => { + crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| { + (benchfn.clone())(harness) + }); + } + DynTestFn(f) => { + let cb = move || __rust_begin_short_backtrace(f); + run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb), concurrency) + } + StaticTestFn(f) => run_test_inner( + desc, + monitor_ch, + opts.nocapture, + Box::new(move || __rust_begin_short_backtrace(f)), + concurrency, + ), + } +} + +/// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`. +#[inline(never)] +fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) { + f() +} + +fn calc_result(desc: &TestDesc, task_result: Result<(), Box<dyn Any + Send>>) -> TestResult { + match (&desc.should_panic, task_result) { + (&ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TrOk, + (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => { + if err + .downcast_ref::<String>() + .map(|e| &**e) + .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) + .map(|e| e.contains(msg)) + .unwrap_or(false) + { + TrOk + } else { + if desc.allow_fail { + TrAllowedFail + } else { + TrFailedMsg(format!("Panic did not include expected string '{}'", msg)) + } + } + } + _ if desc.allow_fail => TrAllowedFail, + _ => TrFailed, + } +} + +#[derive(Clone, PartialEq)] +pub struct MetricMap(BTreeMap<String, Metric>); + +impl MetricMap { + pub fn new() -> MetricMap { + MetricMap(BTreeMap::new()) + } + + /// Insert a named `value` (+/- `noise`) metric into the map. The value + /// must be non-negative. The `noise` indicates the uncertainty of the + /// metric, which doubles as the "noise range" of acceptable + /// pairwise-regressions on this named value, when comparing from one + /// metric to the next using `compare_to_old`. + /// + /// If `noise` is positive, then it means this metric is of a value + /// you want to see grow smaller, so a change larger than `noise` in the + /// positive direction represents a regression. + /// + /// If `noise` is negative, then it means this metric is of a value + /// you want to see grow larger, so a change larger than `noise` in the + /// negative direction represents a regression. + pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) { + let m = Metric { value, noise }; + self.0.insert(name.to_owned(), m); + } + + pub fn fmt_metrics(&self) -> String { + let v = self + .0 + .iter() + .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) + .collect::<Vec<_>>(); + v.join(", ") + } +} + +// Benchmarking pub use std::hint::black_box; +impl Bencher { + /// Callback for benchmark functions to run in their body. + pub fn iter<T, F>(&mut self, mut inner: F) + where + F: FnMut() -> T, + { + if self.mode == BenchMode::Single { + ns_iter_inner(&mut inner, 1); + return; + } + + self.summary = Some(iter(&mut inner)); + } + + pub fn bench<F>(&mut self, mut f: F) -> Option<stats::Summary> + where + F: FnMut(&mut Bencher), + { + f(self); + return self.summary; + } +} + +fn ns_from_dur(dur: Duration) -> u64 { + dur.as_secs() * 1_000_000_000 + (dur.subsec_nanos() as u64) +} + +fn ns_iter_inner<T, F>(inner: &mut F, k: u64) -> u64 +where + F: FnMut() -> T, +{ + let start = Instant::now(); + for _ in 0..k { + black_box(inner()); + } + return ns_from_dur(start.elapsed()); +} + +pub fn iter<T, F>(inner: &mut F) -> stats::Summary +where + F: FnMut() -> T, +{ + // Initial bench run to get ballpark figure. + let ns_single = ns_iter_inner(inner, 1); + + // Try to estimate iter count for 1ms falling back to 1m + // iterations if first run took < 1ns. + let ns_target_total = 1_000_000; // 1ms + let mut n = ns_target_total / cmp::max(1, ns_single); + + // if the first run took more than 1ms we don't want to just + // be left doing 0 iterations on every loop. The unfortunate + // side effect of not being able to do as many runs is + // automatically handled by the statistical analysis below + // (i.e., larger error bars). + n = cmp::max(1, n); + + let mut total_run = Duration::new(0, 0); + let samples: &mut [f64] = &mut [0.0_f64; 50]; + loop { + let loop_start = Instant::now(); + + for p in &mut *samples { + *p = ns_iter_inner(inner, n) as f64 / n as f64; + } + + stats::winsorize(samples, 5.0); + let summ = stats::Summary::new(samples); + + for p in &mut *samples { + let ns = ns_iter_inner(inner, 5 * n); + *p = ns as f64 / (5 * n) as f64; + } + + stats::winsorize(samples, 5.0); + let summ5 = stats::Summary::new(samples); + + let loop_run = loop_start.elapsed(); + + // If we've run for 100ms and seem to have converged to a + // stable median. + if loop_run > Duration::from_millis(100) + && summ.median_abs_dev_pct < 1.0 + && summ.median - summ5.median < summ5.median_abs_dev + { + return summ5; + } + + total_run = total_run + loop_run; + // Longest we ever run for is 3s. + if total_run > Duration::from_secs(3) { + return summ5; + } + + // If we overflow here just return the results so far. We check a + // multiplier of 10 because we're about to multiply by 2 and the + // next iteration of the loop will also multiply by 5 (to calculate + // the summ5 result) + n = match n.checked_mul(10) { + Some(_) => n * 2, + None => { + return summ5; + } + }; + } +} + +pub mod bench { + use super::{BenchMode, BenchSamples, Bencher, MonitorMsg, Sender, Sink, TestDesc, TestResult}; + use crate::stats; + use std::cmp; + use std::io; + use std::panic::{catch_unwind, AssertUnwindSafe}; + use std::sync::{Arc, Mutex}; + + pub fn benchmark<F>(desc: TestDesc, monitor_ch: Sender<MonitorMsg>, nocapture: bool, f: F) + where + F: FnMut(&mut Bencher), + { + let mut bs = Bencher { + mode: BenchMode::Auto, + summary: None, + bytes: 0, + }; + + let data = Arc::new(Mutex::new(Vec::new())); + let data2 = data.clone(); + + let oldio = if !nocapture { + Some(( + io::set_print(Some(Box::new(Sink(data2.clone())))), + io::set_panic(Some(Box::new(Sink(data2)))), + )) + } else { + None + }; + + let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f))); + + if let Some((printio, panicio)) = oldio { + io::set_print(printio); + io::set_panic(panicio); + }; + + let test_result = match result { + //bs.bench(f) { + Ok(Some(ns_iter_summ)) => { + let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); + let mb_s = bs.bytes * 1000 / ns_iter; + + let bs = BenchSamples { + ns_iter_summ, + mb_s: mb_s as usize, + }; + TestResult::TrBench(bs) + } + Ok(None) => { + // iter not called, so no data. + // FIXME: error in this case? + let samples: &mut [f64] = &mut [0.0_f64; 1]; + let bs = BenchSamples { + ns_iter_summ: stats::Summary::new(samples), + mb_s: 0, + }; + TestResult::TrBench(bs) + } + Err(_) => TestResult::TrFailed, + }; + + let stdout = data.lock().unwrap().to_vec(); + monitor_ch.send((desc, test_result, stdout)).unwrap(); + } + + pub fn run_once<F>(f: F) + where + F: FnMut(&mut Bencher), + { + let mut bs = Bencher { + mode: BenchMode::Single, + summary: None, + bytes: 0, + }; + bs.bench(f); + } +} + #[cfg(test)] mod tests { + use crate::bench; + use crate::test::{ + filter_tests, parse_opts, run_test, DynTestFn, DynTestName, MetricMap, RunIgnored, + ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TrFailed, TrFailedMsg, + TrIgnored, TrOk, + }; use crate::Bencher; - use libtest::stats::Stats; + use crate::Concurrent; + use std::sync::mpsc::channel; - #[bench] - pub fn sum_three_items(b: &mut Bencher) { - b.iter(|| { - [1e20f64, 1.5f64, -1e20f64].sum(); - }) + fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> { + vec![ + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("1"), + ignore: true, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(move || {})), + }, + TestDescAndFn { + desc: TestDesc { + name: StaticTestName("2"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(move || {})), + }, + ] } - #[bench] - pub fn sum_many_f64(b: &mut Bencher) { - let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60]; - let v = (0..500).map(|i| nums[i % 5]).collect::<Vec<_>>(); - b.iter(|| { - v.sum(); - }) + #[test] + pub fn do_not_run_ignored_tests() { + fn f() { + panic!(); + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: true, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx, Concurrent::No); + let (_, res, _) = rx.recv().unwrap(); + assert!(res != TrOk); + } + + #[test] + pub fn ignored_tests_result_in_ignored() { + fn f() {} + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: true, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx, Concurrent::No); + let (_, res, _) = rx.recv().unwrap(); + assert!(res == TrIgnored); + } + + #[test] + fn test_should_panic() { + fn f() { + panic!(); + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_panic: ShouldPanic::Yes, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx, Concurrent::No); + let (_, res, _) = rx.recv().unwrap(); + assert!(res == TrOk); + } + + #[test] + fn test_should_panic_good_message() { + fn f() { + panic!("an error message"); + } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_panic: ShouldPanic::YesWithMessage("error message"), + allow_fail: false, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx, Concurrent::No); + let (_, res, _) = rx.recv().unwrap(); + assert!(res == TrOk); + } + + #[test] + fn test_should_panic_bad_message() { + fn f() { + panic!("an error message"); + } + let expected = "foobar"; + let failed_msg = "Panic did not include expected string"; + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_panic: ShouldPanic::YesWithMessage(expected), + allow_fail: false, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx, Concurrent::No); + let (_, res, _) = rx.recv().unwrap(); + assert!(res == TrFailedMsg(format!("{} '{}'", failed_msg, expected))); + } + + #[test] + fn test_should_panic_but_succeeds() { + fn f() {} + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_panic: ShouldPanic::Yes, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(f)), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx, Concurrent::No); + let (_, res, _) = rx.recv().unwrap(); + assert!(res == TrFailed); + } + + #[test] + fn parse_ignored_flag() { + let args = vec![ + "progname".to_string(), + "filter".to_string(), + "--ignored".to_string(), + ]; + let opts = parse_opts(&args).unwrap().unwrap(); + assert_eq!(opts.run_ignored, RunIgnored::Only); + } + + #[test] + fn parse_include_ignored_flag() { + let args = vec![ + "progname".to_string(), + "filter".to_string(), + "-Zunstable-options".to_string(), + "--include-ignored".to_string(), + ]; + let opts = parse_opts(&args).unwrap().unwrap(); + assert_eq!(opts.run_ignored, RunIgnored::Yes); + } + + #[test] + pub fn filter_for_ignored_option() { + // When we run ignored tests the test filter should filter out all the + // unignored tests and flip the ignore flag on the rest to false + + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.run_ignored = RunIgnored::Only; + + let tests = one_ignored_one_unignored_test(); + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].desc.name.to_string(), "1"); + assert!(!filtered[0].desc.ignore); } - #[bench] - pub fn no_iter(_: &mut Bencher) {} + #[test] + pub fn run_include_ignored_option() { + // When we "--include-ignored" tests, the ignore flag should be set to false on + // all tests and no test filtered out + + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.run_ignored = RunIgnored::Yes; + + let tests = one_ignored_one_unignored_test(); + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 2); + assert!(!filtered[0].desc.ignore); + assert!(!filtered[1].desc.ignore); + } + + #[test] + pub fn exclude_should_panic_option() { + let mut opts = TestOpts::new(); + opts.run_tests = true; + opts.exclude_should_panic = true; + + let mut tests = one_ignored_one_unignored_test(); + tests.push(TestDescAndFn { + desc: TestDesc { + name: StaticTestName("3"), + ignore: false, + should_panic: ShouldPanic::Yes, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(move || {})), + }); + + let filtered = filter_tests(&opts, tests); + + assert_eq!(filtered.len(), 2); + assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No)); + } + + #[test] + pub fn exact_filter_match() { + fn tests() -> Vec<TestDescAndFn> { + vec!["base", "base::test", "base::test1", "base::test2"] + .into_iter() + .map(|name| TestDescAndFn { + desc: TestDesc { + name: StaticTestName(name), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(move || {})), + }) + .collect() + } + + let substr = filter_tests( + &TestOpts { + filter: Some("base".into()), + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(substr.len(), 4); + + let substr = filter_tests( + &TestOpts { + filter: Some("bas".into()), + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(substr.len(), 4); + + let substr = filter_tests( + &TestOpts { + filter: Some("::test".into()), + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(substr.len(), 3); + + let substr = filter_tests( + &TestOpts { + filter: Some("base::test".into()), + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(substr.len(), 3); + + let exact = filter_tests( + &TestOpts { + filter: Some("base".into()), + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(exact.len(), 1); + + let exact = filter_tests( + &TestOpts { + filter: Some("bas".into()), + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(exact.len(), 0); + + let exact = filter_tests( + &TestOpts { + filter: Some("::test".into()), + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(exact.len(), 0); + + let exact = filter_tests( + &TestOpts { + filter: Some("base::test".into()), + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); + assert_eq!(exact.len(), 1); + } + + #[test] + pub fn sort_tests() { + let mut opts = TestOpts::new(); + opts.run_tests = true; + + let names = vec![ + "sha1::test".to_string(), + "isize::test_to_str".to_string(), + "isize::test_pow".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::parse_ignored_flag".to_string(), + "test::parse_include_ignored_flag".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::run_include_ignored_option".to_string(), + "test::sort_tests".to_string(), + ]; + let tests = { + fn testfn() {} + let mut tests = Vec::new(); + for name in &names { + let test = TestDescAndFn { + desc: TestDesc { + name: DynTestName((*name).clone()), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(testfn)), + }; + tests.push(test); + } + tests + }; + let filtered = filter_tests(&opts, tests); + + let expected = vec![ + "isize::test_pow".to_string(), + "isize::test_to_str".to_string(), + "sha1::test".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::parse_ignored_flag".to_string(), + "test::parse_include_ignored_flag".to_string(), + "test::run_include_ignored_option".to_string(), + "test::sort_tests".to_string(), + ]; + + for (a, b) in expected.iter().zip(filtered) { + assert!(*a == b.desc.name.to_string()); + } + } + + #[test] + pub fn test_metricmap_compare() { + let mut m1 = MetricMap::new(); + let mut m2 = MetricMap::new(); + m1.insert_metric("in-both-noise", 1000.0, 200.0); + m2.insert_metric("in-both-noise", 1100.0, 200.0); + + m1.insert_metric("in-first-noise", 1000.0, 2.0); + m2.insert_metric("in-second-noise", 1000.0, 2.0); + + m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0); + m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0); + + m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0); + m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0); + + m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0); + m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0); + + m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0); + m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0); + } + + #[test] + pub fn test_bench_once_no_iter() { + fn f(_: &mut Bencher) {} + bench::run_once(f); + } + + #[test] + pub fn test_bench_once_iter() { + fn f(b: &mut Bencher) { + b.iter(|| {}) + } + bench::run_once(f); + } + + #[test] + pub fn test_bench_no_iter() { + fn f(_: &mut Bencher) {} + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + crate::bench::benchmark(desc, tx, true, f); + rx.recv().unwrap(); + } + + #[test] + pub fn test_bench_iter() { + fn f(b: &mut Bencher) { + b.iter(|| {}) + } + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + crate::bench::benchmark(desc, tx, true, f); + rx.recv().unwrap(); + } } diff --git a/src/libtest/stats.rs b/src/libtest/stats.rs new file mode 100644 index 00000000000..5c9421d5ea4 --- /dev/null +++ b/src/libtest/stats.rs @@ -0,0 +1,922 @@ +#![allow(missing_docs)] +#![allow(deprecated)] // Float + +use std::cmp::Ordering::{self, Equal, Greater, Less}; +use std::mem; + +fn local_cmp(x: f64, y: f64) -> Ordering { + // arbitrarily decide that NaNs are larger than everything. + if y.is_nan() { + Less + } else if x.is_nan() { + Greater + } else if x < y { + Less + } else if x == y { + Equal + } else { + Greater + } +} + +fn local_sort(v: &mut [f64]) { + v.sort_by(|x: &f64, y: &f64| local_cmp(*x, *y)); +} + +/// Trait that provides simple descriptive statistics on a univariate set of numeric samples. +pub trait Stats { + /// Sum of the samples. + /// + /// Note: this method sacrifices performance at the altar of accuracy + /// Depends on IEEE-754 arithmetic guarantees. See proof of correctness at: + /// ["Adaptive Precision Floating-Point Arithmetic and Fast Robust Geometric + /// Predicates"][paper] + /// + /// [paper]: http://www.cs.cmu.edu/~quake-papers/robust-arithmetic.ps + fn sum(&self) -> f64; + + /// Minimum value of the samples. + fn min(&self) -> f64; + + /// Maximum value of the samples. + fn max(&self) -> f64; + + /// Arithmetic mean (average) of the samples: sum divided by sample-count. + /// + /// See: <https://en.wikipedia.org/wiki/Arithmetic_mean> + fn mean(&self) -> f64; + + /// Median of the samples: value separating the lower half of the samples from the higher half. + /// Equal to `self.percentile(50.0)`. + /// + /// See: <https://en.wikipedia.org/wiki/Median> + fn median(&self) -> f64; + + /// Variance of the samples: bias-corrected mean of the squares of the differences of each + /// sample from the sample mean. Note that this calculates the _sample variance_ rather than the + /// population variance, which is assumed to be unknown. It therefore corrects the `(n-1)/n` + /// bias that would appear if we calculated a population variance, by dividing by `(n-1)` rather + /// than `n`. + /// + /// See: <https://en.wikipedia.org/wiki/Variance> + fn var(&self) -> f64; + + /// Standard deviation: the square root of the sample variance. + /// + /// Note: this is not a robust statistic for non-normal distributions. Prefer the + /// `median_abs_dev` for unknown distributions. + /// + /// See: <https://en.wikipedia.org/wiki/Standard_deviation> + fn std_dev(&self) -> f64; + + /// Standard deviation as a percent of the mean value. See `std_dev` and `mean`. + /// + /// Note: this is not a robust statistic for non-normal distributions. Prefer the + /// `median_abs_dev_pct` for unknown distributions. + fn std_dev_pct(&self) -> f64; + + /// Scaled median of the absolute deviations of each sample from the sample median. This is a + /// robust (distribution-agnostic) estimator of sample variability. Use this in preference to + /// `std_dev` if you cannot assume your sample is normally distributed. Note that this is scaled + /// by the constant `1.4826` to allow its use as a consistent estimator for the standard + /// deviation. + /// + /// See: <http://en.wikipedia.org/wiki/Median_absolute_deviation> + fn median_abs_dev(&self) -> f64; + + /// Median absolute deviation as a percent of the median. See `median_abs_dev` and `median`. + fn median_abs_dev_pct(&self) -> f64; + + /// Percentile: the value below which `pct` percent of the values in `self` fall. For example, + /// percentile(95.0) will return the value `v` such that 95% of the samples `s` in `self` + /// satisfy `s <= v`. + /// + /// Calculated by linear interpolation between closest ranks. + /// + /// See: <http://en.wikipedia.org/wiki/Percentile> + fn percentile(&self, pct: f64) -> f64; + + /// Quartiles of the sample: three values that divide the sample into four equal groups, each + /// with 1/4 of the data. The middle value is the median. See `median` and `percentile`. This + /// function may calculate the 3 quartiles more efficiently than 3 calls to `percentile`, but + /// is otherwise equivalent. + /// + /// See also: <https://en.wikipedia.org/wiki/Quartile> + fn quartiles(&self) -> (f64, f64, f64); + + /// Inter-quartile range: the difference between the 25th percentile (1st quartile) and the 75th + /// percentile (3rd quartile). See `quartiles`. + /// + /// See also: <https://en.wikipedia.org/wiki/Interquartile_range> + fn iqr(&self) -> f64; +} + +/// Extracted collection of all the summary statistics of a sample set. +#[derive(Clone, PartialEq, Copy)] +#[allow(missing_docs)] +pub struct Summary { + pub sum: f64, + pub min: f64, + pub max: f64, + pub mean: f64, + pub median: f64, + pub var: f64, + pub std_dev: f64, + pub std_dev_pct: f64, + pub median_abs_dev: f64, + pub median_abs_dev_pct: f64, + pub quartiles: (f64, f64, f64), + pub iqr: f64, +} + +impl Summary { + /// Construct a new summary of a sample set. + pub fn new(samples: &[f64]) -> Summary { + Summary { + sum: samples.sum(), + min: samples.min(), + max: samples.max(), + mean: samples.mean(), + median: samples.median(), + var: samples.var(), + std_dev: samples.std_dev(), + std_dev_pct: samples.std_dev_pct(), + median_abs_dev: samples.median_abs_dev(), + median_abs_dev_pct: samples.median_abs_dev_pct(), + quartiles: samples.quartiles(), + iqr: samples.iqr(), + } + } +} + +impl Stats for [f64] { + // FIXME #11059 handle NaN, inf and overflow + fn sum(&self) -> f64 { + let mut partials = vec![]; + + for &x in self { + let mut x = x; + let mut j = 0; + // This inner loop applies `hi`/`lo` summation to each + // partial so that the list of partial sums remains exact. + for i in 0..partials.len() { + let mut y: f64 = partials[i]; + if x.abs() < y.abs() { + mem::swap(&mut x, &mut y); + } + // Rounded `x+y` is stored in `hi` with round-off stored in + // `lo`. Together `hi+lo` are exactly equal to `x+y`. + let hi = x + y; + let lo = y - (hi - x); + if lo != 0.0 { + partials[j] = lo; + j += 1; + } + x = hi; + } + if j >= partials.len() { + partials.push(x); + } else { + partials[j] = x; + partials.truncate(j + 1); + } + } + let zero: f64 = 0.0; + partials.iter().fold(zero, |p, q| p + *q) + } + + fn min(&self) -> f64 { + assert!(!self.is_empty()); + self.iter().fold(self[0], |p, q| p.min(*q)) + } + + fn max(&self) -> f64 { + assert!(!self.is_empty()); + self.iter().fold(self[0], |p, q| p.max(*q)) + } + + fn mean(&self) -> f64 { + assert!(!self.is_empty()); + self.sum() / (self.len() as f64) + } + + fn median(&self) -> f64 { + self.percentile(50 as f64) + } + + fn var(&self) -> f64 { + if self.len() < 2 { + 0.0 + } else { + let mean = self.mean(); + let mut v: f64 = 0.0; + for s in self { + let x = *s - mean; + v = v + x * x; + } + // N.B., this is _supposed to be_ len-1, not len. If you + // change it back to len, you will be calculating a + // population variance, not a sample variance. + let denom = (self.len() - 1) as f64; + v / denom + } + } + + fn std_dev(&self) -> f64 { + self.var().sqrt() + } + + fn std_dev_pct(&self) -> f64 { + let hundred = 100 as f64; + (self.std_dev() / self.mean()) * hundred + } + + fn median_abs_dev(&self) -> f64 { + let med = self.median(); + let abs_devs: Vec<f64> = self.iter().map(|&v| (med - v).abs()).collect(); + // This constant is derived by smarter statistics brains than me, but it is + // consistent with how R and other packages treat the MAD. + let number = 1.4826; + abs_devs.median() * number + } + + fn median_abs_dev_pct(&self) -> f64 { + let hundred = 100 as f64; + (self.median_abs_dev() / self.median()) * hundred + } + + fn percentile(&self, pct: f64) -> f64 { + let mut tmp = self.to_vec(); + local_sort(&mut tmp); + percentile_of_sorted(&tmp, pct) + } + + fn quartiles(&self) -> (f64, f64, f64) { + let mut tmp = self.to_vec(); + local_sort(&mut tmp); + let first = 25f64; + let a = percentile_of_sorted(&tmp, first); + let second = 50f64; + let b = percentile_of_sorted(&tmp, second); + let third = 75f64; + let c = percentile_of_sorted(&tmp, third); + (a, b, c) + } + + fn iqr(&self) -> f64 { + let (a, _, c) = self.quartiles(); + c - a + } +} + +// Helper function: extract a value representing the `pct` percentile of a sorted sample-set, using +// linear interpolation. If samples are not sorted, return nonsensical value. +fn percentile_of_sorted(sorted_samples: &[f64], pct: f64) -> f64 { + assert!(!sorted_samples.is_empty()); + if sorted_samples.len() == 1 { + return sorted_samples[0]; + } + let zero: f64 = 0.0; + assert!(zero <= pct); + let hundred = 100f64; + assert!(pct <= hundred); + if pct == hundred { + return sorted_samples[sorted_samples.len() - 1]; + } + let length = (sorted_samples.len() - 1) as f64; + let rank = (pct / hundred) * length; + let lrank = rank.floor(); + let d = rank - lrank; + let n = lrank as usize; + let lo = sorted_samples[n]; + let hi = sorted_samples[n + 1]; + lo + (hi - lo) * d +} + +/// Winsorize a set of samples, replacing values above the `100-pct` percentile +/// and below the `pct` percentile with those percentiles themselves. This is a +/// way of minimizing the effect of outliers, at the cost of biasing the sample. +/// It differs from trimming in that it does not change the number of samples, +/// just changes the values of those that are outliers. +/// +/// See: <http://en.wikipedia.org/wiki/Winsorising> +pub fn winsorize(samples: &mut [f64], pct: f64) { + let mut tmp = samples.to_vec(); + local_sort(&mut tmp); + let lo = percentile_of_sorted(&tmp, pct); + let hundred = 100 as f64; + let hi = percentile_of_sorted(&tmp, hundred - pct); + for samp in samples { + if *samp > hi { + *samp = hi + } else if *samp < lo { + *samp = lo + } + } +} + +// Test vectors generated from R, using the script src/etc/stat-test-vectors.r. + +#[cfg(test)] +mod tests { + use crate::stats::Stats; + use crate::stats::Summary; + use std::f64; + use std::io::prelude::*; + use std::io; + + macro_rules! assert_approx_eq { + ($a: expr, $b: expr) => {{ + let (a, b) = (&$a, &$b); + assert!( + (*a - *b).abs() < 1.0e-6, + "{} is not approximately equal to {}", + *a, + *b + ); + }}; + } + + fn check(samples: &[f64], summ: &Summary) { + let summ2 = Summary::new(samples); + + let mut w = io::sink(); + let w = &mut w; + (write!(w, "\n")).unwrap(); + + assert_eq!(summ.sum, summ2.sum); + assert_eq!(summ.min, summ2.min); + assert_eq!(summ.max, summ2.max); + assert_eq!(summ.mean, summ2.mean); + assert_eq!(summ.median, summ2.median); + + // We needed a few more digits to get exact equality on these + // but they're within float epsilon, which is 1.0e-6. + assert_approx_eq!(summ.var, summ2.var); + assert_approx_eq!(summ.std_dev, summ2.std_dev); + assert_approx_eq!(summ.std_dev_pct, summ2.std_dev_pct); + assert_approx_eq!(summ.median_abs_dev, summ2.median_abs_dev); + assert_approx_eq!(summ.median_abs_dev_pct, summ2.median_abs_dev_pct); + + assert_eq!(summ.quartiles, summ2.quartiles); + assert_eq!(summ.iqr, summ2.iqr); + } + + #[test] + fn test_min_max_nan() { + let xs = &[1.0, 2.0, f64::NAN, 3.0, 4.0]; + let summary = Summary::new(xs); + assert_eq!(summary.min, 1.0); + assert_eq!(summary.max, 4.0); + } + + #[test] + fn test_norm2() { + let val = &[958.0000000000, 924.0000000000]; + let summ = &Summary { + sum: 1882.0000000000, + min: 924.0000000000, + max: 958.0000000000, + mean: 941.0000000000, + median: 941.0000000000, + var: 578.0000000000, + std_dev: 24.0416305603, + std_dev_pct: 2.5549022912, + median_abs_dev: 25.2042000000, + median_abs_dev_pct: 2.6784484591, + quartiles: (932.5000000000, 941.0000000000, 949.5000000000), + iqr: 17.0000000000, + }; + check(val, summ); + } + #[test] + fn test_norm10narrow() { + let val = &[ + 966.0000000000, + 985.0000000000, + 1110.0000000000, + 848.0000000000, + 821.0000000000, + 975.0000000000, + 962.0000000000, + 1157.0000000000, + 1217.0000000000, + 955.0000000000, + ]; + let summ = &Summary { + sum: 9996.0000000000, + min: 821.0000000000, + max: 1217.0000000000, + mean: 999.6000000000, + median: 970.5000000000, + var: 16050.7111111111, + std_dev: 126.6914010938, + std_dev_pct: 12.6742097933, + median_abs_dev: 102.2994000000, + median_abs_dev_pct: 10.5408964451, + quartiles: (956.7500000000, 970.5000000000, 1078.7500000000), + iqr: 122.0000000000, + }; + check(val, summ); + } + #[test] + fn test_norm10medium() { + let val = &[ + 954.0000000000, + 1064.0000000000, + 855.0000000000, + 1000.0000000000, + 743.0000000000, + 1084.0000000000, + 704.0000000000, + 1023.0000000000, + 357.0000000000, + 869.0000000000, + ]; + let summ = &Summary { + sum: 8653.0000000000, + min: 357.0000000000, + max: 1084.0000000000, + mean: 865.3000000000, + median: 911.5000000000, + var: 48628.4555555556, + std_dev: 220.5186059170, + std_dev_pct: 25.4846418487, + median_abs_dev: 195.7032000000, + median_abs_dev_pct: 21.4704552935, + quartiles: (771.0000000000, 911.5000000000, 1017.2500000000), + iqr: 246.2500000000, + }; + check(val, summ); + } + #[test] + fn test_norm10wide() { + let val = &[ + 505.0000000000, + 497.0000000000, + 1591.0000000000, + 887.0000000000, + 1026.0000000000, + 136.0000000000, + 1580.0000000000, + 940.0000000000, + 754.0000000000, + 1433.0000000000, + ]; + let summ = &Summary { + sum: 9349.0000000000, + min: 136.0000000000, + max: 1591.0000000000, + mean: 934.9000000000, + median: 913.5000000000, + var: 239208.9888888889, + std_dev: 489.0899599142, + std_dev_pct: 52.3146817750, + median_abs_dev: 611.5725000000, + median_abs_dev_pct: 66.9482758621, + quartiles: (567.2500000000, 913.5000000000, 1331.2500000000), + iqr: 764.0000000000, + }; + check(val, summ); + } + #[test] + fn test_norm25verynarrow() { + let val = &[ + 991.0000000000, + 1018.0000000000, + 998.0000000000, + 1013.0000000000, + 974.0000000000, + 1007.0000000000, + 1014.0000000000, + 999.0000000000, + 1011.0000000000, + 978.0000000000, + 985.0000000000, + 999.0000000000, + 983.0000000000, + 982.0000000000, + 1015.0000000000, + 1002.0000000000, + 977.0000000000, + 948.0000000000, + 1040.0000000000, + 974.0000000000, + 996.0000000000, + 989.0000000000, + 1015.0000000000, + 994.0000000000, + 1024.0000000000, + ]; + let summ = &Summary { + sum: 24926.0000000000, + min: 948.0000000000, + max: 1040.0000000000, + mean: 997.0400000000, + median: 998.0000000000, + var: 393.2066666667, + std_dev: 19.8294393937, + std_dev_pct: 1.9888308788, + median_abs_dev: 22.2390000000, + median_abs_dev_pct: 2.2283567134, + quartiles: (983.0000000000, 998.0000000000, 1013.0000000000), + iqr: 30.0000000000, + }; + check(val, summ); + } + #[test] + fn test_exp10a() { + let val = &[ + 23.0000000000, + 11.0000000000, + 2.0000000000, + 57.0000000000, + 4.0000000000, + 12.0000000000, + 5.0000000000, + 29.0000000000, + 3.0000000000, + 21.0000000000, + ]; + let summ = &Summary { + sum: 167.0000000000, + min: 2.0000000000, + max: 57.0000000000, + mean: 16.7000000000, + median: 11.5000000000, + var: 287.7888888889, + std_dev: 16.9643416875, + std_dev_pct: 101.5828843560, + median_abs_dev: 13.3434000000, + median_abs_dev_pct: 116.0295652174, + quartiles: (4.2500000000, 11.5000000000, 22.5000000000), + iqr: 18.2500000000, + }; + check(val, summ); + } + #[test] + fn test_exp10b() { + let val = &[ + 24.0000000000, + 17.0000000000, + 6.0000000000, + 38.0000000000, + 25.0000000000, + 7.0000000000, + 51.0000000000, + 2.0000000000, + 61.0000000000, + 32.0000000000, + ]; + let summ = &Summary { + sum: 263.0000000000, + min: 2.0000000000, + max: 61.0000000000, + mean: 26.3000000000, + median: 24.5000000000, + var: 383.5666666667, + std_dev: 19.5848580967, + std_dev_pct: 74.4671410520, + median_abs_dev: 22.9803000000, + median_abs_dev_pct: 93.7971428571, + quartiles: (9.5000000000, 24.5000000000, 36.5000000000), + iqr: 27.0000000000, + }; + check(val, summ); + } + #[test] + fn test_exp10c() { + let val = &[ + 71.0000000000, + 2.0000000000, + 32.0000000000, + 1.0000000000, + 6.0000000000, + 28.0000000000, + 13.0000000000, + 37.0000000000, + 16.0000000000, + 36.0000000000, + ]; + let summ = &Summary { + sum: 242.0000000000, + min: 1.0000000000, + max: 71.0000000000, + mean: 24.2000000000, + median: 22.0000000000, + var: 458.1777777778, + std_dev: 21.4050876611, + std_dev_pct: 88.4507754589, + median_abs_dev: 21.4977000000, + median_abs_dev_pct: 97.7168181818, + quartiles: (7.7500000000, 22.0000000000, 35.0000000000), + iqr: 27.2500000000, + }; + check(val, summ); + } + #[test] + fn test_exp25() { + let val = &[ + 3.0000000000, + 24.0000000000, + 1.0000000000, + 19.0000000000, + 7.0000000000, + 5.0000000000, + 30.0000000000, + 39.0000000000, + 31.0000000000, + 13.0000000000, + 25.0000000000, + 48.0000000000, + 1.0000000000, + 6.0000000000, + 42.0000000000, + 63.0000000000, + 2.0000000000, + 12.0000000000, + 108.0000000000, + 26.0000000000, + 1.0000000000, + 7.0000000000, + 44.0000000000, + 25.0000000000, + 11.0000000000, + ]; + let summ = &Summary { + sum: 593.0000000000, + min: 1.0000000000, + max: 108.0000000000, + mean: 23.7200000000, + median: 19.0000000000, + var: 601.0433333333, + std_dev: 24.5161851301, + std_dev_pct: 103.3565983562, + median_abs_dev: 19.2738000000, + median_abs_dev_pct: 101.4410526316, + quartiles: (6.0000000000, 19.0000000000, 31.0000000000), + iqr: 25.0000000000, + }; + check(val, summ); + } + #[test] + fn test_binom25() { + let val = &[ + 18.0000000000, + 17.0000000000, + 27.0000000000, + 15.0000000000, + 21.0000000000, + 25.0000000000, + 17.0000000000, + 24.0000000000, + 25.0000000000, + 24.0000000000, + 26.0000000000, + 26.0000000000, + 23.0000000000, + 15.0000000000, + 23.0000000000, + 17.0000000000, + 18.0000000000, + 18.0000000000, + 21.0000000000, + 16.0000000000, + 15.0000000000, + 31.0000000000, + 20.0000000000, + 17.0000000000, + 15.0000000000, + ]; + let summ = &Summary { + sum: 514.0000000000, + min: 15.0000000000, + max: 31.0000000000, + mean: 20.5600000000, + median: 20.0000000000, + var: 20.8400000000, + std_dev: 4.5650848842, + std_dev_pct: 22.2037202539, + median_abs_dev: 5.9304000000, + median_abs_dev_pct: 29.6520000000, + quartiles: (17.0000000000, 20.0000000000, 24.0000000000), + iqr: 7.0000000000, + }; + check(val, summ); + } + #[test] + fn test_pois25lambda30() { + let val = &[ + 27.0000000000, + 33.0000000000, + 34.0000000000, + 34.0000000000, + 24.0000000000, + 39.0000000000, + 28.0000000000, + 27.0000000000, + 31.0000000000, + 28.0000000000, + 38.0000000000, + 21.0000000000, + 33.0000000000, + 36.0000000000, + 29.0000000000, + 37.0000000000, + 32.0000000000, + 34.0000000000, + 31.0000000000, + 39.0000000000, + 25.0000000000, + 31.0000000000, + 32.0000000000, + 40.0000000000, + 24.0000000000, + ]; + let summ = &Summary { + sum: 787.0000000000, + min: 21.0000000000, + max: 40.0000000000, + mean: 31.4800000000, + median: 32.0000000000, + var: 26.5933333333, + std_dev: 5.1568724372, + std_dev_pct: 16.3814245145, + median_abs_dev: 5.9304000000, + median_abs_dev_pct: 18.5325000000, + quartiles: (28.0000000000, 32.0000000000, 34.0000000000), + iqr: 6.0000000000, + }; + check(val, summ); + } + #[test] + fn test_pois25lambda40() { + let val = &[ + 42.0000000000, + 50.0000000000, + 42.0000000000, + 46.0000000000, + 34.0000000000, + 45.0000000000, + 34.0000000000, + 49.0000000000, + 39.0000000000, + 28.0000000000, + 40.0000000000, + 35.0000000000, + 37.0000000000, + 39.0000000000, + 46.0000000000, + 44.0000000000, + 32.0000000000, + 45.0000000000, + 42.0000000000, + 37.0000000000, + 48.0000000000, + 42.0000000000, + 33.0000000000, + 42.0000000000, + 48.0000000000, + ]; + let summ = &Summary { + sum: 1019.0000000000, + min: 28.0000000000, + max: 50.0000000000, + mean: 40.7600000000, + median: 42.0000000000, + var: 34.4400000000, + std_dev: 5.8685603004, + std_dev_pct: 14.3978417577, + median_abs_dev: 5.9304000000, + median_abs_dev_pct: 14.1200000000, + quartiles: (37.0000000000, 42.0000000000, 45.0000000000), + iqr: 8.0000000000, + }; + check(val, summ); + } + #[test] + fn test_pois25lambda50() { + let val = &[ + 45.0000000000, + 43.0000000000, + 44.0000000000, + 61.0000000000, + 51.0000000000, + 53.0000000000, + 59.0000000000, + 52.0000000000, + 49.0000000000, + 51.0000000000, + 51.0000000000, + 50.0000000000, + 49.0000000000, + 56.0000000000, + 42.0000000000, + 52.0000000000, + 51.0000000000, + 43.0000000000, + 48.0000000000, + 48.0000000000, + 50.0000000000, + 42.0000000000, + 43.0000000000, + 42.0000000000, + 60.0000000000, + ]; + let summ = &Summary { + sum: 1235.0000000000, + min: 42.0000000000, + max: 61.0000000000, + mean: 49.4000000000, + median: 50.0000000000, + var: 31.6666666667, + std_dev: 5.6273143387, + std_dev_pct: 11.3913245723, + median_abs_dev: 4.4478000000, + median_abs_dev_pct: 8.8956000000, + quartiles: (44.0000000000, 50.0000000000, 52.0000000000), + iqr: 8.0000000000, + }; + check(val, summ); + } + #[test] + fn test_unif25() { + let val = &[ + 99.0000000000, + 55.0000000000, + 92.0000000000, + 79.0000000000, + 14.0000000000, + 2.0000000000, + 33.0000000000, + 49.0000000000, + 3.0000000000, + 32.0000000000, + 84.0000000000, + 59.0000000000, + 22.0000000000, + 86.0000000000, + 76.0000000000, + 31.0000000000, + 29.0000000000, + 11.0000000000, + 41.0000000000, + 53.0000000000, + 45.0000000000, + 44.0000000000, + 98.0000000000, + 98.0000000000, + 7.0000000000, + ]; + let summ = &Summary { + sum: 1242.0000000000, + min: 2.0000000000, + max: 99.0000000000, + mean: 49.6800000000, + median: 45.0000000000, + var: 1015.6433333333, + std_dev: 31.8691595957, + std_dev_pct: 64.1488719719, + median_abs_dev: 45.9606000000, + median_abs_dev_pct: 102.1346666667, + quartiles: (29.0000000000, 45.0000000000, 79.0000000000), + iqr: 50.0000000000, + }; + check(val, summ); + } + + #[test] + fn test_sum_f64s() { + assert_eq!([0.5f64, 3.2321f64, 1.5678f64].sum(), 5.2999); + } + #[test] + fn test_sum_f64_between_ints_that_sum_to_0() { + assert_eq!([1e30f64, 1.2f64, -1e30f64].sum(), 1.2); + } +} + +#[cfg(test)] +mod bench { + extern crate test; + use self::test::Bencher; + use crate::stats::Stats; + + #[bench] + pub fn sum_three_items(b: &mut Bencher) { + b.iter(|| { + [1e20f64, 1.5f64, -1e20f64].sum(); + }) + } + #[bench] + pub fn sum_many_f64(b: &mut Bencher) { + let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60]; + let v = (0..500).map(|i| nums[i % 5]).collect::<Vec<_>>(); + + b.iter(|| { + v.sum(); + }) + } + + #[bench] + pub fn no_iter(_: &mut Bencher) {} +} |
