From 616af6eb83630b76bb3a9bde57a00f5ebe5dbd6c Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Thu, 4 Dec 2014 23:02:36 -0800 Subject: Allow message specification for should_fail The test harness will make sure that the panic message contains the specified string. This is useful to help make `#[should_fail]` tests a bit less brittle by decreasing the chance that the test isn't "accidentally" passing due to a panic occurring earlier than expected. The behavior is in some ways similar to JUnit's `expected` feature: `@Test(expected=NullPointerException.class)`. Without the message assertion, this test would pass even though it's not actually reaching the intended part of the code: ```rust #[test] #[should_fail(message = "out of bounds")] fn test_oob_array_access() { let idx: uint = from_str("13o").unwrap(); // oops, this will panic [1i32, 2, 3][idx]; } ``` --- src/libtest/lib.rs | 93 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 23 deletions(-) (limited to 'src/libtest') diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index c943d8706e5..5cdd346f981 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -47,6 +47,7 @@ use self::TestEvent::*; use self::NamePadding::*; use self::OutputLocation::*; +use std::any::{Any, AnyRefExt}; use std::collections::TreeMap; use stats::Stats; use getopts::{OptGroup, optflag, optopt}; @@ -78,7 +79,7 @@ pub mod test { MetricChange, Improvement, Regression, LikelyNoise, StaticTestFn, StaticTestName, DynTestName, DynTestFn, run_test, test_main, test_main_static, filter_tests, - parse_opts, StaticBenchFn}; + parse_opts, StaticBenchFn, ShouldFail}; } pub mod stats; @@ -184,13 +185,19 @@ pub struct Bencher { pub bytes: u64, } +#[deriving(Clone, Show, PartialEq, Eq, Hash)] +pub enum ShouldFail { + No, + Yes(Option<&'static str>) +} + // The definition of a single test. A test runner will run a list of // these. #[deriving(Clone, Show, PartialEq, Eq, Hash)] pub struct TestDesc { pub name: TestName, pub ignore: bool, - pub should_fail: bool, + pub should_fail: ShouldFail, } #[deriving(Show)] @@ -346,7 +353,7 @@ fn optgroups() -> Vec { fn usage(binary: &str) { let message = format!("Usage: {} [OPTIONS] [FILTER]", binary); - println!(r"{usage} + println!(r#"{usage} The FILTER regex is tested against the name of all tests to run, and only those tests that match are run. @@ -366,10 +373,12 @@ Test Attributes: function takes one argument (test::Bencher). #[should_fail] - This function (also labeled with #[test]) will only pass if the code causes a failure (an assertion failure or panic!) + A message may be provided, which the failure string must + contain: #[should_fail(message = "foo")]. #[ignore] - When applied to a function which is already attributed as a test, then the test runner will ignore these tests during normal test runs. Running with --ignored will run these - tests.", + tests."#, usage = getopts::usage(message.as_slice(), optgroups().as_slice())); } @@ -902,13 +911,13 @@ fn should_sort_failures_before_printing_them() { let test_a = TestDesc { name: StaticTestName("a"), ignore: false, - should_fail: false + should_fail: ShouldFail::No }; let test_b = TestDesc { name: StaticTestName("b"), ignore: false, - should_fail: false + should_fail: ShouldFail::No }; let mut st = ConsoleTestState { @@ -1114,7 +1123,7 @@ pub fn run_test(opts: &TestOpts, let stdout = reader.read_to_end().unwrap().into_iter().collect(); let task_result = result_future.into_inner(); - let test_result = calc_result(&desc, task_result.is_ok()); + let test_result = calc_result(&desc, task_result); monitor_ch.send((desc.clone(), test_result, stdout)); }) } @@ -1148,13 +1157,17 @@ pub fn run_test(opts: &TestOpts, } } -fn calc_result(desc: &TestDesc, task_succeeded: bool) -> TestResult { - if task_succeeded { - if desc.should_fail { TrFailed } - else { TrOk } - } else { - if desc.should_fail { TrOk } - else { TrFailed } +fn calc_result(desc: &TestDesc, task_result: Result<(), Box>) -> TestResult { + match (&desc.should_fail, task_result) { + (&ShouldFail::No, Ok(())) | + (&ShouldFail::Yes(None), Err(_)) => TrOk, + (&ShouldFail::Yes(Some(msg)), Err(ref err)) + if err.downcast_ref::() + .map(|e| &**e) + .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) + .map(|e| e.contains(msg)) + .unwrap_or(false) => TrOk, + _ => TrFailed, } } @@ -1437,7 +1450,7 @@ mod tests { TestDesc, TestDescAndFn, TestOpts, run_test, Metric, MetricMap, MetricAdded, MetricRemoved, Improvement, Regression, LikelyNoise, - StaticTestName, DynTestName, DynTestFn}; + StaticTestName, DynTestName, DynTestFn, ShouldFail}; use std::io::TempDir; #[test] @@ -1447,7 +1460,7 @@ mod tests { desc: TestDesc { name: StaticTestName("whatever"), ignore: true, - should_fail: false + should_fail: ShouldFail::No, }, testfn: DynTestFn(proc() f()), }; @@ -1464,7 +1477,7 @@ mod tests { desc: TestDesc { name: StaticTestName("whatever"), ignore: true, - should_fail: false + should_fail: ShouldFail::No, }, testfn: DynTestFn(proc() f()), }; @@ -1481,7 +1494,24 @@ mod tests { desc: TestDesc { name: StaticTestName("whatever"), ignore: false, - should_fail: true + should_fail: ShouldFail::Yes(None) + }, + testfn: DynTestFn(proc() f()), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx); + let (_, res, _) = rx.recv(); + assert!(res == TrOk); + } + + #[test] + fn test_should_fail_good_message() { + fn f() { panic!("an error message"); } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_fail: ShouldFail::Yes(Some("error message")) }, testfn: DynTestFn(proc() f()), }; @@ -1491,6 +1521,23 @@ mod tests { assert!(res == TrOk); } + #[test] + fn test_should_fail_bad_message() { + fn f() { panic!("an error message"); } + let desc = TestDescAndFn { + desc: TestDesc { + name: StaticTestName("whatever"), + ignore: false, + should_fail: ShouldFail::Yes(Some("foobar")) + }, + testfn: DynTestFn(proc() f()), + }; + let (tx, rx) = channel(); + run_test(&TestOpts::new(), false, desc, tx); + let (_, res, _) = rx.recv(); + assert!(res == TrFailed); + } + #[test] fn test_should_fail_but_succeeds() { fn f() { } @@ -1498,7 +1545,7 @@ mod tests { desc: TestDesc { name: StaticTestName("whatever"), ignore: false, - should_fail: true + should_fail: ShouldFail::Yes(None) }, testfn: DynTestFn(proc() f()), }; @@ -1544,7 +1591,7 @@ mod tests { desc: TestDesc { name: StaticTestName("1"), ignore: true, - should_fail: false, + should_fail: ShouldFail::No, }, testfn: DynTestFn(proc() {}), }, @@ -1552,7 +1599,7 @@ mod tests { desc: TestDesc { name: StaticTestName("2"), ignore: false, - should_fail: false + should_fail: ShouldFail::No, }, testfn: DynTestFn(proc() {}), }); @@ -1588,7 +1635,7 @@ mod tests { desc: TestDesc { name: DynTestName((*name).clone()), ignore: false, - should_fail: false + should_fail: ShouldFail::No, }, testfn: DynTestFn(testfn), }; @@ -1629,7 +1676,7 @@ mod tests { desc: TestDesc { name: DynTestName(name.to_string()), ignore: false, - should_fail: false + should_fail: ShouldFail::No, }, testfn: DynTestFn(test_fn) } -- cgit 1.4.1-3-g733a5 From 3246d4f36979658c61c9e2bcfe524b1d45b3257a Mon Sep 17 00:00:00 2001 From: Steven Fackler Date: Sat, 6 Dec 2014 15:16:38 -0800 Subject: Change from message to expected --- src/libsyntax/test.rs | 2 +- src/libtest/lib.rs | 2 +- src/test/run-fail/test-should-fail-bad-message.rs | 2 +- src/test/run-pass/test-should-fail-good-message.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libtest') diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs index 3d2cb988434..ca2f190ce76 100644 --- a/src/libsyntax/test.rs +++ b/src/libsyntax/test.rs @@ -369,7 +369,7 @@ fn should_fail(i: &ast::Item) -> ShouldFail { match i.attrs.iter().find(|attr| attr.check_name("should_fail")) { Some(attr) => { let msg = attr.meta_item_list() - .and_then(|list| list.iter().find(|mi| mi.check_name("message"))) + .and_then(|list| list.iter().find(|mi| mi.check_name("expected"))) .and_then(|mi| mi.value_str()); ShouldFail::Yes(msg) } diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 5cdd346f981..0c71dcee2e5 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -374,7 +374,7 @@ Test Attributes: #[should_fail] - This function (also labeled with #[test]) will only pass if the code causes a failure (an assertion failure or panic!) A message may be provided, which the failure string must - contain: #[should_fail(message = "foo")]. + contain: #[should_fail(expected = "foo")]. #[ignore] - When applied to a function which is already attributed as a test, then the test runner will ignore these tests during normal test runs. Running with --ignored will run these diff --git a/src/test/run-fail/test-should-fail-bad-message.rs b/src/test/run-fail/test-should-fail-bad-message.rs index dc703de489c..76a5022e3be 100644 --- a/src/test/run-fail/test-should-fail-bad-message.rs +++ b/src/test/run-fail/test-should-fail-bad-message.rs @@ -14,7 +14,7 @@ // ignore-pretty: does not work well with `--test` #[test] -#[should_fail(message = "foobar")] +#[should_fail(expected = "foobar")] fn test_foo() { panic!("blah") } diff --git a/src/test/run-pass/test-should-fail-good-message.rs b/src/test/run-pass/test-should-fail-good-message.rs index 9d422700fbf..dcb2fe6dfc0 100644 --- a/src/test/run-pass/test-should-fail-good-message.rs +++ b/src/test/run-pass/test-should-fail-good-message.rs @@ -12,13 +12,13 @@ // ignore-pretty: does not work well with `--test` #[test] -#[should_fail(message = "foo")] +#[should_fail(expected = "foo")] fn test_foo() { panic!("foo bar") } #[test] -#[should_fail(message = "foo")] +#[should_fail(expected = "foo")] fn test_foo_dynamic() { panic!("{} bar", "foo") } -- cgit 1.4.1-3-g733a5