add `allow_fail` test attribute

This change allows the user to add an `#[allow_fail]` attribute to
tests that will cause the test to compile & run, but if the test fails
it will not cause the entire test run to fail. The test output will
show the failure, but in yellow instead of red, and also indicate that
it was an allowed failure.

7 files changed, 81 insertions, 25 deletions

diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs
index bea13397eca..5a56c33b806 100644
--- a/src/librustdoc/html/markdown.rs
+++ b/src/librustdoc/html/markdown.rs
@@ -769,7 +769,7 @@ pub fn old_find_testable_code(doc: &str, tests: &mut ::test::Collector, position
                                block_info.should_panic, block_info.no_run,
                                block_info.ignore, block_info.test_harness,
                                block_info.compile_fail, block_info.error_codes,
-                               line, filename);
+                               line, filename, block_info.allow_fail);
             } else {
                 tests.add_old_test(text, filename);
             }
@@ -859,7 +859,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector, position: Sp
                                block_info.should_panic, block_info.no_run,
                                block_info.ignore, block_info.test_harness,
                                block_info.compile_fail, block_info.error_codes,
-                               line, filename);
+                               line, filename, block_info.allow_fail);
                 prev_offset = offset;
             }
             Event::Start(Tag::Header(level)) => {
@@ -889,6 +889,7 @@ struct LangString {
     test_harness: bool,
     compile_fail: bool,
     error_codes: Vec<String>,
+    allow_fail: bool,
 }
 
 impl LangString {
@@ -902,6 +903,7 @@ impl LangString {
             test_harness: false,
             compile_fail: false,
             error_codes: Vec::new(),
+            allow_fail: false,
         }
     }
 
@@ -930,6 +932,7 @@ impl LangString {
                 }
                 "no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; }
                 "ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; }
+                "allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; }
                 "rust" => { data.rust = true; seen_rust_tags = true; }
                 "test_harness" => {
                     data.test_harness = true;
@@ -1118,7 +1121,7 @@ mod tests {
     fn test_lang_string_parse() {
         fn t(s: &str,
             should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
-            compile_fail: bool, error_codes: Vec<String>) {
+            compile_fail: bool, allow_fail: bool, error_codes: Vec<String>) {
             assert_eq!(LangString::parse(s), LangString {
                 should_panic: should_panic,
                 no_run: no_run,
@@ -1128,25 +1131,27 @@ mod tests {
                 compile_fail: compile_fail,
                 error_codes: error_codes,
                 original: s.to_owned(),
+                allow_fail: allow_fail,
             })
         }
 
         // marker                | should_panic| no_run| ignore| rust | test_harness| compile_fail
-        //                       | error_codes
-        t("",                      false,        false,  false,  true,  false, false, Vec::new());
-        t("rust",                  false,        false,  false,  true,  false, false, Vec::new());
-        t("sh",                    false,        false,  false,  false, false, false, Vec::new());
-        t("ignore",                false,        false,  true,   true,  false, false, Vec::new());
-        t("should_panic",          true,         false,  false,  true,  false, false, Vec::new());
-        t("no_run",                false,        true,   false,  true,  false, false, Vec::new());
-        t("test_harness",          false,        false,  false,  true,  true,  false, Vec::new());
-        t("compile_fail",          false,        true,   false,  true,  false, true,  Vec::new());
-        t("{.no_run .example}",    false,        true,   false,  true,  false, false, Vec::new());
-        t("{.sh .should_panic}",   true,         false,  false,  false, false, false, Vec::new());
-        t("{.example .rust}",      false,        false,  false,  true,  false, false, Vec::new());
-        t("{.test_harness .rust}", false,        false,  false,  true,  true,  false, Vec::new());
-        t("text, no_run",          false,        true,   false,  false, false, false, Vec::new());
-        t("text,no_run",           false,        true,   false,  false, false, false, Vec::new());
+        //                       | allow_fail | error_codes
+        t("",                      false,        false,  false,  true,  false, false, false, Vec::new());
+        t("rust",                  false,        false,  false,  true,  false, false, false, Vec::new());
+        t("sh",                    false,        false,  false,  false, false, false, false, Vec::new());
+        t("ignore",                false,        false,  true,   true,  false, false, false, Vec::new());
+        t("should_panic",          true,         false,  false,  true,  false, false, false, Vec::new());
+        t("no_run",                false,        true,   false,  true,  false, false, false, Vec::new());
+        t("test_harness",          false,        false,  false,  true,  true,  false, false, Vec::new());
+        t("compile_fail",          false,        true,   false,  true,  false, true,  false, Vec::new());
+        t("allow_fail",            false,        false,  false,  true,  false, false, true, Vec::new());
+        t("{.no_run .example}",    false,        true,   false,  true,  false, false, false, Vec::new());
+        t("{.sh .should_panic}",   true,         false,  false,  false, false, false, false, Vec::new());
+        t("{.example .rust}",      false,        false,  false,  true,  false, false, false, Vec::new());
+        t("{.test_harness .rust}", false,        false,  false,  true,  true,  false, false, Vec::new());
+        t("text, no_run",          false,        true,   false,  false, false, false, false, Vec::new());
+        t("text,no_run",           false,        true,   false,  false, false, false, false, Vec::new());
     }
 
     #[test]
diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs
index cfe2fad0fa4..4766778eed1 100644
--- a/src/librustdoc/test.rs
+++ b/src/librustdoc/test.rs
@@ -467,7 +467,7 @@ impl Collector {
     pub fn add_test(&mut self, test: String,
                     should_panic: bool, no_run: bool, should_ignore: bool,
                     as_test_harness: bool, compile_fail: bool, error_codes: Vec<String>,
-                    line: usize, filename: String) {
+                    line: usize, filename: String, allow_fail: bool) {
         let name = self.generate_name(line, &filename);
         // to be removed when hoedown is removed
         if self.render_type == RenderType::Pulldown {
@@ -499,6 +499,7 @@ impl Collector {
                 ignore: should_ignore,
                 // compiler failures are test failures
                 should_panic: testing::ShouldPanic::No,
+                allow_fail: allow_fail,
             },
             testfn: testing::DynTestFn(box move |()| {
                 let panic = io::set_panic(None);
diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs
index d7d3a70f3c7..07db5b83331 100644
--- a/src/libsyntax/feature_gate.rs
+++ b/src/libsyntax/feature_gate.rs
@@ -534,6 +534,7 @@ pub const BUILTIN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeG
     ("derive", Normal, Ungated),
     ("should_panic", Normal, Ungated),
     ("ignore", Normal, Ungated),
+    ("allow_fail", Normal, Ungated),
     ("no_implicit_prelude", Normal, Ungated),
     ("reexport_test_harness_main", Normal, Ungated),
     ("link_args", Normal, Ungated),
diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs
index a0d1785c6ff..86f5f42eac7 100644
--- a/src/libsyntax/test.rs
+++ b/src/libsyntax/test.rs
@@ -52,7 +52,8 @@ struct Test {
     path: Vec<Ident> ,
     bench: bool,
     ignore: bool,
-    should_panic: ShouldPanic
+    should_panic: ShouldPanic,
+    allow_fail: bool,
 }
 
 struct TestCtxt<'a> {
@@ -133,7 +134,8 @@ impl<'a> fold::Folder for TestHarnessGenerator<'a> {
                         path: self.cx.path.clone(),
                         bench: is_bench_fn(&self.cx, &i),
                         ignore: is_ignored(&i),
-                        should_panic: should_panic(&i, &self.cx)
+                        should_panic: should_panic(&i, &self.cx),
+                        allow_fail: is_allowed_fail(&i),
                     };
                     self.cx.testfns.push(test);
                     self.tests.push(i.ident);
@@ -383,6 +385,10 @@ fn is_ignored(i: &ast::Item) -> bool {
     i.attrs.iter().any(|attr| attr.check_name("ignore"))
 }
 
+fn is_allowed_fail(i: &ast::Item) -> bool {
+    i.attrs.iter().any(|attr| attr.check_name("allow_fail"))
+}
+
 fn should_panic(i: &ast::Item, cx: &TestCtxt) -> ShouldPanic {
     match i.attrs.iter().find(|attr| attr.check_name("should_panic")) {
         Some(attr) => {
@@ -668,6 +674,7 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
             }
         }
     };
+    let allow_fail_expr = ecx.expr_bool(span, test.allow_fail);
 
     // self::test::TestDesc { ... }
     let desc_expr = ecx.expr_struct(
@@ -675,7 +682,8 @@ fn mk_test_desc_and_fn_rec(cx: &TestCtxt, test: &Test) -> P<ast::Expr> {
         test_path("TestDesc"),
         vec![field("name", name_expr),
              field("ignore", ignore_expr),
-             field("should_panic", fail_expr)]);
+             field("should_panic", fail_expr),
+             field("allow_fail", allow_fail_expr)]);
 
 
     let mut visible_path = match cx.toplevel_reexport {
diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs
index 2094fd8898d..c68039f21ec 100644
--- a/src/libtest/lib.rs
+++ b/src/libtest/lib.rs
@@ -212,6 +212,7 @@ pub struct TestDesc {
     pub name: TestName,
     pub ignore: bool,
     pub should_panic: ShouldPanic,
+    pub allow_fail: bool,
 }
 
 #[derive(Clone)]
@@ -523,6 +524,7 @@ pub enum TestResult {
     TrFailed,
     TrFailedMsg(String),
     TrIgnored,
+    TrAllowedFail,
     TrMetrics(MetricMap),
     TrBench(BenchSamples),
 }
@@ -543,6 +545,7 @@ struct ConsoleTestState<T> {
     passed: usize,
     failed: usize,
     ignored: usize,
+    allowed_fail: usize,
     filtered_out: usize,
     measured: usize,
     metrics: MetricMap,
@@ -572,6 +575,7 @@ impl<T: Write> ConsoleTestState<T> {
             passed: 0,
             failed: 0,
             ignored: 0,
+            allowed_fail: 0,
             filtered_out: 0,
             measured: 0,
             metrics: MetricMap::new(),
@@ -594,6 +598,10 @@ impl<T: Write> ConsoleTestState<T> {
         self.write_short_result("ignored", "i", term::color::YELLOW)
     }
 
+    pub fn write_allowed_fail(&mut self) -> io::Result<()> {
+        self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW)
+    }
+
     pub fn write_metric(&mut self) -> io::Result<()> {
         self.write_pretty("metric", term::color::CYAN)
     }
@@ -669,6 +677,7 @@ impl<T: Write> ConsoleTestState<T> {
             TrOk => self.write_ok(),
             TrFailed | TrFailedMsg(_) => self.write_failed(),
             TrIgnored => self.write_ignored(),
+            TrAllowedFail => self.write_allowed_fail(),
             TrMetrics(ref mm) => {
                 self.write_metric()?;
                 self.write_plain(&format!(": {}\n", mm.fmt_metrics()))
@@ -702,6 +711,7 @@ impl<T: Write> ConsoleTestState<T> {
                         TrFailed => "failed".to_owned(),
                         TrFailedMsg(ref msg) => format!("failed: {}", msg),
                         TrIgnored => "ignored".to_owned(),
+                        TrAllowedFail => "failed (allowed)".to_owned(),
                         TrMetrics(ref mm) => mm.fmt_metrics(),
                         TrBench(ref bs) => fmt_bench_samples(bs),
                     },
@@ -761,7 +771,7 @@ impl<T: Write> ConsoleTestState<T> {
     }
 
     pub fn write_run_finish(&mut self) -> io::Result<bool> {
-        assert!(self.passed + self.failed + self.ignored + self.measured == self.total);
+        assert!(self.passed + self.failed + self.ignored + self.measured + self.allowed_fail == self.total);
 
         if self.options.display_output {
             self.write_outputs()?;
@@ -778,9 +788,10 @@ impl<T: Write> ConsoleTestState<T> {
         } else {
             self.write_pretty("FAILED", term::color::RED)?;
         }
-        let s = format!(". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n",
+        let s = format!(". {} passed; {} failed; {} allowed to fail; {} ignored; {} measured; {} filtered out\n\n",
                         self.passed,
                         self.failed,
+                        self.allowed_fail,
                         self.ignored,
                         self.measured,
                         self.filtered_out);
@@ -891,6 +902,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Resu
                         st.not_failures.push((test, stdout));
                     }
                     TrIgnored => st.ignored += 1,
+                    TrAllowedFail => st.allowed_fail += 1,
                     TrMetrics(mm) => {
                         let tname = test.name;
                         let MetricMap(mm) = mm;
@@ -1471,8 +1483,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box<Any + Send>>) -> Tes
                   .unwrap_or(false) {
                 TrOk
             } else {
-                TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
+                if desc.allow_fail {
+                    TrAllowedFail
+                } else {
+                    TrFailedMsg(format!("Panic did not include expected string '{}'", msg))
+                }
             },
+        _ if desc.allow_fail => TrAllowedFail,
         _ => TrFailed,
     }
 }
diff --git a/src/test/run-pass/test-allow-fail-attr.rs b/src/test/run-pass/test-allow-fail-attr.rs
new file mode 100644
index 00000000000..7d750d51dcd
--- /dev/null
+++ b/src/test/run-pass/test-allow-fail-attr.rs
@@ -0,0 +1,23 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: --test
+
+#[test]
+#[allow_fail]
+fn test1() {
+    panic!();
+}
+
+#[test]
+#[allow_fail]
+fn test2() {
+    assert!(true);
+}
diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs
index c88ffba357a..b4663b0ee6c 100644
--- a/src/tools/compiletest/src/main.rs
+++ b/src/tools/compiletest/src/main.rs
@@ -476,6 +476,7 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn
             name: make_test_name(config, testpaths),
             ignore: ignore,
             should_panic: should_panic,
+            allow_fail: false,
         },
         testfn: make_test_closure(config, testpaths),
     }