Add the directive `compare-output-by-lines`

14 files changed, 53 insertions, 19 deletions

diff --git a/src/doc/rustc-dev-guide/src/tests/directives.md b/src/doc/rustc-dev-guide/src/tests/directives.md
index f4ba9a044e6..fbbeb7e97d3 100644
--- a/src/doc/rustc-dev-guide/src/tests/directives.md
+++ b/src/doc/rustc-dev-guide/src/tests/directives.md
@@ -111,6 +111,7 @@ for more details.
 | `forbid-output`                   | A pattern which must not appear in stderr/`cfail` output                                                                 | `ui`, `incremental`                          | Regex pattern                                                                           |
 | `run-flags`                       | Flags passed to the test executable                                                                                      | `ui`                                         | Arbitrary flags                                                                         |
 | `known-bug`                       | No error annotation needed due to known bug                                                                              | `ui`, `crashes`, `incremental`               | Issue number `#123456`                                                                  |
+| `compare-output-by-lines`         | Compare the output by lines, rather than as a single string                                                              | All                                          | N/A                                                                                     |
 
 [^check_stdout]: presently <!-- date-check: Oct 2024 --> this has a weird quirk
     where the test binary's stdout and stderr gets concatenated and then
diff --git a/src/doc/rustc-dev-guide/src/tests/ui.md b/src/doc/rustc-dev-guide/src/tests/ui.md
index 25dd5814cf6..d3a2c406402 100644
--- a/src/doc/rustc-dev-guide/src/tests/ui.md
+++ b/src/doc/rustc-dev-guide/src/tests/ui.md
@@ -95,6 +95,7 @@ will check for output files:
   [Normalization](#normalization)).
 - `dont-check-compiler-stderr` — Ignores stderr from the compiler.
 - `dont-check-compiler-stdout` — Ignores stdout from the compiler.
+- `compare-output-by-lines` — Some tests have non-deterministic orders of output, so we need to compare by lines.
 
 UI tests run with `-Zdeduplicate-diagnostics=no` flag which disables rustc's
 built-in diagnostic deduplication mechanism. This means you may see some
diff --git a/src/tools/compiletest/src/directives.rs b/src/tools/compiletest/src/directives.rs
index 13e694b7f03..027ee063a49 100644
--- a/src/tools/compiletest/src/directives.rs
+++ b/src/tools/compiletest/src/directives.rs
@@ -205,6 +205,8 @@ pub struct TestProps {
     pub dont_require_annotations: HashSet<ErrorKind>,
     /// Whether pretty printers should be disabled in gdb.
     pub disable_gdb_pretty_printers: bool,
+    /// Compare the output by lines, rather than as a single string.
+    pub compare_output_by_lines: bool,
 }
 
 mod directives {
@@ -254,6 +256,7 @@ mod directives {
     // This isn't a real directive, just one that is probably mistyped often
     pub const INCORRECT_COMPILER_FLAGS: &'static str = "compiler-flags";
     pub const DISABLE_GDB_PRETTY_PRINTERS: &'static str = "disable-gdb-pretty-printers";
+    pub const COMPARE_OUTPUT_BY_LINES: &'static str = "compare-output-by-lines";
 }
 
 impl TestProps {
@@ -310,6 +313,7 @@ impl TestProps {
             add_core_stubs: false,
             dont_require_annotations: Default::default(),
             disable_gdb_pretty_printers: false,
+            compare_output_by_lines: false,
         }
     }
 
@@ -664,6 +668,11 @@ impl TestProps {
                         DISABLE_GDB_PRETTY_PRINTERS,
                         &mut self.disable_gdb_pretty_printers,
                     );
+                    config.set_name_directive(
+                        ln,
+                        COMPARE_OUTPUT_BY_LINES,
+                        &mut self.compare_output_by_lines,
+                    );
                 },
             );
 
diff --git a/src/tools/compiletest/src/directives/directive_names.rs b/src/tools/compiletest/src/directives/directive_names.rs
index f7955429d83..9bdd8e9fa11 100644
--- a/src/tools/compiletest/src/directives/directive_names.rs
+++ b/src/tools/compiletest/src/directives/directive_names.rs
@@ -17,6 +17,7 @@ pub(crate) const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "check-run-results",
     "check-stdout",
     "check-test-line-numbers-match",
+    "compare-output-by-lines",
     "compile-flags",
     "disable-gdb-pretty-printers",
     "doc-flags",
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index be4663fffbe..af35302b55b 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -2749,7 +2749,11 @@ impl<'test> TestCx<'test> {
         // Wrapper tools set by `runner` might provide extra output on failure,
         // for example a WebAssembly runtime might print the stack trace of an
         // `unreachable` instruction by default.
-        let compare_output_by_lines = self.config.runner.is_some();
+        //
+        // Also, some tests like `ui/parallel-rustc` have non-deterministic
+        // orders of output, so we need to compare by lines.
+        let compare_output_by_lines =
+            self.props.compare_output_by_lines || self.config.runner.is_some();
 
         let tmp;
         let (expected, actual): (&str, &str) = if compare_output_by_lines {
diff --git a/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.rs b/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.rs
index 73d173022f6..a357040a4e4 100644
--- a/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.rs
+++ b/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.rs
@@ -1,16 +1,18 @@
+// Test for #111528, the ice issue cause waiting on a query that panicked
+//
 //@ compile-flags: -Z threads=16
 //@ build-fail
+//@ compare-output-by-lines
 
-#![crate_type="rlib"]
+#![crate_type = "rlib"]
 #![allow(warnings)]
 
-#[export_name="fail"]
-pub fn a() {
-}
+#[export_name = "fail"]
+pub fn a() {}
 
-#[export_name="fail"]
+#[export_name = "fail"]
 pub fn b() {
-//~^ ERROR symbol `fail` is already defined
+    //~^ ERROR symbol `fail` is already defined
 }
 
 fn main() {}
diff --git a/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.stderr b/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.stderr
index 7963165e31b..80f63733fb3 100644
--- a/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.stderr
+++ b/tests/ui/parallel-rustc/cache-after-waiting-issue-111528.stderr
@@ -1,5 +1,5 @@
 error: symbol `fail` is already defined
-  --> $DIR/cache-after-waiting-issue-111528.rs:12:1
+  --> $DIR/cache-after-waiting-issue-111528.rs:14:1
    |
 LL | pub fn b() {
    | ^^^^^^^^^^
diff --git a/tests/ui/parallel-rustc/cycle_crash-issue-135870.rs b/tests/ui/parallel-rustc/cycle_crash-issue-135870.rs
new file mode 100644
index 00000000000..4407e3aca80
--- /dev/null
+++ b/tests/ui/parallel-rustc/cycle_crash-issue-135870.rs
@@ -0,0 +1,8 @@
+// Test for #135870, which causes a deadlock bug
+//
+//@ compile-flags: -Z threads=2
+//@ compare-output-by-lines
+
+const FOO: usize = FOO; //~ ERROR cycle detected when simplifying constant for the type system `FOO`
+
+fn main() {}
diff --git a/tests/ui/parallel-rustc/cycle_crash.stderr b/tests/ui/parallel-rustc/cycle_crash-issue-135870.stderr
index 7af3b8ee532..6e588d1f894 100644
--- a/tests/ui/parallel-rustc/cycle_crash.stderr
+++ b/tests/ui/parallel-rustc/cycle_crash-issue-135870.stderr
@@ -1,11 +1,11 @@
 error[E0391]: cycle detected when simplifying constant for the type system `FOO`
-  --> $DIR/cycle_crash.rs:3:1
+  --> $DIR/cycle_crash-issue-135870.rs:6:1
    |
 LL | const FOO: usize = FOO;
    | ^^^^^^^^^^^^^^^^
    |
 note: ...which requires const-evaluating + checking `FOO`...
-  --> $DIR/cycle_crash.rs:3:20
+  --> $DIR/cycle_crash-issue-135870.rs:6:20
    |
 LL | const FOO: usize = FOO;
    |                    ^^^
diff --git a/tests/ui/parallel-rustc/cycle_crash.rs b/tests/ui/parallel-rustc/cycle_crash.rs
deleted file mode 100644
index 94ae11aef39..00000000000
--- a/tests/ui/parallel-rustc/cycle_crash.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-//@ compile-flags: -Z threads=2
-
-const FOO: usize = FOO; //~ERROR cycle detected when simplifying constant for the type system `FOO`
-
-fn main() {}
diff --git a/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205-2.rs b/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205-2.rs
index 024df728736..523b1b2d1f3 100644
--- a/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205-2.rs
+++ b/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205-2.rs
@@ -1,6 +1,10 @@
+// Test for #118205, which causes a deadlock bug
+//
 //@ compile-flags:-C extra-filename=-1 -Z threads=16
 //@ no-prefer-dynamic
 //@ build-pass
+//@ compare-output-by-lines
+
 #![crate_name = "crateresolve1"]
 #![crate_type = "lib"]
 
diff --git a/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205.rs b/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205.rs
index 3ccc1ea5f10..65f99c30643 100644
--- a/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205.rs
+++ b/tests/ui/parallel-rustc/export-symbols-deadlock-issue-118205.rs
@@ -1,5 +1,8 @@
+// Test for #118205, which causes a deadlock bug
+//
 //@ compile-flags: -Z threads=16
 //@ build-pass
+//@ compare-output-by-lines
 
 pub static GLOBAL: isize = 3;
 
diff --git a/tests/ui/parallel-rustc/hello_world.rs b/tests/ui/parallel-rustc/hello_world.rs
index 56698fe2489..57891b92da0 100644
--- a/tests/ui/parallel-rustc/hello_world.rs
+++ b/tests/ui/parallel-rustc/hello_world.rs
@@ -1,5 +1,8 @@
+// Test for the basic function of parallel front end
+//
 //@ compile-flags: -Z threads=8
 //@ run-pass
+//@ compare-output-by-lines
 
 fn main() {
     println!("Hello world!");
diff --git a/tests/ui/parallel-rustc/read-stolen-value-issue-111520.rs b/tests/ui/parallel-rustc/read-stolen-value-issue-111520.rs
index ea8ecb67859..a6b37e62913 100644
--- a/tests/ui/parallel-rustc/read-stolen-value-issue-111520.rs
+++ b/tests/ui/parallel-rustc/read-stolen-value-issue-111520.rs
@@ -1,18 +1,21 @@
+// Test for #111520, which causes an ice bug cause of reading stolen value
+//
 //@ compile-flags: -Z threads=16
 //@ run-pass
+//@ compare-output-by-lines
 
 #[repr(transparent)]
 struct Sched {
     i: i32,
 }
 impl Sched {
-    extern "C" fn get(self) -> i32 { self.i }
+    extern "C" fn get(self) -> i32 {
+        self.i
+    }
 }
 
 fn main() {
     let s = Sched { i: 4 };
-    let f = || -> i32 {
-        s.get()
-    };
+    let f = || -> i32 { s.get() };
     println!("f: {}", f());
 }