From 4a5aa1a104a9a85cc765af863f64297f9e7e73f6 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Wed, 6 Mar 2024 12:43:00 -0800
Subject: [PATCH] compiletest: Automatically compare output by subset with
 runners

This commit updates compiletest to automatically compare test output
with subsets if a `--runner` argument is configured. Runners might
inject extra information on failures, for example a WebAssembly runtime
printing a wasm stack trace, which won't be in the output of a native
runtime. The output with a `--runner` argument, however, should still
have all the native output present.
---
 src/tools/compiletest/src/runtest.rs | 69 ++++++++--------------------
 1 file changed, 20 insertions(+), 49 deletions(-)

diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 3e0021bf7c65..7be0571b1111 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -493,12 +493,8 @@ fn run_coverage_map_test(&self) {
         let expected_coverage_dump = self.load_expected_output(kind);
         let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]);
 
-        let coverage_dump_errors = self.compare_output(
-            kind,
-            &actual_coverage_dump,
-            &expected_coverage_dump,
-            self.props.compare_output_lines_by_subset,
-        );
+        let coverage_dump_errors =
+            self.compare_output(kind, &actual_coverage_dump, &expected_coverage_dump);
 
         if coverage_dump_errors > 0 {
             self.fatal_proc_rec(
@@ -591,12 +587,8 @@ fn run_coverage_run_test(&self) {
                 self.fatal_proc_rec(&err, &proc_res);
             });
 
-        let coverage_errors = self.compare_output(
-            kind,
-            &normalized_actual_coverage,
-            &expected_coverage,
-            self.props.compare_output_lines_by_subset,
-        );
+        let coverage_errors =
+            self.compare_output(kind, &normalized_actual_coverage, &expected_coverage);
 
         if coverage_errors > 0 {
             self.fatal_proc_rec(
@@ -4051,35 +4043,17 @@ fn load_compare_outputs(
         match output_kind {
             TestOutput::Compile => {
                 if !self.props.dont_check_compiler_stdout {
-                    errors += self.compare_output(
-                        stdout_kind,
-                        &normalized_stdout,
-                        &expected_stdout,
-                        self.props.compare_output_lines_by_subset,
-                    );
+                    errors +=
+                        self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout);
                 }
                 if !self.props.dont_check_compiler_stderr {
-                    errors += self.compare_output(
-                        stderr_kind,
-                        &normalized_stderr,
-                        &expected_stderr,
-                        self.props.compare_output_lines_by_subset,
-                    );
+                    errors +=
+                        self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr);
                 }
             }
             TestOutput::Run => {
-                errors += self.compare_output(
-                    stdout_kind,
-                    &normalized_stdout,
-                    &expected_stdout,
-                    self.props.compare_output_lines_by_subset,
-                );
-                errors += self.compare_output(
-                    stderr_kind,
-                    &normalized_stderr,
-                    &expected_stderr,
-                    self.props.compare_output_lines_by_subset,
-                );
+                errors += self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout);
+                errors += self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr);
             }
         }
         errors
@@ -4173,12 +4147,7 @@ fn run_ui_test(&self) {
                 )
             });
 
-            errors += self.compare_output(
-                "fixed",
-                &fixed_code,
-                &expected_fixed,
-                self.props.compare_output_lines_by_subset,
-            );
+            errors += self.compare_output("fixed", &fixed_code, &expected_fixed);
         } else if !expected_fixed.is_empty() {
             panic!(
                 "the `//@ run-rustfix` directive wasn't found but a `*.fixed` \
@@ -4673,17 +4642,19 @@ fn delete_file(&self, file: &PathBuf) {
         }
     }
 
-    fn compare_output(
-        &self,
-        kind: &str,
-        actual: &str,
-        expected: &str,
-        compare_output_by_lines: bool,
-    ) -> usize {
+    fn compare_output(&self, kind: &str, actual: &str, expected: &str) -> usize {
         if actual == expected {
             return 0;
         }
 
+        // If `compare-output-lines-by-subset` is not explicitly enabled then
+        // auto-enable it when a `runner` is in use since wrapper tools might
+        // provide extra output on failure, for example a WebAssembly runtime
+        // might print the stack trace of an `unreachable` instruction by
+        // default.
+        let compare_output_by_lines =
+            self.props.compare_output_lines_by_subset || self.config.runner.is_some();
+
         let tmp;
         let (expected, actual): (&str, &str) = if compare_output_by_lines {
             let actual_lines: HashSet<_> = actual.lines().collect();