diff --git a/src/tools/compiletest/src/directives.rs b/src/tools/compiletest/src/directives.rs
index 51eac58c971e..0318ed2b3d11 100644
--- a/src/tools/compiletest/src/directives.rs
+++ b/src/tools/compiletest/src/directives.rs
@@ -18,7 +18,7 @@
 use crate::directives::needs::CachedNeedsConditions;
 use crate::edition::{Edition, parse_edition};
 use crate::errors::ErrorKind;
-use crate::executor::{CollectedTestDesc, ShouldPanic};
+use crate::executor::{CollectedTestDesc, ShouldFail};
 use crate::util::static_regex;
 use crate::{fatal, help};
 
@@ -1366,10 +1366,10 @@ macro_rules! decision {
     // The `should-fail` annotation doesn't apply to pretty tests,
     // since we run the pretty printer across all tests by default.
     // If desired, we could add a `should-fail-pretty` annotation.
-    let should_panic = match config.mode {
-        TestMode::Pretty => ShouldPanic::No,
-        _ if should_fail => ShouldPanic::Yes,
-        _ => ShouldPanic::No,
+    let should_fail = if should_fail && config.mode != TestMode::Pretty {
+        ShouldFail::Yes
+    } else {
+        ShouldFail::No
     };
 
     CollectedTestDesc {
@@ -1377,7 +1377,7 @@ macro_rules! decision {
         filterable_path: filterable_path.to_owned(),
         ignore,
         ignore_message,
-        should_panic,
+        should_fail,
     }
 }
 
diff --git a/src/tools/compiletest/src/directives/tests.rs b/src/tools/compiletest/src/directives/tests.rs
index 98249e69601b..0bf2bcd3af43 100644
--- a/src/tools/compiletest/src/directives/tests.rs
+++ b/src/tools/compiletest/src/directives/tests.rs
@@ -7,7 +7,7 @@
     extract_llvm_version, extract_version_range, iter_directives, line_directive, parse_edition,
     parse_normalize_rule,
 };
-use crate::executor::{CollectedTestDesc, ShouldPanic};
+use crate::executor::{CollectedTestDesc, ShouldFail};
 
 fn make_test_description(
     config: &Config,
@@ -247,9 +247,9 @@ fn should_fail() {
     let p = Utf8Path::new("a.rs");
 
     let d = make_test_description(&config, tn.clone(), p, p, "", None);
-    assert_eq!(d.should_panic, ShouldPanic::No);
+    assert_eq!(d.should_fail, ShouldFail::No);
     let d = make_test_description(&config, tn, p, p, "//@ should-fail", None);
-    assert_eq!(d.should_panic, ShouldPanic::Yes);
+    assert_eq!(d.should_fail, ShouldFail::Yes);
 }
 
 #[test]
diff --git a/src/tools/compiletest/src/executor.rs b/src/tools/compiletest/src/executor.rs
index d6dbdbf8639e..5a2136c55b05 100644
--- a/src/tools/compiletest/src/executor.rs
+++ b/src/tools/compiletest/src/executor.rs
@@ -112,7 +112,7 @@ fn spawn_test_thread(
         config: Arc::clone(&test.config),
         testpaths: test.testpaths.clone(),
         revision: test.revision.clone(),
-        should_panic: test.desc.should_panic,
+        should_fail: test.desc.should_fail,
         completion_sender,
     };
     let thread_builder = thread::Builder::new().name(test.desc.name.clone());
@@ -127,7 +127,7 @@ struct TestThreadArgs {
     config: Arc<Config>,
     testpaths: TestPaths,
     revision: Option<String>,
-    should_panic: ShouldPanic,
+    should_fail: ShouldFail,
 
     completion_sender: mpsc::Sender<TestCompletion>,
 }
@@ -170,11 +170,11 @@ fn test_thread_main(args: TestThreadArgs) {
     }
 
     // Interpret the presence/absence of a panic as test failure/success.
-    let outcome = match (args.should_panic, panic_payload) {
-        (ShouldPanic::No, None) | (ShouldPanic::Yes, Some(_)) => TestOutcome::Succeeded,
-        (ShouldPanic::No, Some(_)) => TestOutcome::Failed { message: None },
-        (ShouldPanic::Yes, None) => {
-            TestOutcome::Failed { message: Some("test did not panic as expected") }
+    let outcome = match (args.should_fail, panic_payload) {
+        (ShouldFail::No, None) | (ShouldFail::Yes, Some(_)) => TestOutcome::Succeeded,
+        (ShouldFail::No, Some(_)) => TestOutcome::Failed { message: None },
+        (ShouldFail::Yes, None) => {
+            TestOutcome::Failed { message: Some("`//@ should-fail` test did not fail as expected") }
         }
     };
 
@@ -338,7 +338,7 @@ pub(crate) struct CollectedTestDesc {
     pub(crate) filterable_path: Utf8PathBuf,
     pub(crate) ignore: bool,
     pub(crate) ignore_message: Option<Cow<'static, str>>,
-    pub(crate) should_panic: ShouldPanic,
+    pub(crate) should_fail: ShouldFail,
 }
 
 /// Whether console output should be colored or not.
@@ -350,9 +350,10 @@ pub enum ColorConfig {
     NeverColor,
 }
 
-/// Whether test is expected to panic or not.
+/// Tests with `//@ should-fail` are tests of compiletest itself, and should
+/// be reported as successful if and only if they would have _failed_.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-pub(crate) enum ShouldPanic {
+pub(crate) enum ShouldFail {
     No,
     Yes,
 }