Rollup merge of #142983 - compiler-errors:taint-invalid-call-abi, r=workingjubilee

Taint body on invalid call ABI

Fixes https://github.com/rust-lang/rust/issues/142969

I'm not certain if there are any other paths that should be tainted, but they would operate similarly. Perhaps pointer coercion.

Introduces `extern "rust-invalid"` for testing purposes.

r? ```@workingjubilee``` or ```@oli-obk``` (or anyone)

13 files changed, 70 insertions, 3 deletions

diff --git a/compiler/rustc_abi/src/extern_abi.rs b/compiler/rustc_abi/src/extern_abi.rs
index 7457ae1f033..1b8c2de1588 100644
--- a/compiler/rustc_abi/src/extern_abi.rs
+++ b/compiler/rustc_abi/src/extern_abi.rs
@@ -36,6 +36,10 @@ pub enum ExternAbi {
     /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
     RustCold,
 
+    /// An always-invalid ABI that's used to test "this ABI is not supported by this platform"
+    /// in a platform-agnostic way.
+    RustInvalid,
+
     /// Unstable impl detail that directly uses Rust types to describe the ABI to LLVM.
     /// Even normally-compatible Rust types can become ABI-incompatible with this ABI!
     Unadjusted,
@@ -157,6 +161,7 @@ abi_impls! {
             RiscvInterruptS =><= "riscv-interrupt-s",
             RustCall =><= "rust-call",
             RustCold =><= "rust-cold",
+            RustInvalid =><= "rust-invalid",
             Stdcall { unwind: false } =><= "stdcall",
             Stdcall { unwind: true } =><= "stdcall-unwind",
             System { unwind: false } =><= "system",
diff --git a/compiler/rustc_ast_lowering/src/stability.rs b/compiler/rustc_ast_lowering/src/stability.rs
index b8fa2dd3dd6..9b60807e650 100644
--- a/compiler/rustc_ast_lowering/src/stability.rs
+++ b/compiler/rustc_ast_lowering/src/stability.rs
@@ -96,6 +96,9 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
         ExternAbi::RustCold => {
             Err(UnstableAbi { abi, feature: sym::rust_cold_cc, explain: GateReason::Experimental })
         }
+        ExternAbi::RustInvalid => {
+            Err(UnstableAbi { abi, feature: sym::rustc_attrs, explain: GateReason::ImplDetail })
+        }
         ExternAbi::GpuKernel => Err(UnstableAbi {
             abi,
             feature: sym::abi_gpu_kernel,
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index 7a3647df0c4..f790c51f8f1 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -156,7 +156,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub(crate) fn check_call_abi(&self, abi: ExternAbi, span: Span) {
         let canon_abi = match AbiMap::from_target(&self.sess().target).canonize_abi(abi, false) {
             AbiMapping::Direct(canon_abi) | AbiMapping::Deprecated(canon_abi) => canon_abi,
-            AbiMapping::Invalid => return,
+            AbiMapping::Invalid => {
+                // This should be reported elsewhere, but we want to taint this body
+                // so that we don't try to evaluate calls to ABIs that are invalid.
+                let guar = self.dcx().span_delayed_bug(
+                    span,
+                    format!("invalid abi for platform should have reported an error: {abi}"),
+                );
+                self.set_tainted_by_errors(guar);
+                return;
+            }
         };
 
         let valid = match canon_abi {
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 13c281a6182..5cb943b0d8c 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1253,7 +1253,8 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)
         | CCmseNonSecureCall
         | CCmseNonSecureEntry
         | Custom
-        | Unadjusted => false,
+        | Unadjusted
+        | RustInvalid => false,
         Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,
     }
 }
diff --git a/compiler/rustc_smir/src/rustc_internal/internal.rs b/compiler/rustc_smir/src/rustc_internal/internal.rs
index a4c6f186222..c0d9937e34d 100644
--- a/compiler/rustc_smir/src/rustc_internal/internal.rs
+++ b/compiler/rustc_smir/src/rustc_internal/internal.rs
@@ -494,6 +494,7 @@ impl RustcInternal for Abi {
             Abi::RustCall => rustc_abi::ExternAbi::RustCall,
             Abi::Unadjusted => rustc_abi::ExternAbi::Unadjusted,
             Abi::RustCold => rustc_abi::ExternAbi::RustCold,
+            Abi::RustInvalid => rustc_abi::ExternAbi::RustInvalid,
             Abi::RiscvInterruptM => rustc_abi::ExternAbi::RiscvInterruptM,
             Abi::RiscvInterruptS => rustc_abi::ExternAbi::RiscvInterruptS,
             Abi::Custom => rustc_abi::ExternAbi::Custom,
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
index b4239ddd896..2c652c7546e 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -877,6 +877,7 @@ impl<'tcx> Stable<'tcx> for rustc_abi::ExternAbi {
             ExternAbi::RustCall => Abi::RustCall,
             ExternAbi::Unadjusted => Abi::Unadjusted,
             ExternAbi::RustCold => Abi::RustCold,
+            ExternAbi::RustInvalid => Abi::RustInvalid,
             ExternAbi::RiscvInterruptM => Abi::RiscvInterruptM,
             ExternAbi::RiscvInterruptS => Abi::RiscvInterruptS,
             ExternAbi::Custom => Abi::Custom,
diff --git a/compiler/rustc_smir/src/stable_mir/ty.rs b/compiler/rustc_smir/src/stable_mir/ty.rs
index 4415cd6e2e3..1ae79491642 100644
--- a/compiler/rustc_smir/src/stable_mir/ty.rs
+++ b/compiler/rustc_smir/src/stable_mir/ty.rs
@@ -1126,6 +1126,7 @@ pub enum Abi {
     RustCold,
     RiscvInterruptM,
     RiscvInterruptS,
+    RustInvalid,
     Custom,
 }
 
diff --git a/compiler/rustc_target/src/spec/abi_map.rs b/compiler/rustc_target/src/spec/abi_map.rs
index 42ec10a8e15..ce1bdcbb8ac 100644
--- a/compiler/rustc_target/src/spec/abi_map.rs
+++ b/compiler/rustc_target/src/spec/abi_map.rs
@@ -156,7 +156,8 @@ impl AbiMap {
                 | ExternAbi::Msp430Interrupt
                 | ExternAbi::RiscvInterruptM
                 | ExternAbi::RiscvInterruptS
-                | ExternAbi::X86Interrupt,
+                | ExternAbi::X86Interrupt
+                | ExternAbi::RustInvalid,
                 _,
             ) => return AbiMapping::Invalid,
         };
diff --git a/tests/ui/abi/invalid-call-abi-ctfe.rs b/tests/ui/abi/invalid-call-abi-ctfe.rs
new file mode 100644
index 00000000000..343cc728fe3
--- /dev/null
+++ b/tests/ui/abi/invalid-call-abi-ctfe.rs
@@ -0,0 +1,14 @@
+// Fix for #142969 where an invalid ABI in a signature still had its call ABI computed
+// because CTFE tried to evaluate it, despite previous errors during AST-to-HIR lowering.
+
+#![feature(rustc_attrs)]
+
+const extern "rust-invalid" fn foo() {
+    //~^ ERROR "rust-invalid" is not a supported ABI for the current target
+    panic!()
+}
+
+const _: () = foo();
+
+
+fn main() {}
diff --git a/tests/ui/abi/invalid-call-abi-ctfe.stderr b/tests/ui/abi/invalid-call-abi-ctfe.stderr
new file mode 100644
index 00000000000..402de4b69b9
--- /dev/null
+++ b/tests/ui/abi/invalid-call-abi-ctfe.stderr
@@ -0,0 +1,9 @@
+error[E0570]: "rust-invalid" is not a supported ABI for the current target
+  --> $DIR/invalid-call-abi-ctfe.rs:6:14
+   |
+LL | const extern "rust-invalid" fn foo() {
+   |              ^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0570`.
diff --git a/tests/ui/abi/invalid-call-abi.rs b/tests/ui/abi/invalid-call-abi.rs
new file mode 100644
index 00000000000..076ddd91ab0
--- /dev/null
+++ b/tests/ui/abi/invalid-call-abi.rs
@@ -0,0 +1,12 @@
+// Tests the `"rustc-invalid"` ABI, which is never canonizable.
+
+#![feature(rustc_attrs)]
+
+const extern "rust-invalid" fn foo() {
+    //~^ ERROR "rust-invalid" is not a supported ABI for the current target
+    panic!()
+}
+
+fn main() {
+    foo();
+}
diff --git a/tests/ui/abi/invalid-call-abi.stderr b/tests/ui/abi/invalid-call-abi.stderr
new file mode 100644
index 00000000000..c4a90158dcf
--- /dev/null
+++ b/tests/ui/abi/invalid-call-abi.stderr
@@ -0,0 +1,9 @@
+error[E0570]: "rust-invalid" is not a supported ABI for the current target
+  --> $DIR/invalid-call-abi.rs:5:14
+   |
+LL | const extern "rust-invalid" fn foo() {
+   |              ^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0570`.
diff --git a/tests/ui/print-calling-conventions.stdout b/tests/ui/print-calling-conventions.stdout
index 7b5ae495660..4df6bd27f45 100644
--- a/tests/ui/print-calling-conventions.stdout
+++ b/tests/ui/print-calling-conventions.stdout
@@ -20,6 +20,7 @@ riscv-interrupt-m
 riscv-interrupt-s
 rust-call
 rust-cold
+rust-invalid
 stdcall
 stdcall-unwind
 system